Skip to content

Commit 91731d0

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#133279 from ffromani/pod-level-resource-managers
[PodLevelResources] handle pod-level resource manager alignment
2 parents 3e3b244 + a3a767b commit 91731d0

File tree

9 files changed

+492
-32
lines changed

9 files changed

+492
-32
lines changed

‎pkg/kubelet/cm/cpumanager/cpu_manager_test.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,22 @@ func makePod(podUID, containerName, cpuRequest, cpuLimit string) *v1.Pod {
172172
return pod
173173
}
174174

175+
func makePodWithPodLevelResources(podUID, podCPURequest, podCPULimit, containerName, cpuRequest, cpuLimit string) *v1.Pod {
176+
pod := makePod(podUID, containerName, cpuRequest, cpuLimit)
177+
pod.Spec.Resources = &v1.ResourceRequirements{
178+
Requests: v1.ResourceList{
179+
v1.ResourceName(v1.ResourceCPU): resource.MustParse(podCPURequest),
180+
v1.ResourceName(v1.ResourceMemory): resource.MustParse("1G"),
181+
},
182+
Limits: v1.ResourceList{
183+
v1.ResourceName(v1.ResourceCPU): resource.MustParse(podCPULimit),
184+
v1.ResourceName(v1.ResourceMemory): resource.MustParse("1G"),
185+
},
186+
}
187+
188+
return pod
189+
}
190+
175191
func makeMultiContainerPod(initCPUs, appCPUs []struct{ request, limit string }) *v1.Pod {
176192
pod := &v1.Pod{
177193
ObjectMeta: metav1.ObjectMeta{

‎pkg/kubelet/cm/cpumanager/policy_static.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222

2323
v1 "k8s.io/api/core/v1"
2424
utilfeature "k8s.io/apiserver/pkg/util/feature"
25+
resourcehelper "k8s.io/component-helpers/resource"
2526
"k8s.io/klog/v2"
2627
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
2728
v1qos "k8s.io/kubernetes/pkg/apis/core/v1/helper/qos"
@@ -320,6 +321,11 @@ func (p *staticPolicy) Allocate(s state.State, pod *v1.Pod, container *v1.Contai
320321
return nil
321322
}
322323

324+
if utilfeature.DefaultFeatureGate.Enabled(features.PodLevelResources) && resourcehelper.IsPodLevelResourcesSet(pod) {
325+
klog.V(2).InfoS("CPU Manager allocation skipped, pod is using pod-level resources which are not supported by the static CPU manager policy", "pod", klog.KObj(pod), "podUID", pod.UID)
326+
return nil
327+
}
328+
323329
klog.InfoS("Static policy: Allocate", "pod", klog.KObj(pod), "containerName", container.Name)
324330
// container belongs in an exclusively allocated pool
325331
metrics.CPUManagerPinningRequestsTotal.Inc()
@@ -557,6 +563,11 @@ func (p *staticPolicy) GetTopologyHints(s state.State, pod *v1.Pod, container *v
557563
return nil
558564
}
559565

566+
if utilfeature.DefaultFeatureGate.Enabled(features.PodLevelResources) && resourcehelper.IsPodLevelResourcesSet(pod) {
567+
klog.V(3).InfoS("CPU Manager hint generation skipped, pod is using pod-level resources which are not supported by the static CPU manager policy", "pod", klog.KObj(pod), "podUID", pod.UID)
568+
return nil
569+
}
570+
560571
// Short circuit to regenerate the same hints if there are already
561572
// guaranteed CPUs allocated to the Container. This might happen after a
562573
// kubelet restart, for example.
@@ -604,6 +615,11 @@ func (p *staticPolicy) GetPodTopologyHints(s state.State, pod *v1.Pod) map[strin
604615
return nil
605616
}
606617

618+
if utilfeature.DefaultFeatureGate.Enabled(features.PodLevelResources) && resourcehelper.IsPodLevelResourcesSet(pod) {
619+
klog.V(3).InfoS("CPU Manager pod hint generation skipped, pod is using pod-level resources which are not supported by the static CPU manager policy", "pod", klog.KObj(pod), "podUID", pod.UID)
620+
return nil
621+
}
622+
607623
assignedCPUs := cpuset.New()
608624
for _, container := range append(pod.Spec.InitContainers, pod.Spec.Containers...) {
609625
requestedByContainer := p.guaranteedCPUs(pod, &container)

‎pkg/kubelet/cm/cpumanager/topology_hints_test.go‎

Lines changed: 25 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ import (
2626
"k8s.io/apimachinery/pkg/types"
2727
utilfeature "k8s.io/apiserver/pkg/util/feature"
2828
featuregatetesting "k8s.io/component-base/featuregate/testing"
29-
pkgfeatures "k8s.io/kubernetes/pkg/features"
29+
"k8s.io/kubernetes/pkg/features"
3030
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/state"
3131
"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
3232
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
@@ -35,12 +35,13 @@ import (
3535
)
3636

3737
type testCase struct {
38-
name string
39-
pod v1.Pod
40-
container v1.Container
41-
assignments state.ContainerCPUAssignments
42-
defaultCPUSet cpuset.CPUSet
43-
expectedHints []topologymanager.TopologyHint
38+
name string
39+
pod v1.Pod
40+
container v1.Container
41+
assignments state.ContainerCPUAssignments
42+
defaultCPUSet cpuset.CPUSet
43+
expectedHints []topologymanager.TopologyHint
44+
podLevelResourcesEnabled bool
4445
}
4546

4647
func returnMachineInfo() cadvisorapi.MachineInfo {
@@ -210,9 +211,10 @@ func TestPodGuaranteedCPUs(t *testing.T) {
210211

211212
func TestGetTopologyHints(t *testing.T) {
212213
machineInfo := returnMachineInfo()
213-
tcases := returnTestCases()
214214

215-
for _, tc := range tcases {
215+
for _, tc := range returnTestCases() {
216+
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodLevelResources, tc.podLevelResourcesEnabled)
217+
216218
topology, _ := topology.Discover(&machineInfo)
217219

218220
var activePods []*v1.Pod
@@ -261,6 +263,8 @@ func TestGetPodTopologyHints(t *testing.T) {
261263
machineInfo := returnMachineInfo()
262264

263265
for _, tc := range returnTestCases() {
266+
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodLevelResources, tc.podLevelResourcesEnabled)
267+
264268
topology, _ := topology.Discover(&machineInfo)
265269

266270
var activePods []*v1.Pod
@@ -442,7 +446,7 @@ func TestGetPodTopologyHintsWithPolicyOptions(t *testing.T) {
442446

443447
for _, testCase := range testCases {
444448
t.Run(testCase.description, func(t *testing.T) {
445-
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, pkgfeatures.CPUManagerPolicyAlphaOptions, true)
449+
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.CPUManagerPolicyAlphaOptions, true)
446450

447451
var activePods []*v1.Pod
448452
for p := range testCase.assignments {
@@ -495,6 +499,9 @@ func returnTestCases() []testCase {
495499
testPod4 := makePod("fakePod", "fakeContainer", "11", "11")
496500
testContainer4 := &testPod4.Spec.Containers[0]
497501

502+
testPod5 := makePodWithPodLevelResources("fakePod", "5", "5", "fakeContainer", "4", "4")
503+
testContainer5 := &testPod5.Spec.Containers[0]
504+
498505
firstSocketMask, _ := bitmask.NewBitMask(0)
499506
secondSocketMask, _ := bitmask.NewBitMask(1)
500507
crossSocketMask, _ := bitmask.NewBitMask(0, 1)
@@ -657,5 +664,13 @@ func returnTestCases() []testCase {
657664
defaultCPUSet: cpuset.New(),
658665
expectedHints: []topologymanager.TopologyHint{},
659666
},
667+
{
668+
name: "Pod has pod level resources, no hint generation",
669+
pod: *testPod5,
670+
container: *testContainer5,
671+
defaultCPUSet: cpuset.New(0, 1, 2, 3),
672+
expectedHints: nil,
673+
podLevelResourcesEnabled: true,
674+
},
660675
}
661676
}

‎pkg/kubelet/cm/memorymanager/memory_manager_test.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,13 @@ func getPod(podUID string, containerName string, requirements *v1.ResourceRequir
151151
}
152152
}
153153

154+
func getPodWithPodLevelResources(podUID string, podRequirements *v1.ResourceRequirements, containerName string, containerRequirements *v1.ResourceRequirements) *v1.Pod {
155+
pod := getPod(podUID, containerName, containerRequirements)
156+
pod.Spec.Resources = podRequirements
157+
158+
return pod
159+
}
160+
154161
func getPodWithInitContainers(podUID string, containers []v1.Container, initContainers []v1.Container) *v1.Pod {
155162
return &v1.Pod{
156163
ObjectMeta: metav1.ObjectMeta{

‎pkg/kubelet/cm/memorymanager/policy_static.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
v1 "k8s.io/api/core/v1"
2828
"k8s.io/apimachinery/pkg/api/resource"
2929
utilfeature "k8s.io/apiserver/pkg/util/feature"
30+
resourcehelper "k8s.io/component-helpers/resource"
3031
"k8s.io/klog/v2"
3132
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
3233
corehelper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
@@ -108,6 +109,11 @@ func (p *staticPolicy) Allocate(ctx context.Context, s state.State, pod *v1.Pod,
108109
}
109110

110111
podUID := string(pod.UID)
112+
if utilfeature.DefaultFeatureGate.Enabled(features.PodLevelResources) && resourcehelper.IsPodLevelResourcesSet(pod) {
113+
logger.V(2).Info("Allocation skipped, pod is using pod-level resources which are not supported by the static Memory manager policy", "podUID", podUID)
114+
return nil
115+
}
116+
111117
logger.Info("Allocate")
112118
// container belongs in an exclusively allocated pool
113119
metrics.MemoryManagerPinningRequestTotal.Inc()
@@ -412,6 +418,11 @@ func (p *staticPolicy) GetPodTopologyHints(ctx context.Context, s state.State, p
412418
return nil
413419
}
414420

421+
if utilfeature.DefaultFeatureGate.Enabled(features.PodLevelResources) && resourcehelper.IsPodLevelResourcesSet(pod) {
422+
logger.V(3).Info("Topology hints generation skipped, pod is using pod-level resources which are not supported by the static Memory manager policy", "podUID", pod.UID)
423+
return nil
424+
}
425+
415426
for _, ctn := range append(pod.Spec.InitContainers, pod.Spec.Containers...) {
416427
containerBlocks := s.GetMemoryBlocks(string(pod.UID), ctn.Name)
417428
// Short circuit to regenerate the same hints if there are already
@@ -442,6 +453,11 @@ func (p *staticPolicy) GetTopologyHints(ctx context.Context, s state.State, pod
442453
return nil
443454
}
444455

456+
if utilfeature.DefaultFeatureGate.Enabled(features.PodLevelResources) && resourcehelper.IsPodLevelResourcesSet(pod) {
457+
logger.V(3).Info("Topology hints generation skipped, pod is using pod-level resources which are not supported by the static Memory manager policy", "podUID", pod.UID)
458+
return nil
459+
}
460+
445461
containerBlocks := s.GetMemoryBlocks(string(pod.UID), container.Name)
446462
// Short circuit to regenerate the same hints if there are already
447463
// memory allocated for the container. This might happen after a

‎pkg/kubelet/cm/memorymanager/policy_static_test.go‎

Lines changed: 120 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ import (
2626

2727
v1 "k8s.io/api/core/v1"
2828
"k8s.io/apimachinery/pkg/api/resource"
29+
utilfeature "k8s.io/apiserver/pkg/util/feature"
30+
featuregatetesting "k8s.io/component-base/featuregate/testing"
31+
"k8s.io/kubernetes/pkg/features"
2932
"k8s.io/kubernetes/pkg/kubelet/cm/memorymanager/state"
3033
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager"
3134
"k8s.io/kubernetes/pkg/kubelet/cm/topologymanager/bitmask"
@@ -42,6 +45,17 @@ const (
4245
var (
4346
containerRestartPolicyAlways = v1.ContainerRestartPolicyAlways
4447

48+
podLevelRequirementsGuaranteed = &v1.ResourceRequirements{
49+
Limits: v1.ResourceList{
50+
v1.ResourceCPU: resource.MustParse("1000Mi"),
51+
v1.ResourceMemory: resource.MustParse("1Gi"),
52+
},
53+
Requests: v1.ResourceList{
54+
v1.ResourceCPU: resource.MustParse("1000Mi"),
55+
v1.ResourceMemory: resource.MustParse("1Gi"),
56+
},
57+
}
58+
4559
requirementsGuaranteed = &v1.ResourceRequirements{
4660
Limits: v1.ResourceList{
4761
v1.ResourceCPU: resource.MustParse("1000Mi"),
@@ -131,6 +145,7 @@ type testStaticPolicy struct {
131145
topologyHint *topologymanager.TopologyHint
132146
expectedTopologyHints map[string][]topologymanager.TopologyHint
133147
initContainersReusableMemory reusableMemory
148+
podLevelResourcesEnabled bool
134149
}
135150

136151
func initTests(t *testing.T, testCase *testStaticPolicy, hint *topologymanager.TopologyHint, initContainersReusableMemory reusableMemory) (Policy, state.State, error) {
@@ -2006,18 +2021,76 @@ func TestStaticPolicyAllocate(t *testing.T) {
20062021
topologyHint: &topologymanager.TopologyHint{NUMANodeAffinity: newNUMAAffinity(0, 1), Preferred: true},
20072022
expectedError: fmt.Errorf("[memorymanager] preferred hint violates NUMA node allocation"),
20082023
},
2024+
{
2025+
description: "should do nothing for guaranteed pod with pod level resources",
2026+
expectedAssignments: state.ContainerMemoryAssignments{},
2027+
machineState: state.NUMANodeMap{
2028+
0: &state.NUMANodeState{
2029+
MemoryMap: map[v1.ResourceName]*state.MemoryTable{
2030+
v1.ResourceMemory: {
2031+
Allocatable: 1536 * mb,
2032+
Free: 1536 * mb,
2033+
Reserved: 0,
2034+
SystemReserved: 512 * mb,
2035+
TotalMemSize: 2 * gb,
2036+
},
2037+
hugepages1Gi: {
2038+
Allocatable: gb,
2039+
Free: gb,
2040+
Reserved: 0,
2041+
SystemReserved: 0,
2042+
TotalMemSize: gb,
2043+
},
2044+
},
2045+
Cells: []int{},
2046+
},
2047+
},
2048+
expectedMachineState: state.NUMANodeMap{
2049+
0: &state.NUMANodeState{
2050+
MemoryMap: map[v1.ResourceName]*state.MemoryTable{
2051+
v1.ResourceMemory: {
2052+
Allocatable: 1536 * mb,
2053+
Free: 1536 * mb,
2054+
Reserved: 0,
2055+
SystemReserved: 512 * mb,
2056+
TotalMemSize: 2 * gb,
2057+
},
2058+
hugepages1Gi: {
2059+
Allocatable: gb,
2060+
Free: gb,
2061+
Reserved: 0,
2062+
SystemReserved: 0,
2063+
TotalMemSize: gb,
2064+
},
2065+
},
2066+
Cells: []int{},
2067+
},
2068+
},
2069+
systemReserved: systemReservedMemory{
2070+
0: map[v1.ResourceName]uint64{
2071+
v1.ResourceMemory: 512 * mb,
2072+
},
2073+
},
2074+
pod: getPodWithPodLevelResources("pod1", podLevelRequirementsGuaranteed, "container1", requirementsGuaranteed),
2075+
expectedTopologyHints: nil,
2076+
topologyHint: &topologymanager.TopologyHint{},
2077+
expectedError: nil,
2078+
podLevelResourcesEnabled: true,
2079+
},
20092080
}
20102081

20112082
for _, testCase := range testCases {
20122083
t.Run(testCase.description, func(t *testing.T) {
2084+
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodLevelResources, testCase.podLevelResourcesEnabled)
2085+
20132086
t.Logf("TestStaticPolicyAllocate %s", testCase.description)
20142087
p, s, err := initTests(t, &testCase, testCase.topologyHint, nil)
20152088
if err != nil {
20162089
t.Fatalf("Unexpected error: %v", err)
20172090
}
20182091

20192092
err = p.Allocate(tCtx, s, testCase.pod, &testCase.pod.Spec.Containers[0])
2020-
if !reflect.DeepEqual(err, testCase.expectedError) {
2093+
if (err == nil) != (testCase.expectedError == nil) || (err != nil && testCase.expectedError != nil && err.Error() != testCase.expectedError.Error()) {
20212094
t.Fatalf("The actual error %v is different from the expected one %v", err, testCase.expectedError)
20222095
}
20232096

@@ -3732,10 +3805,23 @@ func TestStaticPolicyGetTopologyHints(t *testing.T) {
37323805
},
37333806
},
37343807
},
3808+
{
3809+
description: "should not provide topology hints for guaranteed pod with pod level resources",
3810+
pod: getPodWithPodLevelResources("pod1", podLevelRequirementsGuaranteed, "container1", requirementsGuaranteed),
3811+
systemReserved: systemReservedMemory{
3812+
0: map[v1.ResourceName]uint64{
3813+
v1.ResourceMemory: 1024 * mb,
3814+
},
3815+
},
3816+
expectedTopologyHints: nil,
3817+
podLevelResourcesEnabled: true,
3818+
},
37353819
}
37363820

37373821
for _, testCase := range testCases {
37383822
t.Run(testCase.description, func(t *testing.T) {
3823+
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodLevelResources, testCase.podLevelResourcesEnabled)
3824+
37393825
p, s, err := initTests(t, &testCase, nil, nil)
37403826
if err != nil {
37413827
t.Fatalf("Unexpected error: %v", err)
@@ -3749,6 +3835,39 @@ func TestStaticPolicyGetTopologyHints(t *testing.T) {
37493835
}
37503836
}
37513837

3838+
func TestStaticPolicyGetPodTopologyHints(t *testing.T) {
3839+
_, tCtx := ktesting.NewTestContext(t)
3840+
testCases := []testStaticPolicy{
3841+
{
3842+
description: "should not provide pod topology hints for guaranteed pod with pod level resources",
3843+
pod: getPodWithPodLevelResources("pod1", podLevelRequirementsGuaranteed, "container1", requirementsGuaranteed),
3844+
systemReserved: systemReservedMemory{
3845+
0: map[v1.ResourceName]uint64{
3846+
v1.ResourceMemory: 1024 * mb,
3847+
},
3848+
},
3849+
expectedTopologyHints: nil,
3850+
podLevelResourcesEnabled: true,
3851+
},
3852+
}
3853+
3854+
for _, testCase := range testCases {
3855+
t.Run(testCase.description, func(t *testing.T) {
3856+
featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.PodLevelResources, testCase.podLevelResourcesEnabled)
3857+
3858+
p, s, err := initTests(t, &testCase, nil, nil)
3859+
if err != nil {
3860+
t.Fatalf("Unexpected error: %v", err)
3861+
}
3862+
3863+
topologyHints := p.GetPodTopologyHints(tCtx, s, testCase.pod)
3864+
if !reflect.DeepEqual(topologyHints, testCase.expectedTopologyHints) {
3865+
t.Fatalf("The actual topology hints: '%+v' are different from the expected one: '%+v'", topologyHints, testCase.expectedTopologyHints)
3866+
}
3867+
})
3868+
}
3869+
}
3870+
37523871
func Test_getPodRequestedResources(t *testing.T) {
37533872
testCases := []struct {
37543873
description string

0 commit comments

Comments
 (0)