In-Place Pod Vertical Scaling feature

api/api-rules/violation_exceptions.list

@@ -80,6 +80,7 @@ API rule violation: list_type_missing,k8s.io/api/core/v1,Container,Args
 API rule violation: list_type_missing,k8s.io/api/core/v1,Container,Command
 API rule violation: list_type_missing,k8s.io/api/core/v1,Container,Env
 API rule violation: list_type_missing,k8s.io/api/core/v1,Container,EnvFrom
+API rule violation: list_type_missing,k8s.io/api/core/v1,Container,ResizePolicy
 API rule violation: list_type_missing,k8s.io/api/core/v1,Container,VolumeDevices
 API rule violation: list_type_missing,k8s.io/api/core/v1,Container,VolumeMounts
 API rule violation: list_type_missing,k8s.io/api/core/v1,ContainerImage,Names
@@ -94,6 +95,7 @@ API rule violation: list_type_missing,k8s.io/api/core/v1,EphemeralContainerCommo
 API rule violation: list_type_missing,k8s.io/api/core/v1,EphemeralContainerCommon,Env
 API rule violation: list_type_missing,k8s.io/api/core/v1,EphemeralContainerCommon,EnvFrom
 API rule violation: list_type_missing,k8s.io/api/core/v1,EphemeralContainerCommon,Ports
+API rule violation: list_type_missing,k8s.io/api/core/v1,EphemeralContainerCommon,ResizePolicy
 API rule violation: list_type_missing,k8s.io/api/core/v1,EphemeralContainerCommon,VolumeDevices
 API rule violation: list_type_missing,k8s.io/api/core/v1,EphemeralContainerCommon,VolumeMounts
 API rule violation: list_type_missing,k8s.io/api/core/v1,EphemeralContainers,EphemeralContainers

pkg/api/pod/util.go

@@ -444,6 +444,14 @@ func dropDisabledFields(
 		podSpec.SetHostnameAsFQDN = nil
 	}
 
+	if !utilfeature.DefaultFeatureGate.Enabled(features.InPlacePodVerticalScaling) && !inPlacePodVerticalScalingInUse(oldPodSpec) {
+		// Drop ResourcesAllocated and ResizePolicy fields. Don't drop updates to Resources field because
+		// template spec Resources field is mutable for certain controllers. Let ValidatePodUpdate handle it.
+		for i := range podSpec.Containers {
+			podSpec.Containers[i].ResourcesAllocated = nil
+			podSpec.Containers[i].ResizePolicy = nil
+		}
+	}
 }
 
 // dropDisabledRunAsGroupField removes disabled fields from PodSpec related
@@ -559,6 +567,22 @@ func overheadInUse(podSpec *api.PodSpec) bool {
 	return false
 }
 
+// inPlacePodVerticalScalingInUse returns true if the pod spec is non-nil and has ResizePolicy or ResourcesAllocated set
+func inPlacePodVerticalScalingInUse(podSpec *api.PodSpec) bool {
+	if podSpec == nil {
+		return false
+	}
+	var inUse bool
+	VisitContainers(podSpec, AllContainers, func(c *api.Container, containerType ContainerType) bool {
+		if (c.ResourcesAllocated != nil && len(c.ResourcesAllocated) > 0) || len(c.ResizePolicy) > 0 {
+			inUse = true
+			return false
+		}
+		return true
+	})
+	return inUse
+}
+
 // procMountInUse returns true if the pod spec is non-nil and has a SecurityContext's ProcMount field set to a non-default value
 func procMountInUse(podSpec *api.PodSpec) bool {
 	if podSpec == nil {

pkg/api/pod/util_test.go

@@ -1740,3 +1740,115 @@ func TestDropEphemeralContainers(t *testing.T) {
 		}
 	}
 }
+
+func TestDropInPlacePodVerticalScaling(t *testing.T) {
+	podWithInPlaceVerticalScaling := func() *api.Pod {
+		return &api.Pod{
+			Spec: api.PodSpec{
+				Containers: []api.Container{
+					{
+						Name:  "c1",
+						Image: "image",
+						Resources: api.ResourceRequirements{
+							Requests: api.ResourceList{api.ResourceCPU: resource.MustParse("100m")},
+							Limits:   api.ResourceList{api.ResourceCPU: resource.MustParse("200m")},
+						},
+						ResourcesAllocated: api.ResourceList{api.ResourceCPU: resource.MustParse("100m")},
+						ResizePolicy: []api.ResizePolicy{
+							{ResourceName: api.ResourceCPU, Policy: api.NoRestart},
+							{ResourceName: api.ResourceMemory, Policy: api.RestartContainer},
+						},
+					},
+				},
+			},
+		}
+	}
+	podWithoutInPlaceVerticalScaling := func() *api.Pod {
+		return &api.Pod{
+			Spec: api.PodSpec{
+				Containers: []api.Container{
+					{
+						Name:  "c1",
+						Image: "image",
+						Resources: api.ResourceRequirements{
+							Requests: api.ResourceList{api.ResourceCPU: resource.MustParse("100m")},
+							Limits:   api.ResourceList{api.ResourceCPU: resource.MustParse("200m")},
+						},
+					},
+				},
+			},
+		}
+	}
+
+	podInfo := []struct {
+		description               string
+		hasInPlaceVerticalScaling bool
+		pod                       func() *api.Pod
+	}{
+		{
+			description:               "has in-place vertical scaling enabled with resources",
+			hasInPlaceVerticalScaling: true,
+			pod:                       podWithInPlaceVerticalScaling,
+		},
+		{
+			description:               "has in-place vertical scaling disabled",
+			hasInPlaceVerticalScaling: false,
+			pod:                       podWithoutInPlaceVerticalScaling,
+		},
+		{
+			description:               "is nil",
+			hasInPlaceVerticalScaling: false,
+			pod:                       func() *api.Pod { return nil },
+		},
+	}
+
+	for _, enabled := range []bool{true, false} {
+		for _, oldPodInfo := range podInfo {
+			for _, newPodInfo := range podInfo {
+				oldPodHasInPlaceVerticalScaling, oldPod := oldPodInfo.hasInPlaceVerticalScaling, oldPodInfo.pod()
+				newPodHasInPlaceVerticalScaling, newPod := newPodInfo.hasInPlaceVerticalScaling, newPodInfo.pod()
+				if newPod == nil {
+					continue
+				}
+
+				t.Run(fmt.Sprintf("feature enabled=%v, old pod %v, new pod %v", enabled, oldPodInfo.description, newPodInfo.description), func(t *testing.T) {
+					defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.InPlacePodVerticalScaling, enabled)()
+
+					var oldPodSpec *api.PodSpec
+					if oldPod != nil {
+						oldPodSpec = &oldPod.Spec
+					}
+					dropDisabledFields(&newPod.Spec, nil, oldPodSpec, nil)
+
+					// old pod should never be changed
+					if !reflect.DeepEqual(oldPod, oldPodInfo.pod()) {
+						t.Errorf("old pod changed: %v", diff.ObjectReflectDiff(oldPod, oldPodInfo.pod()))
+					}
+
+					switch {
+					case enabled || oldPodHasInPlaceVerticalScaling:
+						// new pod should not be changed if the feature is enabled, or if the old pod had
+						// ResizePolicy or ResourcesAllocated fields set
+						if !reflect.DeepEqual(newPod, newPodInfo.pod()) {
+							t.Errorf("new pod changed: %v", diff.ObjectReflectDiff(newPod, newPodInfo.pod()))
+						}
+					case newPodHasInPlaceVerticalScaling:
+						// new pod should be changed
+						if reflect.DeepEqual(newPod, newPodInfo.pod()) {
+							t.Errorf("new pod was not changed")
+						}
+						// new pod should not have ResourcesAllocated or ResizePolicy
+						if !reflect.DeepEqual(newPod, podWithoutInPlaceVerticalScaling()) {
+							t.Errorf("new pod had ResourcesAllocated or ResizePolicy: %v", diff.ObjectReflectDiff(newPod, podWithoutInPlaceVerticalScaling()))
+						}
+					default:
+						// new pod should not need to be changed
+						if !reflect.DeepEqual(newPod, newPodInfo.pod()) {
+							t.Errorf("new pod changed: %v", diff.ObjectReflectDiff(newPod, newPodInfo.pod()))
+						}
+					}
+				})
+			}
+		}
+	}
+}

pkg/api/v1/resource/helpers.go

@@ -86,47 +86,90 @@ func PodRequestsAndLimits(pod *v1.Pod) (reqs, limits v1.ResourceList) {
 	return
 }
 
-// GetResourceRequestQuantity finds and returns the request quantity for a specific resource.
-func GetResourceRequestQuantity(pod *v1.Pod, resourceName v1.ResourceName) resource.Quantity {
-	requestQuantity := resource.Quantity{}
+// PodResourceAllocations returns a dictionary of resources allocated to the containers of pod.
+func PodResourceAllocations(pod *v1.Pod) (allocations v1.ResourceList) {
+	allocations = v1.ResourceList{}
+	for _, container := range pod.Spec.Containers {
+		addResourceList(allocations, container.ResourcesAllocated)
+	}
+	// init containers define the minimum of any resource
+	for _, container := range pod.Spec.InitContainers {
+		maxResourceList(allocations, container.Resources.Requests)
+	}
+	if pod.Spec.Overhead != nil && utilfeature.DefaultFeatureGate.Enabled(features.PodOverhead) {
+		addResourceList(allocations, pod.Spec.Overhead)
+	}
+	return
+}
 
+func getFormattedResourceQuantity(resourceName v1.ResourceName) resource.Quantity {
+	resourceQuantity := resource.Quantity{}
 	switch resourceName {
 	case v1.ResourceCPU:
-		requestQuantity = resource.Quantity{Format: resource.DecimalSI}
+		resourceQuantity = resource.Quantity{Format: resource.DecimalSI}
 	case v1.ResourceMemory, v1.ResourceStorage, v1.ResourceEphemeralStorage:
-		requestQuantity = resource.Quantity{Format: resource.BinarySI}
+		resourceQuantity = resource.Quantity{Format: resource.BinarySI}
 	default:
-		requestQuantity = resource.Quantity{Format: resource.DecimalSI}
+		resourceQuantity = resource.Quantity{Format: resource.DecimalSI}
 	}
+	return resourceQuantity
+}
 
+func addPodOverheadQuantity(pod *v1.Pod, resourceName v1.ResourceName, resourceQuantity resource.Quantity) resource.Quantity {
+	// if PodOverhead feature is supported, add overhead for running a pod
+	// to the total requests if the resource total is non-zero
+	if pod.Spec.Overhead != nil && utilfeature.DefaultFeatureGate.Enabled(features.PodOverhead) {
+		if podOverhead, ok := pod.Spec.Overhead[resourceName]; ok && !resourceQuantity.IsZero() {
+			resourceQuantity.Add(podOverhead)
+		}
+	}
+	return resourceQuantity
+}
+
+// GetResourceRequestQuantity finds and returns the request quantity for a specific resource.
+func GetResourceRequestQuantity(pod *v1.Pod, resourceName v1.ResourceName) resource.Quantity {
+	requestQuantity := getFormattedResourceQuantity(resourceName)
 	if resourceName == v1.ResourceEphemeralStorage && !utilfeature.DefaultFeatureGate.Enabled(features.LocalStorageCapacityIsolation) {
 		// if the local storage capacity isolation feature gate is disabled, pods request 0 disk
 		return requestQuantity
 	}
-
 	for _, container := range pod.Spec.Containers {
 		if rQuantity, ok := container.Resources.Requests[resourceName]; ok {
 			requestQuantity.Add(rQuantity)
 		}
 	}
-
 	for _, container := range pod.Spec.InitContainers {
 		if rQuantity, ok := container.Resources.Requests[resourceName]; ok {
 			if requestQuantity.Cmp(rQuantity) < 0 {
 				requestQuantity = rQuantity.DeepCopy()
 			}
 		}
 	}
+	requestQuantity = addPodOverheadQuantity(pod, resourceName, requestQuantity)
+	return requestQuantity
+}
 
-	// if PodOverhead feature is supported, add overhead for running a pod
-	// to the total requests if the resource total is non-zero
-	if pod.Spec.Overhead != nil && utilfeature.DefaultFeatureGate.Enabled(features.PodOverhead) {
-		if podOverhead, ok := pod.Spec.Overhead[resourceName]; ok && !requestQuantity.IsZero() {
-			requestQuantity.Add(podOverhead)
+// GetResourceAllocationQuantity finds and returns the resourcesAllocated quantity for a specific resource.
+func GetResourceAllocationQuantity(pod *v1.Pod, resourceName v1.ResourceName) resource.Quantity {
+	allocQuantity := getFormattedResourceQuantity(resourceName)
+	if resourceName == v1.ResourceEphemeralStorage && !utilfeature.DefaultFeatureGate.Enabled(features.LocalStorageCapacityIsolation) {
+		// if the local storage capacity isolation feature gate is disabled, pods request 0 disk
+		return allocQuantity
+	}
+	for _, container := range pod.Spec.Containers {
+		if rQuantity, ok := container.ResourcesAllocated[resourceName]; ok {
+			allocQuantity.Add(rQuantity)
 		}
 	}
-
-	return requestQuantity
+	for _, container := range pod.Spec.InitContainers {
+		if rQuantity, ok := container.Resources.Requests[resourceName]; ok {
+			if allocQuantity.Cmp(rQuantity) < 0 {
+				allocQuantity = rQuantity.DeepCopy()
+			}
+		}
+	}
+	allocQuantity = addPodOverheadQuantity(pod, resourceName, allocQuantity)
+	return allocQuantity
 }
 
 // GetResourceRequest finds and returns the request value for a specific resource.
@@ -144,6 +187,18 @@ func GetResourceRequest(pod *v1.Pod, resource v1.ResourceName) int64 {
 	return requestQuantity.Value()
 }
 
+// GetResourceAllocation finds and returns resource allocation for a specific resource.
+func GetResourceAllocation(pod *v1.Pod, resource v1.ResourceName) int64 {
+	if resource == v1.ResourcePods {
+		return 1
+	}
+	allocQuantity := GetResourceAllocationQuantity(pod, resource)
+	if resource == v1.ResourceCPU {
+		return allocQuantity.MilliValue()
+	}
+	return allocQuantity.Value()
+}
+
 // ExtractResourceValueByContainerName extracts the value of a resource
 // by providing container name
 func ExtractResourceValueByContainerName(fs *v1.ResourceFieldSelector, pod *v1.Pod, containerName string) (string, error) {
@@ -192,6 +247,12 @@ func ExtractContainerResourceValue(fs *v1.ResourceFieldSelector, container *v1.C
 		return convertResourceMemoryToString(container.Resources.Requests.Memory(), divisor)
 	case "requests.ephemeral-storage":
 		return convertResourceEphemeralStorageToString(container.Resources.Requests.StorageEphemeral(), divisor)
+	case "resourcesAllocated.cpu":
+		return convertResourceCPUToString(container.ResourcesAllocated.Cpu(), divisor)
+	case "resourcesAllocated.memory":
+		return convertResourceMemoryToString(container.ResourcesAllocated.Memory(), divisor)
+	case "resourcesAllocated.ephemeral-storage":
+		return convertResourceEphemeralStorageToString(container.ResourcesAllocated.StorageEphemeral(), divisor)
 	}
 
 	return "", fmt.Errorf("unsupported container resource : %v", fs.Resource)