Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add namespace label selector #1501

Draft
wants to merge 9 commits into
base: master
Choose a base branch
from
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ The Default Evictor Plugin is used by default for filtering pods before processi
| `evictSystemCriticalPods` |`bool`| `false` | [Warning: Will evict Kubernetes system pods] allows eviction of pods with any priority, including system pods like kube-dns |
| `ignorePvcPods` |`bool`| `false` | set whether PVC pods should be evicted or ignored |
| `evictFailedBarePods` |`bool`| `false` | allow eviction of pods without owner references and in failed phase |
| `namespaceLabelSelector` |`metav1.LabelSelector`|| limiting the pods which are processed by namespace (see [label filtering](#label-filtering)) |
| `labelSelector` |`metav1.LabelSelector`|| (see [label filtering](#label-filtering)) |
| `priorityThreshold` |`priorityThreshold`|| (see [priority filtering](#priority-filtering)) |
| `nodeFit` |`bool`|`false`| (see [node fit filtering](#node-fit-filtering)) |
Expand Down
57 changes: 57 additions & 0 deletions pkg/framework/plugins/defaultevictor/defaultevictor.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
utilerrors "k8s.io/apimachinery/pkg/util/errors"
"k8s.io/client-go/tools/cache"
"k8s.io/klog/v2"

nodeutil "sigs.k8s.io/descheduler/pkg/descheduler/node"
podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod"
frameworktypes "sigs.k8s.io/descheduler/pkg/framework/types"
Expand Down Expand Up @@ -228,8 +229,28 @@ func (d *DefaultEvictor) PreEvictionFilter(pod *v1.Pod) bool {
klog.InfoS("pod does not fit on any other node because of nodeSelector(s), Taint(s), or nodes marked as unschedulable", "pod", klog.KObj(pod))
return false
}
}

if d.args.NamespaceLabelSelector == nil {
return true
}

// check pod by namespace label filter
indexName := "namespaceWithLabelSelector"
indexer, err := getNamespacesListByLabelSelector(indexName, d.args.NamespaceLabelSelector, d.handle)
Copy link
Contributor

@ingvagabund ingvagabund Dec 6, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The indexer needs to be created before any of the plugins start running. Otherwise, the indexer cache might not get populated properly. In New function. In the same manner as getPodIndexerByOwnerRefs is.

if err != nil {
klog.ErrorS(err, "unable to list namespaces", "pod", klog.KObj(pod))
return false
}
objs, err := indexer.ByIndex(indexName, pod.Namespace)
if err != nil {
klog.ErrorS(err, "unable to list namespaces for namespaceLabelSelector filter in the policy parameter", "pod", klog.KObj(pod))
return false
}
if len(objs) == 0 {
klog.InfoS("pod namespace do not match the namespaceLabelSelector filter in the policy parameter", "pod", klog.KObj(pod))
return false
}
return true
}

Expand Down Expand Up @@ -292,3 +313,39 @@ func getPodIndexerByOwnerRefs(indexName string, handle frameworktypes.Handle) (c

return indexer, nil
}

func getNamespacesListByLabelSelector(indexName string, labelSelector *metav1.LabelSelector, handle frameworktypes.Handle) (cache.Indexer, error) {
nsInformer := handle.SharedInformerFactory().Core().V1().Namespaces().Informer()
indexer := nsInformer.GetIndexer()

// do not reinitialize the indexer, if it's been defined already
for name := range indexer.GetIndexers() {
if name == indexName {
return indexer, nil
}
}

if err := nsInformer.AddIndexers(cache.Indexers{
indexName: func(obj interface{}) ([]string, error) {
RomanenkoDenys marked this conversation as resolved.
Show resolved Hide resolved
ns, ok := obj.(*v1.Namespace)
if !ok {
return []string{}, errors.New("unexpected object")
}

selector, err := metav1.LabelSelectorAsSelector(labelSelector)
if err != nil {
return []string{}, errors.New("could not get selector from label selector")
}
if labelSelector != nil && !selector.Empty() {
if !selector.Matches(labels.Set(ns.Labels)) {
return []string{}, nil
}
}
return []string{ns.GetName()}, nil
},
}); err != nil {
return nil, err
}

return indexer, nil
}
87 changes: 86 additions & 1 deletion pkg/framework/plugins/defaultevictor/defaultevictor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"k8s.io/apimachinery/pkg/util/uuid"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes/fake"

"sigs.k8s.io/descheduler/pkg/api"
podutil "sigs.k8s.io/descheduler/pkg/descheduler/pod"
frameworkfake "sigs.k8s.io/descheduler/pkg/framework/fake"
Expand All @@ -40,18 +41,27 @@ type testCase struct {
description string
pods []*v1.Pod
nodes []*v1.Node
namespaces []*v1.Namespace
pdbs []*policyv1.PodDisruptionBudget
evictFailedBarePods bool
evictLocalStoragePods bool
evictSystemCriticalPods bool
priorityThreshold *int32
nodeFit bool
useNamespaceSelector bool
minReplicas uint
minPodAge *metav1.Duration
result bool
ignorePodsWithoutPDB bool
}

var namespace = "test"
var namespaceSelector = &metav1.LabelSelector{
MatchLabels: map[string]string{
"kubernetes.io/metadata.name": namespace,
},
}

func TestDefaultEvictorPreEvictionFilter(t *testing.T) {
n1 := test.BuildTestNode("node1", 1000, 2000, 13, nil)

Expand Down Expand Up @@ -308,6 +318,71 @@ func TestDefaultEvictorPreEvictionFilter(t *testing.T) {
nodeFit: false,
result: true,
},
{
description: "Pod with namespace matched namespace selector, should be evicted",
pods: []*v1.Pod{
test.BuildTestPod("p1", 400, 0, n1.Name, func(pod *v1.Pod) {
pod.ObjectMeta.Namespace = namespace
pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList()
pod.Spec.NodeSelector = map[string]string{
nodeLabelKey: nodeLabelValue,
}
}),
},
nodes: []*v1.Node{
test.BuildTestNode("node2", 1000, 2000, 13, func(node *v1.Node) {
node.ObjectMeta.Labels = map[string]string{
nodeLabelKey: nodeLabelValue,
}
}),
test.BuildTestNode("node3", 1000, 2000, 13, func(node *v1.Node) {
node.ObjectMeta.Labels = map[string]string{
nodeLabelKey: nodeLabelValue,
}
}),
},
namespaces: []*v1.Namespace{
test.BuildTestNamespace("default"),
test.BuildTestNamespace(namespace),
},
evictLocalStoragePods: false,
evictSystemCriticalPods: false,
nodeFit: true,
useNamespaceSelector: true,
result: true,
},
{
description: "Pod with namespace does not matched namespace selector, should not be evicted",
pods: []*v1.Pod{
test.BuildTestPod("p1", 400, 0, n1.Name, func(pod *v1.Pod) {
pod.ObjectMeta.OwnerReferences = test.GetNormalPodOwnerRefList()
pod.Spec.NodeSelector = map[string]string{
nodeLabelKey: "fail",
}
}),
},
nodes: []*v1.Node{
test.BuildTestNode("node2", 1000, 2000, 13, func(node *v1.Node) {
node.ObjectMeta.Labels = map[string]string{
nodeLabelKey: nodeLabelValue,
}
}),
test.BuildTestNode("node3", 1000, 2000, 13, func(node *v1.Node) {
node.ObjectMeta.Labels = map[string]string{
nodeLabelKey: nodeLabelValue,
}
}),
},
namespaces: []*v1.Namespace{
test.BuildTestNamespace("default"),
test.BuildTestNamespace(namespace),
},
evictLocalStoragePods: false,
evictSystemCriticalPods: false,
nodeFit: true,
useNamespaceSelector: true,
result: false,
},
}

for _, test := range testCases {
Expand Down Expand Up @@ -835,22 +910,28 @@ func TestReinitialization(t *testing.T) {

func initializePlugin(ctx context.Context, test testCase) (frameworktypes.Plugin, error) {
var objs []runtime.Object

for _, node := range test.nodes {
objs = append(objs, node)
}

for _, pod := range test.pods {
objs = append(objs, pod)
}
for _, pdb := range test.pdbs {
objs = append(objs, pdb)
}

for _, ns := range test.namespaces {
objs = append(objs, ns)
}

fakeClient := fake.NewSimpleClientset(objs...)

sharedInformerFactory := informers.NewSharedInformerFactory(fakeClient, 0)
podInformer := sharedInformerFactory.Core().V1().Pods().Informer()
_ = sharedInformerFactory.Policy().V1().PodDisruptionBudgets().Lister()

_ = sharedInformerFactory.Core().V1().Namespaces().Lister()
getPodsAssignedToNode, err := podutil.BuildGetPodsAssignedToNodeFunc(podInformer)
if err != nil {
return nil, fmt.Errorf("build get pods assigned to node function error: %v", err)
Expand All @@ -873,6 +954,10 @@ func initializePlugin(ctx context.Context, test testCase) (frameworktypes.Plugin
IgnorePodsWithoutPDB: test.ignorePodsWithoutPDB,
}

if test.useNamespaceSelector {
defaultEvictorArgs.NamespaceLabelSelector = namespaceSelector
}

evictorPlugin, err := New(
defaultEvictorArgs,
&frameworkfake.HandleImpl{
Expand Down
3 changes: 3 additions & 0 deletions pkg/framework/plugins/defaultevictor/defaults.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,9 @@ func SetDefaults_DefaultEvictorArgs(obj runtime.Object) {
if !args.EvictFailedBarePods {
args.EvictFailedBarePods = false
}
if args.NamespaceLabelSelector == nil {
args.NamespaceLabelSelector = nil
}
if args.LabelSelector == nil {
args.LabelSelector = nil
}
Expand Down
2 changes: 2 additions & 0 deletions pkg/framework/plugins/defaultevictor/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ package defaultevictor

import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"sigs.k8s.io/descheduler/pkg/api"
)

Expand All @@ -31,6 +32,7 @@ type DefaultEvictorArgs struct {
EvictSystemCriticalPods bool `json:"evictSystemCriticalPods,omitempty"`
IgnorePvcPods bool `json:"ignorePvcPods,omitempty"`
EvictFailedBarePods bool `json:"evictFailedBarePods,omitempty"`
NamespaceLabelSelector *metav1.LabelSelector `json:"namespaceLabelSelector,omitempty"`
LabelSelector *metav1.LabelSelector `json:"labelSelector,omitempty"`
PriorityThreshold *api.PriorityThreshold `json:"priorityThreshold,omitempty"`
NodeFit bool `json:"nodeFit,omitempty"`
Expand Down
6 changes: 6 additions & 0 deletions pkg/framework/plugins/defaultevictor/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

16 changes: 16 additions & 0 deletions test/test_utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,22 @@ import (
utilptr "k8s.io/utils/ptr"
)

// BuildTestNamespace creates a test namespace with given parameters.
func BuildTestNamespace(name string) *v1.Namespace {
namespace := &v1.Namespace{
TypeMeta: metav1.TypeMeta{
Kind: "Namespace",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: name,
UID: uuid.NewUUID(),
Labels: map[string]string{"kubernetes.io/metadata.name": name},
},
}
return namespace
}

// BuildTestPod creates a test pod with given parameters.
func BuildTestPod(name string, cpu, memory int64, nodeName string, apply func(*v1.Pod)) *v1.Pod {
pod := &v1.Pod{
Expand Down