From 9f4313dbce1c290d5cff3cc422db0cf4fce2ec24 Mon Sep 17 00:00:00 2001 From: Jay2006sawant Date: Mon, 27 Jul 2026 09:12:00 +0530 Subject: [PATCH 1/3] fix webhook to deny privileged containers Signed-off-by: Jay2006sawant --- pkg/scheduler/webhook.go | 3 +- pkg/scheduler/webhook_test.go | 105 ++++++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 1 deletion(-) diff --git a/pkg/scheduler/webhook.go b/pkg/scheduler/webhook.go index 00b4578e7f..49d69fb9e1 100644 --- a/pkg/scheduler/webhook.go +++ b/pkg/scheduler/webhook.go @@ -19,6 +19,7 @@ package scheduler import ( "context" "encoding/json" + "fmt" "net/http" corev1 "k8s.io/api/core/v1" @@ -74,7 +75,7 @@ func (h *webhook) Handle(_ context.Context, req admission.Request) admission.Res if ctr.SecurityContext != nil { if ctr.SecurityContext.Privileged != nil && *ctr.SecurityContext.Privileged { klog.Warningf(template+" - Denying admission as container %s is privileged", pod.Namespace, pod.Name, pod.UID, c.Name) - continue + return admission.Denied(fmt.Sprintf("container %s is privileged", c.Name)) } } for _, val := range device.GetDevices() { diff --git a/pkg/scheduler/webhook_test.go b/pkg/scheduler/webhook_test.go index bb5e9f94df..70c84c5169 100644 --- a/pkg/scheduler/webhook_test.go +++ b/pkg/scheduler/webhook_test.go @@ -642,3 +642,108 @@ func TestSchedulerNameEmptyNoOverwrite(t *testing.T) { t.Fatalf("Expected schedulerName patch to %q, got patches: %+v", config.SchedulerName, resp.Patches) } } + +func TestPrivilegedContainerDenied(t *testing.T) { + prevSchedulerName := config.SchedulerName + t.Cleanup(func() { config.SchedulerName = prevSchedulerName }) + + config.SchedulerName = "hami-scheduler" + sConfig := &config.Config{ + NvidiaConfig: nvidia.NvidiaConfig{ + ResourceCountName: "hami.io/gpu", + ResourceMemoryName: "hami.io/gpumem", + ResourceMemoryPercentageName: "hami.io/gpumem-percentage", + ResourceCoreName: "hami.io/gpucores", + DefaultMemory: 0, + DefaultCores: 0, + DefaultGPUNum: 1, + }, + } + if err := config.InitDevicesWithConfig(sConfig); err != nil { + t.Fatalf("Failed to initialize devices with config: %v", err) + } + + privileged := true + testCases := []struct { + name string + pod *corev1.Pod + }{ + { + name: "privileged container only", + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "privileged-pod", Namespace: "default"}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "privileged", + SecurityContext: &corev1.SecurityContext{ + Privileged: &privileged, + }, + }, + }, + }, + }, + }, + { + name: "privileged sidecar with gpu workload", + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "mixed-pod", Namespace: "default"}, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "privileged-sidecar", + SecurityContext: &corev1.SecurityContext{ + Privileged: &privileged, + }, + }, + { + Name: "gpu-workload", + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "hami.io/gpu": resource.MustParse("1"), + }, + }, + }, + }, + }, + }, + }, + } + + wh, err := NewWebHook() + if err != nil { + t.Fatalf("Error creating WebHook: %v", err) + } + + scheme := runtime.NewScheme() + corev1.AddToScheme(scheme) + codec := serializer.NewCodecFactory(scheme).LegacyCodec(corev1.SchemeGroupVersion) + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + podBytes, err := runtime.Encode(codec, tc.pod) + if err != nil { + t.Fatalf("Error encoding pod: %v", err) + } + + req := admission.Request{ + AdmissionRequest: admissionv1.AdmissionRequest{ + UID: "test-uid", + Namespace: tc.pod.Namespace, + Name: tc.pod.Name, + Object: runtime.RawExtension{ + Raw: podBytes, + }, + }, + } + + resp := wh.Handle(context.Background(), req) + if resp.Allowed { + t.Fatalf("Expected denied response for privileged pod, but got allowed with %d patches", len(resp.Patches)) + } + if resp.Result == nil || resp.Result.Message == "" { + t.Fatalf("Expected denial message, got: %+v", resp.Result) + } + }) + } +} From db52c72d84244b6e77f0ed8ffba69d9e7e8e1c22 Mon Sep 17 00:00:00 2001 From: Jay2006sawant Date: Mon, 27 Jul 2026 09:12:00 +0530 Subject: [PATCH 2/3] check privileged init containers before scheduler bypass Signed-off-by: Jay2006sawant --- pkg/scheduler/webhook.go | 32 +++++++++++++++++----- pkg/scheduler/webhook_test.go | 51 ++++++++++++++++++++++++++++++++++- 2 files changed, 75 insertions(+), 8 deletions(-) diff --git a/pkg/scheduler/webhook.go b/pkg/scheduler/webhook.go index 49d69fb9e1..cd20532332 100644 --- a/pkg/scheduler/webhook.go +++ b/pkg/scheduler/webhook.go @@ -62,6 +62,10 @@ func (h *webhook) Handle(_ context.Context, req admission.Request) admission.Res klog.Warningf(template+" - Denying admission as pod has no containers", pod.Namespace, pod.Name, pod.UID) return admission.Denied("pod has no containers") } + if name, privileged := privilegedContainerName(pod); privileged { + klog.Warningf(template+" - Denying admission as container %s is privileged", pod.Namespace, pod.Name, pod.UID, name) + return admission.Denied(fmt.Sprintf("container %s is privileged", name)) + } if pod.Spec.SchedulerName != "" && (pod.Spec.SchedulerName != corev1.DefaultSchedulerName || !config.ForceOverwriteDefaultScheduler) && (len(config.SchedulerName) == 0 || pod.Spec.SchedulerName != config.SchedulerName) { @@ -70,14 +74,8 @@ func (h *webhook) Handle(_ context.Context, req admission.Request) admission.Res } klog.V(5).Infof(template, pod.Namespace, pod.Name, pod.UID) hasResource := false - for idx, ctr := range pod.Spec.Containers { + for idx := range pod.Spec.Containers { c := &pod.Spec.Containers[idx] - if ctr.SecurityContext != nil { - if ctr.SecurityContext.Privileged != nil && *ctr.SecurityContext.Privileged { - klog.Warningf(template+" - Denying admission as container %s is privileged", pod.Namespace, pod.Name, pod.UID, c.Name) - return admission.Denied(fmt.Sprintf("container %s is privileged", c.Name)) - } - } for _, val := range device.GetDevices() { found, err := val.MutateAdmission(c, pod) if err != nil { @@ -109,6 +107,26 @@ func (h *webhook) Handle(_ context.Context, req admission.Request) admission.Res return admission.PatchResponseFromRaw(req.Object.Raw, marshaledPod) } +func privilegedContainerName(pod *corev1.Pod) (string, bool) { + for _, ctr := range pod.Spec.InitContainers { + if isPrivilegedContainer(&ctr) { + return ctr.Name, true + } + } + for _, ctr := range pod.Spec.Containers { + if isPrivilegedContainer(&ctr) { + return ctr.Name, true + } + } + return "", false +} + +func isPrivilegedContainer(ctr *corev1.Container) bool { + return ctr.SecurityContext != nil && + ctr.SecurityContext.Privileged != nil && + *ctr.SecurityContext.Privileged +} + func fitResourceQuota(pod *corev1.Pod) bool { for deviceName, dev := range device.GetDevices() { // Only supports NVIDIA diff --git a/pkg/scheduler/webhook_test.go b/pkg/scheduler/webhook_test.go index 70c84c5169..8f622ed3fd 100644 --- a/pkg/scheduler/webhook_test.go +++ b/pkg/scheduler/webhook_test.go @@ -645,7 +645,13 @@ func TestSchedulerNameEmptyNoOverwrite(t *testing.T) { func TestPrivilegedContainerDenied(t *testing.T) { prevSchedulerName := config.SchedulerName - t.Cleanup(func() { config.SchedulerName = prevSchedulerName }) + prevDevicesMap := device.DevicesMap + prevDevicesToHandle := device.DevicesToHandle + t.Cleanup(func() { + config.SchedulerName = prevSchedulerName + device.DevicesMap = prevDevicesMap + device.DevicesToHandle = prevDevicesToHandle + }) config.SchedulerName = "hami-scheduler" sConfig := &config.Config{ @@ -708,6 +714,49 @@ func TestPrivilegedContainerDenied(t *testing.T) { }, }, }, + { + name: "privileged init container with gpu workload", + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "init-privileged-pod", Namespace: "default"}, + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{ + { + Name: "privileged-init", + SecurityContext: &corev1.SecurityContext{ + Privileged: &privileged, + }, + }, + }, + Containers: []corev1.Container{ + { + Name: "gpu-workload", + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + "hami.io/gpu": resource.MustParse("1"), + }, + }, + }, + }, + }, + }, + }, + { + name: "privileged pod with different scheduler", + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "other-scheduler-pod", Namespace: "default"}, + Spec: corev1.PodSpec{ + SchedulerName: "other-scheduler", + Containers: []corev1.Container{ + { + Name: "privileged", + SecurityContext: &corev1.SecurityContext{ + Privileged: &privileged, + }, + }, + }, + }, + }, + }, } wh, err := NewWebHook() From d9bb61e3b3de5f9dd203028234fd6bb36f9acb4e Mon Sep 17 00:00:00 2001 From: Jay2006sawant Date: Mon, 27 Jul 2026 14:07:41 +0530 Subject: [PATCH 3/3] deny privileged containers only when gpu resources are requested Signed-off-by: Jay2006sawant --- pkg/scheduler/webhook.go | 9 +++++---- pkg/scheduler/webhook_test.go | 25 ++++++++++++++++++++----- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/pkg/scheduler/webhook.go b/pkg/scheduler/webhook.go index cd20532332..3f749e8f46 100644 --- a/pkg/scheduler/webhook.go +++ b/pkg/scheduler/webhook.go @@ -62,10 +62,6 @@ func (h *webhook) Handle(_ context.Context, req admission.Request) admission.Res klog.Warningf(template+" - Denying admission as pod has no containers", pod.Namespace, pod.Name, pod.UID) return admission.Denied("pod has no containers") } - if name, privileged := privilegedContainerName(pod); privileged { - klog.Warningf(template+" - Denying admission as container %s is privileged", pod.Namespace, pod.Name, pod.UID, name) - return admission.Denied(fmt.Sprintf("container %s is privileged", name)) - } if pod.Spec.SchedulerName != "" && (pod.Spec.SchedulerName != corev1.DefaultSchedulerName || !config.ForceOverwriteDefaultScheduler) && (len(config.SchedulerName) == 0 || pod.Spec.SchedulerName != config.SchedulerName) { @@ -73,6 +69,7 @@ func (h *webhook) Handle(_ context.Context, req admission.Request) admission.Res return admission.Allowed("pod already has different scheduler assigned") } klog.V(5).Infof(template, pod.Namespace, pod.Name, pod.UID) + privilegedName, hasPrivileged := privilegedContainerName(pod) hasResource := false for idx := range pod.Spec.Containers { c := &pod.Spec.Containers[idx] @@ -85,6 +82,10 @@ func (h *webhook) Handle(_ context.Context, req admission.Request) admission.Res hasResource = hasResource || found } } + if hasPrivileged && hasResource { + klog.Warningf(template+" - Denying admission as container %s is privileged", pod.Namespace, pod.Name, pod.UID, privilegedName) + return admission.Denied(fmt.Sprintf("container %s is privileged", privilegedName)) + } if !hasResource { klog.V(3).Infof(template+" - Allowing admission: no GPU resource found", pod.Namespace, pod.Name, pod.UID) diff --git a/pkg/scheduler/webhook_test.go b/pkg/scheduler/webhook_test.go index 8f622ed3fd..a03e488310 100644 --- a/pkg/scheduler/webhook_test.go +++ b/pkg/scheduler/webhook_test.go @@ -18,6 +18,7 @@ package scheduler import ( "context" + "strings" "testing" admissionv1 "k8s.io/api/admission/v1" @@ -671,11 +672,12 @@ func TestPrivilegedContainerDenied(t *testing.T) { privileged := true testCases := []struct { - name string - pod *corev1.Pod + name string + pod *corev1.Pod + allowed bool }{ { - name: "privileged container only", + name: "privileged container only without gpu", pod: &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{Name: "privileged-pod", Namespace: "default"}, Spec: corev1.PodSpec{ @@ -689,6 +691,7 @@ func TestPrivilegedContainerDenied(t *testing.T) { }, }, }, + allowed: true, }, { name: "privileged sidecar with gpu workload", @@ -713,6 +716,7 @@ func TestPrivilegedContainerDenied(t *testing.T) { }, }, }, + allowed: false, }, { name: "privileged init container with gpu workload", @@ -739,6 +743,7 @@ func TestPrivilegedContainerDenied(t *testing.T) { }, }, }, + allowed: false, }, { name: "privileged pod with different scheduler", @@ -756,6 +761,7 @@ func TestPrivilegedContainerDenied(t *testing.T) { }, }, }, + allowed: true, }, } @@ -787,11 +793,20 @@ func TestPrivilegedContainerDenied(t *testing.T) { } resp := wh.Handle(context.Background(), req) + if tc.allowed { + if !resp.Allowed { + t.Fatalf("Expected allowed response, but got denied: %+v", resp.Result) + } + return + } if resp.Allowed { t.Fatalf("Expected denied response for privileged pod, but got allowed with %d patches", len(resp.Patches)) } - if resp.Result == nil || resp.Result.Message == "" { - t.Fatalf("Expected denial message, got: %+v", resp.Result) + if len(resp.Patches) != 0 { + t.Fatalf("Expected no patches for privileged pod, got %d", len(resp.Patches)) + } + if resp.Result == nil || !strings.Contains(resp.Result.Message, "is privileged") { + t.Fatalf("Expected privilege denial message, got: %+v", resp.Result) } }) }