Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -138,3 +138,4 @@ spec:
- hostPath:
path: /var/log/kube-apiserver
name: audit-dir

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor

Remove trailing blank line.

The YAML file has an extra blank line at the end of the volumes section that violates linting rules.

Apply this diff to remove the trailing blank line:

  - hostPath:
      path: /var/log/kube-apiserver
    name: audit-dir
-
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
- hostPath:
path: /var/log/kube-apiserver
name: audit-dir
🧰 Tools
🪛 YAMLlint (1.37.1)

[warning] 141-141: too many blank lines (1 > 0)

(empty-lines)

🤖 Prompt for AI Agents
In bindata/bootkube/bootstrap-manifests/kube-apiserver-pod.yaml around line 141,
there is an extra trailing blank line at the end of the volumes section; remove
that blank line so the file ends immediately after the last volume entry (no
extra newline-only line), ensuring the YAML has no blank line violation per
linting rules.

2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -135,3 +135,5 @@ require (
)

replace github.com/onsi/ginkgo/v2 => github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20241205171354-8006f302fd12

replace github.com/openshift/library-go => github.com/ardaguclu/library-go v0.0.0-20251105061910-519a2c27c78d
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8
github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
github.com/apparentlymart/go-cidr v1.0.1 h1:NmIwLZ/KdsjIUlhf+/Np40atNXm/+lZ5txfTJ/SpF+U=
github.com/apparentlymart/go-cidr v1.0.1/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc=
github.com/ardaguclu/library-go v0.0.0-20251105061910-519a2c27c78d h1:nLyySCXMIChM6B3mcIIoErpirIvKZDvR+yCIR+p28Mc=
github.com/ardaguclu/library-go v0.0.0-20251105061910-519a2c27c78d/go.mod h1:UdWv/P+Rw/TrLB6qpwLsUIf5urKlbAqEoK9qnP1NVmQ=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
Expand Down Expand Up @@ -165,8 +167,6 @@ github.com/openshift/build-machinery-go v0.0.0-20250530140348-dc5b2804eeee h1:+S
github.com/openshift/build-machinery-go v0.0.0-20250530140348-dc5b2804eeee/go.mod h1:8jcm8UPtg2mCAsxfqKil1xrmRMI3a+XU2TZ9fF8A7TE=
github.com/openshift/client-go v0.0.0-20251015124057-db0dee36e235 h1:9JBeIXmnHlpXTQPi7LPmu1jdxznBhAE7bb1K+3D8gxY=
github.com/openshift/client-go v0.0.0-20251015124057-db0dee36e235/go.mod h1:L49W6pfrZkfOE5iC1PqEkuLkXG4W0BX4w8b+L2Bv7fM=
github.com/openshift/library-go v0.0.0-20251015151611-6fc7a74b67c5 h1:bANtDc8SgetSK4nQehf59x3+H9FqVJCprgjs49/OTg0=
github.com/openshift/library-go v0.0.0-20251015151611-6fc7a74b67c5/go.mod h1:OlFFws1AO51uzfc48MsStGE4SFMWlMZD0+f5a/zCtKI=
github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20241205171354-8006f302fd12 h1:AKx/w1qpS8We43bsRgf8Nll3CGlDHpr/WAXvuedTNZI=
github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20241205171354-8006f302fd12/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0=
Expand Down
2 changes: 2 additions & 0 deletions pkg/operator/starter.go
Original file line number Diff line number Diff line change
Expand Up @@ -281,9 +281,11 @@ func RunOperator(ctx context.Context, controllerContext *controllercmd.Controlle
os.Getenv("IMAGE"),
os.Getenv("OPERATOR_IMAGE"),
os.Getenv("OPERATOR_IMAGE_VERSION"),
os.Getenv("KMS_PLUGIN_IMAGE"),
operatorClient,
kubeInformersForNamespaces.InformersFor(operatorclient.TargetNamespace),
kubeInformersForNamespaces,
configInformers.Config().V1().APIServers(),
kubeClient,
startupmonitorreadiness.IsStartupMonitorEnabledFunction(configInformers.Config().V1().Infrastructures().Lister(), operatorClient),
requireMultipleEtcdEndpoints,
Expand Down
96 changes: 96 additions & 0 deletions pkg/operator/targetconfigcontroller/kms_plugin.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
package targetconfigcontroller

import (
"context"
"fmt"

configv1 "github.com/openshift/api/config/v1"
configv1listers "github.com/openshift/client-go/config/listers/config/v1"
"github.com/openshift/library-go/pkg/operator/encryption/kms"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/klog/v2"
)

// copied from https://github.com/flavianmissi/cluster-kube-apiserver-operator/tree/kms-plugin-sidecars

const (
// KMSPluginImageEnvVar is the environment variable that specifies the KMS plugin container image
// This should be set by the operator deployment
KMSPluginImageEnvVar = "KMS_PLUGIN_IMAGE"

// DefaultKMSPluginImage is the fallback image if KMS_PLUGIN_IMAGE is not set
DefaultKMSPluginImage = "quay.io/fmissi/aws-kms-plugin:0.1.0"
)

// getKMSEncryptionConfig checks if KMS encryption is enabled and returns the configuration
// Returns:
// - kmsConfig: the KMS configuration if enabled, nil otherwise
// - enabled: true if KMS encryption is enabled
// - error: any error encountered while reading the config
func getKMSEncryptionConfig(ctx context.Context, apiserverLister configv1listers.APIServerLister) (*configv1.KMSConfig, bool, error) {
apiserver, err := apiserverLister.Get("cluster")
if err != nil {
if apierrors.IsNotFound(err) {
klog.V(4).Info("APIServer config.openshift.io/cluster not found, KMS encryption not enabled")
return nil, false, nil
}
return nil, false, fmt.Errorf("failed to get APIServer config: %w", err)
}

// Check if encryption is configured
if apiserver.Spec.Encryption.Type != configv1.EncryptionTypeKMS {
klog.V(4).Infof("Encryption type is %q, not KMS - skipping KMS plugin injection", apiserver.Spec.Encryption.Type)
return nil, false, nil
}

// KMS type is set, must have KMS config
if apiserver.Spec.Encryption.KMS == nil {
return nil, false, fmt.Errorf("encryption type is KMS but kms config is nil")
}

klog.Infof("KMS encryption enabled with type=%s, region=%s, keyARN=%s",
apiserver.Spec.Encryption.KMS.Type,
apiserver.Spec.Encryption.KMS.AWS.Region,
apiserver.Spec.Encryption.KMS.AWS.KeyARN)
Comment on lines +52 to +55
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🔴 Critical

Guard against nil AWS KMS config before logging.

apiserver.Spec.Encryption.KMS.AWS is dereferenced unconditionally, so any cluster using a non-AWS KMS provider (or with an incomplete config during rollout) will panic the operator here. Please gate the logging on KMS.AWS != nil (or branch by provider) before touching those fields.

-	klog.Infof("KMS encryption enabled with type=%s, region=%s, keyARN=%s",
-		apiserver.Spec.Encryption.KMS.Type,
-		apiserver.Spec.Encryption.KMS.AWS.Region,
-		apiserver.Spec.Encryption.KMS.AWS.KeyARN)
+	klog.Infof("KMS encryption enabled with type=%s", apiserver.Spec.Encryption.KMS.Type)
+	if awsConfig := apiserver.Spec.Encryption.KMS.AWS; awsConfig != nil {
+		klog.Infof("KMS AWS config: region=%s, keyARN=%s", awsConfig.Region, awsConfig.KeyARN)
+	}
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
klog.Infof("KMS encryption enabled with type=%s, region=%s, keyARN=%s",
apiserver.Spec.Encryption.KMS.Type,
apiserver.Spec.Encryption.KMS.AWS.Region,
apiserver.Spec.Encryption.KMS.AWS.KeyARN)
klog.Infof("KMS encryption enabled with type=%s", apiserver.Spec.Encryption.KMS.Type)
if awsConfig := apiserver.Spec.Encryption.KMS.AWS; awsConfig != nil {
klog.Infof("KMS AWS config: region=%s, keyARN=%s", awsConfig.Region, awsConfig.KeyARN)
}
🤖 Prompt for AI Agents
In pkg/operator/targetconfigcontroller/kms_plugin.go around lines 52 to 55, the
code unconditionally dereferences apiserver.Spec.Encryption.KMS.AWS which will
panic if AWS KMS config is nil; update the logging to first check that
apiserver.Spec.Encryption.KMS != nil and apiserver.Spec.Encryption.KMS.AWS !=
nil (or branch by provider) and only include region/keyARN in the message when
AWS config exists, otherwise log a provider-only message or omit AWS fields to
avoid nil dereference.


return apiserver.Spec.Encryption.KMS, true, nil
}

// injectKMSPlugin adds the KMS plugin sidecar container to the kube-apiserver pod
// if KMS encryption is enabled in the cluster APIServer config
func injectKMSPlugin(ctx context.Context, pod *corev1.Pod, apiserverLister configv1listers.APIServerLister, kmsPluginImage string) error {
// Check if KMS encryption is enabled
kmsConfig, enabled, err := getKMSEncryptionConfig(ctx, apiserverLister)
if err != nil {
return fmt.Errorf("failed to check KMS encryption config: %w", err)
}

if !enabled {
klog.V(4).Info("KMS encryption not enabled, skipping sidecar injection")
return nil
}

// Validate the image is set
if kmsPluginImage == "" {
kmsPluginImage = DefaultKMSPluginImage
}

klog.Infof("Injecting KMS plugin sidecar container (image: %s)", kmsPluginImage)

// Create container config for kube-apiserver
// kube-apiserver uses hostNetwork: true, so it accesses AWS credentials via IMDS
containerConfig := &kms.ContainerConfig{
Image: kmsPluginImage,
UseHostNetwork: true, // Static pod with hostNetwork uses EC2 IMDS for AWS credentials
KMSConfig: kmsConfig,
}

// Inject the KMS plugin sidecar container and volumes into the pod spec
if err := kms.AddKMSPluginToPodSpec(&pod.Spec, kmsConfig, containerConfig, true); err != nil {
return fmt.Errorf("failed to inject KMS plugin sidecar: %w", err)
}

klog.Infof("Successfully injected KMS plugin sidecar container")
return nil
}
44 changes: 37 additions & 7 deletions pkg/operator/targetconfigcontroller/targetconfigcontroller.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ import (

kubecontrolplanev1 "github.com/openshift/api/kubecontrolplane/v1"
operatorv1 "github.com/openshift/api/operator/v1"
configv1informers "github.com/openshift/client-go/config/informers/externalversions/config/v1"
configv1listers "github.com/openshift/client-go/config/listers/config/v1"
"github.com/openshift/cluster-kube-apiserver-operator/bindata"
"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/configobservation/node"
"github.com/openshift/cluster-kube-apiserver-operator/pkg/operator/operatorclient"
Expand Down Expand Up @@ -51,21 +53,24 @@ type TargetConfigController struct {
targetImagePullSpec string
operatorImagePullSpec string
operatorImageVersion string
kmsPluginImage string

operatorClient v1helpers.StaticPodOperatorClient

kubeClient kubernetes.Interface
configMapLister corev1listers.ConfigMapLister
apiserverLister configv1listers.APIServerLister

isStartupMonitorEnabledFn func() (bool, error)
requireMultipleEtcdEndpointsFn func() bool
}

func NewTargetConfigController(
targetImagePullSpec, operatorImagePullSpec, operatorImageVersion string,
targetImagePullSpec, operatorImagePullSpec, operatorImageVersion, kmsPluginImage string,
operatorClient v1helpers.StaticPodOperatorClient,
kubeInformersForOpenshiftKubeAPIServerNamespace informers.SharedInformerFactory,
kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces,
apiserverInformer configv1informers.APIServerInformer,
kubeClient kubernetes.Interface,
isStartupMonitorEnabledFn func() (bool, error),
requireMultipleEtcdEndpointsFn func() bool,
Expand All @@ -75,9 +80,11 @@ func NewTargetConfigController(
targetImagePullSpec: targetImagePullSpec,
operatorImagePullSpec: operatorImagePullSpec,
operatorImageVersion: operatorImageVersion,
kmsPluginImage: kmsPluginImage,
operatorClient: operatorClient,
kubeClient: kubeClient,
configMapLister: kubeInformersForNamespaces.ConfigMapLister(),
apiserverLister: apiserverInformer.Lister(),
isStartupMonitorEnabledFn: isStartupMonitorEnabledFn,
requireMultipleEtcdEndpointsFn: requireMultipleEtcdEndpointsFn,
}
Expand All @@ -91,6 +98,7 @@ func NewTargetConfigController(
kubeInformersForNamespaces.InformersFor(operatorclient.GlobalMachineSpecifiedConfigNamespace).Core().V1().ConfigMaps().Informer(),
kubeInformersForNamespaces.InformersFor(operatorclient.OperatorNamespace).Core().V1().ConfigMaps().Informer(),
kubeInformersForNamespaces.InformersFor(operatorclient.TargetNamespace).Core().V1().ConfigMaps().Informer(),
apiserverInformer.Informer(),
).WithSync(c.sync).ResyncEvery(time.Minute).ToController("TargetConfigController", eventRecorder.WithComponentSuffix("target-config-controller"))
}

Expand Down Expand Up @@ -219,7 +227,18 @@ func createTargetConfig(ctx context.Context, c TargetConfigController, recorder
if err != nil {
errors = append(errors, fmt.Errorf("%q: %v", "configmap/config", err))
}
_, _, err = managePods(ctx, c.kubeClient.CoreV1(), c.isStartupMonitorEnabledFn, recorder, operatorSpec, c.targetImagePullSpec, c.operatorImagePullSpec, c.operatorImageVersion)
_, _, err = managePods(
ctx,
c.kubeClient.CoreV1(),
c.apiserverLister,
c.isStartupMonitorEnabledFn,
recorder,
operatorSpec,
c.targetImagePullSpec,
c.operatorImagePullSpec,
c.operatorImageVersion,
c.kmsPluginImage,
)
if err != nil {
errors = append(errors, fmt.Errorf("%q: %v", "configmap/kube-apiserver-pod", err))
}
Expand Down Expand Up @@ -303,11 +322,17 @@ func manageKubeAPIServerConfig(ctx context.Context, client coreclientv1.ConfigMa
return resourceapply.ApplyConfigMap(ctx, client, recorder, requiredConfigMap)
}

func managePods(ctx context.Context, client coreclientv1.ConfigMapsGetter, isStartupMonitorEnabledFn func() (bool, error), recorder events.Recorder, operatorSpec *operatorv1.StaticPodOperatorSpec, imagePullSpec, operatorImagePullSpec, operatorImageVersion string) (*corev1.ConfigMap, bool, error) {
appliedPodTemplate, err := manageTemplate(string(bindata.MustAsset("assets/kube-apiserver/pod.yaml")), imagePullSpec, operatorImagePullSpec, operatorImageVersion, operatorSpec)
if err != nil {
return nil, false, err
}
func managePods(
ctx context.Context,
client coreclientv1.ConfigMapsGetter,
apiserverLister configv1listers.APIServerLister,
isStartupMonitorEnabledFn func() (bool, error),
recorder events.Recorder,
operatorSpec *operatorv1.StaticPodOperatorSpec,
imagePullSpec, operatorImagePullSpec, operatorImageVersion, kmsPluginImage string,
) (*corev1.ConfigMap, bool, error) {
appliedPodTemplate, err := manageTemplate(
string(bindata.MustAsset("assets/kube-apiserver/pod.yaml")), imagePullSpec, operatorImagePullSpec, operatorImageVersion, operatorSpec)
required := resourceread.ReadPodV1OrDie([]byte(appliedPodTemplate))

var observedConfig map[string]interface{}
Expand All @@ -324,6 +349,11 @@ func managePods(ctx context.Context, client coreclientv1.ConfigMapsGetter, isSta
required.Spec.Containers[i].Env = append(container.Env, proxyEnvVars...)
}

// Inject KMS plugin sidecar if KMS encryption is enabled
if err := injectKMSPlugin(ctx, required, apiserverLister, kmsPluginImage); err != nil {
return nil, false, fmt.Errorf("failed to inject KMS plugin: %v", err)
}

configMap := resourceread.ReadConfigMapV1OrDie(bindata.MustAsset("assets/kube-apiserver/pod-cm.yaml"))
configMap.Data["pod.yaml"] = resourceread.WritePodV1OrDie(required)
configMap.Data["forceRedeploymentReason"] = operatorSpec.ForceRedeploymentReason
Expand Down
Loading