From 61136794179184141da35bdc1c7e34d521e7811d Mon Sep 17 00:00:00 2001 From: Greg Kurz Date: Tue, 2 Sep 2025 13:42:15 +0200 Subject: [PATCH 1/3] Move peer-pods code to its own file $ wc -l controllers/openshift_controller.go 2764 controllers/openshift_controller.go The file is huge already. The peer-pods logic needs some heavy fixing that will require to add some more peer-pods specific code. Let's move peer-pods to a separate file. It will be easier to read and maintain. This is consistent with what we already do for other more recent features. This doesn't change behavior. Signed-off-by: Greg Kurz --- controllers/openshift_controller.go | 481 +------------------------- controllers/peerpods.go | 506 ++++++++++++++++++++++++++++ 2 files changed, 511 insertions(+), 476 deletions(-) create mode 100644 controllers/peerpods.go diff --git a/controllers/openshift_controller.go b/controllers/openshift_controller.go index 2e7e0bde8..3f7aceb6d 100644 --- a/controllers/openshift_controller.go +++ b/controllers/openshift_controller.go @@ -20,7 +20,6 @@ import ( "encoding/json" "fmt" "os" - "path/filepath" "reflect" "strings" "time" @@ -28,7 +27,6 @@ import ( appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/intstr" ignTypes "github.com/coreos/ignition/v2/config/v3_2/types" "github.com/go-logr/logr" @@ -70,30 +68,17 @@ type KataConfigOpenShiftReconciler struct { } const ( - OperatorNamespace = "openshift-sandboxed-containers-operator" - dashboard_configmap_name = "grafana-dashboard-sandboxed-containers" - dashboard_configmap_namespace = "openshift-config-managed" - container_runtime_config_name = "kata-crio-config" - extension_mc_name = "50-enable-sandboxed-containers-extension" - DEFAULT_PEER_PODS = "10" - peerpodConfigCrdName = "peerpodconfig-openshift" - peerpodsMachineConfigPathLocation = "/config/peerpods" - peerpodsCrioMachineConfig = "50-kata-remote" - peerpodsCrioMachineConfigYaml = "mc-50-crio-config.yaml" - peerpodsKataRemoteMachineConfig = "40-worker-kata-remote-config" - peerpodsKataRemoteMachineConfigYaml = "mc-40-kata-remote-config.yaml" + OperatorNamespace = "openshift-sandboxed-containers-operator" + dashboard_configmap_name = "grafana-dashboard-sandboxed-containers" + dashboard_configmap_namespace = "openshift-config-managed" + container_runtime_config_name = "kata-crio-config" + extension_mc_name = "50-enable-sandboxed-containers-extension" // Use same Pod Overhead as upstream kata-deploy using, see // https://github.com/kata-containers/kata-containers/blob/main/tools/packaging/kata-deploy/runtimeclasses/kata-qemu.yaml#L7 kataRuntimeClassName = "kata" kataRuntimeClassCpuOverhead = "0.25" // We need a higher value than upstream (see https://github.com/openshift/sandboxed-containers-operator/pull/84) kataRuntimeClassMemOverhead = "350Mi" - // https://github.com/kata-containers/kata-containers/blob/main/tools/packaging/kata-deploy/runtimeclasses/kata-remote.yaml#L7 - peerpodsRuntimeClassName = "kata-remote" - peerpodsRuntimeClassCpuOverhead = "0.25" - peerpodsRuntimeClassMemOverhead = "120Mi" - // cloud-api-adaptor (CAA) daemonset name - caaDsName = "osc-caa-ds" ) // +kubebuilder:rbac:groups=kataconfiguration.openshift.io,resources=kataconfigs;kataconfigs/finalizers,verbs=get;list;watch;create;update;patch;delete @@ -361,209 +346,6 @@ func (r *KataConfigOpenShiftReconciler) removeLogLevel() error { return nil } -func MountProgagationRef(mode corev1.MountPropagationMode) *corev1.MountPropagationMode { - return &mode -} - -func (r *KataConfigOpenShiftReconciler) processDaemonsetForCAA() *appsv1.DaemonSet { - var ( - runPrivileged = true - runAsUser int64 = 0 - defaultMode int32 = 0600 - sshSecretOptional = true - authJsonSecretOptional = true - nodeSelector = r.getNodeSelectorAsMap() - ) - - dsLabelSelectors := map[string]string{ - "name": caaDsName, - } - - imageString := os.Getenv("RELATED_IMAGE_CAA") - if imageString == "" { - r.Log.Info("RELATED_IMAGE_CAA env var is unset or empty, cloud-api-adaptor pods will not run") - } - - return &appsv1.DaemonSet{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "apps/v1", - Kind: "DaemonSet", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: caaDsName, - Namespace: os.Getenv("PEERPODS_NAMESPACE"), - }, - Spec: appsv1.DaemonSetSpec{ - Selector: &metav1.LabelSelector{ - MatchLabels: dsLabelSelectors, - }, - UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ - Type: "RollingUpdate", - RollingUpdate: &appsv1.RollingUpdateDaemonSet{ - MaxUnavailable: &intstr.IntOrString{ - Type: intstr.Int, - IntVal: 1, - }, - }, - }, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: dsLabelSelectors, - }, - Spec: corev1.PodSpec{ - ServiceAccountName: "default", - NodeSelector: nodeSelector, - HostNetwork: true, - Containers: []corev1.Container{ - { - Name: "caa-pod", - Image: imageString, - ImagePullPolicy: "Always", - SecurityContext: &corev1.SecurityContext{ - // TODO - do we really need to run as root? - Privileged: &runPrivileged, - RunAsUser: &runAsUser, - }, - Command: []string{"/usr/local/bin/entrypoint.sh"}, - Env: []corev1.EnvVar{ - { - Name: "NODE_NAME", - ValueFrom: &corev1.EnvVarSource{ - FieldRef: &corev1.ObjectFieldSelector{ - FieldPath: "spec.nodeName", - }, - }, - }, - }, - EnvFrom: []corev1.EnvFromSource{ - { - SecretRef: &corev1.SecretEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "peer-pods-secret", - }, - }, - }, - { - ConfigMapRef: &corev1.ConfigMapEnvSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: "peer-pods-cm", - }, - }, - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "auth-json-volume", - MountPath: "/root/containers/", - ReadOnly: true, - }, { - Name: "ssh", - MountPath: "/root/.ssh", - ReadOnly: true, - }, - { - MountPath: "/run/peerpod", - Name: "pods-dir", - }, - { - MountPath: "/run/netns", - MountPropagation: MountProgagationRef(corev1.MountPropagationHostToContainer), - Name: "netns", - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "auth-json-volume", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "auth-json-secret", - DefaultMode: &defaultMode, - Optional: &authJsonSecretOptional, - }, - }, - }, { - Name: "ssh", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: "ssh-key-secret", - DefaultMode: &defaultMode, - Optional: &sshSecretOptional, - }, - }, - }, - { - Name: "pods-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/run/peerpod", - }, - }, - }, - { - Name: "netns", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/run/netns", - }, - }, - }, - }, - }, - }, - }, - } -} - -// Handles provider specific parts of the CAA Ds -// Modifies the DaemonSet if needed -func (r *KataConfigOpenShiftReconciler) processProviderConfigCAA(ds *appsv1.DaemonSet) error { - r.Log.Info("Getting cloud provider from infra") - provider, err := getCloudProviderFromInfra(r.Client) - if err != nil { - return fmt.Errorf("failed to get cloud provider from infra: %w", err) - } - - switch provider { - case IBMCloudProvider: - var expSecs int64 = 3600 - vaultTokenVolume := corev1.Volume{ - Name: "vault-token", - VolumeSource: corev1.VolumeSource{ - Projected: &corev1.ProjectedVolumeSource{ - Sources: []corev1.VolumeProjection{ - { - ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ - Path: "vault-token", - ExpirationSeconds: &expSecs, - Audience: "iam", - }, - }, - }, - }, - }, - } - - vaultTokenVolumeMount := corev1.VolumeMount{ - MountPath: "/var/run/secrets/tokens", - Name: "vault-token", - } - - ds.Spec.Template.Spec.Volumes = append(ds.Spec.Template.Spec.Volumes, vaultTokenVolume) - for i := range ds.Spec.Template.Spec.Containers { - container := &ds.Spec.Template.Spec.Containers[i] - if container.Name == "caa-pod" { - container.VolumeMounts = append(container.VolumeMounts, vaultTokenVolumeMount) - } - } - - return nil - default: - return nil - } -} - func (r *KataConfigOpenShiftReconciler) processDaemonsetForMonitor() *appsv1.DaemonSet { var ( runPrivileged = false @@ -2330,174 +2112,6 @@ func (r *KataConfigOpenShiftReconciler) isUpdating() bool { return cond.Status == corev1.ConditionTrue && cond.Reason == "Updating" } -func (r *KataConfigOpenShiftReconciler) createAuthJsonSecret() error { - var err error - - pullSecret := &corev1.Secret{} - err = r.Client.Get(context.TODO(), types.NamespacedName{Name: "pull-secret", Namespace: "openshift-config"}, pullSecret) - if err != nil { - r.Log.Info("Error fetching pull-secret", "err", err) - return err - } - - authJsonSecret := corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: "auth-json-secret", - Namespace: OperatorNamespace, - }, - Data: map[string][]byte{ - "auth.json": pullSecret.Data[".dockerconfigjson"], - }, - Type: corev1.SecretTypeOpaque, - } - - err = r.Client.Create(context.TODO(), &authJsonSecret) - if err != nil { - if k8serrors.IsAlreadyExists(err) { - err = r.Client.Update(context.TODO(), &authJsonSecret) - if err != nil { - r.Log.Info("Error updating auth-json-secret", "err", err) - return err - } - } else { - r.Log.Info("Error creating auth-json-secret", "err", err) - return err - } - } - - return err -} - -// Create the MachineConfigs for PeerPod -// We do it before kata-oc creation to optimise the reboots required for MC creation -func (r *KataConfigOpenShiftReconciler) enablePeerPodsMc() error { - - //Create MachineConfig for kata-remote hyp CRIO config - crioMachineConfigFilePath := filepath.Join(peerpodsMachineConfigPathLocation, peerpodsCrioMachineConfigYaml) - err := r.createMcFromFile(crioMachineConfigFilePath) - if err != nil { - r.Log.Info("Error in creating CRIO MachineConfig", "err", err) - return err - } - - //Create MachineConfig for kata-remote hyp config toml - kataConfigMachineConfigFilePath := filepath.Join(peerpodsMachineConfigPathLocation, peerpodsKataRemoteMachineConfigYaml) - err = r.createMcFromFile(kataConfigMachineConfigFilePath) - if err != nil { - r.Log.Info("Error in creating kata remote configuration.toml MachineConfig", "err", err) - return err - } - - return nil -} - -// Create the PeerPodConfig CRDs and misc configs required for peer-pods -func (r *KataConfigOpenShiftReconciler) enablePeerPodsMiscConfigs() error { - // Create the CAA daemonset - ds := r.processDaemonsetForCAA() - if err := r.processProviderConfigCAA(ds); err != nil { - r.Log.Error(err, "Failed setting cloud provider specific configuration for cloud-api-adaptor DS") - return err - } - r.Log.Info("Got CAA ds manifest", "ds", ds) - - if err := controllerutil.SetControllerReference(r.kataConfig, ds, r.Scheme); err != nil { - r.Log.Error(err, "Failed setting ControllerReference for cloud-api-adaptor DS") - return err - } - - err := r.Client.Update(context.TODO(), ds) - if err != nil && k8serrors.IsNotFound(err) { - r.Log.Error(err, "cloud-api-adaptor daemonset doesn't exist. Creating") - err = r.Client.Create(context.TODO(), ds) - if err != nil { - r.Log.Error(err, "failed to create cloud-api-adaptor daemonset") - return err - } - } - - // Create the mutating webhook deployment - err = r.createMutatingWebhookDeployment() - if err != nil { - r.Log.Info("Error in creating mutating webhook deployment for peerpods", "err", err) - return err - } - - // Create the mutating webhook service - err = r.createMutatingWebhookService() - if err != nil { - r.Log.Info("Error in creating mutating webhook service for peerpods", "err", err) - return err - } - - // Create the mutating webhook - err = r.createMutatingWebhookConfig() - if err != nil { - r.Log.Info("Error in creating mutating webhook for peerpods", "err", err) - return err - } - - // Create runtimeClass config for peer-pods - err = r.createRuntimeClass(peerpodsRuntimeClassName, peerpodsRuntimeClassCpuOverhead, peerpodsRuntimeClassMemOverhead) - if err != nil { - r.Log.Info("Error in creating kata remote runtimeclass", "err", err) - return err - } - return nil -} - -func (r *KataConfigOpenShiftReconciler) disablePeerPods() error { - ds := r.processDaemonsetForCAA() - err := r.Client.Delete(context.TODO(), ds) - if err != nil { - if k8serrors.IsNotFound(err) { - r.Log.Info("cloud-api-adaptor daemonset was already deleted") - } else { - r.Log.Error(err, "error when deleting cloud-api-adaptor Daemonset, try again") - return err - } - } - - if r.DeploymentMode == MachineConfigMode { - // We are explicitly ignoring any errors in peerpodconfig and related machineconfigs removal as - // these can be removed manually if needed and this is not in the critical path - // of operator functionality - _ = r.deletePeerPodsMC() - } - - if r.DeploymentMode == DaemonSetMode { - // We want to make sure that the osc-config-sync ds removal successfully started - // So we will try again in case of an error - err = r.deletePeedPodsConfigDaemonSet() - if err != nil { - return err - } - } - - // Delete mutating webhook deployment - err = r.deleteMutatingWebhookDeployment() - if err != nil { - r.Log.Info("Error in deleting mutating webhook deployment for peerpods", "err", err) - return err - } - - // Delete mutating webhook service - err = r.deleteMutatingWebhookService() - if err != nil { - r.Log.Info("Error in deleting mutating webhook service for peerpods", "err", err) - return err - } - - // Delete the mutating webhook - err = r.deleteMutatingWebhookConfig() - if err != nil { - r.Log.Info("Error in deleting mutating webhook for peerpods", "err", err) - return err - } - - return nil -} - // Create the MachineConfigs from file // Full path of the file should be provided func (r *KataConfigOpenShiftReconciler) createMcFromFile(machineConfigYamlFile string) error { @@ -2576,53 +2190,6 @@ func (r *KataConfigOpenShiftReconciler) deleteDaemonsetForMonitor() error { return nil } -func (r *KataConfigOpenShiftReconciler) deletePodVMImage() (*ctrl.Result, error) { - // Handle podvm image deletion - // Since we want to declaratively reach the final state, we need to reconcile when there are errors - // as we want the system to give a chance of fixing the error. - // For cases we don't want to reconcile, ie for ImageDeletedSuccessfully and UnsupportedPodVMImageProvider - // we should just log the message and let the code continue without explicitly returning from the method - - // Following are returned statuses: - // ImageDeletedSuccessfully - // UnsupportedPodVMImageProvider - // ImageDeletionFailed - // RequeueNeeded - // ImageDeletionStatusUnknown - status, err := ImageDelete(r.Client) - switch status { - case ImageDeletedSuccessfully: - r.setInProgressConditionToPodVMImageDeleted() - r.Log.Info("PodVM Image deleted successfully") - - case UnsupportedPodVMImageProvider: - r.setInProgressConditionToPodVMImageUnsupportedProvider() - r.Log.Info("unsupported cloud provider, skipping image deletion") - - case RequeueNeeded: - r.setInProgressConditionToPodVMImageDeleting() - return &ctrl.Result{Requeue: true, RequeueAfter: 15 * time.Second}, err - - case ImageDeletionFailed: - r.setInProgressConditionToPodVMImageDeletionFailed() - if err != nil { - // We requeue only if there is an error. - return &ctrl.Result{Requeue: true, RequeueAfter: 15 * time.Second}, err - } - // If there's no error, log and continue - r.Log.Info("Image deletion failed. Check logs for more details") - - case ImageDeletionStatusUnknown: - r.setInProgressConditionToPodVMImageDeletionUnknown() - return &ctrl.Result{Requeue: true, RequeueAfter: 15 * time.Second}, err - - default: - // For all other statuses, just log and continue - r.Log.Info("PodVM Image deletion status and error", "status", status, "error", err) - } - return nil, nil -} - func (r *KataConfigOpenShiftReconciler) deleteScc() error { scc := GetScc() err := r.Client.Delete(context.TODO(), scc) @@ -2724,41 +2291,3 @@ func (r *KataConfigOpenShiftReconciler) postKataInstallation() (*ctrl.Result, er } return nil, nil } - -func (r *KataConfigOpenShiftReconciler) deletePeerPodsMC() error { - mc := mcfgv1.MachineConfig{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "machineconfiguration.openshift.io/v1", - Kind: "MachineConfig", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: peerpodsKataRemoteMachineConfig, - }, - } - - err := r.Client.Delete(context.TODO(), &mc) - if err != nil { - // error during removing mc. Just log the error and move on. - r.Log.Info("Error found deleting mc. If the MachineConfig object exists after uninstallation it can be safely deleted manually", - "mc", mc.Name, "err", err) - } - - mc = mcfgv1.MachineConfig{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "machineconfiguration.openshift.io/v1", - Kind: "MachineConfig", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: peerpodsCrioMachineConfig, - }, - } - - err = r.Client.Delete(context.TODO(), &mc) - if err != nil { - // error during removing mc. Just log the error and move on. - r.Log.Info("Error found deleting mc. If the MachineConfig object exists after uninstallation it can be safely deleted manually", - "mc", mc.Name, "err", err) - } - - return nil -} diff --git a/controllers/peerpods.go b/controllers/peerpods.go new file mode 100644 index 000000000..36938c0c3 --- /dev/null +++ b/controllers/peerpods.go @@ -0,0 +1,506 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + mcfgv1 "github.com/openshift/api/machineconfiguration/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +const ( + DEFAULT_PEER_PODS = "10" + peerpodConfigCrdName = "peerpodconfig-openshift" + peerpodsMachineConfigPathLocation = "/config/peerpods" + peerpodsCrioMachineConfig = "50-kata-remote" + peerpodsCrioMachineConfigYaml = "mc-50-crio-config.yaml" + peerpodsKataRemoteMachineConfig = "40-worker-kata-remote-config" + peerpodsKataRemoteMachineConfigYaml = "mc-40-kata-remote-config.yaml" + // https://github.com/kata-containers/kata-containers/blob/main/tools/packaging/kata-deploy/runtimeclasses/kata-remote.yaml#L7 + peerpodsRuntimeClassName = "kata-remote" + peerpodsRuntimeClassCpuOverhead = "0.25" + peerpodsRuntimeClassMemOverhead = "120Mi" + // cloud-api-adaptor (CAA) daemonset name + caaDsName = "osc-caa-ds" +) + +func MountProgagationRef(mode corev1.MountPropagationMode) *corev1.MountPropagationMode { + return &mode +} + +func (r *KataConfigOpenShiftReconciler) processDaemonsetForCAA() *appsv1.DaemonSet { + var ( + runPrivileged = true + runAsUser int64 = 0 + defaultMode int32 = 0600 + sshSecretOptional = true + authJsonSecretOptional = true + nodeSelector = r.getNodeSelectorAsMap() + ) + + dsLabelSelectors := map[string]string{ + "name": caaDsName, + } + + imageString := os.Getenv("RELATED_IMAGE_CAA") + if imageString == "" { + r.Log.Info("RELATED_IMAGE_CAA env var is unset or empty, cloud-api-adaptor pods will not run") + } + + return &appsv1.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "DaemonSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: caaDsName, + Namespace: os.Getenv("PEERPODS_NAMESPACE"), + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: dsLabelSelectors, + }, + UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ + Type: "RollingUpdate", + RollingUpdate: &appsv1.RollingUpdateDaemonSet{ + MaxUnavailable: &intstr.IntOrString{ + Type: intstr.Int, + IntVal: 1, + }, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: dsLabelSelectors, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: "default", + NodeSelector: nodeSelector, + HostNetwork: true, + Containers: []corev1.Container{ + { + Name: "caa-pod", + Image: imageString, + ImagePullPolicy: "Always", + SecurityContext: &corev1.SecurityContext{ + // TODO - do we really need to run as root? + Privileged: &runPrivileged, + RunAsUser: &runAsUser, + }, + Command: []string{"/usr/local/bin/entrypoint.sh"}, + Env: []corev1.EnvVar{ + { + Name: "NODE_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + }, + EnvFrom: []corev1.EnvFromSource{ + { + SecretRef: &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "peer-pods-secret", + }, + }, + }, + { + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "peer-pods-cm", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "auth-json-volume", + MountPath: "/root/containers/", + ReadOnly: true, + }, { + Name: "ssh", + MountPath: "/root/.ssh", + ReadOnly: true, + }, + { + MountPath: "/run/peerpod", + Name: "pods-dir", + }, + { + MountPath: "/run/netns", + MountPropagation: MountProgagationRef(corev1.MountPropagationHostToContainer), + Name: "netns", + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "auth-json-volume", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "auth-json-secret", + DefaultMode: &defaultMode, + Optional: &authJsonSecretOptional, + }, + }, + }, { + Name: "ssh", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "ssh-key-secret", + DefaultMode: &defaultMode, + Optional: &sshSecretOptional, + }, + }, + }, + { + Name: "pods-dir", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/run/peerpod", + }, + }, + }, + { + Name: "netns", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/run/netns", + }, + }, + }, + }, + }, + }, + }, + } +} + +// Handles provider specific parts of the CAA Ds +// Modifies the DaemonSet if needed +func (r *KataConfigOpenShiftReconciler) processProviderConfigCAA(ds *appsv1.DaemonSet) error { + r.Log.Info("Getting cloud provider from infra") + provider, err := getCloudProviderFromInfra(r.Client) + if err != nil { + return fmt.Errorf("failed to get cloud provider from infra: %w", err) + } + + switch provider { + case IBMCloudProvider: + var expSecs int64 = 3600 + vaultTokenVolume := corev1.Volume{ + Name: "vault-token", + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: []corev1.VolumeProjection{ + { + ServiceAccountToken: &corev1.ServiceAccountTokenProjection{ + Path: "vault-token", + ExpirationSeconds: &expSecs, + Audience: "iam", + }, + }, + }, + }, + }, + } + + vaultTokenVolumeMount := corev1.VolumeMount{ + MountPath: "/var/run/secrets/tokens", + Name: "vault-token", + } + + ds.Spec.Template.Spec.Volumes = append(ds.Spec.Template.Spec.Volumes, vaultTokenVolume) + for i := range ds.Spec.Template.Spec.Containers { + container := &ds.Spec.Template.Spec.Containers[i] + if container.Name == "caa-pod" { + container.VolumeMounts = append(container.VolumeMounts, vaultTokenVolumeMount) + } + } + + return nil + default: + return nil + } +} + +func (r *KataConfigOpenShiftReconciler) createAuthJsonSecret() error { + var err error + + pullSecret := &corev1.Secret{} + err = r.Client.Get(context.TODO(), types.NamespacedName{Name: "pull-secret", Namespace: "openshift-config"}, pullSecret) + if err != nil { + r.Log.Info("Error fetching pull-secret", "err", err) + return err + } + + authJsonSecret := corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "auth-json-secret", + Namespace: OperatorNamespace, + }, + Data: map[string][]byte{ + "auth.json": pullSecret.Data[".dockerconfigjson"], + }, + Type: corev1.SecretTypeOpaque, + } + + err = r.Client.Create(context.TODO(), &authJsonSecret) + if err != nil { + if k8serrors.IsAlreadyExists(err) { + err = r.Client.Update(context.TODO(), &authJsonSecret) + if err != nil { + r.Log.Info("Error updating auth-json-secret", "err", err) + return err + } + } else { + r.Log.Info("Error creating auth-json-secret", "err", err) + return err + } + } + + return err +} + +// Create the MachineConfigs for PeerPod +// We do it before kata-oc creation to optimise the reboots required for MC creation +func (r *KataConfigOpenShiftReconciler) enablePeerPodsMc() error { + + //Create MachineConfig for kata-remote hyp CRIO config + crioMachineConfigFilePath := filepath.Join(peerpodsMachineConfigPathLocation, peerpodsCrioMachineConfigYaml) + err := r.createMcFromFile(crioMachineConfigFilePath) + if err != nil { + r.Log.Info("Error in creating CRIO MachineConfig", "err", err) + return err + } + + //Create MachineConfig for kata-remote hyp config toml + kataConfigMachineConfigFilePath := filepath.Join(peerpodsMachineConfigPathLocation, peerpodsKataRemoteMachineConfigYaml) + err = r.createMcFromFile(kataConfigMachineConfigFilePath) + if err != nil { + r.Log.Info("Error in creating kata remote configuration.toml MachineConfig", "err", err) + return err + } + + return nil +} + +// Create the PeerPodConfig CRDs and misc configs required for peer-pods +func (r *KataConfigOpenShiftReconciler) enablePeerPodsMiscConfigs() error { + // Create the CAA daemonset + ds := r.processDaemonsetForCAA() + if err := r.processProviderConfigCAA(ds); err != nil { + r.Log.Error(err, "Failed setting cloud provider specific configuration for cloud-api-adaptor DS") + return err + } + r.Log.Info("Got CAA ds manifest", "ds", ds) + + if err := controllerutil.SetControllerReference(r.kataConfig, ds, r.Scheme); err != nil { + r.Log.Error(err, "Failed setting ControllerReference for cloud-api-adaptor DS") + return err + } + + err := r.Client.Update(context.TODO(), ds) + if err != nil && k8serrors.IsNotFound(err) { + r.Log.Error(err, "cloud-api-adaptor daemonset doesn't exist. Creating") + err = r.Client.Create(context.TODO(), ds) + if err != nil { + r.Log.Error(err, "failed to create cloud-api-adaptor daemonset") + return err + } + } + + // Create the mutating webhook deployment + err = r.createMutatingWebhookDeployment() + if err != nil { + r.Log.Info("Error in creating mutating webhook deployment for peerpods", "err", err) + return err + } + + // Create the mutating webhook service + err = r.createMutatingWebhookService() + if err != nil { + r.Log.Info("Error in creating mutating webhook service for peerpods", "err", err) + return err + } + + // Create the mutating webhook + err = r.createMutatingWebhookConfig() + if err != nil { + r.Log.Info("Error in creating mutating webhook for peerpods", "err", err) + return err + } + + // Create runtimeClass config for peer-pods + err = r.createRuntimeClass(peerpodsRuntimeClassName, peerpodsRuntimeClassCpuOverhead, peerpodsRuntimeClassMemOverhead) + if err != nil { + r.Log.Info("Error in creating kata remote runtimeclass", "err", err) + return err + } + return nil +} + +func (r *KataConfigOpenShiftReconciler) disablePeerPods() error { + ds := r.processDaemonsetForCAA() + err := r.Client.Delete(context.TODO(), ds) + if err != nil { + if k8serrors.IsNotFound(err) { + r.Log.Info("cloud-api-adaptor daemonset was already deleted") + } else { + r.Log.Error(err, "error when deleting cloud-api-adaptor Daemonset, try again") + return err + } + } + + if r.DeploymentMode == MachineConfigMode { + // We are explicitly ignoring any errors in peerpodconfig and related machineconfigs removal as + // these can be removed manually if needed and this is not in the critical path + // of operator functionality + _ = r.deletePeerPodsMC() + } + + if r.DeploymentMode == DaemonSetMode { + // We want to make sure that the osc-config-sync ds removal successfully started + // So we will try again in case of an error + err = r.deletePeedPodsConfigDaemonSet() + if err != nil { + return err + } + } + + // Delete mutating webhook deployment + err = r.deleteMutatingWebhookDeployment() + if err != nil { + r.Log.Info("Error in deleting mutating webhook deployment for peerpods", "err", err) + return err + } + + // Delete mutating webhook service + err = r.deleteMutatingWebhookService() + if err != nil { + r.Log.Info("Error in deleting mutating webhook service for peerpods", "err", err) + return err + } + + // Delete the mutating webhook + err = r.deleteMutatingWebhookConfig() + if err != nil { + r.Log.Info("Error in deleting mutating webhook for peerpods", "err", err) + return err + } + + return nil +} + +func (r *KataConfigOpenShiftReconciler) deletePodVMImage() (*ctrl.Result, error) { + // Handle podvm image deletion + // Since we want to declaratively reach the final state, we need to reconcile when there are errors + // as we want the system to give a chance of fixing the error. + // For cases we don't want to reconcile, ie for ImageDeletedSuccessfully and UnsupportedPodVMImageProvider + // we should just log the message and let the code continue without explicitly returning from the method + + // Following are returned statuses: + // ImageDeletedSuccessfully + // UnsupportedPodVMImageProvider + // ImageDeletionFailed + // RequeueNeeded + // ImageDeletionStatusUnknown + status, err := ImageDelete(r.Client) + switch status { + case ImageDeletedSuccessfully: + r.setInProgressConditionToPodVMImageDeleted() + r.Log.Info("PodVM Image deleted successfully") + + case UnsupportedPodVMImageProvider: + r.setInProgressConditionToPodVMImageUnsupportedProvider() + r.Log.Info("unsupported cloud provider, skipping image deletion") + + case RequeueNeeded: + r.setInProgressConditionToPodVMImageDeleting() + return &ctrl.Result{Requeue: true, RequeueAfter: 15 * time.Second}, err + + case ImageDeletionFailed: + r.setInProgressConditionToPodVMImageDeletionFailed() + if err != nil { + // We requeue only if there is an error. + return &ctrl.Result{Requeue: true, RequeueAfter: 15 * time.Second}, err + } + // If there's no error, log and continue + r.Log.Info("Image deletion failed. Check logs for more details") + + case ImageDeletionStatusUnknown: + r.setInProgressConditionToPodVMImageDeletionUnknown() + return &ctrl.Result{Requeue: true, RequeueAfter: 15 * time.Second}, err + + default: + // For all other statuses, just log and continue + r.Log.Info("PodVM Image deletion status and error", "status", status, "error", err) + } + return nil, nil +} + +func (r *KataConfigOpenShiftReconciler) deletePeerPodsMC() error { + mc := mcfgv1.MachineConfig{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "machineconfiguration.openshift.io/v1", + Kind: "MachineConfig", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: peerpodsKataRemoteMachineConfig, + }, + } + + err := r.Client.Delete(context.TODO(), &mc) + if err != nil { + // error during removing mc. Just log the error and move on. + r.Log.Info("Error found deleting mc. If the MachineConfig object exists after uninstallation it can be safely deleted manually", + "mc", mc.Name, "err", err) + } + + mc = mcfgv1.MachineConfig{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "machineconfiguration.openshift.io/v1", + Kind: "MachineConfig", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: peerpodsCrioMachineConfig, + }, + } + + err = r.Client.Delete(context.TODO(), &mc) + if err != nil { + // error during removing mc. Just log the error and move on. + r.Log.Info("Error found deleting mc. If the MachineConfig object exists after uninstallation it can be safely deleted manually", + "mc", mc.Name, "err", err) + } + + return nil +} From 439695e54a3230eb100fb34465873a4f35578c18 Mon Sep 17 00:00:00 2001 From: Greg Kurz Date: Tue, 2 Sep 2025 14:34:04 +0200 Subject: [PATCH 2/3] Refactor peer pods enablement code Consolidate the peer pods enablement code in a separate function. This doesn't change behavior. Signed-off-by: Greg Kurz --- controllers/openshift_controller.go | 65 ++------------------------- controllers/peerpods.go | 70 +++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+), 62 deletions(-) diff --git a/controllers/openshift_controller.go b/controllers/openshift_controller.go index 3f7aceb6d..496f92f8e 100644 --- a/controllers/openshift_controller.go +++ b/controllers/openshift_controller.go @@ -2225,69 +2225,10 @@ func (r *KataConfigOpenShiftReconciler) postKataInstallation() (*ctrl.Result, er // create Pod VM image CRD and runtimeclass for peerpods if r.kataConfig.Spec.EnablePeerPods { - //Get pull-secret from openshift-config ns and save it as auth-json-secret in our ns - //This will be used by the podvm image provider to pull the pause image for embedding - err = r.createAuthJsonSecret() - if err != nil { - r.Log.Info("Error in creating auth-json-secret", "err", err) - return &ctrl.Result{Requeue: true, RequeueAfter: 15 * time.Second}, err - } - - // Create the podvm image - // Since we want to declaratively reach the final state, we need to reconcile when there are errors - // as we want the system to give a chance of fixing the error. - // For cases we don't want to reconcile, ie for ImageCreatedSuccessfully and UnsupportedPodVMImageProvider - // we should just log the message and let the code continue without explicitly returning from the method - - // Following are the returned statuses: - // ImageCreatedSuccessfully - // UnsupportedPodVMImageProvider - // ImageCreationFailed - // RequeueNeeded - // ImageCreationStatusUnknown - - status, err := ImageCreate(r.Client) - switch status { - case ImageCreatedSuccessfully: - r.setInProgressConditionToPodVMImageCreated() - r.Log.Info("PodVM Image created successfully") - - case UnsupportedPodVMImageProvider: - r.setInProgressConditionToPodVMImageUnsupportedProvider() - r.Log.Info("unsupported cloud provider, skipping image creation") - - case RequeueNeeded: - r.setInProgressConditionToPodVMImageCreating() - return &ctrl.Result{Requeue: true, RequeueAfter: 15 * time.Second}, err - - case ImageCreationFailed: - r.setInProgressConditionToPodVMImageCreationFailed() - if err != nil { - // We requeue only if there is an error. - return &ctrl.Result{Requeue: true, RequeueAfter: 15 * time.Second}, err - } - // If there's no error, log and continue - r.Log.Info("Image creation failed. Check logs for more details") - - case ImageCreationStatusUnknown: - r.setInProgressConditionToPodVMImageCreationUnknown() - - // Reconcile with error - return &ctrl.Result{Requeue: true, RequeueAfter: 15 * time.Second}, err - - default: - // For all other statuses, just log and continue - r.Log.Info("PodVM Image creation status and error", "status", status, "error", err) - } - - err = r.enablePeerPodsMiscConfigs() - if err != nil { - r.Log.Info("Enabling peerpodconfig CR, runtimeclass etc", "err", err) - return &ctrl.Result{Requeue: true, RequeueAfter: 15 * time.Second}, err + res, err := r.enablePeerPods() + if res != nil || err != nil { + return res, err } - - // Reset the in progress condition - r.resetInProgressCondition() } return nil, nil } diff --git a/controllers/peerpods.go b/controllers/peerpods.go index 36938c0c3..961b4b17b 100644 --- a/controllers/peerpods.go +++ b/controllers/peerpods.go @@ -504,3 +504,73 @@ func (r *KataConfigOpenShiftReconciler) deletePeerPodsMC() error { return nil } + +func (r *KataConfigOpenShiftReconciler) enablePeerPods() (*ctrl.Result, error) { + //Get pull-secret from openshift-config ns and save it as auth-json-secret in our ns + //This will be used by the podvm image provider to pull the pause image for embedding + err := r.createAuthJsonSecret() + if err != nil { + r.Log.Info("Error in creating auth-json-secret", "err", err) + return &ctrl.Result{Requeue: true, RequeueAfter: 15 * time.Second}, err + } + + // Create the podvm image + // Since we want to declaratively reach the final state, we need to reconcile when there are errors + // as we want the system to give a chance of fixing the error. + // For cases we don't want to reconcile, ie for ImageCreatedSuccessfully and UnsupportedPodVMImageProvider + // we should just log the message and let the code continue without explicitly returning from the method + + // Following are the returned statuses: + // ImageCreatedSuccessfully + // UnsupportedPodVMImageProvider + // ImageCreationFailed + // RequeueNeeded + // ImageCreationStatusUnknown + + status, err := ImageCreate(r.Client) + switch status { + case ImageCreatedSuccessfully: + r.setInProgressConditionToPodVMImageCreated() + r.Log.Info("PodVM Image created successfully") + + case UnsupportedPodVMImageProvider: + r.setInProgressConditionToPodVMImageUnsupportedProvider() + r.Log.Info("unsupported cloud provider, skipping image creation") + + case RequeueNeeded: + r.setInProgressConditionToPodVMImageCreating() + return &ctrl.Result{Requeue: true, RequeueAfter: 15 * time.Second}, err + + case ImageCreationFailed: + r.setInProgressConditionToPodVMImageCreationFailed() + if err != nil { + // We requeue only if there is an error. + return &ctrl.Result{Requeue: true, RequeueAfter: 15 * time.Second}, err + } + // If there's no error, log and continue + r.Log.Info("Image creation failed. Check logs for more details") + + case ImageCreationStatusUnknown: + r.setInProgressConditionToPodVMImageCreationUnknown() + + // Reconcile with error + return &ctrl.Result{Requeue: true, RequeueAfter: time.Second * 15}, err + + default: + // For all other statuses, just log and continue + r.Log.Info("PodVM Image creation status and error", "status", status, "error", err) + } + + err = r.enablePeerPodsMiscConfigs() + if err != nil { + r.Log.Info("Enabling peerpodconfig CR, runtimeclass etc", "err", err) + // Give sometime for the error to go away before reconciling again + return &ctrl.Result{Requeue: true, RequeueAfter: 15 * time.Second}, err + + } + + // Reset the in progress condition + r.resetInProgressCondition() + + return nil, nil +} From 46ccd5296436a9e8d039346ba25de6f37c830f80 Mon Sep 17 00:00:00 2001 From: Greg Kurz Date: Thu, 4 Sep 2025 07:49:12 +0200 Subject: [PATCH 3/3] Refactor peer pods disablement code Consolidate the peer pods disablement code in a separate function. For the sake of symmetry with enablePeerPods, name it disablePeerPods. The existing disablePeerPods is arbitrary renamed to disablePeerPodsMiscConfigs. Signed-off-by: Greg Kurz --- controllers/daemonset_reconcile.go | 2 +- controllers/openshift_controller.go | 14 ++------------ controllers/peerpods.go | 21 ++++++++++++++++++++- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/controllers/daemonset_reconcile.go b/controllers/daemonset_reconcile.go index dd6a86391..a35c07351 100644 --- a/controllers/daemonset_reconcile.go +++ b/controllers/daemonset_reconcile.go @@ -163,7 +163,7 @@ func (r *KataConfigOpenShiftReconciler) processKataConfigDeleteRequestDaemonSet( if r.kataConfig.Spec.EnablePeerPods { // We want to make sure that the osc-config-sync daemonset removal successfully started // So we will try again in case of an error - err = r.disablePeerPods() + err = r.disablePeerPodsMiscConfigs() if err != nil { return ctrl.Result{Requeue: true, RequeueAfter: time.Second * 15}, err } diff --git a/controllers/openshift_controller.go b/controllers/openshift_controller.go index 496f92f8e..17b24decf 100644 --- a/controllers/openshift_controller.go +++ b/controllers/openshift_controller.go @@ -1020,20 +1020,10 @@ func (r *KataConfigOpenShiftReconciler) processKataConfigDeleteRequest() (ctrl.R } if r.kataConfig.Spec.EnablePeerPods { - // We are explicitly ignoring any errors in peerpodconfig and related machineconfigs removal as - // these can be removed manually if needed and this is not in the critical path - // of operator functionality - _ = r.disablePeerPods() - - // Handle podvm image deletion - res, err := r.deletePodVMImage() - if res != nil { + res, err := r.disablePeerPods() + if res != nil || err != nil { return *res, err } - // FIXME : dead code, revisit deletePodVMImage() return paths - if err != nil { - return ctrl.Result{Requeue: true, RequeueAfter: 15 * time.Second}, err - } } err = r.deleteScc() diff --git a/controllers/peerpods.go b/controllers/peerpods.go index 961b4b17b..9fc4ab166 100644 --- a/controllers/peerpods.go +++ b/controllers/peerpods.go @@ -368,7 +368,7 @@ func (r *KataConfigOpenShiftReconciler) enablePeerPodsMiscConfigs() error { return nil } -func (r *KataConfigOpenShiftReconciler) disablePeerPods() error { +func (r *KataConfigOpenShiftReconciler) disablePeerPodsMiscConfigs() error { ds := r.processDaemonsetForCAA() err := r.Client.Delete(context.TODO(), ds) if err != nil { @@ -574,3 +574,22 @@ func (r *KataConfigOpenShiftReconciler) enablePeerPods() (*ctrl.Result, error) { return nil, nil } + +func (r *KataConfigOpenShiftReconciler) disablePeerPods() (*ctrl.Result, error) { + // We are explicitly ignoring any errors in peerpodconfig and related machineconfigs removal as + // these can be removed manually if needed and this is not in the critical path + // of operator functionality + _ = r.disablePeerPodsMiscConfigs() + + // Handle podvm image deletion + res, err := r.deletePodVMImage() + if res != nil { + return res, err + } + // FIXME : dead code, revisit deletePodVMImage() return paths + if err != nil { + return &ctrl.Result{Requeue: true, RequeueAfter: 15 * time.Second}, err + } + + return nil, nil +}