From 27341f3b2b1c420c5a39d11965fadaef98292667 Mon Sep 17 00:00:00 2001 From: Jason Parraga Date: Wed, 24 Jul 2024 06:23:08 -0700 Subject: [PATCH] fix: Reference service accounts in deployments (#316) * Reference service accounts in deployments Signed-off-by: Jason Parraga * Handle custom service accounts Signed-off-by: Jason Parraga --------- Signed-off-by: Jason Parraga --- .../install/armadaserver_controller.go | 22 +++++++++----- .../install/eventingester_controller.go | 22 ++++++++++---- .../controller/install/lookout_controller.go | 22 ++++++++++---- .../install/lookoutingester_controller.go | 26 +++++++++++----- .../install/scheduler_controller.go | 22 ++++++++++---- .../install/scheduleringester_controller.go | 30 ++++++++++++------- 6 files changed, 100 insertions(+), 44 deletions(-) diff --git a/internal/controller/install/armadaserver_controller.go b/internal/controller/install/armadaserver_controller.go index 7c0019a1..5966a173 100644 --- a/internal/controller/install/armadaserver_controller.go +++ b/internal/controller/install/armadaserver_controller.go @@ -233,7 +233,17 @@ func generateArmadaServerInstallComponents(as *installv1alpha1.ArmadaServer, sch return nil, err } - deployment, err := createArmadaServerDeployment(as) + var serviceAccount *corev1.ServiceAccount + serviceAccountName := as.Spec.CustomServiceAccount + if serviceAccountName == "" { + serviceAccount = builders.CreateServiceAccount(as.Name, as.Namespace, AllLabels(as.Name, as.Labels), as.Spec.ServiceAccount) + if err = controllerutil.SetOwnerReference(as, serviceAccount, scheme); err != nil { + return nil, errors.WithStack(err) + } + serviceAccountName = serviceAccount.Name + } + + deployment, err := createArmadaServerDeployment(as, serviceAccountName) if err != nil { return nil, err } @@ -266,11 +276,6 @@ func generateArmadaServerInstallComponents(as *installv1alpha1.ArmadaServer, sch return nil, err } - svcAcct := builders.CreateServiceAccount(as.Name, as.Namespace, AllLabels(as.Name, as.Labels), as.Spec.ServiceAccount) - if err := controllerutil.SetOwnerReference(as, svcAcct, scheme); err != nil { - return nil, err - } - pdb := createPodDisruptionBudget(as) if err := controllerutil.SetOwnerReference(as, pdb, scheme); err != nil { return nil, err @@ -304,7 +309,7 @@ func generateArmadaServerInstallComponents(as *installv1alpha1.ArmadaServer, sch IngressGrpc: ingressGrpc, IngressHttp: ingressHttp, Service: service, - ServiceAccount: svcAcct, + ServiceAccount: serviceAccount, Secret: secret, PodDisruptionBudget: pdb, PrometheusRule: pr, @@ -483,7 +488,7 @@ func createArmadaServerMigrationJobs(as *installv1alpha1.ArmadaServer) ([]*batch return []*batchv1.Job{&pulsarWaitJob, &initPulsarJob}, nil } -func createArmadaServerDeployment(as *installv1alpha1.ArmadaServer) (*appsv1.Deployment, error) { +func createArmadaServerDeployment(as *installv1alpha1.ArmadaServer, serviceAccountName string) (*appsv1.Deployment, error) { var replicas int32 = 1 var runAsUser int64 = 1000 var runAsGroup int64 = 2000 @@ -519,6 +524,7 @@ func createArmadaServerDeployment(as *installv1alpha1.ArmadaServer) (*appsv1.Dep }, }, Spec: corev1.PodSpec{ + ServiceAccountName: serviceAccountName, TerminationGracePeriodSeconds: as.DeletionGracePeriodSeconds, SecurityContext: &corev1.PodSecurityContext{ RunAsUser: &runAsUser, diff --git a/internal/controller/install/eventingester_controller.go b/internal/controller/install/eventingester_controller.go index ff05d692..280e3a99 100644 --- a/internal/controller/install/eventingester_controller.go +++ b/internal/controller/install/eventingester_controller.go @@ -20,6 +20,8 @@ import ( "context" "time" + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -134,7 +136,18 @@ func (r *EventIngesterReconciler) generateEventIngesterComponents(eventIngester if err := controllerutil.SetOwnerReference(eventIngester, secret, scheme); err != nil { return nil, err } - deployment, err := r.createDeployment(eventIngester) + + var serviceAccount *corev1.ServiceAccount + serviceAccountName := eventIngester.Spec.CustomServiceAccount + if serviceAccountName == "" { + serviceAccount = builders.CreateServiceAccount(eventIngester.Name, eventIngester.Namespace, AllLabels(eventIngester.Name, eventIngester.Labels), eventIngester.Spec.ServiceAccount) + if err = controllerutil.SetOwnerReference(eventIngester, serviceAccount, scheme); err != nil { + return nil, errors.WithStack(err) + } + serviceAccountName = serviceAccount.Name + } + + deployment, err := r.createDeployment(eventIngester, serviceAccountName) if err != nil { return nil, err } @@ -142,10 +155,6 @@ func (r *EventIngesterReconciler) generateEventIngesterComponents(eventIngester return nil, err } - serviceAccount := builders.CreateServiceAccount(eventIngester.Name, eventIngester.Namespace, AllLabels(eventIngester.Name, eventIngester.Labels), eventIngester.Spec.ServiceAccount) - if err := controllerutil.SetOwnerReference(eventIngester, serviceAccount, scheme); err != nil { - return nil, err - } return &CommonComponents{ Deployment: deployment, ServiceAccount: serviceAccount, @@ -153,7 +162,7 @@ func (r *EventIngesterReconciler) generateEventIngesterComponents(eventIngester }, nil } -func (r *EventIngesterReconciler) createDeployment(eventIngester *installv1alpha1.EventIngester) (*appsv1.Deployment, error) { +func (r *EventIngesterReconciler) createDeployment(eventIngester *installv1alpha1.EventIngester, serviceAccountName string) (*appsv1.Deployment, error) { var runAsUser int64 = 1000 var runAsGroup int64 = 2000 allowPrivilegeEscalation := false @@ -188,6 +197,7 @@ func (r *EventIngesterReconciler) createDeployment(eventIngester *installv1alpha Annotations: map[string]string{"checksum/config": GenerateChecksumConfig(eventIngester.Spec.ApplicationConfig.Raw)}, }, Spec: corev1.PodSpec{ + ServiceAccountName: serviceAccountName, TerminationGracePeriodSeconds: eventIngester.Spec.TerminationGracePeriodSeconds, SecurityContext: &corev1.PodSecurityContext{ RunAsUser: &runAsUser, diff --git a/internal/controller/install/lookout_controller.go b/internal/controller/install/lookout_controller.go index d60c09e8..e1817e4b 100644 --- a/internal/controller/install/lookout_controller.go +++ b/internal/controller/install/lookout_controller.go @@ -17,6 +17,8 @@ import ( "context" "time" + "github.com/pkg/errors" + "k8s.io/utils/ptr" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" @@ -203,7 +205,18 @@ func generateLookoutInstallComponents(lookout *installv1alpha1.Lookout, scheme * if err := controllerutil.SetOwnerReference(lookout, secret, scheme); err != nil { return nil, err } - deployment, err := createLookoutDeployment(lookout) + + var serviceAccount *corev1.ServiceAccount + serviceAccountName := lookout.Spec.CustomServiceAccount + if serviceAccountName == "" { + serviceAccount = builders.CreateServiceAccount(lookout.Name, lookout.Namespace, AllLabels(lookout.Name, lookout.Labels), lookout.Spec.ServiceAccount) + if err = controllerutil.SetOwnerReference(lookout, serviceAccount, scheme); err != nil { + return nil, errors.WithStack(err) + } + serviceAccountName = serviceAccount.Name + } + + deployment, err := createLookoutDeployment(lookout, serviceAccountName) if err != nil { return nil, err } @@ -215,10 +228,6 @@ func generateLookoutInstallComponents(lookout *installv1alpha1.Lookout, scheme * if err := controllerutil.SetOwnerReference(lookout, service, scheme); err != nil { return nil, err } - serviceAccount := builders.CreateServiceAccount(lookout.Name, lookout.Namespace, AllLabels(lookout.Name, lookout.Labels), lookout.Spec.ServiceAccount) - if err := controllerutil.SetOwnerReference(lookout, serviceAccount, scheme); err != nil { - return nil, err - } var serviceMonitor *monitoringv1.ServiceMonitor if lookout.Spec.Prometheus != nil && lookout.Spec.Prometheus.Enabled { @@ -290,7 +299,7 @@ func createLookoutServiceMonitor(lookout *installv1alpha1.Lookout) *monitoringv1 // Function to build the deployment object for Lookout. // This should be changing from CRD to CRD. Not sure if generailize this helps much -func createLookoutDeployment(lookout *installv1alpha1.Lookout) (*appsv1.Deployment, error) { +func createLookoutDeployment(lookout *installv1alpha1.Lookout, serviceAccountName string) (*appsv1.Deployment, error) { var runAsUser int64 = 1000 var runAsGroup int64 = 2000 allowPrivilegeEscalation := false @@ -313,6 +322,7 @@ func createLookoutDeployment(lookout *installv1alpha1.Lookout) (*appsv1.Deployme Annotations: map[string]string{"checksum/config": GenerateChecksumConfig(lookout.Spec.ApplicationConfig.Raw)}, }, Spec: corev1.PodSpec{ + ServiceAccountName: serviceAccountName, TerminationGracePeriodSeconds: lookout.DeletionGracePeriodSeconds, SecurityContext: &corev1.PodSecurityContext{ RunAsUser: &runAsUser, diff --git a/internal/controller/install/lookoutingester_controller.go b/internal/controller/install/lookoutingester_controller.go index 9bae84a4..b6af1cba 100644 --- a/internal/controller/install/lookoutingester_controller.go +++ b/internal/controller/install/lookoutingester_controller.go @@ -21,6 +21,8 @@ import ( "fmt" "time" + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -70,7 +72,7 @@ func (r *LookoutIngesterReconciler) Reconcile(ctx context.Context, req ctrl.Requ } lookoutIngester.Spec.PortConfig = pc - components, err := r.generateInstallComponents(&lookoutIngester) + components, err := r.generateInstallComponents(&lookoutIngester, r.Scheme) if err != nil { return ctrl.Result{}, err } @@ -128,7 +130,7 @@ func (r *LookoutIngesterReconciler) SetupWithManager(mgr ctrl.Manager) error { Complete(r) } -func (r *LookoutIngesterReconciler) generateInstallComponents(lookoutIngester *installv1alpha1.LookoutIngester) (*CommonComponents, error) { +func (r *LookoutIngesterReconciler) generateInstallComponents(lookoutIngester *installv1alpha1.LookoutIngester, scheme *runtime.Scheme) (*CommonComponents, error) { secret, err := builders.CreateSecret(lookoutIngester.Spec.ApplicationConfig, lookoutIngester.Name, lookoutIngester.Namespace, GetConfigFilename(lookoutIngester.Name)) if err != nil { return nil, err @@ -136,17 +138,24 @@ func (r *LookoutIngesterReconciler) generateInstallComponents(lookoutIngester *i if err := controllerutil.SetOwnerReference(lookoutIngester, secret, r.Scheme); err != nil { return nil, err } - deployment, err := r.createDeployment(lookoutIngester) + + var serviceAccount *corev1.ServiceAccount + serviceAccountName := lookoutIngester.Spec.CustomServiceAccount + if serviceAccountName == "" { + serviceAccount = builders.CreateServiceAccount(lookoutIngester.Name, lookoutIngester.Namespace, AllLabels(lookoutIngester.Name, lookoutIngester.Labels), lookoutIngester.Spec.ServiceAccount) + if err = controllerutil.SetOwnerReference(lookoutIngester, serviceAccount, scheme); err != nil { + return nil, errors.WithStack(err) + } + serviceAccountName = serviceAccount.Name + } + + deployment, err := r.createDeployment(lookoutIngester, serviceAccountName) if err != nil { return nil, err } if err := controllerutil.SetOwnerReference(lookoutIngester, deployment, r.Scheme); err != nil { return nil, err } - serviceAccount := builders.CreateServiceAccount(lookoutIngester.Name, lookoutIngester.Namespace, AllLabels(lookoutIngester.Name, lookoutIngester.Labels), lookoutIngester.Spec.ServiceAccount) - if err := controllerutil.SetOwnerReference(lookoutIngester, serviceAccount, r.Scheme); err != nil { - return nil, err - } return &CommonComponents{ Deployment: deployment, @@ -156,7 +165,7 @@ func (r *LookoutIngesterReconciler) generateInstallComponents(lookoutIngester *i } // TODO: Flesh this out for lookoutingester -func (r *LookoutIngesterReconciler) createDeployment(lookoutIngester *installv1alpha1.LookoutIngester) (*appsv1.Deployment, error) { +func (r *LookoutIngesterReconciler) createDeployment(lookoutIngester *installv1alpha1.LookoutIngester, serviceAccountName string) (*appsv1.Deployment, error) { var replicas int32 = 1 var runAsUser int64 = 1000 var runAsGroup int64 = 2000 @@ -187,6 +196,7 @@ func (r *LookoutIngesterReconciler) createDeployment(lookoutIngester *installv1a Annotations: map[string]string{"checksum/config": GenerateChecksumConfig(lookoutIngester.Spec.ApplicationConfig.Raw)}, }, Spec: corev1.PodSpec{ + ServiceAccountName: serviceAccountName, TerminationGracePeriodSeconds: lookoutIngester.Spec.TerminationGracePeriodSeconds, SecurityContext: &corev1.PodSecurityContext{ RunAsUser: &runAsUser, diff --git a/internal/controller/install/scheduler_controller.go b/internal/controller/install/scheduler_controller.go index 56c91fc9..c0f980eb 100644 --- a/internal/controller/install/scheduler_controller.go +++ b/internal/controller/install/scheduler_controller.go @@ -18,6 +18,8 @@ import ( "fmt" "time" + "github.com/pkg/errors" + "k8s.io/utils/ptr" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" @@ -197,7 +199,18 @@ func generateSchedulerInstallComponents(scheduler *installv1alpha1.Scheduler, sc if err := controllerutil.SetOwnerReference(scheduler, secret, scheme); err != nil { return nil, err } - deployment, err := createSchedulerDeployment(scheduler) + + var serviceAccount *corev1.ServiceAccount + serviceAccountName := scheduler.Spec.CustomServiceAccount + if serviceAccountName == "" { + serviceAccount = builders.CreateServiceAccount(scheduler.Name, scheduler.Namespace, AllLabels(scheduler.Name, scheduler.Labels), scheduler.Spec.ServiceAccount) + if err = controllerutil.SetOwnerReference(scheduler, serviceAccount, scheme); err != nil { + return nil, errors.WithStack(err) + } + serviceAccountName = serviceAccount.Name + } + + deployment, err := createSchedulerDeployment(scheduler, serviceAccountName) if err != nil { return nil, err } @@ -209,10 +222,6 @@ func generateSchedulerInstallComponents(scheduler *installv1alpha1.Scheduler, sc if err := controllerutil.SetOwnerReference(scheduler, service, scheme); err != nil { return nil, err } - serviceAccount := builders.CreateServiceAccount(scheduler.Name, scheduler.Namespace, AllLabels(scheduler.Name, scheduler.Labels), scheduler.Spec.ServiceAccount) - if err := controllerutil.SetOwnerReference(scheduler, serviceAccount, scheme); err != nil { - return nil, err - } var serviceMonitor *monitoringv1.ServiceMonitor if scheduler.Spec.Prometheus != nil && scheduler.Spec.Prometheus.Enabled { @@ -284,7 +293,7 @@ func createSchedulerServiceMonitor(scheduler *installv1alpha1.Scheduler) *monito // Function to build the deployment object for Scheduler. // This should be changing from CRD to CRD. Not sure if generailize this helps much -func createSchedulerDeployment(scheduler *installv1alpha1.Scheduler) (*appsv1.Deployment, error) { +func createSchedulerDeployment(scheduler *installv1alpha1.Scheduler, serviceAccountName string) (*appsv1.Deployment, error) { var runAsUser int64 = 1000 var runAsGroup int64 = 2000 allowPrivilegeEscalation := false @@ -307,6 +316,7 @@ func createSchedulerDeployment(scheduler *installv1alpha1.Scheduler) (*appsv1.De Annotations: map[string]string{"checksum/config": GenerateChecksumConfig(scheduler.Spec.ApplicationConfig.Raw)}, }, Spec: corev1.PodSpec{ + ServiceAccountName: serviceAccountName, TerminationGracePeriodSeconds: scheduler.DeletionGracePeriodSeconds, SecurityContext: &corev1.PodSecurityContext{ RunAsUser: &runAsUser, diff --git a/internal/controller/install/scheduleringester_controller.go b/internal/controller/install/scheduleringester_controller.go index 57c95d8f..744cd5fe 100644 --- a/internal/controller/install/scheduleringester_controller.go +++ b/internal/controller/install/scheduleringester_controller.go @@ -20,6 +20,8 @@ import ( "context" "time" + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -126,26 +128,33 @@ func (r *SchedulerIngesterReconciler) SetupWithManager(mgr ctrl.Manager) error { Complete(r) } -func (r *SchedulerIngesterReconciler) generateSchedulerIngesterComponents(scheduleringester *installv1alpha1.SchedulerIngester, scheme *runtime.Scheme) (*CommonComponents, error) { - secret, err := builders.CreateSecret(scheduleringester.Spec.ApplicationConfig, scheduleringester.Name, scheduleringester.Namespace, GetConfigFilename(scheduleringester.Name)) +func (r *SchedulerIngesterReconciler) generateSchedulerIngesterComponents(schedulerIngester *installv1alpha1.SchedulerIngester, scheme *runtime.Scheme) (*CommonComponents, error) { + secret, err := builders.CreateSecret(schedulerIngester.Spec.ApplicationConfig, schedulerIngester.Name, schedulerIngester.Namespace, GetConfigFilename(schedulerIngester.Name)) if err != nil { return nil, err } - if err := controllerutil.SetOwnerReference(scheduleringester, secret, scheme); err != nil { + if err := controllerutil.SetOwnerReference(schedulerIngester, secret, scheme); err != nil { return nil, err } - deployment, err := r.createDeployment(scheduleringester) + + var serviceAccount *corev1.ServiceAccount + serviceAccountName := schedulerIngester.Spec.CustomServiceAccount + if serviceAccountName == "" { + serviceAccount = builders.CreateServiceAccount(schedulerIngester.Name, schedulerIngester.Namespace, AllLabels(schedulerIngester.Name, schedulerIngester.Labels), schedulerIngester.Spec.ServiceAccount) + if err = controllerutil.SetOwnerReference(schedulerIngester, serviceAccount, scheme); err != nil { + return nil, errors.WithStack(err) + } + serviceAccountName = serviceAccount.Name + } + + deployment, err := r.createDeployment(schedulerIngester, serviceAccountName) if err != nil { return nil, err } - if err := controllerutil.SetOwnerReference(scheduleringester, deployment, scheme); err != nil { + if err := controllerutil.SetOwnerReference(schedulerIngester, deployment, scheme); err != nil { return nil, err } - serviceAccount := builders.CreateServiceAccount(scheduleringester.Name, scheduleringester.Namespace, AllLabels(scheduleringester.Name, scheduleringester.Labels), scheduleringester.Spec.ServiceAccount) - if err := controllerutil.SetOwnerReference(scheduleringester, serviceAccount, scheme); err != nil { - return nil, err - } return &CommonComponents{ Deployment: deployment, ServiceAccount: serviceAccount, @@ -153,7 +162,7 @@ func (r *SchedulerIngesterReconciler) generateSchedulerIngesterComponents(schedu }, nil } -func (r *SchedulerIngesterReconciler) createDeployment(scheduleringester *installv1alpha1.SchedulerIngester) (*appsv1.Deployment, error) { +func (r *SchedulerIngesterReconciler) createDeployment(scheduleringester *installv1alpha1.SchedulerIngester, serviceAccountName string) (*appsv1.Deployment, error) { var runAsUser int64 = 1000 var runAsGroup int64 = 2000 allowPrivilegeEscalation := false @@ -188,6 +197,7 @@ func (r *SchedulerIngesterReconciler) createDeployment(scheduleringester *instal Annotations: map[string]string{"checksum/config": GenerateChecksumConfig(scheduleringester.Spec.ApplicationConfig.Raw)}, }, Spec: corev1.PodSpec{ + ServiceAccountName: serviceAccountName, TerminationGracePeriodSeconds: scheduleringester.Spec.TerminationGracePeriodSeconds, SecurityContext: &corev1.PodSecurityContext{ RunAsUser: &runAsUser,