Skip to content

Commit

Permalink
fix: Reference service accounts in deployments (#316)
Browse files Browse the repository at this point in the history 
* Reference service accounts in deployments

Signed-off-by: Jason Parraga <[email protected]>

* Handle custom service accounts

Signed-off-by: Jason Parraga <[email protected]>

---------

Signed-off-by: Jason Parraga <[email protected]>
  • Loading branch information
@Sovietaced
Sovietaced authored Jul 24, 2024
1 parent 251ffab commit 27341f3
Show file tree
Hide file tree
Showing 6 changed files with 100 additions and 44 deletions.
22 changes: 14 additions & 8 deletions internal/controller/install/armadaserver_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,17 @@ func generateArmadaServerInstallComponents(as *installv1alpha1.ArmadaServer, sch
return nil, err
}

deployment, err := createArmadaServerDeployment(as)
var serviceAccount *corev1.ServiceAccount
serviceAccountName := as.Spec.CustomServiceAccount
if serviceAccountName == "" {
serviceAccount = builders.CreateServiceAccount(as.Name, as.Namespace, AllLabels(as.Name, as.Labels), as.Spec.ServiceAccount)
if err = controllerutil.SetOwnerReference(as, serviceAccount, scheme); err != nil {
return nil, errors.WithStack(err)
}
serviceAccountName = serviceAccount.Name
}

deployment, err := createArmadaServerDeployment(as, serviceAccountName)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -266,11 +276,6 @@ func generateArmadaServerInstallComponents(as *installv1alpha1.ArmadaServer, sch
return nil, err
}

svcAcct := builders.CreateServiceAccount(as.Name, as.Namespace, AllLabels(as.Name, as.Labels), as.Spec.ServiceAccount)
if err := controllerutil.SetOwnerReference(as, svcAcct, scheme); err != nil {
return nil, err
}

pdb := createPodDisruptionBudget(as)
if err := controllerutil.SetOwnerReference(as, pdb, scheme); err != nil {
return nil, err
Expand Down Expand Up @@ -304,7 +309,7 @@ func generateArmadaServerInstallComponents(as *installv1alpha1.ArmadaServer, sch
IngressGrpc: ingressGrpc,
IngressHttp: ingressHttp,
Service: service,
ServiceAccount: svcAcct,
ServiceAccount: serviceAccount,
Secret: secret,
PodDisruptionBudget: pdb,
PrometheusRule: pr,
Expand Down Expand Up @@ -483,7 +488,7 @@ func createArmadaServerMigrationJobs(as *installv1alpha1.ArmadaServer) ([]*batch
return []*batchv1.Job{&pulsarWaitJob, &initPulsarJob}, nil
}

func createArmadaServerDeployment(as *installv1alpha1.ArmadaServer) (*appsv1.Deployment, error) {
func createArmadaServerDeployment(as *installv1alpha1.ArmadaServer, serviceAccountName string) (*appsv1.Deployment, error) {
var replicas int32 = 1
var runAsUser int64 = 1000
var runAsGroup int64 = 2000
Expand Down Expand Up @@ -519,6 +524,7 @@ func createArmadaServerDeployment(as *installv1alpha1.ArmadaServer) (*appsv1.Dep
},
},
Spec: corev1.PodSpec{
ServiceAccountName: serviceAccountName,
TerminationGracePeriodSeconds: as.DeletionGracePeriodSeconds,
SecurityContext: &corev1.PodSecurityContext{
RunAsUser: &runAsUser,
Expand Down
22 changes: 16 additions & 6 deletions internal/controller/install/eventingester_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ import (
"context"
"time"

"github.com/pkg/errors"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
Expand Down Expand Up @@ -134,26 +136,33 @@ func (r *EventIngesterReconciler) generateEventIngesterComponents(eventIngester
if err := controllerutil.SetOwnerReference(eventIngester, secret, scheme); err != nil {
return nil, err
}
deployment, err := r.createDeployment(eventIngester)

var serviceAccount *corev1.ServiceAccount
serviceAccountName := eventIngester.Spec.CustomServiceAccount
if serviceAccountName == "" {
serviceAccount = builders.CreateServiceAccount(eventIngester.Name, eventIngester.Namespace, AllLabels(eventIngester.Name, eventIngester.Labels), eventIngester.Spec.ServiceAccount)
if err = controllerutil.SetOwnerReference(eventIngester, serviceAccount, scheme); err != nil {
return nil, errors.WithStack(err)
}
serviceAccountName = serviceAccount.Name
}

deployment, err := r.createDeployment(eventIngester, serviceAccountName)
if err != nil {
return nil, err
}
if err := controllerutil.SetOwnerReference(eventIngester, deployment, scheme); err != nil {
return nil, err
}

serviceAccount := builders.CreateServiceAccount(eventIngester.Name, eventIngester.Namespace, AllLabels(eventIngester.Name, eventIngester.Labels), eventIngester.Spec.ServiceAccount)
if err := controllerutil.SetOwnerReference(eventIngester, serviceAccount, scheme); err != nil {
return nil, err
}
return &CommonComponents{
Deployment: deployment,
ServiceAccount: serviceAccount,
Secret: secret,
}, nil
}

func (r *EventIngesterReconciler) createDeployment(eventIngester *installv1alpha1.EventIngester) (*appsv1.Deployment, error) {
func (r *EventIngesterReconciler) createDeployment(eventIngester *installv1alpha1.EventIngester, serviceAccountName string) (*appsv1.Deployment, error) {
var runAsUser int64 = 1000
var runAsGroup int64 = 2000
allowPrivilegeEscalation := false
Expand Down Expand Up @@ -188,6 +197,7 @@ func (r *EventIngesterReconciler) createDeployment(eventIngester *installv1alpha
Annotations: map[string]string{"checksum/config": GenerateChecksumConfig(eventIngester.Spec.ApplicationConfig.Raw)},
},
Spec: corev1.PodSpec{
ServiceAccountName: serviceAccountName,
TerminationGracePeriodSeconds: eventIngester.Spec.TerminationGracePeriodSeconds,
SecurityContext: &corev1.PodSecurityContext{
RunAsUser: &runAsUser,
Expand Down
22 changes: 16 additions & 6 deletions internal/controller/install/lookout_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ import (
"context"
"time"

"github.com/pkg/errors"

"k8s.io/utils/ptr"

monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
Expand Down Expand Up @@ -203,7 +205,18 @@ func generateLookoutInstallComponents(lookout *installv1alpha1.Lookout, scheme *
if err := controllerutil.SetOwnerReference(lookout, secret, scheme); err != nil {
return nil, err
}
deployment, err := createLookoutDeployment(lookout)

var serviceAccount *corev1.ServiceAccount
serviceAccountName := lookout.Spec.CustomServiceAccount
if serviceAccountName == "" {
serviceAccount = builders.CreateServiceAccount(lookout.Name, lookout.Namespace, AllLabels(lookout.Name, lookout.Labels), lookout.Spec.ServiceAccount)
if err = controllerutil.SetOwnerReference(lookout, serviceAccount, scheme); err != nil {
return nil, errors.WithStack(err)
}
serviceAccountName = serviceAccount.Name
}

deployment, err := createLookoutDeployment(lookout, serviceAccountName)
if err != nil {
return nil, err
}
Expand All @@ -215,10 +228,6 @@ func generateLookoutInstallComponents(lookout *installv1alpha1.Lookout, scheme *
if err := controllerutil.SetOwnerReference(lookout, service, scheme); err != nil {
return nil, err
}
serviceAccount := builders.CreateServiceAccount(lookout.Name, lookout.Namespace, AllLabels(lookout.Name, lookout.Labels), lookout.Spec.ServiceAccount)
if err := controllerutil.SetOwnerReference(lookout, serviceAccount, scheme); err != nil {
return nil, err
}

var serviceMonitor *monitoringv1.ServiceMonitor
if lookout.Spec.Prometheus != nil && lookout.Spec.Prometheus.Enabled {
Expand Down Expand Up @@ -290,7 +299,7 @@ func createLookoutServiceMonitor(lookout *installv1alpha1.Lookout) *monitoringv1

// Function to build the deployment object for Lookout.
// This should be changing from CRD to CRD. Not sure if generailize this helps much
func createLookoutDeployment(lookout *installv1alpha1.Lookout) (*appsv1.Deployment, error) {
func createLookoutDeployment(lookout *installv1alpha1.Lookout, serviceAccountName string) (*appsv1.Deployment, error) {
var runAsUser int64 = 1000
var runAsGroup int64 = 2000
allowPrivilegeEscalation := false
Expand All @@ -313,6 +322,7 @@ func createLookoutDeployment(lookout *installv1alpha1.Lookout) (*appsv1.Deployme
Annotations: map[string]string{"checksum/config": GenerateChecksumConfig(lookout.Spec.ApplicationConfig.Raw)},
},
Spec: corev1.PodSpec{
ServiceAccountName: serviceAccountName,
TerminationGracePeriodSeconds: lookout.DeletionGracePeriodSeconds,
SecurityContext: &corev1.PodSecurityContext{
RunAsUser: &runAsUser,
Expand Down
26 changes: 18 additions & 8 deletions internal/controller/install/lookoutingester_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ import (
"fmt"
"time"

"github.com/pkg/errors"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
Expand Down Expand Up @@ -70,7 +72,7 @@ func (r *LookoutIngesterReconciler) Reconcile(ctx context.Context, req ctrl.Requ
}
lookoutIngester.Spec.PortConfig = pc

components, err := r.generateInstallComponents(&lookoutIngester)
components, err := r.generateInstallComponents(&lookoutIngester, r.Scheme)
if err != nil {
return ctrl.Result{}, err
}
Expand Down Expand Up @@ -128,25 +130,32 @@ func (r *LookoutIngesterReconciler) SetupWithManager(mgr ctrl.Manager) error {
Complete(r)
}

func (r *LookoutIngesterReconciler) generateInstallComponents(lookoutIngester *installv1alpha1.LookoutIngester) (*CommonComponents, error) {
func (r *LookoutIngesterReconciler) generateInstallComponents(lookoutIngester *installv1alpha1.LookoutIngester, scheme *runtime.Scheme) (*CommonComponents, error) {
secret, err := builders.CreateSecret(lookoutIngester.Spec.ApplicationConfig, lookoutIngester.Name, lookoutIngester.Namespace, GetConfigFilename(lookoutIngester.Name))
if err != nil {
return nil, err
}
if err := controllerutil.SetOwnerReference(lookoutIngester, secret, r.Scheme); err != nil {
return nil, err
}
deployment, err := r.createDeployment(lookoutIngester)

var serviceAccount *corev1.ServiceAccount
serviceAccountName := lookoutIngester.Spec.CustomServiceAccount
if serviceAccountName == "" {
serviceAccount = builders.CreateServiceAccount(lookoutIngester.Name, lookoutIngester.Namespace, AllLabels(lookoutIngester.Name, lookoutIngester.Labels), lookoutIngester.Spec.ServiceAccount)
if err = controllerutil.SetOwnerReference(lookoutIngester, serviceAccount, scheme); err != nil {
return nil, errors.WithStack(err)
}
serviceAccountName = serviceAccount.Name
}

deployment, err := r.createDeployment(lookoutIngester, serviceAccountName)
if err != nil {
return nil, err
}
if err := controllerutil.SetOwnerReference(lookoutIngester, deployment, r.Scheme); err != nil {
return nil, err
}
serviceAccount := builders.CreateServiceAccount(lookoutIngester.Name, lookoutIngester.Namespace, AllLabels(lookoutIngester.Name, lookoutIngester.Labels), lookoutIngester.Spec.ServiceAccount)
if err := controllerutil.SetOwnerReference(lookoutIngester, serviceAccount, r.Scheme); err != nil {
return nil, err
}

return &CommonComponents{
Deployment: deployment,
Expand All @@ -156,7 +165,7 @@ func (r *LookoutIngesterReconciler) generateInstallComponents(lookoutIngester *i
}

// TODO: Flesh this out for lookoutingester
func (r *LookoutIngesterReconciler) createDeployment(lookoutIngester *installv1alpha1.LookoutIngester) (*appsv1.Deployment, error) {
func (r *LookoutIngesterReconciler) createDeployment(lookoutIngester *installv1alpha1.LookoutIngester, serviceAccountName string) (*appsv1.Deployment, error) {
var replicas int32 = 1
var runAsUser int64 = 1000
var runAsGroup int64 = 2000
Expand Down Expand Up @@ -187,6 +196,7 @@ func (r *LookoutIngesterReconciler) createDeployment(lookoutIngester *installv1a
Annotations: map[string]string{"checksum/config": GenerateChecksumConfig(lookoutIngester.Spec.ApplicationConfig.Raw)},
},
Spec: corev1.PodSpec{
ServiceAccountName: serviceAccountName,
TerminationGracePeriodSeconds: lookoutIngester.Spec.TerminationGracePeriodSeconds,
SecurityContext: &corev1.PodSecurityContext{
RunAsUser: &runAsUser,
Expand Down
22 changes: 16 additions & 6 deletions internal/controller/install/scheduler_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ import (
"fmt"
"time"

"github.com/pkg/errors"

"k8s.io/utils/ptr"

monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
Expand Down Expand Up @@ -197,7 +199,18 @@ func generateSchedulerInstallComponents(scheduler *installv1alpha1.Scheduler, sc
if err := controllerutil.SetOwnerReference(scheduler, secret, scheme); err != nil {
return nil, err
}
deployment, err := createSchedulerDeployment(scheduler)

var serviceAccount *corev1.ServiceAccount
serviceAccountName := scheduler.Spec.CustomServiceAccount
if serviceAccountName == "" {
serviceAccount = builders.CreateServiceAccount(scheduler.Name, scheduler.Namespace, AllLabels(scheduler.Name, scheduler.Labels), scheduler.Spec.ServiceAccount)
if err = controllerutil.SetOwnerReference(scheduler, serviceAccount, scheme); err != nil {
return nil, errors.WithStack(err)
}
serviceAccountName = serviceAccount.Name
}

deployment, err := createSchedulerDeployment(scheduler, serviceAccountName)
if err != nil {
return nil, err
}
Expand All @@ -209,10 +222,6 @@ func generateSchedulerInstallComponents(scheduler *installv1alpha1.Scheduler, sc
if err := controllerutil.SetOwnerReference(scheduler, service, scheme); err != nil {
return nil, err
}
serviceAccount := builders.CreateServiceAccount(scheduler.Name, scheduler.Namespace, AllLabels(scheduler.Name, scheduler.Labels), scheduler.Spec.ServiceAccount)
if err := controllerutil.SetOwnerReference(scheduler, serviceAccount, scheme); err != nil {
return nil, err
}

var serviceMonitor *monitoringv1.ServiceMonitor
if scheduler.Spec.Prometheus != nil && scheduler.Spec.Prometheus.Enabled {
Expand Down Expand Up @@ -284,7 +293,7 @@ func createSchedulerServiceMonitor(scheduler *installv1alpha1.Scheduler) *monito

// Function to build the deployment object for Scheduler.
// This should be changing from CRD to CRD. Not sure if generailize this helps much
func createSchedulerDeployment(scheduler *installv1alpha1.Scheduler) (*appsv1.Deployment, error) {
func createSchedulerDeployment(scheduler *installv1alpha1.Scheduler, serviceAccountName string) (*appsv1.Deployment, error) {
var runAsUser int64 = 1000
var runAsGroup int64 = 2000
allowPrivilegeEscalation := false
Expand All @@ -307,6 +316,7 @@ func createSchedulerDeployment(scheduler *installv1alpha1.Scheduler) (*appsv1.De
Annotations: map[string]string{"checksum/config": GenerateChecksumConfig(scheduler.Spec.ApplicationConfig.Raw)},
},
Spec: corev1.PodSpec{
ServiceAccountName: serviceAccountName,
TerminationGracePeriodSeconds: scheduler.DeletionGracePeriodSeconds,
SecurityContext: &corev1.PodSecurityContext{
RunAsUser: &runAsUser,
Expand Down
30 changes: 20 additions & 10 deletions internal/controller/install/scheduleringester_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ import (
"context"
"time"

"github.com/pkg/errors"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
Expand Down Expand Up @@ -126,34 +128,41 @@ func (r *SchedulerIngesterReconciler) SetupWithManager(mgr ctrl.Manager) error {
Complete(r)
}

func (r *SchedulerIngesterReconciler) generateSchedulerIngesterComponents(scheduleringester *installv1alpha1.SchedulerIngester, scheme *runtime.Scheme) (*CommonComponents, error) {
secret, err := builders.CreateSecret(scheduleringester.Spec.ApplicationConfig, scheduleringester.Name, scheduleringester.Namespace, GetConfigFilename(scheduleringester.Name))
func (r *SchedulerIngesterReconciler) generateSchedulerIngesterComponents(schedulerIngester *installv1alpha1.SchedulerIngester, scheme *runtime.Scheme) (*CommonComponents, error) {
secret, err := builders.CreateSecret(schedulerIngester.Spec.ApplicationConfig, schedulerIngester.Name, schedulerIngester.Namespace, GetConfigFilename(schedulerIngester.Name))
if err != nil {
return nil, err
}
if err := controllerutil.SetOwnerReference(scheduleringester, secret, scheme); err != nil {
if err := controllerutil.SetOwnerReference(schedulerIngester, secret, scheme); err != nil {
return nil, err
}
deployment, err := r.createDeployment(scheduleringester)

var serviceAccount *corev1.ServiceAccount
serviceAccountName := schedulerIngester.Spec.CustomServiceAccount
if serviceAccountName == "" {
serviceAccount = builders.CreateServiceAccount(schedulerIngester.Name, schedulerIngester.Namespace, AllLabels(schedulerIngester.Name, schedulerIngester.Labels), schedulerIngester.Spec.ServiceAccount)
if err = controllerutil.SetOwnerReference(schedulerIngester, serviceAccount, scheme); err != nil {
return nil, errors.WithStack(err)
}
serviceAccountName = serviceAccount.Name
}

deployment, err := r.createDeployment(schedulerIngester, serviceAccountName)
if err != nil {
return nil, err
}
if err := controllerutil.SetOwnerReference(scheduleringester, deployment, scheme); err != nil {
if err := controllerutil.SetOwnerReference(schedulerIngester, deployment, scheme); err != nil {
return nil, err
}

serviceAccount := builders.CreateServiceAccount(scheduleringester.Name, scheduleringester.Namespace, AllLabels(scheduleringester.Name, scheduleringester.Labels), scheduleringester.Spec.ServiceAccount)
if err := controllerutil.SetOwnerReference(scheduleringester, serviceAccount, scheme); err != nil {
return nil, err
}
return &CommonComponents{
Deployment: deployment,
ServiceAccount: serviceAccount,
Secret: secret,
}, nil
}

func (r *SchedulerIngesterReconciler) createDeployment(scheduleringester *installv1alpha1.SchedulerIngester) (*appsv1.Deployment, error) {
func (r *SchedulerIngesterReconciler) createDeployment(scheduleringester *installv1alpha1.SchedulerIngester, serviceAccountName string) (*appsv1.Deployment, error) {
var runAsUser int64 = 1000
var runAsGroup int64 = 2000
allowPrivilegeEscalation := false
Expand Down Expand Up @@ -188,6 +197,7 @@ func (r *SchedulerIngesterReconciler) createDeployment(scheduleringester *instal
Annotations: map[string]string{"checksum/config": GenerateChecksumConfig(scheduleringester.Spec.ApplicationConfig.Raw)},
},
Spec: corev1.PodSpec{
ServiceAccountName: serviceAccountName,
TerminationGracePeriodSeconds: scheduleringester.Spec.TerminationGracePeriodSeconds,
SecurityContext: &corev1.PodSecurityContext{
RunAsUser: &runAsUser,
Expand Down

0 comments on commit 27341f3

Please sign in to comment.