Skip to content

Commit 9e41715

Browse files
SovietacedSovietaced
authored
fix: Delete cronjobs when pruning disabled and actually reconcile scheduler corncobs (#338)
* Delete cronjobs when pruning disabled and actually reconcile scheduler cronjobs Signed-off-by: Jason Parraga <[email protected]> * Address comments Signed-off-by: Jason Parraga <[email protected]> --------- Signed-off-by: Jason Parraga <[email protected]>
1 parent 6633b3f commit 9e41715

File tree

5 files changed

+455
-26
lines changed

5 files changed

+455
-26
lines changed

‎internal/controller/install/common_helpers.go

+27-6
Original file line numberDiff line numberDiff line change
@@ -596,13 +596,13 @@ func upsertObjectIfNeeded(
596596
mutateFn controllerutil.MutateFn,
597597
logger logr.Logger,
598598
) error {
599-
if !isNil(object) {
600-
logger.Info(fmt.Sprintf("Upserting %s %s object", componentName, object.GetObjectKind()))
601-
if _, err := controllerutil.CreateOrUpdate(ctx, client, object, mutateFn); err != nil {
602-
return err
603-
}
599+
if isNil(object) {
600+
return nil
604601
}
605-
return nil
602+
603+
logger.Info(fmt.Sprintf("Upserting %s %s object", componentName, object.GetObjectKind()))
604+
_, err := controllerutil.CreateOrUpdate(ctx, client, object, mutateFn)
605+
return err
606606
}
607607

608608
// Helper function to determine if the object is nil even if it's a pointer to a nil value
@@ -619,6 +619,27 @@ func isNil(i any) bool {
619619
}
620620
}
621621

622+
// deleteObjectIfNeeded will delete the object if it exists.
623+
func deleteObjectIfNeeded(
624+
ctx context.Context,
625+
k8sClient client.Client,
626+
object client.Object,
627+
componentName string,
628+
logger logr.Logger,
629+
) error {
630+
if isNil(object) {
631+
return nil
632+
}
633+
634+
if err := k8sClient.Delete(ctx, object); err != nil {
635+
return client.IgnoreNotFound(err)
636+
}
637+
638+
logger.Info("Successfully deleted %s %s", componentName, object.GetObjectKind())
639+
640+
return nil
641+
}
642+
622643
// getObject will get the object from Kubernetes and return if it is missing or an error.
623644
func getObject(
624645
ctx context.Context,

‎internal/controller/install/lookout_controller.go

+14-11
Original file line numberDiff line numberDiff line change
@@ -124,8 +124,14 @@ func (r *LookoutReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
124124
return ctrl.Result{}, err
125125
}
126126

127-
if err := upsertObjectIfNeeded(ctx, r.Client, components.CronJob, lookout.Kind, mutateFn, logger); err != nil {
128-
return ctrl.Result{}, err
127+
if enabled := lookout.Spec.DbPruningEnabled; enabled != nil && *enabled {
128+
if err := upsertObjectIfNeeded(ctx, r.Client, components.CronJob, lookout.Kind, mutateFn, logger); err != nil {
129+
return ctrl.Result{}, err
130+
}
131+
} else {
132+
if err := deleteObjectIfNeeded(ctx, r.Client, components.CronJob, lookout.Kind, logger); err != nil {
133+
return ctrl.Result{}, err
134+
}
129135
}
130136

131137
if err := upsertObjectIfNeeded(ctx, r.Client, components.ServiceMonitor, lookout.Kind, mutateFn, logger); err != nil {
@@ -210,15 +216,12 @@ func generateLookoutInstallComponents(
210216
return nil, err
211217
}
212218

213-
var cronJob *batchv1.CronJob
214-
if enabled := lookout.Spec.DbPruningEnabled; enabled != nil && *enabled {
215-
cronJob, err = createLookoutCronJob(lookout, serviceAccountName)
216-
if err != nil {
217-
return nil, err
218-
}
219-
if err := controllerutil.SetOwnerReference(lookout, cronJob, scheme); err != nil {
220-
return nil, err
221-
}
219+
cronJob, err := createLookoutCronJob(lookout, serviceAccountName)
220+
if err != nil {
221+
return nil, err
222+
}
223+
if err := controllerutil.SetOwnerReference(lookout, cronJob, scheme); err != nil {
224+
return nil, err
222225
}
223226

224227
ingressHTTP, err := createLookoutIngressHttp(lookout, config)

‎internal/controller/install/lookout_controller_test.go

+182
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,188 @@ func TestLookoutReconciler_Reconcile(t *testing.T) {
218218
}
219219
}
220220

221+
func TestLookoutReconciler_ReconcilePruningDisabled(t *testing.T) {
222+
t.Parallel()
223+
224+
mockCtrl := gomock.NewController(t)
225+
defer mockCtrl.Finish()
226+
227+
scheme, err := v1alpha1.SchemeBuilder.Build()
228+
if err != nil {
229+
t.Fatalf("should not return error when building schema")
230+
}
231+
232+
expectedNamespacedName := types.NamespacedName{Namespace: "default", Name: "lookout"}
233+
dbPruningEnabled := false
234+
dbPruningSchedule := "1d"
235+
terminationGracePeriod := int64(20)
236+
expectedLookout := v1alpha1.Lookout{
237+
TypeMeta: metav1.TypeMeta{
238+
Kind: "Lookout",
239+
APIVersion: "install.armadaproject.io/v1alpha1",
240+
},
241+
ObjectMeta: metav1.ObjectMeta{Namespace: "default", Name: "lookout"},
242+
Spec: v1alpha1.LookoutSpec{
243+
Replicas: ptr.To[int32](2),
244+
CommonSpecBase: installv1alpha1.CommonSpecBase{
245+
Labels: nil,
246+
Image: v1alpha1.Image{
247+
Repository: "testrepo",
248+
Tag: "1.0.0",
249+
},
250+
ApplicationConfig: runtime.RawExtension{},
251+
Resources: &corev1.ResourceRequirements{},
252+
Prometheus: &installv1alpha1.PrometheusConfig{Enabled: true, ScrapeInterval: &metav1.Duration{Duration: 1 * time.Second}},
253+
TerminationGracePeriodSeconds: &terminationGracePeriod,
254+
},
255+
ClusterIssuer: "test",
256+
HostNames: []string{"localhost"},
257+
Ingress: &installv1alpha1.IngressConfig{
258+
IngressClass: "nginx",
259+
Labels: map[string]string{"test": "hello"},
260+
Annotations: map[string]string{"test": "hello"},
261+
},
262+
DbPruningEnabled: &dbPruningEnabled,
263+
DbPruningSchedule: &dbPruningSchedule,
264+
},
265+
}
266+
267+
mockK8sClient := k8sclient.NewMockClient(mockCtrl)
268+
// Lookout
269+
mockK8sClient.
270+
EXPECT().
271+
Get(gomock.Any(), expectedNamespacedName, gomock.AssignableToTypeOf(&v1alpha1.Lookout{})).
272+
Return(nil).
273+
SetArg(2, expectedLookout)
274+
275+
// Finalizer
276+
mockK8sClient.
277+
EXPECT().
278+
Update(gomock.Any(), gomock.AssignableToTypeOf(&installv1alpha1.Lookout{})).
279+
Return(nil)
280+
281+
// ServiceAccount
282+
mockK8sClient.
283+
EXPECT().
284+
Get(gomock.Any(), expectedNamespacedName, gomock.AssignableToTypeOf(&corev1.ServiceAccount{})).
285+
Return(errors.NewNotFound(schema.GroupResource{}, "lookout"))
286+
mockK8sClient.
287+
EXPECT().
288+
Create(gomock.Any(), gomock.AssignableToTypeOf(&corev1.ServiceAccount{})).
289+
Return(nil)
290+
291+
mockK8sClient.
292+
EXPECT().
293+
Get(gomock.Any(), expectedNamespacedName, gomock.AssignableToTypeOf(&corev1.Secret{})).
294+
Return(errors.NewNotFound(schema.GroupResource{}, "lookout"))
295+
mockK8sClient.
296+
EXPECT().
297+
Create(gomock.Any(), gomock.AssignableToTypeOf(&corev1.Secret{})).
298+
Return(nil)
299+
300+
expectedJobName := types.NamespacedName{Namespace: "default", Name: "lookout-migration"}
301+
expectedMigrationJob := &batchv1.Job{
302+
ObjectMeta: metav1.ObjectMeta{
303+
Namespace: "default",
304+
Name: "lookout-migration",
305+
},
306+
Spec: batchv1.JobSpec{
307+
Template: corev1.PodTemplateSpec{
308+
Spec: corev1.PodSpec{
309+
Containers: []corev1.Container{
310+
{
311+
Name: "lookout-migration",
312+
Image: "testrepo:1.0.0",
313+
},
314+
},
315+
},
316+
},
317+
},
318+
Status: batchv1.JobStatus{
319+
Conditions: []batchv1.JobCondition{{
320+
Type: batchv1.JobComplete,
321+
Status: corev1.ConditionTrue,
322+
}},
323+
},
324+
}
325+
326+
mockK8sClient.
327+
EXPECT().
328+
Get(gomock.Any(), expectedJobName, gomock.AssignableToTypeOf(&batchv1.Job{})).
329+
Return(errors.NewNotFound(schema.GroupResource{}, "lookout"))
330+
mockK8sClient.
331+
EXPECT().
332+
Create(gomock.Any(), gomock.AssignableToTypeOf(&batchv1.Job{})).
333+
Return(nil)
334+
mockK8sClient.
335+
EXPECT().
336+
Get(gomock.Any(), expectedJobName, gomock.AssignableToTypeOf(&batchv1.Job{})).
337+
Return(nil).
338+
SetArg(2, *expectedMigrationJob)
339+
340+
mockK8sClient.
341+
EXPECT().
342+
Get(gomock.Any(), expectedNamespacedName, gomock.AssignableToTypeOf(&appsv1.Deployment{})).
343+
Return(errors.NewNotFound(schema.GroupResource{}, "lookout"))
344+
mockK8sClient.
345+
EXPECT().
346+
Create(gomock.Any(), gomock.AssignableToTypeOf(&appsv1.Deployment{})).
347+
Return(nil)
348+
349+
mockK8sClient.
350+
EXPECT().
351+
Get(gomock.Any(), expectedNamespacedName, gomock.AssignableToTypeOf(&corev1.Service{})).
352+
Return(errors.NewNotFound(schema.GroupResource{}, "lookout"))
353+
mockK8sClient.
354+
EXPECT().
355+
Create(gomock.Any(), gomock.AssignableToTypeOf(&corev1.Service{})).
356+
Return(nil)
357+
358+
// IngressHttp
359+
expectedIngressName := expectedNamespacedName
360+
expectedIngressName.Name = expectedIngressName.Name + "-rest"
361+
mockK8sClient.
362+
EXPECT().
363+
Get(gomock.Any(), expectedIngressName, gomock.AssignableToTypeOf(&networkingv1.Ingress{})).
364+
Return(errors.NewNotFound(schema.GroupResource{}, "lookout"))
365+
mockK8sClient.
366+
EXPECT().
367+
Create(gomock.Any(), gomock.AssignableToTypeOf(&networkingv1.Ingress{})).
368+
Return(nil)
369+
370+
// ServiceMonitor
371+
mockK8sClient.
372+
EXPECT().
373+
Get(gomock.Any(), expectedNamespacedName, gomock.AssignableToTypeOf(&monitoringv1.ServiceMonitor{})).
374+
Return(errors.NewNotFound(schema.GroupResource{}, "armadaserver"))
375+
mockK8sClient.
376+
EXPECT().
377+
Create(gomock.Any(), gomock.AssignableToTypeOf(&monitoringv1.ServiceMonitor{})).
378+
Return(nil)
379+
380+
// CronJob should be deleted
381+
expectedCronJobName := expectedNamespacedName
382+
expectedCronJobName.Name = expectedCronJobName.Name + "-db-pruner"
383+
mockK8sClient.
384+
EXPECT().
385+
Delete(gomock.Any(), gomock.AssignableToTypeOf(&batchv1.CronJob{})).
386+
Return(nil)
387+
388+
r := LookoutReconciler{
389+
Client: mockK8sClient,
390+
Scheme: scheme,
391+
}
392+
393+
req := ctrl.Request{
394+
NamespacedName: types.NamespacedName{Namespace: "default", Name: "lookout"},
395+
}
396+
397+
_, err = r.Reconcile(context.Background(), req)
398+
if err != nil {
399+
t.Fatalf("reconcile should not return error")
400+
}
401+
}
402+
221403
func TestLookoutReconciler_ReconcileNoLookout(t *testing.T) {
222404
t.Parallel()
223405

‎internal/controller/install/scheduler_controller.go

+16-9
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,16 @@ func (r *SchedulerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
135135
return ctrl.Result{}, err
136136
}
137137

138+
if scheduler.Spec.Pruner != nil && scheduler.Spec.Pruner.Enabled {
139+
if err := upsertObjectIfNeeded(ctx, r.Client, components.CronJob, scheduler.Kind, mutateFn, logger); err != nil {
140+
return ctrl.Result{}, err
141+
}
142+
} else {
143+
if err := deleteObjectIfNeeded(ctx, r.Client, components.CronJob, scheduler.Kind, logger); err != nil {
144+
return ctrl.Result{}, err
145+
}
146+
}
147+
138148
logger.Info("Successfully reconciled Scheduler object", "durationMillis", time.Since(started).Milliseconds())
139149

140150
return ctrl.Result{}, nil
@@ -218,15 +228,12 @@ func generateSchedulerInstallComponents(
218228
return nil, err
219229
}
220230

221-
var cronJob *batchv1.CronJob
222-
if scheduler.Spec.Pruner != nil && scheduler.Spec.Pruner.Enabled {
223-
cronJob, err = newSchedulerCronJob(scheduler, serviceAccountName)
224-
if err != nil {
225-
return nil, err
226-
}
227-
if err := controllerutil.SetOwnerReference(scheduler, cronJob, scheme); err != nil {
228-
return nil, err
229-
}
231+
cronJob, err := newSchedulerCronJob(scheduler, serviceAccountName)
232+
if err != nil {
233+
return nil, err
234+
}
235+
if err := controllerutil.SetOwnerReference(scheduler, cronJob, scheme); err != nil {
236+
return nil, err
230237
}
231238

232239
ingressGRPC, err := newSchedulerIngressGRPC(scheduler, config)

0 commit comments

Comments
 (0)