Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 26 additions & 7 deletions pkg/controller/clusterdeployment/clusterdeployment_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ import (
"github.com/openshift/hive/pkg/install"
)

// controllerKind contains the schema.GroupVersionKind for this controller type.
var controllerKind = hivev1.SchemeGroupVersion.WithKind("ClusterDeployment")

const (
controllerName = "clusterDeployment"
defaultRequeueTime = 10 * time.Second
Expand Down Expand Up @@ -143,16 +146,23 @@ func Add(mgr manager.Manager) error {

// NewReconciler returns a new reconcile.Reconciler
func NewReconciler(mgr manager.Manager) reconcile.Reconciler {
logger := log.WithField("controller", controllerName)
return &ReconcileClusterDeployment{
Client: controllerutils.NewClientWithMetricsOrDie(mgr, controllerName),
scheme: mgr.GetScheme(),
logger: log.WithField("controller", controllerName),
logger: logger,
expectations: controllerutils.NewExpectations(logger),
remoteClusterAPIClientBuilder: controllerutils.BuildClusterAPIClientFromKubeconfig,
}
}

// AddToManager adds a new Controller to mgr with r as the reconcile.Reconciler
func AddToManager(mgr manager.Manager, r reconcile.Reconciler) error {
cdReconciler, ok := r.(*ReconcileClusterDeployment)
if !ok {
return errors.New("reconciler supplied is not a ReconcileClusterDeployment")
}

c, err := controller.New("clusterdeployment-controller", mgr, controller.Options{Reconciler: r, MaxConcurrentReconciles: controllerutils.GetConcurrentReconciles()})
if err != nil {
log.WithField("controller", controllerName).WithError(err).Error("Error getting new cluster deployment")
Expand All @@ -167,10 +177,7 @@ func AddToManager(mgr manager.Manager, r reconcile.Reconciler) error {
}

// Watch for provisions
if err := c.Watch(&source.Kind{Type: &hivev1.ClusterProvision{}}, &handler.EnqueueRequestForOwner{
IsController: true,
OwnerType: &hivev1.ClusterDeployment{},
}); err != nil {
if err := cdReconciler.watchClusterProvisions(c); err != nil {
return err
}

Expand Down Expand Up @@ -224,6 +231,9 @@ type ReconcileClusterDeployment struct {
scheme *runtime.Scheme
logger log.FieldLogger

// A TTLCache of clusterprovision creates/deletes each clusterdeployment expects to see
expectations controllerutils.ExpectationsInterface

// remoteClusterAPIClientBuilder is a function pointer to the function that builds a client for the
// remote cluster's cluster-api
remoteClusterAPIClientBuilder func(string, string) (client.Client, error)
Expand Down Expand Up @@ -266,6 +276,7 @@ func (r *ReconcileClusterDeployment) Reconcile(request reconcile.Request) (resul
// Object not found, return. Created objects are automatically garbage collected.
// For additional cleanup logic use finalizers.
cdLog.Info("cluster deployment Not Found")
r.expectations.DeleteExpectations(request.NamespacedName.String())
return reconcile.Result{}, nil
}
// Error reading the object - requeue the request.
Expand Down Expand Up @@ -447,6 +458,11 @@ func (r *ReconcileClusterDeployment) reconcile(request reconcile.Request, cd *hi
return r.resolveInstallerImage(cd, imageSet, releaseImage, cdLog)
}

if !r.expectations.SatisfiedExpectations(request.String()) {
cdLog.Debug("waiting for expectations to be satisfied")
return reconcile.Result{}, nil
}

if cd.Status.Provision == nil {
if cd.Status.InstallRestarts > 0 && cd.Annotations[tryInstallOnceAnnotation] == "true" {
cdLog.Debug("not creating new provision since the deployment is set to try install only once")
Expand Down Expand Up @@ -566,8 +582,11 @@ func (r *ReconcileClusterDeployment) startNewProvision(
cdLog.WithError(err).Error("could not set the owner ref on provision")
return reconcile.Result{}, err
}

r.expectations.ExpectCreations(types.NamespacedName{Namespace: cd.Namespace, Name: cd.Name}.String(), 1)
if err := r.Create(context.TODO(), provision); err != nil {
cdLog.WithError(err).Error("could not create provision")
r.expectations.CreationObserved(types.NamespacedName{Namespace: cd.Namespace, Name: cd.Name}.String())
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Something that's confusing me a little bit here, it looks like the intent here is to expect the creation of the cluster provision, but the expectation itself is using the cluster deployment's name. Should this be the provision's name? Or is that an issue with the random component of their names.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The expectations are set up to expect a creation on behalf of the clusterdeployment. The key is always the resource being reconciled, and not the resource being created.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

gotcha thanks.

return reconcile.Result{}, err
}

Expand Down Expand Up @@ -1103,10 +1122,10 @@ func (r *ReconcileClusterDeployment) syncDeletedClusterDeployment(cd *hivev1.Clu
return reconcile.Result{}, err
}
cdLog.Info("deleted outstanding provision")
return reconcile.Result{}, nil
return reconcile.Result{RequeueAfter: defaultRequeueTime}, nil
default:
cdLog.Debug("still waiting for outstanding provision to be removed")
return reconcile.Result{}, nil
return reconcile.Result{RequeueAfter: defaultRequeueTime}, nil
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -104,11 +104,13 @@ func TestClusterDeploymentReconcile(t *testing.T) {
}

tests := []struct {
name string
existing []runtime.Object
expectErr bool
expectedRequeueAfter time.Duration
validate func(client.Client, *testing.T)
name string
existing []runtime.Object
pendingCreation bool
expectErr bool
expectedRequeueAfter time.Duration
expectPendingCreation bool
validate func(client.Client, *testing.T)
}{
{
name: "Add finalizer",
Expand All @@ -132,11 +134,27 @@ func TestClusterDeploymentReconcile(t *testing.T) {
testSecret(corev1.SecretTypeDockerConfigJson, constants.GetMergedPullSecretName(testClusterDeployment()), corev1.DockerConfigJsonKey, "{}"),
testSecret(corev1.SecretTypeOpaque, sshKeySecret, adminSSHKeySecretKey, "fakesshkey"),
},
expectPendingCreation: true,
validate: func(c client.Client, t *testing.T) {
provisions := getProvisions(c)
assert.Len(t, provisions, 1, "expected provision to exist")
},
},
{
name: "Provision not created when pending create",
existing: []runtime.Object{
testClusterDeployment(),
testSecret(corev1.SecretTypeDockerConfigJson, pullSecretSecret, corev1.DockerConfigJsonKey, "{}"),
testSecret(corev1.SecretTypeDockerConfigJson, constants.GetMergedPullSecretName(testClusterDeployment()), corev1.DockerConfigJsonKey, "{}"),
testSecret(corev1.SecretTypeOpaque, sshKeySecret, adminSSHKeySecretKey, "fakesshkey"),
},
pendingCreation: true,
expectPendingCreation: true,
validate: func(c client.Client, t *testing.T) {
provisions := getProvisions(c)
assert.Empty(t, provisions, "expected provision to not exist")
},
},
{
name: "Adopt provision",
existing: []runtime.Object{
Expand Down Expand Up @@ -504,6 +522,7 @@ func TestClusterDeploymentReconcile(t *testing.T) {
testSecret(corev1.SecretTypeDockerConfigJson, constants.GetMergedPullSecretName(testClusterDeployment()), corev1.DockerConfigJsonKey, "{}"),
testSecret(corev1.SecretTypeOpaque, sshKeySecret, adminSSHKeySecretKey, "fakesshkey"),
},
expectPendingCreation: true,
validate: func(c client.Client, t *testing.T) {
provisions := getProvisions(c)
if assert.Len(t, provisions, 1, "expected provision to exist") {
Expand Down Expand Up @@ -616,6 +635,7 @@ func TestClusterDeploymentReconcile(t *testing.T) {
testSecret(corev1.SecretTypeOpaque, sshKeySecret, adminSSHKeySecretKey, "fakesshkey"),
testAvailableDNSZone(),
},
expectPendingCreation: true,
validate: func(c client.Client, t *testing.T) {
provisions := getProvisions(c)
assert.Len(t, provisions, 1, "expected provision to exist")
Expand Down Expand Up @@ -741,6 +761,7 @@ func TestClusterDeploymentReconcile(t *testing.T) {
testFailedProvisionAttempt(2),
testFailedProvisionAttempt(3),
},
expectPendingCreation: true,
validate: func(c client.Client, t *testing.T) {
actualAttempts := []int{}
for _, p := range getProvisions(c) {
Expand Down Expand Up @@ -781,6 +802,7 @@ func TestClusterDeploymentReconcile(t *testing.T) {
testSecret(corev1.SecretTypeOpaque, sshKeySecret, adminSSHKeySecretKey, "fakesshkey"),
testProvision(),
},
expectPendingCreation: true,
validate: func(c client.Client, t *testing.T) {
cd := getCD(c)
if assert.NotNil(t, cd, "missing cluster deployment") {
Expand Down Expand Up @@ -864,6 +886,7 @@ func TestClusterDeploymentReconcile(t *testing.T) {
testSecret(corev1.SecretTypeDockerConfigJson, constants.GetMergedPullSecretName(testClusterDeployment()), corev1.DockerConfigJsonKey, "{}"),
testSecret(corev1.SecretTypeOpaque, sshKeySecret, adminSSHKeySecretKey, "fakesshkey"),
},
expectedRequeueAfter: defaultRequeueTime,
validate: func(c client.Client, t *testing.T) {
provisions := getProvisions(c)
assert.Empty(t, provisions, "expected provision to be deleted")
Expand Down Expand Up @@ -920,20 +943,29 @@ func TestClusterDeploymentReconcile(t *testing.T) {

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
logger := log.WithField("controller", "clusterDeployment")
fakeClient := fake.NewFakeClient(test.existing...)
controllerExpectations := controllerutils.NewExpectations(logger)
rcd := &ReconcileClusterDeployment{
Client: fakeClient,
scheme: scheme.Scheme,
logger: log.WithField("controller", "clusterDeployment"),
logger: logger,
expectations: controllerExpectations,
remoteClusterAPIClientBuilder: testRemoteClusterAPIClientBuilder,
}

result, err := rcd.Reconcile(reconcile.Request{
reconcileRequest := reconcile.Request{
NamespacedName: types.NamespacedName{
Name: testName,
Namespace: testNamespace,
},
})
}

if test.pendingCreation {
controllerExpectations.ExpectCreations(reconcileRequest.String(), 1)
}

result, err := rcd.Reconcile(reconcileRequest)

if test.validate != nil {
test.validate(fakeClient, t)
Expand All @@ -951,6 +983,9 @@ func TestClusterDeploymentReconcile(t *testing.T) {
} else {
assert.InDelta(t, test.expectedRequeueAfter, result.RequeueAfter, float64(10*time.Second), "unexpected requeue after")
}

actualPendingCreation := !controllerExpectations.SatisfiedExpectations(reconcileRequest.String())
assert.Equal(t, test.expectPendingCreation, actualPendingCreation, "unexpected pending creation")
})
}
}
Expand All @@ -974,11 +1009,14 @@ func TestClusterDeploymentReconcileResults(t *testing.T) {

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
logger := log.WithField("controller", "clusterDeployment")
fakeClient := fake.NewFakeClient(test.existing...)
controllerExpectations := controllerutils.NewExpectations(logger)
rcd := &ReconcileClusterDeployment{
Client: fakeClient,
scheme: scheme.Scheme,
logger: log.WithField("controller", "clusterDeployment"),
logger: logger,
expectations: controllerExpectations,
remoteClusterAPIClientBuilder: testRemoteClusterAPIClientBuilder,
}

Expand Down
81 changes: 81 additions & 0 deletions pkg/controller/clusterdeployment/provisionexpectations.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
package clusterdeployment

import (
"context"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/util/workqueue"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/handler"
"sigs.k8s.io/controller-runtime/pkg/source"

hivev1 "github.com/openshift/hive/pkg/apis/hive/v1alpha1"
)

func (r *ReconcileClusterDeployment) watchClusterProvisions(c controller.Controller) error {
handler := &clusterProvisionEventHandler{
EnqueueRequestForOwner: handler.EnqueueRequestForOwner{
IsController: true,
OwnerType: &hivev1.ClusterDeployment{},
},
reconciler: r,
}
return c.Watch(&source.Kind{Type: &hivev1.ClusterProvision{}}, handler)
}

var _ handler.EventHandler = &clusterProvisionEventHandler{}

type clusterProvisionEventHandler struct {
handler.EnqueueRequestForOwner
reconciler *ReconcileClusterDeployment
}

// Create implements handler.EventHandler
func (h *clusterProvisionEventHandler) Create(e event.CreateEvent, q workqueue.RateLimitingInterface) {
h.reconciler.logger.Info("ClusterProvision created")
h.reconciler.trackClusterProvisionAdd(e.Object)
h.EnqueueRequestForOwner.Create(e, q)
}

// resolveControllerRef returns the controller referenced by a ControllerRef,
// or nil if the ControllerRef could not be resolved to a matching controller
// of the correct Kind.
func (r *ReconcileClusterDeployment) resolveControllerRef(namespace string, controllerRef *metav1.OwnerReference) *hivev1.ClusterDeployment {
// We can't look up by UID, so look up by Name and then verify UID.
// Don't even try to look up by Name if it's the wrong Kind.
if controllerRef.Kind != controllerKind.Kind {
return nil
}
cd := &hivev1.ClusterDeployment{}
if err := r.Get(context.TODO(), types.NamespacedName{Namespace: namespace, Name: controllerRef.Name}, cd); err != nil {
return nil
}
if cd.UID != controllerRef.UID {
// The controller we found with this Name is not the same one that the
// ControllerRef points to.
return nil
}
return cd
}

// When a clusterprovision is created, update the expectations of the clusterdeployment that owns the clusterprovision.
func (r *ReconcileClusterDeployment) trackClusterProvisionAdd(obj interface{}) {
provision := obj.(*hivev1.ClusterProvision)
if provision.DeletionTimestamp != nil {
// on a restart of the controller, it's possible a new object shows up in a state that
// is already pending deletion. Prevent the object from being a creation observation.
return
}

// If it has a ControllerRef, that's all that matters.
if controllerRef := metav1.GetControllerOf(provision); controllerRef != nil {
cd := r.resolveControllerRef(provision.Namespace, controllerRef)
if cd == nil {
return
}
cdKey := types.NamespacedName{Namespace: cd.Namespace, Name: cd.Name}.String()
r.expectations.CreationObserved(cdKey)
}
}
Loading