TRT-1193: Revert "IR-373: remove node-ca daemon"

bindata/assets.go

@@ -0,0 +1,19 @@
+package assets
+
+import (
+	"embed"
+)
+
+//go:embed *.yaml
+var f embed.FS
+
+// MustAsset reads and returns the content of the named file or panics
+// if something went wrong.
+func MustAsset(name string) []byte {
+	data, err := f.ReadFile(name)
+	if err != nil {
+		panic(err)
+	}
+
+	return data
+}

bindata/nodecadaemon.yaml

@@ -0,0 +1,79 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: node-ca
+  namespace: openshift-image-registry
+spec:
+  selector:
+    matchLabels:
+      name: node-ca
+  updateStrategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 10%
+  template:
+    metadata:
+      annotations:
+        target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}'
+      labels:
+        name: node-ca
+    spec:
+      nodeSelector:
+        kubernetes.io/os: linux
+      priorityClassName: system-cluster-critical
+      tolerations:
+      - operator: Exists
+      hostNetwork: true # run as host network to tolerate unready networks
+      serviceAccountName: node-ca
+      containers:
+      - name: node-ca
+        securityContext:
+          privileged: true
+          runAsUser: 1001
+          runAsGroup: 0
+        image: docker.io/openshift/origin-cluster-image-registry-operator:latest
+        resources:
+          requests:
+            cpu: 10m
+            memory: 10Mi
+        command:
+        - "/bin/sh"
+        - "-c"
+        - |
+          trap 'jobs -p | xargs -r kill; echo shutting down node-ca; exit 0' TERM
+          while [ true ];
+          do
+            for f in $(ls /tmp/serviceca); do
+                echo $f
+                ca_file_path="/tmp/serviceca/${f}"
+                f=$(echo $f | sed  -r 's/(.*)\.\./\1:/')
+                reg_dir_path="/etc/docker/certs.d/${f}"
+                if [ -e "${reg_dir_path}" ]; then
+                    cp -u $ca_file_path $reg_dir_path/ca.crt
+                else
+                    mkdir $reg_dir_path
+                    cp $ca_file_path $reg_dir_path/ca.crt
+                fi
+            done
+            for d in $(ls /etc/docker/certs.d); do
+                echo $d
+                dp=$(echo $d | sed  -r 's/(.*):/\1\.\./')
+                reg_conf_path="/tmp/serviceca/${dp}"
+                if [ ! -e "${reg_conf_path}" ]; then
+                    rm -rf /etc/docker/certs.d/$d
+                fi
+            done
+            sleep 60 & wait ${!}
+          done
+        volumeMounts:
+        - name: serviceca
+          mountPath: /tmp/serviceca
+        - name: host
+          mountPath: /etc/docker/certs.d
+      volumes:
+      - name: host
+        hostPath:
+          path: /etc/docker/certs.d
+      - name: serviceca
+        configMap:
+          name: image-registry-certificates

pkg/operator/nodecadaemon.go

@@ -5,8 +5,9 @@ import (
 	"fmt"
 	"time"
 
+	appsv1 "k8s.io/api/apps/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	appsv1informers "k8s.io/client-go/informers/apps/v1"
@@ -121,38 +122,65 @@ func (c *NodeCADaemonController) sync() error {
 		Status: operatorv1.ConditionUnknown,
 	}
 
-	_, err := gen.Get()
+	dsObj, err := gen.Get()
 	if errors.IsNotFound(err) {
-		availableCondition.Status = operatorv1.ConditionTrue
-		availableCondition.Reason = "AsExpected"
-		availableCondition.Message = "The daemon set node-ca is removed"
+		availableCondition.Status = operatorv1.ConditionFalse
+		availableCondition.Reason = "NotFound"
+		availableCondition.Message = "The daemon set node-ca does not exist"
 
-		progressingCondition.Status = operatorv1.ConditionFalse
-		progressingCondition.Reason = "AsExpected"
-		progressingCondition.Message = "The daemon set node-ca is removed"
+		progressingCondition.Status = operatorv1.ConditionTrue
+		progressingCondition.Reason = "NotFound"
+		progressingCondition.Message = "The daemon set node-ca does not exist"
 	} else if err != nil {
 		availableCondition.Reason = "Unknown"
-		availableCondition.Message = fmt.Sprintf("Unable to check daemon set existence: %s", err)
+		availableCondition.Message = fmt.Sprintf("Unable to check daemon set availability: %s", err)
 
 		progressingCondition.Reason = "Unknown"
-		progressingCondition.Message = fmt.Sprintf("Unable to check daemon set existence: %s", err)
+		progressingCondition.Message = fmt.Sprintf("Unable to check daemon set progress: %s", err)
 	} else {
-		gracePeriod := int64(0)
-		propagationPolicy := metav1.DeletePropagationForeground
-		opts := metav1.DeleteOptions{
-			GracePeriodSeconds: &gracePeriod,
-			PropagationPolicy:  &propagationPolicy,
+		ds := dsObj.(*appsv1.DaemonSet)
+		if ds.Status.NumberAvailable > 0 {
+			availableCondition.Status = operatorv1.ConditionTrue
+			availableCondition.Reason = "AsExpected"
+			availableCondition.Message = "The daemon set node-ca has available replicas"
+		} else {
+			availableCondition.Status = operatorv1.ConditionFalse
+			availableCondition.Reason = "NoAvailableReplicas"
+			availableCondition.Message = "The daemon set node-ca does not have available replicas"
 		}
-		err := gen.Delete(opts)
-		if err != nil && !errors.IsNotFound(err) {
-			availableCondition.Reason = "Unknown"
-			availableCondition.Message = fmt.Sprintf("Unable to delete daemon set: %s", err)
 
-			progressingCondition.Reason = "Unknown"
-			progressingCondition.Message = fmt.Sprintf("Unable to delete daemon set: %s", err)
+		if ds.Generation != ds.Status.ObservedGeneration {
+			progressingCondition.Status = operatorv1.ConditionTrue
+			progressingCondition.Reason = "Progressing"
+			progressingCondition.Message = "The daemon set node-ca is updating node pods"
+		} else if ds.Status.NumberUnavailable > 0 {
+			progressingCondition.Status = operatorv1.ConditionTrue
+			progressingCondition.Reason = "Unavailable"
+			progressingCondition.Message = "The daemon set node-ca is deploying node pods"
+		} else {
+			progressingCondition.Status = operatorv1.ConditionFalse
+			progressingCondition.Reason = "AsExpected"
+			progressingCondition.Message = "The daemon set node-ca is deployed"
 		}
 	}
 
+	err = resource.ApplyMutator(gen)
+	if err != nil {
+		_, _, updateError := v1helpers.UpdateStatus(
+			ctx,
+			c.operatorClient,
+			v1helpers.UpdateConditionFn(availableCondition),
+			v1helpers.UpdateConditionFn(progressingCondition),
+			v1helpers.UpdateConditionFn(operatorv1.OperatorCondition{
+				Type:    "NodeCADaemonControllerDegraded",
+				Status:  operatorv1.ConditionTrue,
+				Reason:  "Error",
+				Message: err.Error(),
+			}),
+		)
+		return utilerrors.NewAggregate([]error{err, updateError})
+	}
+
 	_, _, err = v1helpers.UpdateStatus(
 		ctx,
 		c.operatorClient,

pkg/resource/nodecadaemon.go

@@ -2,6 +2,7 @@ package resource
 
 import (
 	"context"
+	"os"
 
 	appsv1 "k8s.io/api/apps/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -10,9 +11,14 @@ import (
 	appsv1listers "k8s.io/client-go/listers/apps/v1"
 	corev1listers "k8s.io/client-go/listers/core/v1"
 
+	operatorv1 "github.com/openshift/api/operator/v1"
 	"github.com/openshift/library-go/pkg/operator/events"
+	"github.com/openshift/library-go/pkg/operator/resource/resourceapply"
+	"github.com/openshift/library-go/pkg/operator/resource/resourcemerge"
+	"github.com/openshift/library-go/pkg/operator/resource/resourceread"
 	"github.com/openshift/library-go/pkg/operator/v1helpers"
 
+	assets "github.com/openshift/cluster-image-registry-operator/bindata"
 	"github.com/openshift/cluster-image-registry-operator/pkg/defaults"
 )
 
@@ -52,12 +58,52 @@ func (ds *generatorNodeCADaemonSet) Get() (runtime.Object, error) {
 	return ds.daemonSetLister.Get(ds.GetName())
 }
 
+func (ds *generatorNodeCADaemonSet) expected() *appsv1.DaemonSet {
+	daemonSet := resourceread.ReadDaemonSetV1OrDie(assets.MustAsset("nodecadaemon.yaml"))
+	daemonSet.Spec.Template.Spec.Containers[0].Image = os.Getenv("IMAGE")
+	return daemonSet
+}
+
 func (ds *generatorNodeCADaemonSet) Create() (runtime.Object, error) {
-	return nil, nil
+	dep, _, err := ds.Update(nil)
+	return dep, err
 }
 
 func (ds *generatorNodeCADaemonSet) Update(o runtime.Object) (runtime.Object, bool, error) {
-	return nil, false, nil
+	desiredDaemonSet := ds.expected()
+
+	_, opStatus, _, err := ds.operatorClient.GetOperatorState()
+	if err != nil {
+		return nil, false, err
+	}
+	actualDaemonSet, updated, err := resourceapply.ApplyDaemonSet(
+		context.TODO(),
+		ds.client,
+		ds.eventRecorder,
+		desiredDaemonSet,
+		resourcemerge.ExpectedDaemonSetGeneration(desiredDaemonSet, opStatus.Generations),
+	)
+	if err != nil {
+		return o, updated, err
+	}
+
+	if updated {
+		updateStatusFn := func(newStatus *operatorv1.OperatorStatus) error {
+			resourcemerge.SetDaemonSetGeneration(&newStatus.Generations, actualDaemonSet)
+			return nil
+		}
+
+		_, _, err = v1helpers.UpdateStatus(
+			context.TODO(),
+			ds.operatorClient,
+			updateStatusFn,
+		)
+		if err != nil {
+			return actualDaemonSet, updated, err
+		}
+	}
+
+	return actualDaemonSet, updated, nil
 }
 
 func (ds *generatorNodeCADaemonSet) Delete(opts metav1.DeleteOptions) error {

pkg/resource/nodecadaemon_test.go

@@ -0,0 +1,69 @@
+package resource
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	kfake "k8s.io/client-go/kubernetes/fake"
+
+	imageregistryv1 "github.com/openshift/api/imageregistry/v1"
+	imageregistryfake "github.com/openshift/client-go/imageregistry/clientset/versioned/fake"
+	imageregistryinformers "github.com/openshift/client-go/imageregistry/informers/externalversions"
+	"github.com/openshift/library-go/pkg/operator/events"
+
+	"github.com/openshift/cluster-image-registry-operator/pkg/client"
+)
+
+func findToleration(list []corev1.Toleration, cond func(toleration corev1.Toleration) bool) *corev1.Toleration {
+	for i, t := range list {
+		if cond(t) {
+			return &list[i]
+		}
+	}
+	return nil
+}
+
+func TestNodeCADaemon(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	imageregistryObjects := []runtime.Object{
+		&imageregistryv1.Config{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: "cluster",
+			},
+		},
+	}
+
+	clientset := kfake.NewSimpleClientset()
+	imageregistryClient := imageregistryfake.NewSimpleClientset(imageregistryObjects...)
+
+	imageregistryInformers := imageregistryinformers.NewSharedInformerFactory(imageregistryClient, time.Minute)
+
+	operatorClient := client.NewConfigOperatorClient(
+		imageregistryClient.ImageregistryV1().Configs(),
+		imageregistryInformers.Imageregistry().V1().Configs(),
+	)
+
+	imageregistryInformers.Start(ctx.Done())
+	imageregistryInformers.WaitForCacheSync(ctx.Done())
+
+	g := NewGeneratorNodeCADaemonSet(events.NewInMemoryRecorder("image-registry-operator"), nil, nil, clientset.AppsV1(), operatorClient)
+	obj, err := g.Create()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	ds := obj.(*appsv1.DaemonSet)
+	noScheduleToleration := findToleration(ds.Spec.Template.Spec.Tolerations, func(tol corev1.Toleration) bool {
+		return tol.Key == "" && tol.Operator == "Exists" && tol.Value == "" && tol.Effect == ""
+	})
+	if noScheduleToleration == nil {
+		t.Errorf("unable to find toleration for all taints, %#+v", ds.Spec.Template.Spec.Tolerations)
+	}
+}

test/e2e/aws_test.go

@@ -78,6 +78,7 @@ func TestAWSDefaults(t *testing.T) {
 	framework.EnsureClusterOperatorStatusIsNormal(te)
 	framework.EnsureOperatorIsNotHotLooping(te)
 	framework.EnsureServiceCAConfigMap(te)
+	framework.EnsureNodeCADaemonSetIsAvailable(te)
 
 	s3Driver := storages3.NewDriver(context.Background(), nil, &mockLister.StorageListers)
 	err = s3Driver.UpdateEffectiveConfig()