Bug 1805168: inject global proxy envs,ca into OCM daemonset

bindata/v3.11.0/openshift-controller-manager/ds.yaml

@@ -45,6 +45,8 @@ spec:
           name: client-ca
         - mountPath: /var/run/secrets/serving-cert
           name: serving-cert
+        - mountPath: /etc/pki/ca-trust/extracted/pem
+          name: proxy-ca-bundles
       volumes:
       - name: config
         configMap:
@@ -55,8 +57,13 @@ spec:
       - name: serving-cert
         secret:
           secretName: serving-cert
+      - name: proxy-ca-bundles
+        configMap:
+          name: openshift-global-ca
+          items:
+            - key: ca-bundle.crt
+              path: tls-ca-bundle.pem
       nodeSelector:
         node-role.kubernetes.io/master: ""
-      priorityClassName: "system-cluster-critical"
       tolerations:
       - operator: Exists

pkg/operator/operator.go

@@ -18,6 +18,8 @@ import (
 	"k8s.io/klog"
 
 	operatorapiv1 "github.com/openshift/api/operator/v1"
+	configinformerv1 "github.com/openshift/client-go/config/informers/externalversions/config/v1"
+	proxyvclient1 "github.com/openshift/client-go/config/listers/config/v1"
 	operatorclientv1 "github.com/openshift/client-go/operator/clientset/versioned/typed/operator/v1"
 	operatorinformersv1 "github.com/openshift/client-go/operator/informers/externalversions/operator/v1"
 	"github.com/openshift/cluster-openshift-controller-manager-operator/pkg/util"
@@ -34,6 +36,7 @@ const (
 type OpenShiftControllerManagerOperator struct {
 	targetImagePullSpec  string
 	operatorConfigClient operatorclientv1.OperatorV1Interface
+	proxyLister          proxyvclient1.ProxyLister
 
 	kubeClient kubernetes.Interface
 
@@ -47,6 +50,7 @@ type OpenShiftControllerManagerOperator struct {
 func NewOpenShiftControllerManagerOperator(
 	targetImagePullSpec string,
 	operatorConfigInformer operatorinformersv1.OpenShiftControllerManagerInformer,
+	proxyInformer configinformerv1.ProxyInformer,
 	kubeInformersForOpenshiftControllerManager informers.SharedInformerFactory,
 	operatorConfigClient operatorclientv1.OperatorV1Interface,
 	kubeClient kubernetes.Interface,
@@ -55,13 +59,15 @@ func NewOpenShiftControllerManagerOperator(
 	c := &OpenShiftControllerManagerOperator{
 		targetImagePullSpec:  targetImagePullSpec,
 		operatorConfigClient: operatorConfigClient,
+		proxyLister:          proxyInformer.Lister(),
 		kubeClient:           kubeClient,
 		queue:                workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "KubeApiserverOperator"),
 		rateLimiter:          flowcontrol.NewTokenBucketRateLimiter(0.05 /*3 per minute*/, 4),
 		recorder:             recorder,
 	}
 
 	operatorConfigInformer.Informer().AddEventHandler(c.eventHandler())
+	proxyInformer.Informer().AddEventHandler(c.eventHandler())
 	kubeInformersForOpenshiftControllerManager.Core().V1().ConfigMaps().Informer().AddEventHandler(c.eventHandler())
 	kubeInformersForOpenshiftControllerManager.Core().V1().ServiceAccounts().Informer().AddEventHandler(c.eventHandler())
 	kubeInformersForOpenshiftControllerManager.Core().V1().Services().Informer().AddEventHandler(c.eventHandler())

pkg/operator/starter.go

@@ -44,6 +44,7 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller
 	operator := NewOpenShiftControllerManagerOperator(
 		os.Getenv("IMAGE"),
 		operatorConfigInformers.Operator().V1().OpenShiftControllerManagers(),
+		configInformers.Config().V1().Proxies(),
 		kubeInformersForOpenshiftControllerManagerNamespace,
 		operatorClient.OperatorV1(),
 		kubeClient,

pkg/operator/sync_openshiftcontrollermanager_v311_00.go

@@ -3,6 +3,7 @@ package operator
 import (
 	"fmt"
 	"os"
+	"reflect"
 	"strings"
 
 	appsv1 "k8s.io/api/apps/v1"
@@ -16,6 +17,9 @@ import (
 	coreclientv1 "k8s.io/client-go/kubernetes/typed/core/v1"
 
 	operatorapiv1 "github.com/openshift/api/operator/v1"
+
+	proxyvclient1 "github.com/openshift/client-go/config/listers/config/v1"
+
 	"github.com/openshift/library-go/pkg/operator/events"
 	"github.com/openshift/library-go/pkg/operator/resource/resourceapply"
 	"github.com/openshift/library-go/pkg/operator/resource/resourcehash"
@@ -89,7 +93,7 @@ func syncOpenShiftControllerManager_v311_00_to_latest(c OpenShiftControllerManag
 
 	// our configmaps and secrets are in order, now it is time to create the DS
 	// TODO check basic preconditions here
-	actualDaemonSet, _, err := manageOpenShiftControllerManagerDeployment_v311_00_to_latest(c.kubeClient.AppsV1(), c.recorder, operatorConfig, c.targetImagePullSpec, operatorConfig.Status.Generations, forceRollout)
+	actualDaemonSet, _, err := manageOpenShiftControllerManagerDeployment_v311_00_to_latest(c.kubeClient.AppsV1(), c.recorder, operatorConfig, c.targetImagePullSpec, operatorConfig.Status.Generations, forceRollout, c.proxyLister)
 	if err != nil {
 		errors = append(errors, fmt.Errorf("%q: %v", "deployment", err))
 	}
@@ -196,6 +200,7 @@ func manageOpenShiftControllerManagerConfigMap_v311_00_to_latest(kubeClient kube
 		kubeClient,
 		resourcehash.NewObjectRef().ForConfigMap().InNamespace(util.TargetNamespace).Named("client-ca"),
 		resourcehash.NewObjectRef().ForSecret().InNamespace(util.TargetNamespace).Named("serving-cert"),
+		resourcehash.NewObjectRef().ForConfigMap().InNamespace(util.TargetNamespace).Named("openshift-global-ca"),
 	)
 	if err != nil {
 		return nil, false, err
@@ -271,7 +276,7 @@ func manageOpenShiftGlobalCAConfigMap_v311_00_to_latest(kubeClient kubernetes.In
 	return updated, true, nil
 }
 
-func manageOpenShiftControllerManagerDeployment_v311_00_to_latest(client appsclientv1.DaemonSetsGetter, recorder events.Recorder, options *operatorapiv1.OpenShiftControllerManager, imagePullSpec string, generationStatus []operatorapiv1.GenerationStatus, forceRollout bool) (*appsv1.DaemonSet, bool, error) {
+func manageOpenShiftControllerManagerDeployment_v311_00_to_latest(client appsclientv1.DaemonSetsGetter, recorder events.Recorder, options *operatorapiv1.OpenShiftControllerManager, imagePullSpec string, generationStatus []operatorapiv1.GenerationStatus, forceRollout bool, proxyLister proxyvclient1.ProxyLister) (*appsv1.DaemonSet, bool, error) {
 	required := resourceread.ReadDaemonSetV1OrDie(v311_00_assets.MustAsset("v3.11.0/openshift-controller-manager/ds.yaml"))
 
 	if len(imagePullSpec) > 0 {
@@ -295,5 +300,60 @@ func manageOpenShiftControllerManagerDeployment_v311_00_to_latest(client appscli
 	}
 	required.Annotations[util.VersionAnnotation] = os.Getenv("RELEASE_VERSION")
 
+	proxyCfg, err := proxyLister.Get("cluster")
+	if err != nil {
+		recorder.Eventf("ProxyConfigGetFailed", "Error retrieving global proxy config: %s", err.Error())
+		if !apierrors.IsNotFound(err) {
+			// return daemonset since it is still referenced by caller even with errors
+			return required, false, err
+		}
+	} else {
+		for i, c := range required.Spec.Template.Spec.Containers {
+			newEnvs := []corev1.EnvVar{}
+
+			if len(c.Env) == 0 {
+				if len(proxyCfg.Status.NoProxy) > 0 {
+					newEnvs = append(newEnvs, corev1.EnvVar{Name: "NO_PROXY", Value: proxyCfg.Status.NoProxy})
+				}
+				if len(proxyCfg.Status.HTTPProxy) > 0 {
+					newEnvs = append(newEnvs, corev1.EnvVar{Name: "HTTP_PROXY", Value: proxyCfg.Status.HTTPProxy})
+				}
+				if len(proxyCfg.Status.HTTPSProxy) > 0 {
+					newEnvs = append(newEnvs, corev1.EnvVar{Name: "HTTPS_PROXY", Value: proxyCfg.Status.HTTPSProxy})
+				}
+			}
+
+			for _, env := range c.Env {
+				name := strings.TrimSpace(env.Name)
+				switch name {
+				case "HTTPS_PROXY":
+					if len(proxyCfg.Status.HTTPSProxy) == 0 {
+						continue
+					}
+					env.Value = proxyCfg.Status.HTTPSProxy
+
+				case "HTTP_PROXY":
+					if len(proxyCfg.Status.HTTPProxy) == 0 {
+						continue
+					}
+					env.Value = proxyCfg.Status.HTTPProxy
+
+				case "NO_PROXY":
+					if len(proxyCfg.Status.NoProxy) == 0 {
+						continue
+					}
+					env.Value = proxyCfg.Status.NoProxy
+
+				}
+				newEnvs = append(newEnvs, env)
+			}
+			// reflect.DeepEqual does not consider this case equal
+			envsEqual := c.Env == nil && len(newEnvs) == 0
+			envsEqual = envsEqual || !reflect.DeepEqual(newEnvs, c.Env)
+			forceRollout = forceRollout || !envsEqual
+			required.Spec.Template.Spec.Containers[i].Env = newEnvs
+		}
+	}
+
 	return resourceapply.ApplyDaemonSet(client, recorder, required, resourcemerge.ExpectedDaemonSetGeneration(required, generationStatus), forceRollout)
 }

pkg/operator/sync_openshiftcontrollermanager_v311_00_test.go

@@ -1,6 +1,9 @@
 package operator
 
 import (
+	configv1 "github.com/openshift/api/config/v1"
+	configlistersv1 "github.com/openshift/client-go/config/listers/config/v1"
+	"k8s.io/client-go/tools/cache"
 	"testing"
 
 	appsv1 "k8s.io/api/apps/v1"
@@ -157,6 +160,9 @@ func TestProgressingCondition(t *testing.T) {
 					},
 				})
 
+			indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{})
+			proxyLister := configlistersv1.NewProxyLister(indexer)
+
 			operatorConfig := &operatorv1.OpenShiftControllerManager{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:       "cluster",
@@ -175,6 +181,7 @@ func TestProgressingCondition(t *testing.T) {
 
 			operator := OpenShiftControllerManagerOperator{
 				kubeClient:           kubeClient,
+				proxyLister:          proxyLister,
 				recorder:             events.NewInMemoryRecorder(""),
 				operatorConfigClient: controllerManagerOperatorClient.OperatorV1(),
 			}
@@ -201,3 +208,93 @@ func TestProgressingCondition(t *testing.T) {
 	}
 
 }
+
+func TestDeploymentWithProxy(t *testing.T) {
+	kubeClient := fake.NewSimpleClientset(
+		&appsv1.DaemonSet{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:       "controller-manager",
+				Namespace:  "openshift-controller-manager",
+				Generation: 2,
+			},
+		},
+	)
+	dsClient := kubeClient.AppsV1()
+	indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{})
+	proxyConfig := &configv1.Proxy{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "cluster",
+		},
+		Spec: configv1.ProxySpec{
+			NoProxy:    "no-proxy",
+			HTTPProxy:  "http://my-proxy",
+			HTTPSProxy: "https://my-proxy",
+		},
+		Status: configv1.ProxyStatus{
+			NoProxy:    "no-proxy",
+			HTTPProxy:  "http://my-proxy",
+			HTTPSProxy: "https://my-proxy"},
+	}
+	indexer.Add(proxyConfig)
+	proxyLister := configlistersv1.NewProxyLister(indexer)
+	recorder := events.NewInMemoryRecorder("")
+	operatorConfig := &operatorv1.OpenShiftControllerManager{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "cluster",
+			Generation: 2,
+		},
+		Spec: operatorv1.OpenShiftControllerManagerSpec{
+			OperatorSpec: operatorv1.OperatorSpec{},
+		},
+		Status: operatorv1.OpenShiftControllerManagerStatus{
+			OperatorStatus: operatorv1.OperatorStatus{
+				ObservedGeneration: 2,
+			},
+		},
+	}
+
+	ds, rcBool, err := manageOpenShiftControllerManagerDeployment_v311_00_to_latest(dsClient, recorder, operatorConfig, "my.co/repo/img:latest", operatorConfig.Status.Generations, false, proxyLister)
+
+	if err != nil {
+		t.Fatalf("unexpected error: %s", err.Error())
+	}
+	if !rcBool {
+		t.Fatal("apply daemon set does not think a changes was made")
+	}
+
+	if ds == nil {
+		t.Fatalf("nil daemonset returned")
+	}
+
+	foundNoProxy := false
+	foundHTTPProxy := false
+	foundHTTPSProxy := false
+	for _, c := range ds.Spec.Template.Spec.Containers {
+		for _, e := range c.Env {
+			switch e.Name {
+			case "NO_PROXY":
+				if e.Value == proxyConfig.Status.NoProxy {
+					foundNoProxy = true
+				}
+			case "HTTP_PROXY":
+				if e.Value == proxyConfig.Status.HTTPProxy {
+					foundHTTPProxy = true
+				}
+			case "HTTPS_PROXY":
+				if e.Value == proxyConfig.Status.HTTPSProxy {
+					foundHTTPSProxy = true
+				}
+			}
+		}
+	}
+
+	if !foundNoProxy {
+		t.Fatalf("NO_PROXY not found: %#v", ds.Spec.Template.Spec.Containers)
+	}
+	if !foundHTTPProxy {
+		t.Fatalf("HTTP_PROXY not found: %#v", ds.Spec.Template.Spec.Containers)
+	}
+	if !foundHTTPSProxy {
+		t.Fatalf("HTTPS_PROXY not found: %#v", ds.Spec.Template.Spec.Containers)
+	}
+}