diff --git a/bindata/v3.11.0/openshift-controller-manager/ds.yaml b/bindata/v3.11.0/openshift-controller-manager/ds.yaml index 2ad0fbdd0..8d786f798 100644 --- a/bindata/v3.11.0/openshift-controller-manager/ds.yaml +++ b/bindata/v3.11.0/openshift-controller-manager/ds.yaml @@ -45,6 +45,8 @@ spec: name: client-ca - mountPath: /var/run/secrets/serving-cert name: serving-cert + - mountPath: /etc/pki/ca-trust/extracted/pem + name: proxy-ca-bundles volumes: - name: config configMap: @@ -55,8 +57,13 @@ spec: - name: serving-cert secret: secretName: serving-cert + - name: proxy-ca-bundles + configMap: + name: openshift-global-ca + items: + - key: ca-bundle.crt + path: tls-ca-bundle.pem nodeSelector: node-role.kubernetes.io/master: "" - priorityClassName: "system-cluster-critical" tolerations: - operator: Exists diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index caa10dfce..85a000e42 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -18,6 +18,8 @@ import ( "k8s.io/klog" operatorapiv1 "github.com/openshift/api/operator/v1" + configinformerv1 "github.com/openshift/client-go/config/informers/externalversions/config/v1" + proxyvclient1 "github.com/openshift/client-go/config/listers/config/v1" operatorclientv1 "github.com/openshift/client-go/operator/clientset/versioned/typed/operator/v1" operatorinformersv1 "github.com/openshift/client-go/operator/informers/externalversions/operator/v1" "github.com/openshift/cluster-openshift-controller-manager-operator/pkg/util" @@ -34,6 +36,7 @@ const ( type OpenShiftControllerManagerOperator struct { targetImagePullSpec string operatorConfigClient operatorclientv1.OperatorV1Interface + proxyLister proxyvclient1.ProxyLister kubeClient kubernetes.Interface @@ -47,6 +50,7 @@ type OpenShiftControllerManagerOperator struct { func NewOpenShiftControllerManagerOperator( targetImagePullSpec string, operatorConfigInformer operatorinformersv1.OpenShiftControllerManagerInformer, + proxyInformer configinformerv1.ProxyInformer, kubeInformersForOpenshiftControllerManager informers.SharedInformerFactory, operatorConfigClient operatorclientv1.OperatorV1Interface, kubeClient kubernetes.Interface, @@ -55,6 +59,7 @@ func NewOpenShiftControllerManagerOperator( c := &OpenShiftControllerManagerOperator{ targetImagePullSpec: targetImagePullSpec, operatorConfigClient: operatorConfigClient, + proxyLister: proxyInformer.Lister(), kubeClient: kubeClient, queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "KubeApiserverOperator"), rateLimiter: flowcontrol.NewTokenBucketRateLimiter(0.05 /*3 per minute*/, 4), @@ -62,6 +67,7 @@ func NewOpenShiftControllerManagerOperator( } operatorConfigInformer.Informer().AddEventHandler(c.eventHandler()) + proxyInformer.Informer().AddEventHandler(c.eventHandler()) kubeInformersForOpenshiftControllerManager.Core().V1().ConfigMaps().Informer().AddEventHandler(c.eventHandler()) kubeInformersForOpenshiftControllerManager.Core().V1().ServiceAccounts().Informer().AddEventHandler(c.eventHandler()) kubeInformersForOpenshiftControllerManager.Core().V1().Services().Informer().AddEventHandler(c.eventHandler()) diff --git a/pkg/operator/starter.go b/pkg/operator/starter.go index 29133c786..e3f715738 100644 --- a/pkg/operator/starter.go +++ b/pkg/operator/starter.go @@ -44,6 +44,7 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller operator := NewOpenShiftControllerManagerOperator( os.Getenv("IMAGE"), operatorConfigInformers.Operator().V1().OpenShiftControllerManagers(), + configInformers.Config().V1().Proxies(), kubeInformersForOpenshiftControllerManagerNamespace, operatorClient.OperatorV1(), kubeClient, diff --git a/pkg/operator/sync_openshiftcontrollermanager_v311_00.go b/pkg/operator/sync_openshiftcontrollermanager_v311_00.go index 5b4aade0e..2c3cc9595 100644 --- a/pkg/operator/sync_openshiftcontrollermanager_v311_00.go +++ b/pkg/operator/sync_openshiftcontrollermanager_v311_00.go @@ -3,6 +3,7 @@ package operator import ( "fmt" "os" + "reflect" "strings" appsv1 "k8s.io/api/apps/v1" @@ -16,6 +17,9 @@ import ( coreclientv1 "k8s.io/client-go/kubernetes/typed/core/v1" operatorapiv1 "github.com/openshift/api/operator/v1" + + proxyvclient1 "github.com/openshift/client-go/config/listers/config/v1" + "github.com/openshift/library-go/pkg/operator/events" "github.com/openshift/library-go/pkg/operator/resource/resourceapply" "github.com/openshift/library-go/pkg/operator/resource/resourcehash" @@ -89,7 +93,7 @@ func syncOpenShiftControllerManager_v311_00_to_latest(c OpenShiftControllerManag // our configmaps and secrets are in order, now it is time to create the DS // TODO check basic preconditions here - actualDaemonSet, _, err := manageOpenShiftControllerManagerDeployment_v311_00_to_latest(c.kubeClient.AppsV1(), c.recorder, operatorConfig, c.targetImagePullSpec, operatorConfig.Status.Generations, forceRollout) + actualDaemonSet, _, err := manageOpenShiftControllerManagerDeployment_v311_00_to_latest(c.kubeClient.AppsV1(), c.recorder, operatorConfig, c.targetImagePullSpec, operatorConfig.Status.Generations, forceRollout, c.proxyLister) if err != nil { errors = append(errors, fmt.Errorf("%q: %v", "deployment", err)) } @@ -196,6 +200,7 @@ func manageOpenShiftControllerManagerConfigMap_v311_00_to_latest(kubeClient kube kubeClient, resourcehash.NewObjectRef().ForConfigMap().InNamespace(util.TargetNamespace).Named("client-ca"), resourcehash.NewObjectRef().ForSecret().InNamespace(util.TargetNamespace).Named("serving-cert"), + resourcehash.NewObjectRef().ForConfigMap().InNamespace(util.TargetNamespace).Named("openshift-global-ca"), ) if err != nil { return nil, false, err @@ -271,7 +276,7 @@ func manageOpenShiftGlobalCAConfigMap_v311_00_to_latest(kubeClient kubernetes.In return updated, true, nil } -func manageOpenShiftControllerManagerDeployment_v311_00_to_latest(client appsclientv1.DaemonSetsGetter, recorder events.Recorder, options *operatorapiv1.OpenShiftControllerManager, imagePullSpec string, generationStatus []operatorapiv1.GenerationStatus, forceRollout bool) (*appsv1.DaemonSet, bool, error) { +func manageOpenShiftControllerManagerDeployment_v311_00_to_latest(client appsclientv1.DaemonSetsGetter, recorder events.Recorder, options *operatorapiv1.OpenShiftControllerManager, imagePullSpec string, generationStatus []operatorapiv1.GenerationStatus, forceRollout bool, proxyLister proxyvclient1.ProxyLister) (*appsv1.DaemonSet, bool, error) { required := resourceread.ReadDaemonSetV1OrDie(v311_00_assets.MustAsset("v3.11.0/openshift-controller-manager/ds.yaml")) if len(imagePullSpec) > 0 { @@ -295,5 +300,60 @@ func manageOpenShiftControllerManagerDeployment_v311_00_to_latest(client appscli } required.Annotations[util.VersionAnnotation] = os.Getenv("RELEASE_VERSION") + proxyCfg, err := proxyLister.Get("cluster") + if err != nil { + recorder.Eventf("ProxyConfigGetFailed", "Error retrieving global proxy config: %s", err.Error()) + if !apierrors.IsNotFound(err) { + // return daemonset since it is still referenced by caller even with errors + return required, false, err + } + } else { + for i, c := range required.Spec.Template.Spec.Containers { + newEnvs := []corev1.EnvVar{} + + if len(c.Env) == 0 { + if len(proxyCfg.Status.NoProxy) > 0 { + newEnvs = append(newEnvs, corev1.EnvVar{Name: "NO_PROXY", Value: proxyCfg.Status.NoProxy}) + } + if len(proxyCfg.Status.HTTPProxy) > 0 { + newEnvs = append(newEnvs, corev1.EnvVar{Name: "HTTP_PROXY", Value: proxyCfg.Status.HTTPProxy}) + } + if len(proxyCfg.Status.HTTPSProxy) > 0 { + newEnvs = append(newEnvs, corev1.EnvVar{Name: "HTTPS_PROXY", Value: proxyCfg.Status.HTTPSProxy}) + } + } + + for _, env := range c.Env { + name := strings.TrimSpace(env.Name) + switch name { + case "HTTPS_PROXY": + if len(proxyCfg.Status.HTTPSProxy) == 0 { + continue + } + env.Value = proxyCfg.Status.HTTPSProxy + + case "HTTP_PROXY": + if len(proxyCfg.Status.HTTPProxy) == 0 { + continue + } + env.Value = proxyCfg.Status.HTTPProxy + + case "NO_PROXY": + if len(proxyCfg.Status.NoProxy) == 0 { + continue + } + env.Value = proxyCfg.Status.NoProxy + + } + newEnvs = append(newEnvs, env) + } + // reflect.DeepEqual does not consider this case equal + envsEqual := c.Env == nil && len(newEnvs) == 0 + envsEqual = envsEqual || !reflect.DeepEqual(newEnvs, c.Env) + forceRollout = forceRollout || !envsEqual + required.Spec.Template.Spec.Containers[i].Env = newEnvs + } + } + return resourceapply.ApplyDaemonSet(client, recorder, required, resourcemerge.ExpectedDaemonSetGeneration(required, generationStatus), forceRollout) } diff --git a/pkg/operator/sync_openshiftcontrollermanager_v311_00_test.go b/pkg/operator/sync_openshiftcontrollermanager_v311_00_test.go index 51ac9cdf9..3c4e77472 100644 --- a/pkg/operator/sync_openshiftcontrollermanager_v311_00_test.go +++ b/pkg/operator/sync_openshiftcontrollermanager_v311_00_test.go @@ -1,6 +1,9 @@ package operator import ( + configv1 "github.com/openshift/api/config/v1" + configlistersv1 "github.com/openshift/client-go/config/listers/config/v1" + "k8s.io/client-go/tools/cache" "testing" appsv1 "k8s.io/api/apps/v1" @@ -157,6 +160,9 @@ func TestProgressingCondition(t *testing.T) { }, }) + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) + proxyLister := configlistersv1.NewProxyLister(indexer) + operatorConfig := &operatorv1.OpenShiftControllerManager{ ObjectMeta: metav1.ObjectMeta{ Name: "cluster", @@ -175,6 +181,7 @@ func TestProgressingCondition(t *testing.T) { operator := OpenShiftControllerManagerOperator{ kubeClient: kubeClient, + proxyLister: proxyLister, recorder: events.NewInMemoryRecorder(""), operatorConfigClient: controllerManagerOperatorClient.OperatorV1(), } @@ -201,3 +208,93 @@ func TestProgressingCondition(t *testing.T) { } } + +func TestDeploymentWithProxy(t *testing.T) { + kubeClient := fake.NewSimpleClientset( + &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "controller-manager", + Namespace: "openshift-controller-manager", + Generation: 2, + }, + }, + ) + dsClient := kubeClient.AppsV1() + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) + proxyConfig := &configv1.Proxy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + }, + Spec: configv1.ProxySpec{ + NoProxy: "no-proxy", + HTTPProxy: "http://my-proxy", + HTTPSProxy: "https://my-proxy", + }, + Status: configv1.ProxyStatus{ + NoProxy: "no-proxy", + HTTPProxy: "http://my-proxy", + HTTPSProxy: "https://my-proxy"}, + } + indexer.Add(proxyConfig) + proxyLister := configlistersv1.NewProxyLister(indexer) + recorder := events.NewInMemoryRecorder("") + operatorConfig := &operatorv1.OpenShiftControllerManager{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster", + Generation: 2, + }, + Spec: operatorv1.OpenShiftControllerManagerSpec{ + OperatorSpec: operatorv1.OperatorSpec{}, + }, + Status: operatorv1.OpenShiftControllerManagerStatus{ + OperatorStatus: operatorv1.OperatorStatus{ + ObservedGeneration: 2, + }, + }, + } + + ds, rcBool, err := manageOpenShiftControllerManagerDeployment_v311_00_to_latest(dsClient, recorder, operatorConfig, "my.co/repo/img:latest", operatorConfig.Status.Generations, false, proxyLister) + + if err != nil { + t.Fatalf("unexpected error: %s", err.Error()) + } + if !rcBool { + t.Fatal("apply daemon set does not think a changes was made") + } + + if ds == nil { + t.Fatalf("nil daemonset returned") + } + + foundNoProxy := false + foundHTTPProxy := false + foundHTTPSProxy := false + for _, c := range ds.Spec.Template.Spec.Containers { + for _, e := range c.Env { + switch e.Name { + case "NO_PROXY": + if e.Value == proxyConfig.Status.NoProxy { + foundNoProxy = true + } + case "HTTP_PROXY": + if e.Value == proxyConfig.Status.HTTPProxy { + foundHTTPProxy = true + } + case "HTTPS_PROXY": + if e.Value == proxyConfig.Status.HTTPSProxy { + foundHTTPSProxy = true + } + } + } + } + + if !foundNoProxy { + t.Fatalf("NO_PROXY not found: %#v", ds.Spec.Template.Spec.Containers) + } + if !foundHTTPProxy { + t.Fatalf("HTTP_PROXY not found: %#v", ds.Spec.Template.Spec.Containers) + } + if !foundHTTPSProxy { + t.Fatalf("HTTPS_PROXY not found: %#v", ds.Spec.Template.Spec.Containers) + } +} diff --git a/pkg/operator/v311_00_assets/bindata.go b/pkg/operator/v311_00_assets/bindata.go index fbc2eb64c..407e0aed3 100644 --- a/pkg/operator/v311_00_assets/bindata.go +++ b/pkg/operator/v311_00_assets/bindata.go @@ -153,6 +153,8 @@ spec: name: client-ca - mountPath: /var/run/secrets/serving-cert name: serving-cert + - mountPath: /etc/pki/ca-trust/extracted/pem + name: proxy-ca-bundles volumes: - name: config configMap: @@ -163,9 +165,14 @@ spec: - name: serving-cert secret: secretName: serving-cert + - name: proxy-ca-bundles + configMap: + name: openshift-global-ca + items: + - key: ca-bundle.crt + path: tls-ca-bundle.pem nodeSelector: node-role.kubernetes.io/master: "" - priorityClassName: "system-cluster-critical" tolerations: - operator: Exists `)