From 741876f64cad498db6913922b890d4c723ce574a Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Wed, 20 Nov 2019 10:10:59 -0800 Subject: [PATCH 1/6] Bug 1765044: Adds proxy support to ingress operator --- cmd/ingress-operator/start.go | 20 +++- hack/uninstall.sh | 1 + manifests/01-trusted-ca-configmap.yaml | 11 ++ manifests/02-deployment.yaml | 12 ++ pkg/dns/aws/dns.go | 20 +++- pkg/dns/aws/filewatcher.go | 76 +++++++++++++ pkg/manifests/bindata.go | 32 +++++- pkg/operator/watcher/filewatcher.go | 146 +++++++++++++++++++++++++ pkg/util/certs/certs.go | 55 ++++++++++ 9 files changed, 360 insertions(+), 13 deletions(-) create mode 100644 manifests/01-trusted-ca-configmap.yaml create mode 100644 pkg/dns/aws/filewatcher.go create mode 100644 pkg/operator/watcher/filewatcher.go create mode 100644 pkg/util/certs/certs.go diff --git a/cmd/ingress-operator/start.go b/cmd/ingress-operator/start.go index 9a419e213c..a330e5afd6 100644 --- a/cmd/ingress-operator/start.go +++ b/cmd/ingress-operator/start.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "github.com/openshift/cluster-ingress-operator/pkg/operator/watcher" "os" "github.com/ghodss/yaml" @@ -35,6 +36,8 @@ const ( // operator's namespace that will hold the credentials that the operator // will use to authenticate with the cloud API. cloudCredentialsSecretName = "cloud-credentials" + // The fully qualified path of the trusted CA certificate bundle file. + defaultCABundle = "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem" ) func NewStartCommand() *cobra.Command { @@ -105,6 +108,14 @@ func start() error { os.Exit(1) } + // Setup the file watcher for the trusted ca bundle. + caWatcher, err := watcher.New(defaultCABundle) + if err != nil { + log.Error(err, "failed to create trusted ca bundle watcher") + os.Exit(1) + } + + stop := signals.SetupSignalHandler() operatorConfig := operatorconfig.Config{ OperatorReleaseVersion: releaseVersion, Namespace: operatorNamespace, @@ -112,7 +123,7 @@ func start() error { } // Set up the DNS manager. - dnsProvider, err := createDNSProvider(kubeClient, operatorConfig, dnsConfig, platformStatus) + dnsProvider, err := createDNSProvider(kubeClient, operatorConfig, dnsConfig, platformStatus, caWatcher, stop) if err != nil { log.Error(err, "failed to create DNS manager") os.Exit(1) @@ -123,12 +134,13 @@ func start() error { if err != nil { return fmt.Errorf("failed to create operator: %v", err) } - return op.Start(signals.SetupSignalHandler()) + return op.Start(stop) } // createDNSManager creates a DNS manager compatible with the given cluster // configuration. -func createDNSProvider(cl client.Client, operatorConfig operatorconfig.Config, dnsConfig *configv1.DNS, platformStatus *configv1.PlatformStatus) (dns.Provider, error) { +func createDNSProvider(cl client.Client, operatorConfig operatorconfig.Config, dnsConfig *configv1.DNS, + platformStatus *configv1.PlatformStatus, watcher *watcher.FileWatcher, stop <-chan struct{}) (dns.Provider, error) { var dnsProvider dns.Provider userAgent := fmt.Sprintf("OpenShift/%s (ingress-operator)", operatorConfig.OperatorReleaseVersion) @@ -144,7 +156,7 @@ func createDNSProvider(cl client.Client, operatorConfig operatorconfig.Config, d AccessKey: string(creds.Data["aws_secret_access_key"]), DNS: dnsConfig, Region: platformStatus.AWS.Region, - }, operatorConfig.OperatorReleaseVersion) + }, operatorConfig.OperatorReleaseVersion, watcher, stop) if err != nil { return nil, fmt.Errorf("failed to create AWS DNS manager: %v", err) } diff --git a/hack/uninstall.sh b/hack/uninstall.sh index 51470be58a..01345db700 100755 --- a/hack/uninstall.sh +++ b/hack/uninstall.sh @@ -15,6 +15,7 @@ oc delete clusteroperator.config.openshift.io/ingress oc delete --force --grace-period=0 -n openshift-ingress-operator ingresscontroller/default if [ "$WHAT" == "all" ]; then + oc delete configmap/trusted-ca -n openshift-ingress-operator oc delete namespaces/openshift-ingress-operator else oc delete -n openshift-ingress-operator secrets/router-ca diff --git a/manifests/01-trusted-ca-configmap.yaml b/manifests/01-trusted-ca-configmap.yaml new file mode 100644 index 0000000000..203cc10c16 --- /dev/null +++ b/manifests/01-trusted-ca-configmap.yaml @@ -0,0 +1,11 @@ +# The network operator is responsible for injecting +# the trusted ca bundle into this configmap. +apiVersion: v1 +kind: ConfigMap +metadata: + annotations: + release.openshift.io/create-only: "true" + labels: + config.openshift.io/inject-trusted-cabundle: "true" + name: trusted-ca + namespace: openshift-ingress-operator diff --git a/manifests/02-deployment.yaml b/manifests/02-deployment.yaml index 9ac5cd6209..06c5176eaa 100644 --- a/manifests/02-deployment.yaml +++ b/manifests/02-deployment.yaml @@ -3,6 +3,8 @@ kind: Deployment metadata: name: ingress-operator namespace: openshift-ingress-operator + annotations: + config.openshift.io/inject-proxy: ingress-operator spec: replicas: 1 strategy: @@ -52,6 +54,9 @@ spec: resources: requests: cpu: 10m + volumeMounts: + - name: trusted-ca + mountPath: /etc/pki/ca-trust/extracted/pem/ - name: kube-rbac-proxy image: quay.io/openshift/origin-kube-rbac-proxy:latest args: @@ -76,3 +81,10 @@ spec: - name: metrics-tls secret: secretName: metrics-tls + - name: trusted-ca + configMap: + name: trusted-ca + optional: false + items: + - key: ca-bundle.crt + path: tls-ca-bundle.pem diff --git a/pkg/dns/aws/dns.go b/pkg/dns/aws/dns.go index afec8d0b8f..220f83fdef 100644 --- a/pkg/dns/aws/dns.go +++ b/pkg/dns/aws/dns.go @@ -9,6 +9,7 @@ import ( iov1 "github.com/openshift/cluster-ingress-operator/pkg/api/v1" "github.com/openshift/cluster-ingress-operator/pkg/dns" logf "github.com/openshift/cluster-ingress-operator/pkg/log" + "github.com/openshift/cluster-ingress-operator/pkg/operator/watcher" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/arn" @@ -45,6 +46,10 @@ type Provider struct { config Config + // fileWatcher watches the trusted ca bundle for changes + // and keeps CA certificates of DNS clients updated. + fileWatcher *watcher.FileWatcher + // lock protects access to everything below. lock sync.RWMutex @@ -75,7 +80,7 @@ type Config struct { DNS *configv1.DNS } -func NewProvider(config Config, operatorReleaseVersion string) (*Provider, error) { +func NewProvider(config Config, operatorReleaseVersion string, watcher *watcher.FileWatcher, stop <-chan struct{}) (*Provider, error) { creds := credentials.NewStaticCredentials(config.AccessID, config.AccessKey, "") sess, err := session.NewSessionWithOptions(session.Options{ Config: aws.Config{ @@ -102,9 +107,10 @@ func NewProvider(config Config, operatorReleaseVersion string) (*Provider, error return nil, fmt.Errorf("region is required") } - return &Provider{ - elb: elb.New(sess, aws.NewConfig().WithRegion(region)), - route53: route53.New(sess), + provider := &Provider{ + elb: elb.New(sess, aws.NewConfig().WithRegion(region)), + route53: route53.New(sess), + fileWatcher: watcher, // TODO: This API will only return hostedzone resources (which are global) // when the region is forced to us-east-1. We don't yet understand why. tags: resourcegroupstaggingapi.New(sess, aws.NewConfig().WithRegion("us-east-1")), @@ -112,7 +118,11 @@ func NewProvider(config Config, operatorReleaseVersion string) (*Provider, error idsToTags: map[string]map[string]string{}, lbZones: map[string]string{}, updatedRecords: sets.NewString(), - }, nil + } + if err := provider.startWatcher(stop); err != nil { + return nil, fmt.Errorf("failed to start trusted ca bundle file watcher: %v", err) + } + return provider, nil } // getZoneID finds the ID of given zoneConfig in Route53. If an ID is already diff --git a/pkg/dns/aws/filewatcher.go b/pkg/dns/aws/filewatcher.go new file mode 100644 index 0000000000..e44ce43654 --- /dev/null +++ b/pkg/dns/aws/filewatcher.go @@ -0,0 +1,76 @@ +package aws + +import ( + "bytes" + "io/ioutil" + "time" + + certsutil "github.com/openshift/cluster-ingress-operator/pkg/util/certs" + + "k8s.io/apimachinery/pkg/util/wait" +) + +// startWatcher starts the FileWatcher and periodically ensures DNS clients +// are using the current ca bundle until a message is received on the stop +// or error channels. +func (m *Provider) startWatcher(stop <-chan struct{}) error { + go wait.Until(func() { + if err := m.ensureDNSClientsTLSTransport(); err != nil { + log.Error(err, "failed to ensure dns client tls transport") + } + }, 1*time.Minute, stop) + + errChan := make(chan error) + go func() { + errChan <- m.fileWatcher.Start(stop) + }() + + // Wait for the watcher to exit or an explicit stop. + select { + case <-stop: + return nil + case err := <-errChan: + return err + } +} + +// ensureDNSClientsTLSTransport compares the watched ca bundle with the +// current ca bundle of FileWatcher and updates DNS clients with the current +// ca bundle if the two are not equal. +func (m *Provider) ensureDNSClientsTLSTransport() error { + equal, caBundle, err := m.caBundlesEqual() + if err != nil { + return err + } + if equal { + return nil + } + + certs, err := certsutil.CertsFromPEM(caBundle) + if err != nil { + return err + } + + m.route53.Client.Config.HTTPClient.Transport = certsutil.MakeTLSTransport(certs) + m.elb.Client.Config.HTTPClient.Transport = certsutil.MakeTLSTransport(certs) + m.tags.Client.Config.HTTPClient.Transport = certsutil.MakeTLSTransport(certs) + + return nil +} + +// caBundlesEqual compares the watched ca bundle with the current +// ca bundle of FileWatcher, returning false and the current ca +// bundle if the two are not equal. +func (m *Provider) caBundlesEqual() (bool, []byte, error) { + watchedCAs, err := ioutil.ReadFile(m.fileWatcher.GetFile()) + if err != nil { + return false, nil, err + } + + currentCAs := m.fileWatcher.GetFileData() + if !bytes.Equal(watchedCAs, currentCAs) { + return false, currentCAs, nil + } + + return true, nil, nil +} diff --git a/pkg/manifests/bindata.go b/pkg/manifests/bindata.go index 7df8690578..278db9cb71 100644 --- a/pkg/manifests/bindata.go +++ b/pkg/manifests/bindata.go @@ -24,7 +24,8 @@ // manifests/01-role.yaml (477B) // manifests/01-service-account.yaml (196B) // manifests/01-service.yaml (344B) -// manifests/02-deployment.yaml (2.468kB) +// manifests/01-trusted-ca-configmap.yaml (323B) +// manifests/02-deployment.yaml (2.836kB) // manifests/03-cluster-operator.yaml (357B) // manifests/image-references (433B) @@ -575,7 +576,27 @@ func manifests01ServiceYaml() (*asset, error) { return a, nil } -var _manifests02DeploymentYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x56\x4d\x93\xda\x38\x10\xbd\xcf\xaf\x50\x71\x5e\x81\x99\x64\xb3\x1b\x55\xcd\x81\x25\x4e\x66\xaa\x06\x86\xc2\x53\xc9\x91\x12\x72\x03\x2a\x64\x49\xe9\x6e\x53\xe3\x7f\xbf\x65\x3e\x8d\x61\x48\x0e\xbb\x3e\x51\xea\xf7\x5e\x3f\x59\x4f\x6d\x74\xb4\xdf\x01\xc9\x06\xaf\x84\x8e\x91\x7a\x9b\xfe\xdd\xda\xfa\x5c\x89\x2f\x10\x5d\xa8\x0a\xf0\x7c\x57\x00\xeb\x5c\xb3\x56\x77\x42\x78\x5d\x80\x12\xd6\x2f\x11\x88\x64\x88\x80\x9a\x03\xee\x0b\x14\xb5\x01\x25\x42\x04\x4f\x2b\xbb\x60\x79\x81\xa3\x08\xa6\x96\x41\x88\xce\x1a\x4d\x4a\xf4\xef\x84\x20\x46\xcd\xb0\xac\xea\x8a\x10\x5c\x45\x50\x62\x0a\x06\x41\x33\xd4\x65\x70\x60\x38\xe0\xae\x5c\x68\x36\xab\x67\x3d\x07\x47\xbb\x85\x1b\xa6\x18\x8a\xe8\x34\xc3\x9e\xd9\xd8\x47\xfd\xb8\x33\x91\x1b\x32\x42\x1c\x7c\x6f\x61\x21\x87\xec\xcc\x52\xfd\xac\xcb\x39\xa0\x07\x06\xea\xda\xd0\x0b\xa4\x84\xb3\xbe\x7c\x3b\x89\x87\x1c\x24\x06\x07\xdd\x73\x64\xa1\x89\x01\x95\xe8\x74\xf6\x50\x0e\xae\x6e\x6c\x83\x3f\x5a\x93\x62\x0d\x95\x12\x9d\xdb\x1a\x9d\x63\xaf\x83\x75\x25\x3a\xe9\x9b\x25\xa6\x53\x09\x16\x0b\x30\xac\x44\x67\x1c\x32\xb3\x82\xbc\x74\xd0\xb9\xd2\xa5\xd5\xa0\xf4\x08\xda\xac\xf4\xfc\x84\xfe\xdd\x2e\xe9\x1b\x98\x92\x1b\xb4\xd3\xfe\x32\x30\xc1\xe7\x75\x06\xee\x93\x5f\x7b\xf0\x81\x25\x82\xce\xab\xff\xd7\x01\x01\x6e\xac\x81\x81\x31\xa1\xf4\x3c\x7e\x3f\x12\x42\x44\xb4\x01\x2d\x57\x43\xa7\x89\x76\x48\xaa\x88\xa1\x90\xc6\x95\xf5\x89\x48\x83\x96\xad\xd1\x6e\x4f\x30\xc1\xb3\xb6\x1e\xb0\x11\x3a\x79\x2b\x76\x7b\xbf\x80\x85\xf5\x5b\xc3\x23\x20\xd2\x4b\x98\x04\x67\x4d\xa5\xc4\x57\xed\xdc\x5c\x9b\xf5\x6b\x78\x0e\x4b\x7a\xf1\x29\xe2\x19\xd3\x16\x35\xb8\x74\xee\x40\x78\x5a\x8c\x03\x4f\x10\xa8\xbe\xd7\x2d\x5c\xe3\xe2\xf6\x02\xda\xa5\xf5\xc7\x7d\xb4\xcd\xa9\xfa\x52\x51\x53\xc1\x84\xa2\xd0\x3e\x57\x8d\x25\x79\x6b\x4f\x52\x10\x6b\x6c\x2a\x80\xdf\x34\xd9\xa7\x37\x33\x4d\x9f\xd3\x41\x96\xce\xbe\xa7\xd3\xec\xe9\x65\x7c\x86\x11\x62\xa3\x5d\x09\x4a\x74\x92\x6e\xd2\xed\x4b\xf2\x3a\xd2\x2a\x70\xe7\xaa\xd2\x8f\xc1\xeb\xf0\x71\x36\x1e\x8c\xd2\x6c\x32\x18\xa6\xd7\x94\xbe\x62\x28\x54\xab\x20\xc4\xc2\x82\xcb\xa7\xb0\xb8\xac\xec\x6b\x13\xcd\x2b\x75\x1c\x30\xdd\xe3\x30\xbc\x6a\xe3\x69\x34\xf8\x76\xb5\xf9\x95\x13\x58\xe9\x88\xe1\xad\x92\x18\xca\x7a\x4c\x6c\x3e\x76\x93\x06\x13\x81\x42\x89\x06\xe8\xdc\x18\xc2\xcf\x12\x88\xa9\x6d\xd7\xc4\x52\x89\x7e\x52\x5c\x84\xaf\xbe\x69\x12\xe7\xda\xc8\x6d\xb7\xcb\x64\xfc\x2c\x75\xb5\x9d\x6b\x6d\x7f\x2d\xe6\x65\x30\x34\x2e\xe9\x3c\x15\x52\xba\xb0\xe4\x40\x9c\x03\x62\xab\x42\x60\x4a\x04\xe9\x2c\x31\x78\xa9\xf3\xbc\xce\xcf\x83\xfa\xfc\xe1\xf3\x87\x16\x92\x1d\x49\x63\xe3\x0a\x50\x52\x69\x19\xe8\xe1\xf5\x39\x9b\xa5\xc3\x2f\x8f\xe9\x6c\x9a\x0d\x66\x3f\x9e\x5e\x1f\x67\x83\x34\x9b\xf5\xef\xff\x9e\x7d\x1b\x8e\x66\xd9\xe3\xe0\xfe\xcf\x4f\x7f\x9c\x50\xe9\xf0\xcb\x2f\x70\x17\x3a\xc3\x7f\x86\xbf\xa5\x73\x15\x77\x43\xad\xb5\xb7\x32\x12\x23\xe8\xe2\x61\xc5\x1c\x55\xaf\xd7\xbf\xff\xab\xbb\xcd\xb7\xfa\x94\x24\x49\xd2\xbb\xf6\x2a\x00\x59\x2e\xac\x83\x87\x1e\xb0\xe9\xb1\xa3\x5e\x44\xbb\xd1\x0c\xf5\xef\xae\x39\xbb\x6a\x07\xd2\x1e\x21\xd7\x50\xdd\xe0\xae\xa1\x19\x88\x18\x90\x5b\x07\x7a\x1c\x6b\x93\x80\xac\x44\xeb\xb0\x0e\xdf\xd5\x02\x18\xad\xa1\xff\x30\xbd\xbb\xa7\x80\x22\x60\xa5\xc4\xc7\x64\x64\x1b\xa5\x4d\x70\x65\x01\xa3\x7a\x88\xb7\xec\x16\xf5\xda\xee\xc2\xb6\xf7\xfb\xbe\xed\xfa\x7d\xb5\x5c\xea\xfc\xc5\xbb\x4a\x09\xc6\xf2\x40\xdc\x35\x6d\x7c\xb7\xdf\xd7\xa0\xfa\xff\x0d\x37\x9d\xed\x56\xc6\x17\x8c\x7f\x03\x00\x00\xff\xff\x04\xbf\x99\x13\xa4\x09\x00\x00") +var _manifests01TrustedCaConfigmapYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x54\x90\xbf\x6e\xf3\x30\x0c\xc4\x77\x3d\xc5\x21\x99\x9d\x0f\xdf\xaa\xb5\x73\xb7\xa2\x3b\x23\xd3\x36\x1b\x99\x14\x44\xba\x45\xdf\xbe\x48\xed\xfe\x1b\x79\xe4\x1d\x7f\xb8\x33\x9e\x16\x86\x72\xbc\x59\xbf\xc1\x1a\x77\x0a\xeb\x10\x47\x67\x6f\xa6\x2e\xd7\xca\x98\xee\x92\xbe\x70\x09\xd1\x39\x9d\x11\x0b\x23\xfa\xe6\xc1\x23\x0a\xe1\xba\xe9\x58\x19\xa2\x61\x88\x45\x1c\xc5\x74\x92\x79\xa5\x76\x49\xd4\xe4\x99\xbb\x8b\x69\xc6\xeb\xff\x74\x13\x1d\x33\x1e\x3e\xf7\x8f\xd4\xd2\xca\x41\x23\x05\xe5\x04\x90\xaa\x05\x85\x98\xfa\x7d\x04\x3a\x57\x26\xe7\x8b\x35\x56\x5f\x64\x8a\x8b\xd8\xbf\xd2\x99\x82\x07\xd3\xfa\x9e\x71\x8a\xbe\xf1\x29\x01\x95\xae\x5c\x0f\xdb\xfe\xfd\xaf\x6b\xa7\x1f\x0e\xe8\xa1\xd0\xce\xfc\x2b\x41\x69\xe5\x8c\x9f\x83\x43\xf2\x46\x85\x33\xbe\xc3\x06\xd1\xb9\xb3\xfb\xf0\xd5\x55\xfa\x08\x00\x00\xff\xff\x4c\x3f\xbd\xa8\x43\x01\x00\x00") + +func manifests01TrustedCaConfigmapYamlBytes() ([]byte, error) { + return bindataRead( + _manifests01TrustedCaConfigmapYaml, + "manifests/01-trusted-ca-configmap.yaml", + ) +} + +func manifests01TrustedCaConfigmapYaml() (*asset, error) { + bytes, err := manifests01TrustedCaConfigmapYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "manifests/01-trusted-ca-configmap.yaml", size: 323, mode: os.FileMode(420), modTime: time.Unix(1, 0)} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xe, 0x99, 0xa0, 0xf5, 0x24, 0x3, 0x8, 0xa4, 0xc3, 0xec, 0x80, 0xc0, 0x88, 0xbd, 0x8, 0xb0, 0xfc, 0xf0, 0x3, 0x42, 0x53, 0x21, 0xf3, 0xbb, 0x4a, 0x33, 0xfd, 0x7, 0xa3, 0x23, 0x4, 0xfa}} + return a, nil +} + +var _manifests02DeploymentYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x56\x4d\x6f\xdb\x38\x13\xbe\xe7\x57\x10\x3e\xbf\xb4\x9c\xb4\x6f\x77\x4b\x20\x07\xaf\xab\x36\x01\xe2\x34\x88\x83\xf6\x68\x8c\xa9\xb1\xcd\x35\x45\xb2\x9c\x91\x11\xfd\xfb\x05\xe5\x2f\x59\x76\xdc\x1c\x76\x75\x12\x34\xcf\x3c\xf3\xc1\x67\x46\x84\x60\x7e\x60\x24\xe3\x9d\x12\x10\x02\x65\xeb\xeb\xab\x95\x71\x85\x12\x5f\x30\x58\x5f\x97\xe8\xf8\xaa\x44\x86\x02\x18\xd4\x95\x10\x0e\x4a\x54\xc2\xb8\x45\x44\x22\xe9\x03\x46\x60\x1f\xb7\x06\x0a\xa0\x51\x09\x1f\xd0\xd1\xd2\xcc\x59\x9e\xc1\x81\x73\x9e\x81\x8d\x77\x94\xf8\x84\xd0\xde\xcd\xcd\xa2\xbf\x77\xea\x1b\x9f\x19\xf7\x37\x6a\x96\x21\xfa\xd7\xfa\x4c\x34\x0a\xa8\x93\x73\xc4\x60\x8d\x06\x52\xe2\xfa\x4a\x08\xe2\x08\x8c\x8b\x7a\x43\xcb\x75\x40\x25\x9e\x51\x47\x04\xc6\x64\x46\x8b\x9a\x7d\xdc\x98\x4b\x60\xbd\x7c\x80\x19\xda\x6d\x1a\x17\x4a\x63\x2c\x83\x05\xc6\xad\x67\xab\x1b\xe9\xb1\x47\x24\x17\x68\x84\xd8\xe5\xdd\xc0\x7c\x81\x93\xa3\x94\xd2\xb3\xaa\x66\x18\x1d\x32\x52\x6a\x83\x27\x25\xac\x71\xd5\xeb\x81\xdc\x17\x28\xa3\xb7\xd8\x3f\x46\x96\x40\x8c\x51\x89\x5e\x6f\x0b\x65\x6f\x53\xe0\x43\x9b\x85\x90\x62\x85\xb5\x12\xbd\xcb\x1c\xbd\x7d\xac\x5d\xea\x4a\xf4\xf2\x57\x43\x4c\x07\x13\xce\xe7\xa8\x59\x89\xde\xa3\x9f\xe8\x25\x16\x95\xc5\xde\x99\x28\x9d\x00\x95\x8b\x08\x7a\x09\xb3\x03\xfa\xbd\x51\xf2\x57\xd4\x15\xb7\xdc\x0e\xf5\x4d\x50\x7b\x57\x24\x0d\xdc\x0c\x7e\x9f\x83\xf3\x2c\x23\x42\x51\xff\xb7\x19\x10\xc6\xb5\xd1\x38\xd4\xda\x57\x8e\x1f\xdf\x96\x84\x10\x21\x1a\x1f\x0d\xd7\x23\x0b\x44\x1b\x24\xd5\xc4\x58\x4a\x6d\xab\x74\x22\x52\x47\xc3\x46\x83\xdd\x3a\x68\xef\x18\x8c\xc3\xd8\x12\x9d\xbc\x24\xbb\x6d\xbe\x18\x4b\xe3\x9a\x84\xc7\x48\x04\x0b\x7c\xf2\xd6\xe8\x5a\x89\xaf\x60\xed\x0c\xf4\xea\xc5\x3f\xf8\x05\x7d\x77\x79\x8c\x47\x9e\xa6\x4c\xe0\xca\xda\x9d\xc3\xfd\xfc\xd1\xf3\x53\x44\x4a\xdb\xa1\x83\x6b\x8d\x7f\xe6\xa3\x59\x18\xb7\xaf\xa3\x9b\x9c\x4a\x43\x45\x6d\x06\xed\xcb\x12\x5c\xa1\x5a\x9f\xe4\xa5\x9a\xa4\x20\x86\xd8\x66\x40\xb7\x6e\x7b\x1f\x3a\xf3\x9c\x3f\xe4\xc3\x49\x3e\xfd\x91\x3f\x4f\xee\xbf\x3f\x1e\x61\x84\x58\x83\xad\x50\x89\xde\xa0\x3f\xe8\x5f\x4b\x72\x10\x68\xe9\xb9\x77\x96\xe9\xe7\xf0\x65\x74\x37\x7d\x1c\x8e\xf3\xc9\xd3\x70\x94\x9f\x63\xfa\x1a\x7d\xa9\x3a\x06\x21\xe6\x06\x6d\xf1\x8c\xf3\x53\xcb\xd6\xf6\x04\xbc\x54\xfb\x05\xd3\xdf\xaf\xd4\xb3\x69\xdc\x8f\x87\xdf\xce\x06\x3f\x73\x02\x4b\x68\x56\xa9\x8c\xbe\x4a\x6b\x62\xfd\xb1\x3f\x68\x79\x46\x24\x5f\x45\x8d\x74\x9c\x58\xc4\x5f\x15\x12\x53\x37\x5d\x1d\x2a\x25\xae\x07\x65\xeb\xf3\xda\xdb\xaa\xc4\x71\xd2\x3a\x9d\x6f\x3f\xc7\xa4\x81\x42\x6a\xe8\xb0\x95\xc9\x69\x53\x78\x86\xac\xb3\xb0\x32\x99\x06\xd9\xe0\x33\x7c\xe5\x08\x9a\xb1\xc8\x02\x96\xd9\x89\xda\xd3\x68\xcb\x38\x03\xbd\xf9\x53\x9c\x4a\xf1\x57\x05\x75\xb3\x48\xbb\x0d\xe9\x78\x9e\x2a\x11\xe2\x82\x8e\x65\x28\xa5\xf5\x0b\xf6\xc4\x05\xc6\xd8\xb1\x10\xea\x2a\xa2\xb4\x86\x18\x9d\x84\xa2\x48\x82\xbd\x55\x9f\x3f\x7c\xfe\xd0\x41\xb2\x25\xa9\x4d\x58\x62\x94\x54\x19\x46\xba\x7d\x79\x98\x4c\xf3\xd1\x97\xbb\x7c\xfa\x3c\x19\x4e\x7f\xde\xbf\xdc\x4d\x87\xf9\x64\x7a\x7d\xf3\xe7\xf4\xdb\x68\x3c\x9d\xdc\x0d\x6f\xfe\xff\xe9\x7f\x07\x54\x3e\xfa\xf2\x1b\xdc\x09\xcf\xe8\xaf\xd1\xbb\x78\xce\xe2\x2e\xb0\x75\x6a\xab\x02\x71\x44\x28\x6f\x97\xcc\x41\x65\xd9\xf5\xcd\x1f\xfd\x66\xa0\xd4\xa7\xc1\x60\x30\xc8\xce\xb5\x02\x23\xcb\xb9\xb1\x78\xdb\x1c\x3e\x5b\xca\x42\x34\x6b\x60\x4c\xef\x7d\x7d\x34\xdb\x3b\xa7\x2d\x42\xae\xb0\xbe\xe0\xbb\xc2\xb6\x20\x82\x8f\xdc\x39\xd0\xfd\x1e\x7d\xf2\x91\x95\xe8\x1c\xd6\xee\x47\x5e\x22\x47\xa3\xe9\x5f\x1d\x97\x46\xf7\x58\xfa\x58\x2b\xf1\x71\x30\x36\xef\x98\x24\x79\x32\x28\xad\x7a\xdf\x4e\x3b\xf5\xab\x93\x25\x14\xdf\x9d\xad\x9b\x89\xdc\x39\x6e\x82\xb6\x2e\x0a\x6f\x73\x50\xba\x50\x71\x3b\xb3\xcd\x97\xc7\x37\x3c\x2e\xcc\xff\xe6\xe2\x37\x86\xd0\x66\xbb\xb0\x2d\x7c\x48\xff\x2e\xb0\x4a\xcc\xc1\x52\xbb\x68\xc3\x58\x9e\xec\x9d\xe6\x0a\xa0\x41\xce\x2a\x57\x58\xec\x48\x29\x3d\xa1\xe9\x65\xa3\xc2\x3d\x2a\x60\x79\xf5\x4f\x00\x00\x00\xff\xff\x84\x47\x48\x41\x14\x0b\x00\x00") func manifests02DeploymentYamlBytes() ([]byte, error) { return bindataRead( @@ -590,8 +611,8 @@ func manifests02DeploymentYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "manifests/02-deployment.yaml", size: 2468, mode: os.FileMode(420), modTime: time.Unix(1, 0)} - a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xbf, 0x8d, 0xe3, 0x66, 0x2e, 0xe7, 0xfd, 0x1, 0xc, 0xbf, 0xaf, 0xe9, 0x99, 0xac, 0xb, 0xcd, 0xd4, 0x99, 0xf, 0xbc, 0x4f, 0xd1, 0xcc, 0x9f, 0x1, 0xc3, 0x68, 0xa5, 0xb7, 0x62, 0xe5, 0xeb}} + info := bindataFileInfo{name: "manifests/02-deployment.yaml", size: 2836, mode: os.FileMode(420), modTime: time.Unix(1, 0)} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xad, 0xff, 0x4d, 0x1d, 0x13, 0x30, 0xdd, 0x9e, 0x7b, 0x68, 0x52, 0xa0, 0x87, 0x61, 0xd3, 0xb0, 0xd6, 0x65, 0x9a, 0xaf, 0xf6, 0x86, 0xba, 0x85, 0xb2, 0x95, 0x42, 0xf5, 0x6a, 0xcb, 0xef, 0x8a}} return a, nil } @@ -774,6 +795,8 @@ var _bindata = map[string]func() (*asset, error){ "manifests/01-service.yaml": manifests01ServiceYaml, + "manifests/01-trusted-ca-configmap.yaml": manifests01TrustedCaConfigmapYaml, + "manifests/02-deployment.yaml": manifests02DeploymentYaml, "manifests/03-cluster-operator.yaml": manifests03ClusterOperatorYaml, @@ -853,6 +876,7 @@ var _bintree = &bintree{nil, map[string]*bintree{ "01-role.yaml": {manifests01RoleYaml, map[string]*bintree{}}, "01-service-account.yaml": {manifests01ServiceAccountYaml, map[string]*bintree{}}, "01-service.yaml": {manifests01ServiceYaml, map[string]*bintree{}}, + "01-trusted-ca-configmap.yaml": {manifests01TrustedCaConfigmapYaml, map[string]*bintree{}}, "02-deployment.yaml": {manifests02DeploymentYaml, map[string]*bintree{}}, "03-cluster-operator.yaml": {manifests03ClusterOperatorYaml, map[string]*bintree{}}, "image-references": {manifestsImageReferences, map[string]*bintree{}}, diff --git a/pkg/operator/watcher/filewatcher.go b/pkg/operator/watcher/filewatcher.go new file mode 100644 index 0000000000..f2f7338394 --- /dev/null +++ b/pkg/operator/watcher/filewatcher.go @@ -0,0 +1,146 @@ +package watcher + +import ( + "fmt" + "io/ioutil" + "sync" + + logf "github.com/openshift/cluster-ingress-operator/pkg/log" + "gopkg.in/fsnotify.v1" +) + +var log = logf.Logger.WithName("filewatcher") + +// FileWatcher watches a file for changes. +type FileWatcher struct { + sync.Mutex + file string + currentData []byte + watcher *fsnotify.Watcher +} + +// New returns a new FileWatcher watching the given file. +func New(file string) (*FileWatcher, error) { + var err error + + cw := &FileWatcher{ + file: file, + } + + // Initial read of file. + if err := cw.ReadFile(); err != nil { + return nil, err + } + + cw.watcher, err = fsnotify.NewWatcher() + if err != nil { + return nil, err + } + + return cw, nil +} + +// GetFile fetches the file being watched by FileWatcher. +func (fw *FileWatcher) GetFile() string { + fw.Lock() + defer fw.Unlock() + return fw.file +} + +// GetFileData fetches the currently loaded file data of FileWatcher. +func (fw *FileWatcher) GetFileData() []byte { + fw.Lock() + defer fw.Unlock() + return fw.currentData +} + +// Start starts the FileWatcher. +func (fw *FileWatcher) Start(stopCh <-chan struct{}) error { + if err := fw.watcher.Add(fw.file); err != nil { + return err + } + + go fw.Watch() + + log.Info("Starting file watcher") + + // Block until the stop channel is closed. + <-stopCh + + return fw.watcher.Close() +} + +// Watch reads events from the watcher's channel and reacts to changes. +func (fw *FileWatcher) Watch() { + for { + select { + case event, ok := <-fw.watcher.Events: + // Channel is closed. + if !ok { + return + } + + fw.handleEvent(event) + + case err, ok := <-fw.watcher.Errors: + // Channel is closed. + if !ok { + return + } + + log.Error(err, "file watch error") + } + } +} + +// ReadFile reads the watched file from disk, parses the file, +// and updates FileWatcher current data. +func (fw *FileWatcher) ReadFile() error { + data, err := ioutil.ReadFile(fw.file) + switch { + case err != nil: + return fmt.Errorf("failed to read file %s: %v", fw.file, err) + case len(data) == 0: + return fmt.Errorf("file %s contains no currentData", fw.file) + } + + fw.Lock() + fw.currentData = data + fw.Unlock() + + log.Info("updated file watcher current data") + + return nil +} + +// handleEvent filters events, re-adds and re-reads the watched file +// if removed. +func (fw *FileWatcher) handleEvent(event fsnotify.Event) { + if !(isWrite(event) || isRemove(event) || isCreate(event)) { + return + } + + log.Info("watched file change", "event", event) + + if isRemove(event) { + if err := fw.watcher.Add(event.Name); err != nil { + log.Error(err, "error re-watching file %s", fw.file) + } + } + + if err := fw.ReadFile(); err != nil { + log.Error(err, "error re-reading watched file %s", fw.file) + } +} + +func isWrite(event fsnotify.Event) bool { + return event.Op&fsnotify.Write == fsnotify.Write +} + +func isCreate(event fsnotify.Event) bool { + return event.Op&fsnotify.Create == fsnotify.Create +} + +func isRemove(event fsnotify.Event) bool { + return event.Op&fsnotify.Remove == fsnotify.Remove +} diff --git a/pkg/util/certs/certs.go b/pkg/util/certs/certs.go new file mode 100644 index 0000000000..1ca5953c9f --- /dev/null +++ b/pkg/util/certs/certs.go @@ -0,0 +1,55 @@ +package certs + +import ( + "crypto/tls" + "crypto/x509" + "encoding/pem" + "errors" + "net/http" +) + +// makeTLSConfig returns a TLS configuration with a CA +// pool containing certificates from certs. +func makeTLSConfig(certs []*x509.Certificate) *tls.Config { + caPool := x509.NewCertPool() + for _, cert := range certs { + caPool.AddCert(cert) + } + return &tls.Config{RootCAs: caPool} +} + +// MakeTLSTransport returns an HTTP transport with a TLS +// configuration containing CA certificates from certs. +func MakeTLSTransport(certs []*x509.Certificate) *http.Transport { + cfg := makeTLSConfig(certs) + return &http.Transport{TLSClientConfig: cfg} +} + +// CertsFromPEM parses pemCerts into a list of certificates. +func CertsFromPEM(pemCerts []byte) ([]*x509.Certificate, error) { + ok := false + certs := []*x509.Certificate{} + for len(pemCerts) > 0 { + var block *pem.Block + block, pemCerts = pem.Decode(pemCerts) + if block == nil { + break + } + if block.Type != "CERTIFICATE" || len(block.Headers) != 0 { + continue + } + + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return certs, err + } + + certs = append(certs, cert) + ok = true + } + + if !ok { + return certs, errors.New("could not read any certificates") + } + return certs, nil +} From 1eea37f799c704e3c087b545b08da70fbe5ec317 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Thu, 5 Dec 2019 12:32:03 -0800 Subject: [PATCH 2/6] Refactors ca bundle watcher --- cmd/ingress-operator/start.go | 21 +++---- manifests/02-deployment.yaml | 3 +- pkg/dns/aws/dns.go | 98 ++++++++++++++++++++--------- pkg/dns/aws/filewatcher.go | 71 ++++++++------------- pkg/dns/azure/dns.go | 2 + pkg/dns/dns.go | 10 ++- pkg/dns/gcp/provider.go | 2 + pkg/manifests/bindata.go | 8 +-- pkg/operator/operator.go | 13 +++- pkg/operator/watcher/filewatcher.go | 60 ++++++++++-------- 10 files changed, 165 insertions(+), 123 deletions(-) diff --git a/cmd/ingress-operator/start.go b/cmd/ingress-operator/start.go index a330e5afd6..9beda46f31 100644 --- a/cmd/ingress-operator/start.go +++ b/cmd/ingress-operator/start.go @@ -3,7 +3,6 @@ package main import ( "context" "fmt" - "github.com/openshift/cluster-ingress-operator/pkg/operator/watcher" "os" "github.com/ghodss/yaml" @@ -36,8 +35,6 @@ const ( // operator's namespace that will hold the credentials that the operator // will use to authenticate with the cloud API. cloudCredentialsSecretName = "cloud-credentials" - // The fully qualified path of the trusted CA certificate bundle file. - defaultCABundle = "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem" ) func NewStartCommand() *cobra.Command { @@ -108,13 +105,6 @@ func start() error { os.Exit(1) } - // Setup the file watcher for the trusted ca bundle. - caWatcher, err := watcher.New(defaultCABundle) - if err != nil { - log.Error(err, "failed to create trusted ca bundle watcher") - os.Exit(1) - } - stop := signals.SetupSignalHandler() operatorConfig := operatorconfig.Config{ OperatorReleaseVersion: releaseVersion, @@ -123,7 +113,7 @@ func start() error { } // Set up the DNS manager. - dnsProvider, err := createDNSProvider(kubeClient, operatorConfig, dnsConfig, platformStatus, caWatcher, stop) + dnsProvider, err := createDNSProvider(kubeClient, operatorConfig, dnsConfig, platformStatus) if err != nil { log.Error(err, "failed to create DNS manager") os.Exit(1) @@ -134,18 +124,21 @@ func start() error { if err != nil { return fmt.Errorf("failed to create operator: %v", err) } - return op.Start(stop) + return op.Start(operatorConfig.OperatorReleaseVersion, stop) } // createDNSManager creates a DNS manager compatible with the given cluster // configuration. func createDNSProvider(cl client.Client, operatorConfig operatorconfig.Config, dnsConfig *configv1.DNS, - platformStatus *configv1.PlatformStatus, watcher *watcher.FileWatcher, stop <-chan struct{}) (dns.Provider, error) { + platformStatus *configv1.PlatformStatus) (dns.Provider, error) { var dnsProvider dns.Provider userAgent := fmt.Sprintf("OpenShift/%s (ingress-operator)", operatorConfig.OperatorReleaseVersion) switch platformStatus.Type { case configv1.AWSPlatformType: + if len(platformStatus.AWS.Region) == 0 { + return nil, fmt.Errorf("region is required") + } creds := &corev1.Secret{} err := cl.Get(context.TODO(), types.NamespacedName{Namespace: operatorConfig.Namespace, Name: cloudCredentialsSecretName}, creds) if err != nil { @@ -156,7 +149,7 @@ func createDNSProvider(cl client.Client, operatorConfig operatorconfig.Config, d AccessKey: string(creds.Data["aws_secret_access_key"]), DNS: dnsConfig, Region: platformStatus.AWS.Region, - }, operatorConfig.OperatorReleaseVersion, watcher, stop) + }, operatorConfig.OperatorReleaseVersion) if err != nil { return nil, fmt.Errorf("failed to create AWS DNS manager: %v", err) } diff --git a/manifests/02-deployment.yaml b/manifests/02-deployment.yaml index 06c5176eaa..10012998ba 100644 --- a/manifests/02-deployment.yaml +++ b/manifests/02-deployment.yaml @@ -56,7 +56,7 @@ spec: cpu: 10m volumeMounts: - name: trusted-ca - mountPath: /etc/pki/ca-trust/extracted/pem/ + mountPath: /etc/pki/ca-trust/extracted/pem - name: kube-rbac-proxy image: quay.io/openshift/origin-kube-rbac-proxy:latest args: @@ -84,7 +84,6 @@ spec: - name: trusted-ca configMap: name: trusted-ca - optional: false items: - key: ca-bundle.crt path: tls-ca-bundle.pem diff --git a/pkg/dns/aws/dns.go b/pkg/dns/aws/dns.go index 220f83fdef..3c66822950 100644 --- a/pkg/dns/aws/dns.go +++ b/pkg/dns/aws/dns.go @@ -1,10 +1,15 @@ package aws import ( + "crypto/tls" + "crypto/x509" "fmt" + "net" + "net/http" "reflect" "strings" "sync" + "time" iov1 "github.com/openshift/cluster-ingress-operator/pkg/api/v1" "github.com/openshift/cluster-ingress-operator/pkg/dns" @@ -30,6 +35,9 @@ import ( var ( _ dns.Provider = &Provider{} log = logf.Logger.WithName("dns") + // The fully qualified path of the trusted CA bundle that gets + // mounted from configmap openshift-ingress-operator/trusted-ca. + defaultCABundle = "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem" ) // Provider is a dns.Provider for AWS Route53. It only supports DNSRecords of @@ -47,7 +55,7 @@ type Provider struct { config Config // fileWatcher watches the trusted ca bundle for changes - // and keeps CA certificates of DNS clients updated. + // and keeps AWS sessions updated. fileWatcher *watcher.FileWatcher // lock protects access to everything below. @@ -80,37 +88,24 @@ type Config struct { DNS *configv1.DNS } -func NewProvider(config Config, operatorReleaseVersion string, watcher *watcher.FileWatcher, stop <-chan struct{}) (*Provider, error) { - creds := credentials.NewStaticCredentials(config.AccessID, config.AccessKey, "") - sess, err := session.NewSessionWithOptions(session.Options{ - Config: aws.Config{ - Credentials: creds, - }, - SharedConfigState: session.SharedConfigEnable, - }) +// NewProvider returns a new AWS Provider based on config and adds +// openshift.io/ingress-operator/operatorReleaseVersion to the user-agent +// request header. +func NewProvider(config Config, operatorReleaseVersion string) (*Provider, error) { + caWatcher, err := watcher.New(defaultCABundle) if err != nil { - return nil, fmt.Errorf("couldn't create AWS client session: %v", err) + return nil, err } - sess.Handlers.Build.PushBackNamed(request.NamedHandler{ - Name: "openshift.io/ingress-operator", - Fn: request.MakeAddToUserAgentHandler("openshift.io ingress-operator", operatorReleaseVersion), - }) - region := aws.StringValue(sess.Config.Region) - if len(region) > 0 { - log.Info("using region from shared config", "region name", region) - } else { - region = config.Region - log.Info("using region from operator config", "region name", region) - } - if len(region) == 0 { - return nil, fmt.Errorf("region is required") + sess, err := NewProviderSession(config, operatorReleaseVersion, caWatcher.GetFileData()) + if err != nil { + return nil, err } provider := &Provider{ - elb: elb.New(sess, aws.NewConfig().WithRegion(region)), + elb: elb.New(sess, aws.NewConfig().WithRegion(config.Region)), route53: route53.New(sess), - fileWatcher: watcher, + fileWatcher: caWatcher, // TODO: This API will only return hostedzone resources (which are global) // when the region is forced to us-east-1. We don't yet understand why. tags: resourcegroupstaggingapi.New(sess, aws.NewConfig().WithRegion("us-east-1")), @@ -119,12 +114,59 @@ func NewProvider(config Config, operatorReleaseVersion string, watcher *watcher. lbZones: map[string]string{}, updatedRecords: sets.NewString(), } - if err := provider.startWatcher(stop); err != nil { - return nil, fmt.Errorf("failed to start trusted ca bundle file watcher: %v", err) - } return provider, nil } +// NewProviderSession returns a new AWS Session with credentials from config; +// adding openshift.io/ingress-operator/operatorReleaseVersion to the user-agent +// request header and pemData to the TLS Config CA pool. +func NewProviderSession(config Config, operatorReleaseVersion string, pemData []byte) (*session.Session, error) { + creds := credentials.NewStaticCredentials(config.AccessID, config.AccessKey, "") + trans, err := NewProviderTransport(pemData) + if err != nil { + return nil, fmt.Errorf("failed to create dns provider transport: %v", err) + } + sess, err := session.NewSessionWithOptions(session.Options{ + Config: aws.Config{ + Credentials: creds, + HTTPClient: &http.Client{Transport: trans}, + }, + SharedConfigState: session.SharedConfigEnable, + }) + if err != nil { + return nil, fmt.Errorf("failed to create AWS client session: %v", err) + } + sess.Handlers.Build.PushBackNamed(request.NamedHandler{ + Name: "openshift.io/ingress-operator", + Fn: request.MakeAddToUserAgentHandler("openshift.io ingress-operator", operatorReleaseVersion), + }) + + return sess, nil +} + +// NewProviderTransport returns an http Transport with a TLS Config +// containing root CA's from pemData. +func NewProviderTransport(pemData []byte) (*http.Transport, error) { + caPool := x509.NewCertPool() + if ok := caPool.AppendCertsFromPEM(pemData); !ok { + return nil, fmt.Errorf("failed to append certificates to ca pool") + } + tlsConfig := &tls.Config{RootCAs: caPool} + return &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + TLSClientConfig: tlsConfig, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + }, nil +} + // getZoneID finds the ID of given zoneConfig in Route53. If an ID is already // known, return that; otherwise, use tags to search for the zone. Returns an // error if the zone can't be found. diff --git a/pkg/dns/aws/filewatcher.go b/pkg/dns/aws/filewatcher.go index e44ce43654..73d1fb0e03 100644 --- a/pkg/dns/aws/filewatcher.go +++ b/pkg/dns/aws/filewatcher.go @@ -1,26 +1,29 @@ package aws import ( - "bytes" - "io/ioutil" + "fmt" "time" - certsutil "github.com/openshift/cluster-ingress-operator/pkg/util/certs" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/elb" + "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi" + "github.com/aws/aws-sdk-go/service/route53" "k8s.io/apimachinery/pkg/util/wait" ) -// startWatcher starts the FileWatcher and periodically ensures DNS clients +// StartWatcher starts a file watcher and periodically ensures AWS sessions // are using the current ca bundle until a message is received on the stop // or error channels. -func (m *Provider) startWatcher(stop <-chan struct{}) error { +func (m *Provider) StartWatcher(operatorReleaseVersion string, stop <-chan struct{}) error { + errChan := make(chan error) go wait.Until(func() { - if err := m.ensureDNSClientsTLSTransport(); err != nil { - log.Error(err, "failed to ensure dns client tls transport") + err := m.ensureSessionTransport(operatorReleaseVersion) + if err != nil { + log.Error(err, "failed to ensure dns client transport") } }, 1*time.Minute, stop) - errChan := make(chan error) go func() { errChan <- m.fileWatcher.Start(stop) }() @@ -34,43 +37,21 @@ func (m *Provider) startWatcher(stop <-chan struct{}) error { } } -// ensureDNSClientsTLSTransport compares the watched ca bundle with the -// current ca bundle of FileWatcher and updates DNS clients with the current -// ca bundle if the two are not equal. -func (m *Provider) ensureDNSClientsTLSTransport() error { - equal, caBundle, err := m.caBundlesEqual() - if err != nil { - return err - } - if equal { - return nil - } - - certs, err := certsutil.CertsFromPEM(caBundle) - if err != nil { - return err +// ensureSessionTransport ensures AWS sessions use the current certificates +// from the file watcher. +func (m *Provider) ensureSessionTransport(operatorReleaseVersion string) error { + if m.fileWatcher.FileChanged() { + sess, err := NewProviderSession(m.config, operatorReleaseVersion, m.fileWatcher.GetFileData()) + if err != nil { + return fmt.Errorf("failed to create dns provider session: %v", err) + } + m.lock.Lock() + m.elb = elb.New(sess, aws.NewConfig().WithRegion(m.config.Region)) + m.route53 = route53.New(sess) + m.tags = resourcegroupstaggingapi.New(sess, aws.NewConfig().WithRegion("us-east-1")) + m.lock.Unlock() + m.fileWatcher.SetFileChanged(false) + log.Info("updated dns provider session") } - - m.route53.Client.Config.HTTPClient.Transport = certsutil.MakeTLSTransport(certs) - m.elb.Client.Config.HTTPClient.Transport = certsutil.MakeTLSTransport(certs) - m.tags.Client.Config.HTTPClient.Transport = certsutil.MakeTLSTransport(certs) - return nil } - -// caBundlesEqual compares the watched ca bundle with the current -// ca bundle of FileWatcher, returning false and the current ca -// bundle if the two are not equal. -func (m *Provider) caBundlesEqual() (bool, []byte, error) { - watchedCAs, err := ioutil.ReadFile(m.fileWatcher.GetFile()) - if err != nil { - return false, nil, err - } - - currentCAs := m.fileWatcher.GetFileData() - if !bytes.Equal(watchedCAs, currentCAs) { - return false, currentCAs, nil - } - - return true, nil, nil -} diff --git a/pkg/dns/azure/dns.go b/pkg/dns/azure/dns.go index d701983277..a1574806d8 100644 --- a/pkg/dns/azure/dns.go +++ b/pkg/dns/azure/dns.go @@ -127,3 +127,5 @@ func (m *provider) Delete(record *iov1.DNSRecord, zone configv1.DNSZone) error { func getARecordName(recordDomain string, zoneName string) (string, error) { return strings.TrimSuffix(strings.TrimSuffix(recordDomain, "."), "."+zoneName), nil } + +func (m *provider) StartWatcher(string, <-chan struct{}) error { return nil } diff --git a/pkg/dns/dns.go b/pkg/dns/dns.go index 0f308c2709..482a48488e 100644 --- a/pkg/dns/dns.go +++ b/pkg/dns/dns.go @@ -1,9 +1,8 @@ package dns import ( - iov1 "github.com/openshift/cluster-ingress-operator/pkg/api/v1" - configv1 "github.com/openshift/api/config/v1" + iov1 "github.com/openshift/cluster-ingress-operator/pkg/api/v1" ) // Provider knows how to manage DNS zones only as pertains to routing. @@ -13,6 +12,11 @@ type Provider interface { // Delete will delete record. Delete(record *iov1.DNSRecord, zone configv1.DNSZone) error + + // StartWatcher starts running the FileWatcher. The FileWatcher will stop + // running when the channel is closed. StartWatcher blocks until the + // channel is closed or an error occurs. + StartWatcher(string, <-chan struct{}) error } var _ Provider = &FakeProvider{} @@ -21,3 +25,5 @@ type FakeProvider struct{} func (_ *FakeProvider) Ensure(record *iov1.DNSRecord, zone configv1.DNSZone) error { return nil } func (_ *FakeProvider) Delete(record *iov1.DNSRecord, zone configv1.DNSZone) error { return nil } + +func (_ *FakeProvider) StartWatcher(string, <-chan struct{}) error { return nil } diff --git a/pkg/dns/gcp/provider.go b/pkg/dns/gcp/provider.go index b6fb491759..380792ebae 100644 --- a/pkg/dns/gcp/provider.go +++ b/pkg/dns/gcp/provider.go @@ -75,3 +75,5 @@ func resourceRecordSet(record *iov1.DNSRecord) *gdnsv1.ResourceRecordSet { Ttl: record.Spec.RecordTTL, } } + +func (p *Provider) StartWatcher(string, <-chan struct{}) error { return nil } diff --git a/pkg/manifests/bindata.go b/pkg/manifests/bindata.go index 278db9cb71..1f47c226ab 100644 --- a/pkg/manifests/bindata.go +++ b/pkg/manifests/bindata.go @@ -25,7 +25,7 @@ // manifests/01-service-account.yaml (196B) // manifests/01-service.yaml (344B) // manifests/01-trusted-ca-configmap.yaml (323B) -// manifests/02-deployment.yaml (2.836kB) +// manifests/02-deployment.yaml (2.809kB) // manifests/03-cluster-operator.yaml (357B) // manifests/image-references (433B) @@ -596,7 +596,7 @@ func manifests01TrustedCaConfigmapYaml() (*asset, error) { return a, nil } -var _manifests02DeploymentYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x56\x4d\x6f\xdb\x38\x13\xbe\xe7\x57\x10\x3e\xbf\xb4\x9c\xb4\x6f\x77\x4b\x20\x07\xaf\xab\x36\x01\xe2\x34\x88\x83\xf6\x68\x8c\xa9\xb1\xcd\x35\x45\xb2\x9c\x91\x11\xfd\xfb\x05\xe5\x2f\x59\x76\xdc\x1c\x76\x75\x12\x34\xcf\x3c\xf3\xc1\x67\x46\x84\x60\x7e\x60\x24\xe3\x9d\x12\x10\x02\x65\xeb\xeb\xab\x95\x71\x85\x12\x5f\x30\x58\x5f\x97\xe8\xf8\xaa\x44\x86\x02\x18\xd4\x95\x10\x0e\x4a\x54\xc2\xb8\x45\x44\x22\xe9\x03\x46\x60\x1f\xb7\x06\x0a\xa0\x51\x09\x1f\xd0\xd1\xd2\xcc\x59\x9e\xc1\x81\x73\x9e\x81\x8d\x77\x94\xf8\x84\xd0\xde\xcd\xcd\xa2\xbf\x77\xea\x1b\x9f\x19\xf7\x37\x6a\x96\x21\xfa\xd7\xfa\x4c\x34\x0a\xa8\x93\x73\xc4\x60\x8d\x06\x52\xe2\xfa\x4a\x08\xe2\x08\x8c\x8b\x7a\x43\xcb\x75\x40\x25\x9e\x51\x47\x04\xc6\x64\x46\x8b\x9a\x7d\xdc\x98\x4b\x60\xbd\x7c\x80\x19\xda\x6d\x1a\x17\x4a\x63\x2c\x83\x05\xc6\xad\x67\xab\x1b\xe9\xb1\x47\x24\x17\x68\x84\xd8\xe5\xdd\xc0\x7c\x81\x93\xa3\x94\xd2\xb3\xaa\x66\x18\x1d\x32\x52\x6a\x83\x27\x25\xac\x71\xd5\xeb\x81\xdc\x17\x28\xa3\xb7\xd8\x3f\x46\x96\x40\x8c\x51\x89\x5e\x6f\x0b\x65\x6f\x53\xe0\x43\x9b\x85\x90\x62\x85\xb5\x12\xbd\xcb\x1c\xbd\x7d\xac\x5d\xea\x4a\xf4\xf2\x57\x43\x4c\x07\x13\xce\xe7\xa8\x59\x89\xde\xa3\x9f\xe8\x25\x16\x95\xc5\xde\x99\x28\x9d\x00\x95\x8b\x08\x7a\x09\xb3\x03\xfa\xbd\x51\xf2\x57\xd4\x15\xb7\xdc\x0e\xf5\x4d\x50\x7b\x57\x24\x0d\xdc\x0c\x7e\x9f\x83\xf3\x2c\x23\x42\x51\xff\xb7\x19\x10\xc6\xb5\xd1\x38\xd4\xda\x57\x8e\x1f\xdf\x96\x84\x10\x21\x1a\x1f\x0d\xd7\x23\x0b\x44\x1b\x24\xd5\xc4\x58\x4a\x6d\xab\x74\x22\x52\x47\xc3\x46\x83\xdd\x3a\x68\xef\x18\x8c\xc3\xd8\x12\x9d\xbc\x24\xbb\x6d\xbe\x18\x4b\xe3\x9a\x84\xc7\x48\x04\x0b\x7c\xf2\xd6\xe8\x5a\x89\xaf\x60\xed\x0c\xf4\xea\xc5\x3f\xf8\x05\x7d\x77\x79\x8c\x47\x9e\xa6\x4c\xe0\xca\xda\x9d\xc3\xfd\xfc\xd1\xf3\x53\x44\x4a\xdb\xa1\x83\x6b\x8d\x7f\xe6\xa3\x59\x18\xb7\xaf\xa3\x9b\x9c\x4a\x43\x45\x6d\x06\xed\xcb\x12\x5c\xa1\x5a\x9f\xe4\xa5\x9a\xa4\x20\x86\xd8\x66\x40\xb7\x6e\x7b\x1f\x3a\xf3\x9c\x3f\xe4\xc3\x49\x3e\xfd\x91\x3f\x4f\xee\xbf\x3f\x1e\x61\x84\x58\x83\xad\x50\x89\xde\xa0\x3f\xe8\x5f\x4b\x72\x10\x68\xe9\xb9\x77\x96\xe9\xe7\xf0\x65\x74\x37\x7d\x1c\x8e\xf3\xc9\xd3\x70\x94\x9f\x63\xfa\x1a\x7d\xa9\x3a\x06\x21\xe6\x06\x6d\xf1\x8c\xf3\x53\xcb\xd6\xf6\x04\xbc\x54\xfb\x05\xd3\xdf\xaf\xd4\xb3\x69\xdc\x8f\x87\xdf\xce\x06\x3f\x73\x02\x4b\x68\x56\xa9\x8c\xbe\x4a\x6b\x62\xfd\xb1\x3f\x68\x79\x46\x24\x5f\x45\x8d\x74\x9c\x58\xc4\x5f\x15\x12\x53\x37\x5d\x1d\x2a\x25\xae\x07\x65\xeb\xf3\xda\xdb\xaa\xc4\x71\xd2\x3a\x9d\x6f\x3f\xc7\xa4\x81\x42\x6a\xe8\xb0\x95\xc9\x69\x53\x78\x86\xac\xb3\xb0\x32\x99\x06\xd9\xe0\x33\x7c\xe5\x08\x9a\xb1\xc8\x02\x96\xd9\x89\xda\xd3\x68\xcb\x38\x03\xbd\xf9\x53\x9c\x4a\xf1\x57\x05\x75\xb3\x48\xbb\x0d\xe9\x78\x9e\x2a\x11\xe2\x82\x8e\x65\x28\xa5\xf5\x0b\xf6\xc4\x05\xc6\xd8\xb1\x10\xea\x2a\xa2\xb4\x86\x18\x9d\x84\xa2\x48\x82\xbd\x55\x9f\x3f\x7c\xfe\xd0\x41\xb2\x25\xa9\x4d\x58\x62\x94\x54\x19\x46\xba\x7d\x79\x98\x4c\xf3\xd1\x97\xbb\x7c\xfa\x3c\x19\x4e\x7f\xde\xbf\xdc\x4d\x87\xf9\x64\x7a\x7d\xf3\xe7\xf4\xdb\x68\x3c\x9d\xdc\x0d\x6f\xfe\xff\xe9\x7f\x07\x54\x3e\xfa\xf2\x1b\xdc\x09\xcf\xe8\xaf\xd1\xbb\x78\xce\xe2\x2e\xb0\x75\x6a\xab\x02\x71\x44\x28\x6f\x97\xcc\x41\x65\xd9\xf5\xcd\x1f\xfd\x66\xa0\xd4\xa7\xc1\x60\x30\xc8\xce\xb5\x02\x23\xcb\xb9\xb1\x78\xdb\x1c\x3e\x5b\xca\x42\x34\x6b\x60\x4c\xef\x7d\x7d\x34\xdb\x3b\xa7\x2d\x42\xae\xb0\xbe\xe0\xbb\xc2\xb6\x20\x82\x8f\xdc\x39\xd0\xfd\x1e\x7d\xf2\x91\x95\xe8\x1c\xd6\xee\x47\x5e\x22\x47\xa3\xe9\x5f\x1d\x97\x46\xf7\x58\xfa\x58\x2b\xf1\x71\x30\x36\xef\x98\x24\x79\x32\x28\xad\x7a\xdf\x4e\x3b\xf5\xab\x93\x25\x14\xdf\x9d\xad\x9b\x89\xdc\x39\x6e\x82\xb6\x2e\x0a\x6f\x73\x50\xba\x50\x71\x3b\xb3\xcd\x97\xc7\x37\x3c\x2e\xcc\xff\xe6\xe2\x37\x86\xd0\x66\xbb\xb0\x2d\x7c\x48\xff\x2e\xb0\x4a\xcc\xc1\x52\xbb\x68\xc3\x58\x9e\xec\x9d\xe6\x0a\xa0\x41\xce\x2a\x57\x58\xec\x48\x29\x3d\xa1\xe9\x65\xa3\xc2\x3d\x2a\x60\x79\xf5\x4f\x00\x00\x00\xff\xff\x84\x47\x48\x41\x14\x0b\x00\x00") +var _manifests02DeploymentYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x56\x4f\x6f\xdb\xb8\x13\xbd\xe7\x53\x10\x3e\xff\x68\x39\x69\x7f\xdd\x2d\x81\x1c\xbc\xae\xda\x04\x88\xd3\x20\x0e\xda\xa3\x41\x53\x63\x9b\x6b\x8a\x64\x67\x46\x46\xf4\xed\x17\x94\xff\xc9\xb2\xe3\xe6\xb0\xeb\x93\xa1\x79\xf3\xe6\x71\xf8\x66\x24\x1d\xed\x0f\x40\xb2\xc1\x2b\xa1\x63\xa4\x6c\x7d\x7d\xb5\xb2\xbe\x50\xe2\x0b\x44\x17\xea\x12\x3c\x5f\x95\xc0\xba\xd0\xac\xd5\x95\x10\x5e\x97\xa0\x84\xf5\x0b\x04\x22\x19\x22\xa0\xe6\x80\xdb\x00\x45\x6d\x40\x89\x10\xc1\xd3\xd2\xce\x59\x9e\xc1\x69\xef\x03\x6b\xb6\xc1\x53\xe2\x13\xc2\x04\x3f\xb7\x8b\xfe\x3e\xa9\x6f\x43\x66\xfd\xdf\x60\x58\x46\x0c\xaf\xf5\x99\x6a\x14\xc1\xa4\x64\x84\xe8\xac\xd1\xa4\xc4\xf5\x95\x10\xc4\xa8\x19\x16\xf5\x86\x96\xeb\x08\x4a\x3c\x83\x41\xd0\x0c\x29\x0c\x0e\x0c\x07\xdc\x84\x4b\xcd\x66\xf9\xa0\x67\xe0\xb6\x32\x2e\x1c\x8d\xa1\x8c\x4e\x33\x6c\x33\x5b\xdd\x48\x3f\x77\x44\x72\x81\x46\x88\x9d\xee\x06\x16\x0a\x98\x1c\x49\x4a\xbf\x55\x35\x03\xf4\xc0\x40\xa9\x0d\x81\x94\x70\xd6\x57\xaf\x07\xf2\x50\x80\xc4\xe0\xa0\x7f\x8c\x2c\x35\x31\xa0\x12\xbd\xde\x16\xca\xc1\xa5\xc2\x87\x36\x0b\x21\xc5\x0a\x6a\x25\x7a\x97\x39\x7a\xfb\x5a\x3b\xe9\x4a\xf4\xf2\x57\x4b\x4c\x87\x10\xcc\xe7\x60\x58\x89\xde\x63\x98\x98\x25\x14\x95\x83\xde\x99\x2a\x9d\x02\x95\x47\xd0\x66\xa9\x67\x07\xf4\x7b\xab\xe4\xaf\x60\x2a\x6e\xa5\x1d\xce\x37\x01\x13\x7c\x91\x3c\x70\x33\xf8\xbd\x06\x1f\x58\x22\xe8\xa2\xfe\x6f\x15\x10\xe0\xda\x1a\x18\x1a\x13\x2a\xcf\x8f\x6f\x5b\x42\x88\x88\x36\xa0\xe5\x7a\xe4\x34\xd1\x06\x49\x35\x31\x94\xd2\xb8\x2a\xdd\x88\x34\x68\xd9\x1a\xed\xb6\x09\x26\x78\xd6\xd6\x03\xb6\x4c\x27\x2f\xd9\x6e\xab\x17\xb0\xb4\xbe\x11\x3c\x06\x22\xbd\x80\xa7\xe0\xac\xa9\x95\xf8\xaa\x9d\x9b\x69\xb3\x7a\x09\x0f\x61\x41\xdf\x7d\x8e\x78\x94\x69\xcb\x04\xae\x9c\xdb\x25\xdc\xcf\x1f\x03\x3f\x21\x50\xda\x0e\x1d\x5c\x6b\xfc\xb3\x80\x76\x61\xfd\xfe\x1c\x5d\x71\x2a\x0d\x15\xb5\x19\x4c\x28\x4b\xed\x0b\xd5\x7a\x24\x2f\x9d\x49\x0a\x62\x8d\x6d\x06\xf0\xeb\x76\xf6\xa1\x33\xcf\xf9\x43\x3e\x9c\xe4\xd3\x1f\xf9\xf3\xe4\xfe\xfb\xe3\x11\x46\x88\xb5\x76\x15\x28\xd1\x1b\xf4\x07\xfd\x6b\x49\x5e\x47\x5a\x06\xee\x9d\x65\xfa\x39\x7c\x19\xdd\x4d\x1f\x87\xe3\x7c\xf2\x34\x1c\xe5\xe7\x98\xbe\x62\x28\x55\x27\x20\xc4\xdc\x82\x2b\x9e\x61\x7e\x1a\xd9\xc6\x9e\x34\x2f\xd5\x7e\xc1\xf4\xf7\x2b\xf5\xac\x8c\xfb\xf1\xf0\xdb\xd9\xe2\x67\x6e\x60\xa9\x9b\x55\x2a\x31\x54\x69\x4d\xac\x3f\xf6\x07\xad\x4c\x04\x0a\x15\x1a\xa0\x63\x61\x08\xbf\x2a\x20\xa6\xae\x5c\x13\x2b\x25\xae\x07\x65\xeb\xf1\x3a\xb8\xaa\x84\x71\xf2\x3a\x9d\x6f\x3f\x63\xf2\x40\x21\x8d\xee\xb0\x95\x29\x69\x73\xf0\x0c\xd8\x64\x71\x65\x33\xa3\x65\x83\xcf\xe0\x95\x51\x1b\x86\x22\x8b\x50\x9e\x98\x3d\x4d\xb6\xc4\x99\x36\x9b\x17\xc5\xa9\x13\x7f\x55\xba\x6e\xf6\x68\xb7\x1f\x9d\xcc\x53\x23\x6a\x5c\xd0\xb1\x0b\xa5\x74\x61\xc1\x81\xb8\x00\xc4\x4e\x84\xc0\x54\x08\xd2\x59\x62\xf0\x52\x17\x45\xf2\xeb\xad\xfa\xfc\xe1\xf3\x87\x0e\x92\x1d\x49\x63\xe3\x12\x50\x52\x65\x19\xe8\xf6\xe5\x61\x32\xcd\x47\x5f\xee\xf2\xe9\xf3\x64\x38\xfd\x79\xff\x72\x37\x1d\xe6\x93\xe9\xf5\xcd\x9f\xd3\x6f\xa3\xf1\x74\x72\x37\xbc\xf9\xff\xa7\xff\x1d\x50\xf9\xe8\xcb\x6f\x70\x27\x3c\xa3\xbf\x46\xef\xe2\x39\x8b\xbb\xc0\xd6\x39\x5b\x15\x89\x11\x74\x79\xbb\x64\x8e\x2a\xcb\xae\x6f\xfe\xe8\x37\xf3\xa4\x3e\x0d\x06\x83\x41\x76\xae\x15\x80\x2c\xe7\xd6\xc1\x6d\x73\xf7\xec\x28\x8b\x68\xd7\x9a\x21\xfd\xef\x9b\xa3\xd1\xde\x25\x6d\x11\x72\x05\xf5\x85\xdc\x15\xb4\x0d\x11\x03\x72\xe7\x42\xf7\x6b\xf4\x29\x20\x2b\xd1\xb9\xac\xdd\x7b\xbc\x04\x46\x6b\xe8\x5f\x9d\x96\xc6\xf6\x50\x06\xac\x95\xf8\x38\x18\xdb\x77\x0c\x92\x3c\x99\x93\xd6\x79\xdf\x96\x9d\xfa\xd5\x51\xa9\x8b\xef\xde\xd5\xcd\x40\xee\x12\x37\x45\x5b\xdf\x09\x6f\x73\x50\xfa\x9e\xe2\xb6\xb2\xcd\x93\xc7\x37\x32\x2e\x8c\xff\xe6\xbb\x6f\xac\x63\x9b\xed\xc2\xb2\xb0\x0c\xe5\xc9\x72\x69\xde\xf3\x46\xcb\x59\xe5\x0b\x07\x1d\xc3\xa4\x5f\x6c\x3a\xd6\x78\x6d\x8f\x4a\xbb\xe4\x9f\x00\x00\x00\xff\xff\x2d\xc8\x4a\xbf\xf9\x0a\x00\x00") func manifests02DeploymentYamlBytes() ([]byte, error) { return bindataRead( @@ -611,8 +611,8 @@ func manifests02DeploymentYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "manifests/02-deployment.yaml", size: 2836, mode: os.FileMode(420), modTime: time.Unix(1, 0)} - a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xad, 0xff, 0x4d, 0x1d, 0x13, 0x30, 0xdd, 0x9e, 0x7b, 0x68, 0x52, 0xa0, 0x87, 0x61, 0xd3, 0xb0, 0xd6, 0x65, 0x9a, 0xaf, 0xf6, 0x86, 0xba, 0x85, 0xb2, 0x95, 0x42, 0xf5, 0x6a, 0xcb, 0xef, 0x8a}} + info := bindataFileInfo{name: "manifests/02-deployment.yaml", size: 2809, mode: os.FileMode(420), modTime: time.Unix(1, 0)} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x1f, 0x3, 0x35, 0xf1, 0xb8, 0xea, 0x53, 0xd3, 0xd4, 0xe0, 0x1b, 0x8b, 0xe4, 0x2d, 0xbd, 0x67, 0x5c, 0x4f, 0xda, 0xed, 0xe1, 0x2e, 0x9, 0x72, 0x30, 0xa5, 0xa7, 0x3d, 0x4b, 0x2f, 0x62, 0xf8}} return a, nil } diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index f949114aa6..1504a2286c 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -50,6 +50,8 @@ type Operator struct { manager manager.Manager + dnsProvider dns.Provider + namespace string } @@ -114,7 +116,8 @@ func New(config operatorconfig.Config, dnsProvider dns.Provider, kubeConfig *res } return &Operator{ - manager: mgr, + manager: mgr, + dnsProvider: dnsProvider, // TODO: These are only needed for the default ingress controller stuff, which // should be refactored away. client: mgr.GetClient(), @@ -123,9 +126,10 @@ func New(config operatorconfig.Config, dnsProvider dns.Provider, kubeConfig *res } // Start creates the default IngressController and then starts the operator -// synchronously until a message is received on the stop channel. +// and trusted ca bundle watcher synchronously until a message is received +// on the stop channel. // TODO: Move the default IngressController logic elsewhere. -func (o *Operator) Start(stop <-chan struct{}) error { +func (o *Operator) Start(operatorReleaseVersion string, stop <-chan struct{}) error { // Periodicaly ensure the default controller exists. go wait.Until(func() { if !o.manager.GetCache().WaitForCacheSync(stop) { @@ -142,6 +146,9 @@ func (o *Operator) Start(stop <-chan struct{}) error { go func() { errChan <- o.manager.Start(stop) }() + go func() { + errChan <- o.dnsProvider.StartWatcher(operatorReleaseVersion, stop) + }() // Wait for the manager to exit or an explicit stop. select { diff --git a/pkg/operator/watcher/filewatcher.go b/pkg/operator/watcher/filewatcher.go index f2f7338394..49974ed284 100644 --- a/pkg/operator/watcher/filewatcher.go +++ b/pkg/operator/watcher/filewatcher.go @@ -5,8 +5,9 @@ import ( "io/ioutil" "sync" - logf "github.com/openshift/cluster-ingress-operator/pkg/log" "gopkg.in/fsnotify.v1" + + logf "github.com/openshift/cluster-ingress-operator/pkg/log" ) var log = logf.Logger.WithName("filewatcher") @@ -14,8 +15,9 @@ var log = logf.Logger.WithName("filewatcher") // FileWatcher watches a file for changes. type FileWatcher struct { sync.Mutex - file string + fileName string currentData []byte + changed bool watcher *fsnotify.Watcher } @@ -23,46 +25,57 @@ type FileWatcher struct { func New(file string) (*FileWatcher, error) { var err error - cw := &FileWatcher{ - file: file, + fw := &FileWatcher{ + fileName: file, } // Initial read of file. - if err := cw.ReadFile(); err != nil { + if err := fw.ReadFile(); err != nil { return nil, err } + // Set to false for initial read. + fw.SetFileChanged(false) - cw.watcher, err = fsnotify.NewWatcher() + fw.watcher, err = fsnotify.NewWatcher() if err != nil { return nil, err } - return cw, nil + return fw, nil } -// GetFile fetches the file being watched by FileWatcher. -func (fw *FileWatcher) GetFile() string { +// GetFileData fetches the data of the currently watched file. +func (fw *FileWatcher) GetFileData() []byte { fw.Lock() defer fw.Unlock() - return fw.file + return fw.currentData } -// GetFileData fetches the currently loaded file data of FileWatcher. -func (fw *FileWatcher) GetFileData() []byte { +// FileChanged returns true if FileWatcher changed the current +// data of the watched file. +func (fw *FileWatcher) FileChanged() bool { fw.Lock() defer fw.Unlock() - return fw.currentData + return fw.changed +} + +// SetFileChanged sets the changed flag to changed for the +// watched file. +func (fw *FileWatcher) SetFileChanged(changed bool) { + fw.Lock() + defer fw.Unlock() + fw.changed = changed } // Start starts the FileWatcher. func (fw *FileWatcher) Start(stopCh <-chan struct{}) error { - if err := fw.watcher.Add(fw.file); err != nil { + if err := fw.watcher.Add(fw.fileName); err != nil { return err } go fw.Watch() - log.Info("Starting file watcher") + log.Info("starting file watcher") // Block until the stop channel is closed. <-stopCh @@ -96,19 +109,18 @@ func (fw *FileWatcher) Watch() { // ReadFile reads the watched file from disk, parses the file, // and updates FileWatcher current data. func (fw *FileWatcher) ReadFile() error { - data, err := ioutil.ReadFile(fw.file) + data, err := ioutil.ReadFile(fw.fileName) switch { case err != nil: - return fmt.Errorf("failed to read file %s: %v", fw.file, err) + return fmt.Errorf("failed to read file %s: %v", fw.fileName, err) case len(data) == 0: - return fmt.Errorf("file %s contains no currentData", fw.file) + return fmt.Errorf("file %s contains no data", fw.fileName) } - fw.Lock() fw.currentData = data fw.Unlock() - - log.Info("updated file watcher current data") + fw.SetFileChanged(true) + log.Info("changed file watcher current data") return nil } @@ -119,17 +131,15 @@ func (fw *FileWatcher) handleEvent(event fsnotify.Event) { if !(isWrite(event) || isRemove(event) || isCreate(event)) { return } - log.Info("watched file change", "event", event) if isRemove(event) { if err := fw.watcher.Add(event.Name); err != nil { - log.Error(err, "error re-watching file %s", fw.file) + log.Error(err, "error re-watching file %s", fw.fileName) } } - if err := fw.ReadFile(); err != nil { - log.Error(err, "error re-reading watched file %s", fw.file) + log.Error(err, "error re-reading watched file %s", fw.fileName) } } From db123c1402a93259b991ebb0fa33ac3fc766c26b Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Sun, 8 Dec 2019 22:21:04 -0800 Subject: [PATCH 3/6] Use a reload chan to indicate file watcher updates --- pkg/dns/aws/dns.go | 1 + pkg/dns/aws/filewatcher.go | 43 +++++++++++++---------------- pkg/operator/operator.go | 2 +- pkg/operator/watcher/filewatcher.go | 34 ++++++----------------- 4 files changed, 29 insertions(+), 51 deletions(-) diff --git a/pkg/dns/aws/dns.go b/pkg/dns/aws/dns.go index 3c66822950..8d66a3d64b 100644 --- a/pkg/dns/aws/dns.go +++ b/pkg/dns/aws/dns.go @@ -92,6 +92,7 @@ type Config struct { // openshift.io/ingress-operator/operatorReleaseVersion to the user-agent // request header. func NewProvider(config Config, operatorReleaseVersion string) (*Provider, error) { + //reloadCh := make(chan bool) caWatcher, err := watcher.New(defaultCABundle) if err != nil { return nil, err diff --git a/pkg/dns/aws/filewatcher.go b/pkg/dns/aws/filewatcher.go index 73d1fb0e03..68fff4850e 100644 --- a/pkg/dns/aws/filewatcher.go +++ b/pkg/dns/aws/filewatcher.go @@ -2,14 +2,11 @@ package aws import ( "fmt" - "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/elb" "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi" "github.com/aws/aws-sdk-go/service/route53" - - "k8s.io/apimachinery/pkg/util/wait" ) // StartWatcher starts a file watcher and periodically ensures AWS sessions @@ -17,15 +14,12 @@ import ( // or error channels. func (m *Provider) StartWatcher(operatorReleaseVersion string, stop <-chan struct{}) error { errChan := make(chan error) - go wait.Until(func() { - err := m.ensureSessionTransport(operatorReleaseVersion) - if err != nil { - log.Error(err, "failed to ensure dns client transport") - } - }, 1*time.Minute, stop) - + reloadChan := make(chan bool) + go func() { + errChan <- m.fileWatcher.Start(stop, reloadChan) + }() go func() { - errChan <- m.fileWatcher.Start(stop) + errChan <- m.ensureSessionTransport(operatorReleaseVersion, reloadChan) }() // Wait for the watcher to exit or an explicit stop. @@ -39,19 +33,20 @@ func (m *Provider) StartWatcher(operatorReleaseVersion string, stop <-chan struc // ensureSessionTransport ensures AWS sessions use the current certificates // from the file watcher. -func (m *Provider) ensureSessionTransport(operatorReleaseVersion string) error { - if m.fileWatcher.FileChanged() { - sess, err := NewProviderSession(m.config, operatorReleaseVersion, m.fileWatcher.GetFileData()) - if err != nil { - return fmt.Errorf("failed to create dns provider session: %v", err) +func (m *Provider) ensureSessionTransport(operatorReleaseVersion string, reloadCh chan bool) error { + for { + select { + case <-reloadCh: + sess, err := NewProviderSession(m.config, operatorReleaseVersion, m.fileWatcher.GetFileData()) + if err != nil { + return fmt.Errorf("failed to create dns provider session: %v", err) + } + m.lock.Lock() + m.elb = elb.New(sess, aws.NewConfig().WithRegion(m.config.Region)) + m.route53 = route53.New(sess) + m.tags = resourcegroupstaggingapi.New(sess, aws.NewConfig().WithRegion("us-east-1")) + m.lock.Unlock() + log.Info("dns provider session updated") } - m.lock.Lock() - m.elb = elb.New(sess, aws.NewConfig().WithRegion(m.config.Region)) - m.route53 = route53.New(sess) - m.tags = resourcegroupstaggingapi.New(sess, aws.NewConfig().WithRegion("us-east-1")) - m.lock.Unlock() - m.fileWatcher.SetFileChanged(false) - log.Info("updated dns provider session") } - return nil } diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 1504a2286c..c6ba4b29c0 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -150,7 +150,7 @@ func (o *Operator) Start(operatorReleaseVersion string, stop <-chan struct{}) er errChan <- o.dnsProvider.StartWatcher(operatorReleaseVersion, stop) }() - // Wait for the manager to exit or an explicit stop. + // Wait for the manager or watcher to exit or an explicit stop. select { case <-stop: return nil diff --git a/pkg/operator/watcher/filewatcher.go b/pkg/operator/watcher/filewatcher.go index 49974ed284..07a4d74fa7 100644 --- a/pkg/operator/watcher/filewatcher.go +++ b/pkg/operator/watcher/filewatcher.go @@ -17,7 +17,6 @@ type FileWatcher struct { sync.Mutex fileName string currentData []byte - changed bool watcher *fsnotify.Watcher } @@ -33,8 +32,6 @@ func New(file string) (*FileWatcher, error) { if err := fw.ReadFile(); err != nil { return nil, err } - // Set to false for initial read. - fw.SetFileChanged(false) fw.watcher, err = fsnotify.NewWatcher() if err != nil { @@ -51,29 +48,13 @@ func (fw *FileWatcher) GetFileData() []byte { return fw.currentData } -// FileChanged returns true if FileWatcher changed the current -// data of the watched file. -func (fw *FileWatcher) FileChanged() bool { - fw.Lock() - defer fw.Unlock() - return fw.changed -} - -// SetFileChanged sets the changed flag to changed for the -// watched file. -func (fw *FileWatcher) SetFileChanged(changed bool) { - fw.Lock() - defer fw.Unlock() - fw.changed = changed -} - // Start starts the FileWatcher. -func (fw *FileWatcher) Start(stopCh <-chan struct{}) error { +func (fw *FileWatcher) Start(stopCh <-chan struct{}, reloadCh chan bool) error { if err := fw.watcher.Add(fw.fileName); err != nil { return err } - go fw.Watch() + go fw.Watch(reloadCh) log.Info("starting file watcher") @@ -84,7 +65,7 @@ func (fw *FileWatcher) Start(stopCh <-chan struct{}) error { } // Watch reads events from the watcher's channel and reacts to changes. -func (fw *FileWatcher) Watch() { +func (fw *FileWatcher) Watch(reload chan bool) { for { select { case event, ok := <-fw.watcher.Events: @@ -93,7 +74,7 @@ func (fw *FileWatcher) Watch() { return } - fw.handleEvent(event) + fw.handleEvent(event, reload) case err, ok := <-fw.watcher.Errors: // Channel is closed. @@ -119,15 +100,14 @@ func (fw *FileWatcher) ReadFile() error { fw.Lock() fw.currentData = data fw.Unlock() - fw.SetFileChanged(true) - log.Info("changed file watcher current data") + log.Info("reloaded file watcher current data") return nil } // handleEvent filters events, re-adds and re-reads the watched file // if removed. -func (fw *FileWatcher) handleEvent(event fsnotify.Event) { +func (fw *FileWatcher) handleEvent(event fsnotify.Event, reload chan bool) { if !(isWrite(event) || isRemove(event) || isCreate(event)) { return } @@ -140,6 +120,8 @@ func (fw *FileWatcher) handleEvent(event fsnotify.Event) { } if err := fw.ReadFile(); err != nil { log.Error(err, "error re-reading watched file %s", fw.fileName) + } else { + reload <- true } } From fb97d58bc132efc2cb4a4c8db90889f2d5dfad03 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Mon, 9 Dec 2019 11:43:46 -0800 Subject: [PATCH 4/6] Cleanup of reload chan refactor --- cmd/ingress-operator/start.go | 12 +++-- hack/uninstall.sh | 1 - pkg/dns/aws/dns.go | 81 +++++++++++++++-------------- pkg/dns/aws/filewatcher.go | 31 +++++------ pkg/dns/dns.go | 7 ++- pkg/operator/config/config.go | 4 ++ pkg/operator/operator.go | 11 ++-- pkg/operator/watcher/filewatcher.go | 23 ++++---- pkg/util/certs/certs.go | 55 -------------------- 9 files changed, 84 insertions(+), 141 deletions(-) delete mode 100644 pkg/util/certs/certs.go diff --git a/cmd/ingress-operator/start.go b/cmd/ingress-operator/start.go index 9beda46f31..02c06a6e9e 100644 --- a/cmd/ingress-operator/start.go +++ b/cmd/ingress-operator/start.go @@ -35,6 +35,9 @@ const ( // operator's namespace that will hold the credentials that the operator // will use to authenticate with the cloud API. cloudCredentialsSecretName = "cloud-credentials" + // The fully qualified path of the trusted CA bundle that is + // mounted from configmap openshift-ingress-operator/trusted-ca. + defaultCABundle = "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem" ) func NewStartCommand() *cobra.Command { @@ -105,11 +108,11 @@ func start() error { os.Exit(1) } - stop := signals.SetupSignalHandler() operatorConfig := operatorconfig.Config{ OperatorReleaseVersion: releaseVersion, Namespace: operatorNamespace, IngressControllerImage: ingressControllerImage, + TrustedCABundle: defaultCABundle, } // Set up the DNS manager. @@ -124,13 +127,12 @@ func start() error { if err != nil { return fmt.Errorf("failed to create operator: %v", err) } - return op.Start(operatorConfig.OperatorReleaseVersion, stop) + return op.Start(signals.SetupSignalHandler()) } // createDNSManager creates a DNS manager compatible with the given cluster // configuration. -func createDNSProvider(cl client.Client, operatorConfig operatorconfig.Config, dnsConfig *configv1.DNS, - platformStatus *configv1.PlatformStatus) (dns.Provider, error) { +func createDNSProvider(cl client.Client, operatorConfig operatorconfig.Config, dnsConfig *configv1.DNS, platformStatus *configv1.PlatformStatus) (dns.Provider, error) { var dnsProvider dns.Provider userAgent := fmt.Sprintf("OpenShift/%s (ingress-operator)", operatorConfig.OperatorReleaseVersion) @@ -149,7 +151,7 @@ func createDNSProvider(cl client.Client, operatorConfig operatorconfig.Config, d AccessKey: string(creds.Data["aws_secret_access_key"]), DNS: dnsConfig, Region: platformStatus.AWS.Region, - }, operatorConfig.OperatorReleaseVersion) + }, operatorConfig.OperatorReleaseVersion, operatorConfig.TrustedCABundle) if err != nil { return nil, fmt.Errorf("failed to create AWS DNS manager: %v", err) } diff --git a/hack/uninstall.sh b/hack/uninstall.sh index 01345db700..51470be58a 100755 --- a/hack/uninstall.sh +++ b/hack/uninstall.sh @@ -15,7 +15,6 @@ oc delete clusteroperator.config.openshift.io/ingress oc delete --force --grace-period=0 -n openshift-ingress-operator ingresscontroller/default if [ "$WHAT" == "all" ]; then - oc delete configmap/trusted-ca -n openshift-ingress-operator oc delete namespaces/openshift-ingress-operator else oc delete -n openshift-ingress-operator secrets/router-ca diff --git a/pkg/dns/aws/dns.go b/pkg/dns/aws/dns.go index 8d66a3d64b..4354736715 100644 --- a/pkg/dns/aws/dns.go +++ b/pkg/dns/aws/dns.go @@ -35,9 +35,6 @@ import ( var ( _ dns.Provider = &Provider{} log = logf.Logger.WithName("dns") - // The fully qualified path of the trusted CA bundle that gets - // mounted from configmap openshift-ingress-operator/trusted-ca. - defaultCABundle = "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem" ) // Provider is a dns.Provider for AWS Route53. It only supports DNSRecords of @@ -91,25 +88,23 @@ type Config struct { // NewProvider returns a new AWS Provider based on config and adds // openshift.io/ingress-operator/operatorReleaseVersion to the user-agent // request header. -func NewProvider(config Config, operatorReleaseVersion string) (*Provider, error) { - //reloadCh := make(chan bool) - caWatcher, err := watcher.New(defaultCABundle) +func NewProvider(config Config, operatorReleaseVersion string, trustBundle string) (*Provider, error) { + caWatcher, err := watcher.New(trustBundle) if err != nil { return nil, err } - sess, err := NewProviderSession(config, operatorReleaseVersion, caWatcher.GetFileData()) + sess, err := newProviderSession(config, operatorReleaseVersion, caWatcher.GetFileData()) if err != nil { return nil, err } + elbClient, route53Client, taggingClient := newClients(sess, config.Region) provider := &Provider{ - elb: elb.New(sess, aws.NewConfig().WithRegion(config.Region)), - route53: route53.New(sess), - fileWatcher: caWatcher, - // TODO: This API will only return hostedzone resources (which are global) - // when the region is forced to us-east-1. We don't yet understand why. - tags: resourcegroupstaggingapi.New(sess, aws.NewConfig().WithRegion("us-east-1")), + elb: elbClient, + route53: route53Client, + fileWatcher: caWatcher, + tags: taggingClient, config: config, idsToTags: map[string]map[string]string{}, lbZones: map[string]string{}, @@ -118,14 +113,29 @@ func NewProvider(config Config, operatorReleaseVersion string) (*Provider, error return provider, nil } -// NewProviderSession returns a new AWS Session with credentials from config; +// newProviderSession returns a new AWS Session with credentials from config; // adding openshift.io/ingress-operator/operatorReleaseVersion to the user-agent -// request header and pemData to the TLS Config CA pool. -func NewProviderSession(config Config, operatorReleaseVersion string, pemData []byte) (*session.Session, error) { +// request header and pemData to the TLS Config CA pool. Aside from TLSClientConfig, +// the transport created is identical to DefaultTransport of net/http. +func newProviderSession(config Config, operatorReleaseVersion string, pemData []byte) (*session.Session, error) { creds := credentials.NewStaticCredentials(config.AccessID, config.AccessKey, "") - trans, err := NewProviderTransport(pemData) - if err != nil { - return nil, fmt.Errorf("failed to create dns provider transport: %v", err) + caPool := x509.NewCertPool() + if ok := caPool.AppendCertsFromPEM(pemData); !ok { + return nil, fmt.Errorf("failed to append certificates to ca pool") + } + tlsConfig := &tls.Config{RootCAs: caPool} + trans := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + TLSClientConfig: tlsConfig, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, } sess, err := session.NewSessionWithOptions(session.Options{ Config: aws.Config{ @@ -145,27 +155,18 @@ func NewProviderSession(config Config, operatorReleaseVersion string, pemData [] return sess, nil } -// NewProviderTransport returns an http Transport with a TLS Config -// containing root CA's from pemData. -func NewProviderTransport(pemData []byte) (*http.Transport, error) { - caPool := x509.NewCertPool() - if ok := caPool.AppendCertsFromPEM(pemData); !ok { - return nil, fmt.Errorf("failed to append certificates to ca pool") - } - tlsConfig := &tls.Config{RootCAs: caPool} - return &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - }).DialContext, - TLSClientConfig: tlsConfig, - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - }, nil +// newClients returns new elb, route53 and resourcegroupstagging api clients +// using session and region. Note that region "us-east-1" is used for the +// resourcegroupstagging client. +func newClients(session *session.Session, region string) (*elb.ELB, *route53.Route53, *resourcegroupstaggingapi.ResourceGroupsTaggingAPI) { + elb := elb.New(session, aws.NewConfig().WithRegion(region)) + route53 := route53.New(session) + + // TODO: The resourcegroupstagging API will only return hostedzone resources (which are global) + // when the region is forced to us-east-1. We don't yet understand why. + tags := resourcegroupstaggingapi.New(session, aws.NewConfig().WithRegion("us-east-1")) + + return elb, route53, tags } // getZoneID finds the ID of given zoneConfig in Route53. If an ID is already diff --git a/pkg/dns/aws/filewatcher.go b/pkg/dns/aws/filewatcher.go index 68fff4850e..3ba609c2c5 100644 --- a/pkg/dns/aws/filewatcher.go +++ b/pkg/dns/aws/filewatcher.go @@ -2,24 +2,18 @@ package aws import ( "fmt" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/elb" - "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi" - "github.com/aws/aws-sdk-go/service/route53" ) -// StartWatcher starts a file watcher and periodically ensures AWS sessions -// are using the current ca bundle until a message is received on the stop -// or error channels. +// StartWatcher starts a file watcher and ensures AWS sessions are using the +// current ca bundle until a message is received on the stop or error channels. func (m *Provider) StartWatcher(operatorReleaseVersion string, stop <-chan struct{}) error { errChan := make(chan error) - reloadChan := make(chan bool) + reloadChan := make(chan struct{}) go func() { errChan <- m.fileWatcher.Start(stop, reloadChan) }() go func() { - errChan <- m.ensureSessionTransport(operatorReleaseVersion, reloadChan) + errChan <- m.startSessionReloadHandler(operatorReleaseVersion, reloadChan) }() // Wait for the watcher to exit or an explicit stop. @@ -31,22 +25,23 @@ func (m *Provider) StartWatcher(operatorReleaseVersion string, stop <-chan struc } } -// ensureSessionTransport ensures AWS sessions use the current certificates -// from the file watcher. -func (m *Provider) ensureSessionTransport(operatorReleaseVersion string, reloadCh chan bool) error { +// startSessionReloadHandler creates new route53, elb and resourcegroupstagging +// api client sessions when a value is received on reloadCh. +func (m *Provider) startSessionReloadHandler(operatorReleaseVersion string, reloadCh chan struct{}) error { for { select { case <-reloadCh: - sess, err := NewProviderSession(m.config, operatorReleaseVersion, m.fileWatcher.GetFileData()) + sess, err := newProviderSession(m.config, operatorReleaseVersion, m.fileWatcher.GetFileData()) if err != nil { return fmt.Errorf("failed to create dns provider session: %v", err) } m.lock.Lock() - m.elb = elb.New(sess, aws.NewConfig().WithRegion(m.config.Region)) - m.route53 = route53.New(sess) - m.tags = resourcegroupstaggingapi.New(sess, aws.NewConfig().WithRegion("us-east-1")) + elbClient, route53Client, taggingClient := newClients(sess, m.config.Region) + m.elb = elbClient + m.route53 = route53Client + m.tags = taggingClient m.lock.Unlock() - log.Info("dns provider session updated") + log.Info("recreated aws dns provider sessions") } } } diff --git a/pkg/dns/dns.go b/pkg/dns/dns.go index 482a48488e..e1e114ff2c 100644 --- a/pkg/dns/dns.go +++ b/pkg/dns/dns.go @@ -1,8 +1,9 @@ package dns import ( - configv1 "github.com/openshift/api/config/v1" iov1 "github.com/openshift/cluster-ingress-operator/pkg/api/v1" + + configv1 "github.com/openshift/api/config/v1" ) // Provider knows how to manage DNS zones only as pertains to routing. @@ -13,9 +14,7 @@ type Provider interface { // Delete will delete record. Delete(record *iov1.DNSRecord, zone configv1.DNSZone) error - // StartWatcher starts running the FileWatcher. The FileWatcher will stop - // running when the channel is closed. StartWatcher blocks until the - // channel is closed or an error occurs. + // StartWatcher starts running a file watcher. StartWatcher(string, <-chan struct{}) error } diff --git a/pkg/operator/config/config.go b/pkg/operator/config/config.go index 38aa69e37d..27be9d7674 100644 --- a/pkg/operator/config/config.go +++ b/pkg/operator/config/config.go @@ -11,4 +11,8 @@ type Config struct { // IngressControllerImage is the ingress controller image to manage. IngressControllerImage string + + // TrustedCABundle is the fully qualified path of the operator's trusted + // CA certificate bundle. + TrustedCABundle string } diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index c6ba4b29c0..479d3d2ec6 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -53,6 +53,8 @@ type Operator struct { dnsProvider dns.Provider namespace string + + releaseVersion string } // New creates (but does not start) a new operator from configuration. @@ -116,8 +118,9 @@ func New(config operatorconfig.Config, dnsProvider dns.Provider, kubeConfig *res } return &Operator{ - manager: mgr, - dnsProvider: dnsProvider, + manager: mgr, + dnsProvider: dnsProvider, + releaseVersion: config.OperatorReleaseVersion, // TODO: These are only needed for the default ingress controller stuff, which // should be refactored away. client: mgr.GetClient(), @@ -129,7 +132,7 @@ func New(config operatorconfig.Config, dnsProvider dns.Provider, kubeConfig *res // and trusted ca bundle watcher synchronously until a message is received // on the stop channel. // TODO: Move the default IngressController logic elsewhere. -func (o *Operator) Start(operatorReleaseVersion string, stop <-chan struct{}) error { +func (o *Operator) Start(stop <-chan struct{}) error { // Periodicaly ensure the default controller exists. go wait.Until(func() { if !o.manager.GetCache().WaitForCacheSync(stop) { @@ -147,7 +150,7 @@ func (o *Operator) Start(operatorReleaseVersion string, stop <-chan struct{}) er errChan <- o.manager.Start(stop) }() go func() { - errChan <- o.dnsProvider.StartWatcher(operatorReleaseVersion, stop) + errChan <- o.dnsProvider.StartWatcher(o.releaseVersion, stop) }() // Wait for the manager or watcher to exit or an explicit stop. diff --git a/pkg/operator/watcher/filewatcher.go b/pkg/operator/watcher/filewatcher.go index 07a4d74fa7..58ff6f5db7 100644 --- a/pkg/operator/watcher/filewatcher.go +++ b/pkg/operator/watcher/filewatcher.go @@ -22,8 +22,6 @@ type FileWatcher struct { // New returns a new FileWatcher watching the given file. func New(file string) (*FileWatcher, error) { - var err error - fw := &FileWatcher{ fileName: file, } @@ -33,9 +31,10 @@ func New(file string) (*FileWatcher, error) { return nil, err } - fw.watcher, err = fsnotify.NewWatcher() - if err != nil { + if watcher, err := fsnotify.NewWatcher(); err != nil { return nil, err + } else { + fw.watcher = watcher } return fw, nil @@ -49,15 +48,13 @@ func (fw *FileWatcher) GetFileData() []byte { } // Start starts the FileWatcher. -func (fw *FileWatcher) Start(stopCh <-chan struct{}, reloadCh chan bool) error { +func (fw *FileWatcher) Start(stopCh <-chan struct{}, reloadCh chan struct{}) error { if err := fw.watcher.Add(fw.fileName); err != nil { return err } go fw.Watch(reloadCh) - log.Info("starting file watcher") - // Block until the stop channel is closed. <-stopCh @@ -65,23 +62,21 @@ func (fw *FileWatcher) Start(stopCh <-chan struct{}, reloadCh chan bool) error { } // Watch reads events from the watcher's channel and reacts to changes. -func (fw *FileWatcher) Watch(reload chan bool) { +func (fw *FileWatcher) Watch(reload chan struct{}) { for { select { case event, ok := <-fw.watcher.Events: - // Channel is closed. if !ok { + log.Info("file watch events channel closed") return } - fw.handleEvent(event, reload) case err, ok := <-fw.watcher.Errors: - // Channel is closed. if !ok { + log.Info("file watch error channel closed") return } - log.Error(err, "file watch error") } } @@ -107,7 +102,7 @@ func (fw *FileWatcher) ReadFile() error { // handleEvent filters events, re-adds and re-reads the watched file // if removed. -func (fw *FileWatcher) handleEvent(event fsnotify.Event, reload chan bool) { +func (fw *FileWatcher) handleEvent(event fsnotify.Event, reload chan struct{}) { if !(isWrite(event) || isRemove(event) || isCreate(event)) { return } @@ -121,7 +116,7 @@ func (fw *FileWatcher) handleEvent(event fsnotify.Event, reload chan bool) { if err := fw.ReadFile(); err != nil { log.Error(err, "error re-reading watched file %s", fw.fileName) } else { - reload <- true + reload <- struct{}{} } } diff --git a/pkg/util/certs/certs.go b/pkg/util/certs/certs.go deleted file mode 100644 index 1ca5953c9f..0000000000 --- a/pkg/util/certs/certs.go +++ /dev/null @@ -1,55 +0,0 @@ -package certs - -import ( - "crypto/tls" - "crypto/x509" - "encoding/pem" - "errors" - "net/http" -) - -// makeTLSConfig returns a TLS configuration with a CA -// pool containing certificates from certs. -func makeTLSConfig(certs []*x509.Certificate) *tls.Config { - caPool := x509.NewCertPool() - for _, cert := range certs { - caPool.AddCert(cert) - } - return &tls.Config{RootCAs: caPool} -} - -// MakeTLSTransport returns an HTTP transport with a TLS -// configuration containing CA certificates from certs. -func MakeTLSTransport(certs []*x509.Certificate) *http.Transport { - cfg := makeTLSConfig(certs) - return &http.Transport{TLSClientConfig: cfg} -} - -// CertsFromPEM parses pemCerts into a list of certificates. -func CertsFromPEM(pemCerts []byte) ([]*x509.Certificate, error) { - ok := false - certs := []*x509.Certificate{} - for len(pemCerts) > 0 { - var block *pem.Block - block, pemCerts = pem.Decode(pemCerts) - if block == nil { - break - } - if block.Type != "CERTIFICATE" || len(block.Headers) != 0 { - continue - } - - cert, err := x509.ParseCertificate(block.Bytes) - if err != nil { - return certs, err - } - - certs = append(certs, cert) - ok = true - } - - if !ok { - return certs, errors.New("could not read any certificates") - } - return certs, nil -} From 94f3b938f0225bd085c2ce8780675c3813ac42f9 Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Tue, 10 Dec 2019 12:32:29 -0800 Subject: [PATCH 5/6] Refactors to exit upon watched file change --- cmd/ingress-operator/start.go | 13 ++-- manifests/02-deployment.yaml | 5 +- pkg/dns/aws/dns.go | 100 +++++++--------------------- pkg/dns/aws/filewatcher.go | 47 ------------- pkg/dns/azure/dns.go | 2 - pkg/dns/dns.go | 5 -- pkg/dns/gcp/provider.go | 2 - pkg/manifests/bindata.go | 8 +-- pkg/operator/config/config.go | 4 +- pkg/operator/operator.go | 24 ++++--- pkg/operator/watcher/filewatcher.go | 40 +++-------- 11 files changed, 62 insertions(+), 188 deletions(-) delete mode 100644 pkg/dns/aws/filewatcher.go diff --git a/cmd/ingress-operator/start.go b/cmd/ingress-operator/start.go index 02c06a6e9e..fb4ea9da57 100644 --- a/cmd/ingress-operator/start.go +++ b/cmd/ingress-operator/start.go @@ -35,9 +35,9 @@ const ( // operator's namespace that will hold the credentials that the operator // will use to authenticate with the cloud API. cloudCredentialsSecretName = "cloud-credentials" - // The fully qualified path of the trusted CA bundle that is - // mounted from configmap openshift-ingress-operator/trusted-ca. - defaultCABundle = "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem" + // defaultTrustedCABundle is the fully qualified path of the trusted CA bundle + // that is mounted from configmap openshift-ingress-operator/trusted-ca. + defaultTrustedCABundle = "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem" ) func NewStartCommand() *cobra.Command { @@ -112,7 +112,7 @@ func start() error { OperatorReleaseVersion: releaseVersion, Namespace: operatorNamespace, IngressControllerImage: ingressControllerImage, - TrustedCABundle: defaultCABundle, + TrustedCABundle: defaultTrustedCABundle, } // Set up the DNS manager. @@ -138,9 +138,6 @@ func createDNSProvider(cl client.Client, operatorConfig operatorconfig.Config, d switch platformStatus.Type { case configv1.AWSPlatformType: - if len(platformStatus.AWS.Region) == 0 { - return nil, fmt.Errorf("region is required") - } creds := &corev1.Secret{} err := cl.Get(context.TODO(), types.NamespacedName{Namespace: operatorConfig.Namespace, Name: cloudCredentialsSecretName}, creds) if err != nil { @@ -151,7 +148,7 @@ func createDNSProvider(cl client.Client, operatorConfig operatorconfig.Config, d AccessKey: string(creds.Data["aws_secret_access_key"]), DNS: dnsConfig, Region: platformStatus.AWS.Region, - }, operatorConfig.OperatorReleaseVersion, operatorConfig.TrustedCABundle) + }, operatorConfig.OperatorReleaseVersion) if err != nil { return nil, fmt.Errorf("failed to create AWS DNS manager: %v", err) } diff --git a/manifests/02-deployment.yaml b/manifests/02-deployment.yaml index 10012998ba..1a55158970 100644 --- a/manifests/02-deployment.yaml +++ b/manifests/02-deployment.yaml @@ -55,8 +55,9 @@ spec: requests: cpu: 10m volumeMounts: - - name: trusted-ca - mountPath: /etc/pki/ca-trust/extracted/pem + - name: trusted-ca + mountPath: /etc/pki/ca-trust/extracted/pem + readOnly: true - name: kube-rbac-proxy image: quay.io/openshift/origin-kube-rbac-proxy:latest args: diff --git a/pkg/dns/aws/dns.go b/pkg/dns/aws/dns.go index 4354736715..afec8d0b8f 100644 --- a/pkg/dns/aws/dns.go +++ b/pkg/dns/aws/dns.go @@ -1,20 +1,14 @@ package aws import ( - "crypto/tls" - "crypto/x509" "fmt" - "net" - "net/http" "reflect" "strings" "sync" - "time" iov1 "github.com/openshift/cluster-ingress-operator/pkg/api/v1" "github.com/openshift/cluster-ingress-operator/pkg/dns" logf "github.com/openshift/cluster-ingress-operator/pkg/log" - "github.com/openshift/cluster-ingress-operator/pkg/operator/watcher" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/arn" @@ -51,10 +45,6 @@ type Provider struct { config Config - // fileWatcher watches the trusted ca bundle for changes - // and keeps AWS sessions updated. - fileWatcher *watcher.FileWatcher - // lock protects access to everything below. lock sync.RWMutex @@ -85,88 +75,44 @@ type Config struct { DNS *configv1.DNS } -// NewProvider returns a new AWS Provider based on config and adds -// openshift.io/ingress-operator/operatorReleaseVersion to the user-agent -// request header. -func NewProvider(config Config, operatorReleaseVersion string, trustBundle string) (*Provider, error) { - caWatcher, err := watcher.New(trustBundle) - if err != nil { - return nil, err - } - - sess, err := newProviderSession(config, operatorReleaseVersion, caWatcher.GetFileData()) - if err != nil { - return nil, err - } - - elbClient, route53Client, taggingClient := newClients(sess, config.Region) - provider := &Provider{ - elb: elbClient, - route53: route53Client, - fileWatcher: caWatcher, - tags: taggingClient, - config: config, - idsToTags: map[string]map[string]string{}, - lbZones: map[string]string{}, - updatedRecords: sets.NewString(), - } - return provider, nil -} - -// newProviderSession returns a new AWS Session with credentials from config; -// adding openshift.io/ingress-operator/operatorReleaseVersion to the user-agent -// request header and pemData to the TLS Config CA pool. Aside from TLSClientConfig, -// the transport created is identical to DefaultTransport of net/http. -func newProviderSession(config Config, operatorReleaseVersion string, pemData []byte) (*session.Session, error) { +func NewProvider(config Config, operatorReleaseVersion string) (*Provider, error) { creds := credentials.NewStaticCredentials(config.AccessID, config.AccessKey, "") - caPool := x509.NewCertPool() - if ok := caPool.AppendCertsFromPEM(pemData); !ok { - return nil, fmt.Errorf("failed to append certificates to ca pool") - } - tlsConfig := &tls.Config{RootCAs: caPool} - trans := &http.Transport{ - Proxy: http.ProxyFromEnvironment, - DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - DualStack: true, - }).DialContext, - TLSClientConfig: tlsConfig, - MaxIdleConns: 100, - IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 10 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - } sess, err := session.NewSessionWithOptions(session.Options{ Config: aws.Config{ Credentials: creds, - HTTPClient: &http.Client{Transport: trans}, }, SharedConfigState: session.SharedConfigEnable, }) if err != nil { - return nil, fmt.Errorf("failed to create AWS client session: %v", err) + return nil, fmt.Errorf("couldn't create AWS client session: %v", err) } sess.Handlers.Build.PushBackNamed(request.NamedHandler{ Name: "openshift.io/ingress-operator", Fn: request.MakeAddToUserAgentHandler("openshift.io ingress-operator", operatorReleaseVersion), }) - return sess, nil -} - -// newClients returns new elb, route53 and resourcegroupstagging api clients -// using session and region. Note that region "us-east-1" is used for the -// resourcegroupstagging client. -func newClients(session *session.Session, region string) (*elb.ELB, *route53.Route53, *resourcegroupstaggingapi.ResourceGroupsTaggingAPI) { - elb := elb.New(session, aws.NewConfig().WithRegion(region)) - route53 := route53.New(session) - - // TODO: The resourcegroupstagging API will only return hostedzone resources (which are global) - // when the region is forced to us-east-1. We don't yet understand why. - tags := resourcegroupstaggingapi.New(session, aws.NewConfig().WithRegion("us-east-1")) + region := aws.StringValue(sess.Config.Region) + if len(region) > 0 { + log.Info("using region from shared config", "region name", region) + } else { + region = config.Region + log.Info("using region from operator config", "region name", region) + } + if len(region) == 0 { + return nil, fmt.Errorf("region is required") + } - return elb, route53, tags + return &Provider{ + elb: elb.New(sess, aws.NewConfig().WithRegion(region)), + route53: route53.New(sess), + // TODO: This API will only return hostedzone resources (which are global) + // when the region is forced to us-east-1. We don't yet understand why. + tags: resourcegroupstaggingapi.New(sess, aws.NewConfig().WithRegion("us-east-1")), + config: config, + idsToTags: map[string]map[string]string{}, + lbZones: map[string]string{}, + updatedRecords: sets.NewString(), + }, nil } // getZoneID finds the ID of given zoneConfig in Route53. If an ID is already diff --git a/pkg/dns/aws/filewatcher.go b/pkg/dns/aws/filewatcher.go deleted file mode 100644 index 3ba609c2c5..0000000000 --- a/pkg/dns/aws/filewatcher.go +++ /dev/null @@ -1,47 +0,0 @@ -package aws - -import ( - "fmt" -) - -// StartWatcher starts a file watcher and ensures AWS sessions are using the -// current ca bundle until a message is received on the stop or error channels. -func (m *Provider) StartWatcher(operatorReleaseVersion string, stop <-chan struct{}) error { - errChan := make(chan error) - reloadChan := make(chan struct{}) - go func() { - errChan <- m.fileWatcher.Start(stop, reloadChan) - }() - go func() { - errChan <- m.startSessionReloadHandler(operatorReleaseVersion, reloadChan) - }() - - // Wait for the watcher to exit or an explicit stop. - select { - case <-stop: - return nil - case err := <-errChan: - return err - } -} - -// startSessionReloadHandler creates new route53, elb and resourcegroupstagging -// api client sessions when a value is received on reloadCh. -func (m *Provider) startSessionReloadHandler(operatorReleaseVersion string, reloadCh chan struct{}) error { - for { - select { - case <-reloadCh: - sess, err := newProviderSession(m.config, operatorReleaseVersion, m.fileWatcher.GetFileData()) - if err != nil { - return fmt.Errorf("failed to create dns provider session: %v", err) - } - m.lock.Lock() - elbClient, route53Client, taggingClient := newClients(sess, m.config.Region) - m.elb = elbClient - m.route53 = route53Client - m.tags = taggingClient - m.lock.Unlock() - log.Info("recreated aws dns provider sessions") - } - } -} diff --git a/pkg/dns/azure/dns.go b/pkg/dns/azure/dns.go index a1574806d8..d701983277 100644 --- a/pkg/dns/azure/dns.go +++ b/pkg/dns/azure/dns.go @@ -127,5 +127,3 @@ func (m *provider) Delete(record *iov1.DNSRecord, zone configv1.DNSZone) error { func getARecordName(recordDomain string, zoneName string) (string, error) { return strings.TrimSuffix(strings.TrimSuffix(recordDomain, "."), "."+zoneName), nil } - -func (m *provider) StartWatcher(string, <-chan struct{}) error { return nil } diff --git a/pkg/dns/dns.go b/pkg/dns/dns.go index e1e114ff2c..0f308c2709 100644 --- a/pkg/dns/dns.go +++ b/pkg/dns/dns.go @@ -13,9 +13,6 @@ type Provider interface { // Delete will delete record. Delete(record *iov1.DNSRecord, zone configv1.DNSZone) error - - // StartWatcher starts running a file watcher. - StartWatcher(string, <-chan struct{}) error } var _ Provider = &FakeProvider{} @@ -24,5 +21,3 @@ type FakeProvider struct{} func (_ *FakeProvider) Ensure(record *iov1.DNSRecord, zone configv1.DNSZone) error { return nil } func (_ *FakeProvider) Delete(record *iov1.DNSRecord, zone configv1.DNSZone) error { return nil } - -func (_ *FakeProvider) StartWatcher(string, <-chan struct{}) error { return nil } diff --git a/pkg/dns/gcp/provider.go b/pkg/dns/gcp/provider.go index 380792ebae..b6fb491759 100644 --- a/pkg/dns/gcp/provider.go +++ b/pkg/dns/gcp/provider.go @@ -75,5 +75,3 @@ func resourceRecordSet(record *iov1.DNSRecord) *gdnsv1.ResourceRecordSet { Ttl: record.Spec.RecordTTL, } } - -func (p *Provider) StartWatcher(string, <-chan struct{}) error { return nil } diff --git a/pkg/manifests/bindata.go b/pkg/manifests/bindata.go index 1f47c226ab..94e8c94906 100644 --- a/pkg/manifests/bindata.go +++ b/pkg/manifests/bindata.go @@ -25,7 +25,7 @@ // manifests/01-service-account.yaml (196B) // manifests/01-service.yaml (344B) // manifests/01-trusted-ca-configmap.yaml (323B) -// manifests/02-deployment.yaml (2.809kB) +// manifests/02-deployment.yaml (2.832kB) // manifests/03-cluster-operator.yaml (357B) // manifests/image-references (433B) @@ -596,7 +596,7 @@ func manifests01TrustedCaConfigmapYaml() (*asset, error) { return a, nil } -var _manifests02DeploymentYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x56\x4f\x6f\xdb\xb8\x13\xbd\xe7\x53\x10\x3e\xff\x68\x39\x69\x7f\xdd\x2d\x81\x1c\xbc\xae\xda\x04\x88\xd3\x20\x0e\xda\xa3\x41\x53\x63\x9b\x6b\x8a\x64\x67\x46\x46\xf4\xed\x17\x94\xff\xc9\xb2\xe3\xe6\xb0\xeb\x93\xa1\x79\xf3\xe6\x71\xf8\x66\x24\x1d\xed\x0f\x40\xb2\xc1\x2b\xa1\x63\xa4\x6c\x7d\x7d\xb5\xb2\xbe\x50\xe2\x0b\x44\x17\xea\x12\x3c\x5f\x95\xc0\xba\xd0\xac\xd5\x95\x10\x5e\x97\xa0\x84\xf5\x0b\x04\x22\x19\x22\xa0\xe6\x80\xdb\x00\x45\x6d\x40\x89\x10\xc1\xd3\xd2\xce\x59\x9e\xc1\x69\xef\x03\x6b\xb6\xc1\x53\xe2\x13\xc2\x04\x3f\xb7\x8b\xfe\x3e\xa9\x6f\x43\x66\xfd\xdf\x60\x58\x46\x0c\xaf\xf5\x99\x6a\x14\xc1\xa4\x64\x84\xe8\xac\xd1\xa4\xc4\xf5\x95\x10\xc4\xa8\x19\x16\xf5\x86\x96\xeb\x08\x4a\x3c\x83\x41\xd0\x0c\x29\x0c\x0e\x0c\x07\xdc\x84\x4b\xcd\x66\xf9\xa0\x67\xe0\xb6\x32\x2e\x1c\x8d\xa1\x8c\x4e\x33\x6c\x33\x5b\xdd\x48\x3f\x77\x44\x72\x81\x46\x88\x9d\xee\x06\x16\x0a\x98\x1c\x49\x4a\xbf\x55\x35\x03\xf4\xc0\x40\xa9\x0d\x81\x94\x70\xd6\x57\xaf\x07\xf2\x50\x80\xc4\xe0\xa0\x7f\x8c\x2c\x35\x31\xa0\x12\xbd\xde\x16\xca\xc1\xa5\xc2\x87\x36\x0b\x21\xc5\x0a\x6a\x25\x7a\x97\x39\x7a\xfb\x5a\x3b\xe9\x4a\xf4\xf2\x57\x4b\x4c\x87\x10\xcc\xe7\x60\x58\x89\xde\x63\x98\x98\x25\x14\x95\x83\xde\x99\x2a\x9d\x02\x95\x47\xd0\x66\xa9\x67\x07\xf4\x7b\xab\xe4\xaf\x60\x2a\x6e\xa5\x1d\xce\x37\x01\x13\x7c\x91\x3c\x70\x33\xf8\xbd\x06\x1f\x58\x22\xe8\xa2\xfe\x6f\x15\x10\xe0\xda\x1a\x18\x1a\x13\x2a\xcf\x8f\x6f\x5b\x42\x88\x88\x36\xa0\xe5\x7a\xe4\x34\xd1\x06\x49\x35\x31\x94\xd2\xb8\x2a\xdd\x88\x34\x68\xd9\x1a\xed\xb6\x09\x26\x78\xd6\xd6\x03\xb6\x4c\x27\x2f\xd9\x6e\xab\x17\xb0\xb4\xbe\x11\x3c\x06\x22\xbd\x80\xa7\xe0\xac\xa9\x95\xf8\xaa\x9d\x9b\x69\xb3\x7a\x09\x0f\x61\x41\xdf\x7d\x8e\x78\x94\x69\xcb\x04\xae\x9c\xdb\x25\xdc\xcf\x1f\x03\x3f\x21\x50\xda\x0e\x1d\x5c\x6b\xfc\xb3\x80\x76\x61\xfd\xfe\x1c\x5d\x71\x2a\x0d\x15\xb5\x19\x4c\x28\x4b\xed\x0b\xd5\x7a\x24\x2f\x9d\x49\x0a\x62\x8d\x6d\x06\xf0\xeb\x76\xf6\xa1\x33\xcf\xf9\x43\x3e\x9c\xe4\xd3\x1f\xf9\xf3\xe4\xfe\xfb\xe3\x11\x46\x88\xb5\x76\x15\x28\xd1\x1b\xf4\x07\xfd\x6b\x49\x5e\x47\x5a\x06\xee\x9d\x65\xfa\x39\x7c\x19\xdd\x4d\x1f\x87\xe3\x7c\xf2\x34\x1c\xe5\xe7\x98\xbe\x62\x28\x55\x27\x20\xc4\xdc\x82\x2b\x9e\x61\x7e\x1a\xd9\xc6\x9e\x34\x2f\xd5\x7e\xc1\xf4\xf7\x2b\xf5\xac\x8c\xfb\xf1\xf0\xdb\xd9\xe2\x67\x6e\x60\xa9\x9b\x55\x2a\x31\x54\x69\x4d\xac\x3f\xf6\x07\xad\x4c\x04\x0a\x15\x1a\xa0\x63\x61\x08\xbf\x2a\x20\xa6\xae\x5c\x13\x2b\x25\xae\x07\x65\xeb\xf1\x3a\xb8\xaa\x84\x71\xf2\x3a\x9d\x6f\x3f\x63\xf2\x40\x21\x8d\xee\xb0\x95\x29\x69\x73\xf0\x0c\xd8\x64\x71\x65\x33\xa3\x65\x83\xcf\xe0\x95\x51\x1b\x86\x22\x8b\x50\x9e\x98\x3d\x4d\xb6\xc4\x99\x36\x9b\x17\xc5\xa9\x13\x7f\x55\xba\x6e\xf6\x68\xb7\x1f\x9d\xcc\x53\x23\x6a\x5c\xd0\xb1\x0b\xa5\x74\x61\xc1\x81\xb8\x00\xc4\x4e\x84\xc0\x54\x08\xd2\x59\x62\xf0\x52\x17\x45\xf2\xeb\xad\xfa\xfc\xe1\xf3\x87\x0e\x92\x1d\x49\x63\xe3\x12\x50\x52\x65\x19\xe8\xf6\xe5\x61\x32\xcd\x47\x5f\xee\xf2\xe9\xf3\x64\x38\xfd\x79\xff\x72\x37\x1d\xe6\x93\xe9\xf5\xcd\x9f\xd3\x6f\xa3\xf1\x74\x72\x37\xbc\xf9\xff\xa7\xff\x1d\x50\xf9\xe8\xcb\x6f\x70\x27\x3c\xa3\xbf\x46\xef\xe2\x39\x8b\xbb\xc0\xd6\x39\x5b\x15\x89\x11\x74\x79\xbb\x64\x8e\x2a\xcb\xae\x6f\xfe\xe8\x37\xf3\xa4\x3e\x0d\x06\x83\x41\x76\xae\x15\x80\x2c\xe7\xd6\xc1\x6d\x73\xf7\xec\x28\x8b\x68\xd7\x9a\x21\xfd\xef\x9b\xa3\xd1\xde\x25\x6d\x11\x72\x05\xf5\x85\xdc\x15\xb4\x0d\x11\x03\x72\xe7\x42\xf7\x6b\xf4\x29\x20\x2b\xd1\xb9\xac\xdd\x7b\xbc\x04\x46\x6b\xe8\x5f\x9d\x96\xc6\xf6\x50\x06\xac\x95\xf8\x38\x18\xdb\x77\x0c\x92\x3c\x99\x93\xd6\x79\xdf\x96\x9d\xfa\xd5\x51\xa9\x8b\xef\xde\xd5\xcd\x40\xee\x12\x37\x45\x5b\xdf\x09\x6f\x73\x50\xfa\x9e\xe2\xb6\xb2\xcd\x93\xc7\x37\x32\x2e\x8c\xff\xe6\xbb\x6f\xac\x63\x9b\xed\xc2\xb2\xb0\x0c\xe5\xc9\x72\x69\xde\xf3\x46\xcb\x59\xe5\x0b\x07\x1d\xc3\xa4\x5f\x6c\x3a\xd6\x78\x6d\x8f\x4a\xbb\xe4\x9f\x00\x00\x00\xff\xff\x2d\xc8\x4a\xbf\xf9\x0a\x00\x00") +var _manifests02DeploymentYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x56\x41\x73\xdb\x38\x0f\xbd\xe7\x57\x70\x7c\xfe\x68\xd9\x69\xbf\xee\x96\x33\x39\x78\x5d\xb5\xc9\x4c\xec\x66\xe2\x4c\x7b\xf4\xd0\x14\x6c\x73\x4d\x91\x2c\x00\x79\xa2\x7f\xbf\x43\xd9\xb1\x65\xc5\x71\x73\xd8\xd5\x49\x23\x3c\x3c\x3c\x80\x00\x44\x1d\xed\x0f\x40\xb2\xc1\x2b\xa1\x63\xa4\x6c\x3b\xbc\xda\x58\x5f\x28\xf1\x05\xa2\x0b\x75\x09\x9e\xaf\x4a\x60\x5d\x68\xd6\xea\x4a\x08\xaf\x4b\x50\xc2\xfa\x15\x02\x91\x0c\x11\x50\x73\xc0\xbd\x81\xa2\x36\xa0\x44\x88\xe0\x69\x6d\x97\x2c\xcf\xe0\xb4\xf7\x81\x35\xdb\xe0\x29\xf1\x09\x61\x82\x5f\xda\x55\xff\xe0\xd4\xb7\x21\xb3\xfe\x6f\x30\x2c\x23\x86\xe7\xfa\x4c\x34\x8a\x60\x92\x33\x42\x74\xd6\x68\x52\x62\x78\x25\x04\x31\x6a\x86\x55\xbd\xa3\xe5\x3a\x82\x12\x8f\x60\x10\x34\x43\x32\x83\x03\xc3\x01\x77\xe6\x52\xb3\x59\xdf\xeb\x05\xb8\xbd\x8c\x0b\xa9\x31\x94\xd1\x69\x86\xbd\x67\xab\x1a\xe9\x71\x27\x24\x17\x68\x84\x78\xd1\xdd\xc0\x42\x01\xb3\x13\x49\xe9\xd9\x54\x0b\x40\x0f\x0c\x94\xca\x10\x48\x09\x67\x7d\xf5\x7c\x24\x0f\x05\x48\x0c\x0e\xfa\xa7\xc8\x52\x13\x03\x2a\xd1\xeb\xed\xa1\x1c\x5c\x0a\x7c\x2c\xb3\x10\x52\x6c\xa0\x56\xa2\x77\x99\xa3\x77\x88\xf5\x22\x5d\x89\x5e\xfe\x6c\x89\xe9\x68\x82\xe5\x12\x0c\x2b\xd1\x9b\x86\x99\x59\x43\x51\x39\xe8\x9d\x89\xd2\x09\x50\x79\x04\x6d\xd6\x7a\x71\x44\xbf\x37\x4a\xfe\x0c\xa6\xe2\x96\xdb\x31\xbf\x19\x98\xe0\x8b\xd4\x03\xd7\x83\xdf\x6b\xf0\x81\x25\x82\x2e\xea\xff\x56\x01\x01\x6e\xad\x81\x91\x31\xa1\xf2\x3c\x7d\xbb\x25\x84\x88\x68\x03\x5a\xae\xc7\x4e\x13\xed\x90\x54\x13\x43\x29\x8d\xab\xd2\x89\x48\x83\x96\xad\xd1\x6e\xef\x60\x82\x67\x6d\x3d\x60\xab\xe9\xe4\xa5\xb6\xdb\xeb\x05\x2c\xad\x6f\x04\x4f\x80\x48\xaf\xe0\x21\x38\x6b\x6a\x25\xbe\x6a\xe7\x16\xda\x6c\x9e\xc2\x7d\x58\xd1\x77\x9f\x23\x9e\x78\xda\x32\x81\x2b\xe7\x5e\x1c\xee\x96\xd3\xc0\x0f\x08\x94\xb6\x43\x07\xd7\x1a\xff\x2c\xa0\x5d\x59\x7f\xc8\xa3\x2b\x4e\xa5\xa1\xa2\x36\x83\x09\x65\xa9\x7d\xa1\x5a\x9f\xe4\xa5\x9c\xa4\x20\xd6\xd8\x66\x00\xbf\x6d\x7b\x1f\x2b\xf3\x98\xdf\xe7\xa3\x59\x3e\xff\x91\x3f\xce\xee\xbe\x4f\x4f\x30\x42\x6c\xb5\xab\x40\x89\xde\xa0\x3f\xe8\x0f\x25\x79\x1d\x69\x1d\xb8\x77\x96\xe9\xe7\xe8\x69\x7c\x3b\x9f\x8e\x26\xf9\xec\x61\x34\xce\xcf\x31\x7d\xc5\x50\xaa\x8e\x41\x88\xa5\x05\x57\x3c\xc2\xf2\xb5\x65\x6f\x7b\xd0\xbc\x56\x87\x05\xd3\x3f\xac\xd4\xb3\x32\xee\x26\xa3\x6f\x67\x83\x9f\x39\x81\xb5\x6e\x56\xa9\xc4\x50\xa5\x35\xb1\xfd\xd8\x1f\xb4\x3c\x11\x28\x54\x68\x80\x4e\x85\x21\xfc\xaa\x80\x98\xba\x72\x4d\xac\x94\x18\x0e\xca\xd6\xe7\x6d\x70\x55\x09\x93\xd4\xeb\x74\x7a\x78\x3b\xad\x8c\xa9\x03\x0a\x69\xf4\x09\x57\x99\x1c\x76\x49\x67\xc0\x26\x8b\x1b\x9b\x19\x2d\x1b\x74\x06\xcf\x8c\xda\x30\x14\x59\x84\xb2\x23\x4c\x17\xdf\xbd\xab\x1b\x5e\x78\x35\x03\x69\xe0\x25\x2e\xb4\xd9\xfd\x3f\x5e\x37\xe8\xaf\x4a\xd7\xcd\x7a\xed\x96\xa9\xe3\xf9\xba\x3f\x35\xae\x3a\xf9\x49\xe9\xc2\x8a\x03\x71\x01\x88\x1d\x0b\x81\xa9\x10\xa4\xb3\xc4\xe0\xa5\x2e\x8a\xd4\xc6\x37\xea\xf3\x87\xcf\x1f\x3a\x48\x76\x24\x8d\x8d\x6b\x40\x49\x95\x65\xa0\x9b\xa7\xfb\xd9\x3c\x1f\x7f\xb9\xcd\xe7\x8f\xb3\xd1\xfc\xe7\xdd\xd3\xed\x7c\x94\xcf\xe6\xc3\xeb\x3f\xe7\xdf\xc6\x93\xf9\xec\x76\x74\xfd\xff\x4f\xff\x3b\xa2\xf2\xf1\x97\xdf\xe0\x5e\xf1\x8c\xff\x1a\xbf\x8b\xe7\x2c\xee\x02\x5b\x27\xb7\x2a\x12\x23\xe8\xf2\x66\xcd\x1c\x55\x96\x0d\xaf\xff\xe8\x37\x63\xa6\x3e\x0d\x06\x83\x41\x76\xae\x14\x80\x2c\x97\xd6\xc1\x4d\xd3\x16\xec\x28\x8b\x68\xb7\x9a\x21\xbd\xf7\xcd\xc9\xc4\xbf\x38\xed\x11\x72\x03\xf5\x05\xdf\x0d\xb4\x1b\x22\x06\xec\x36\xec\x61\xbb\x3e\x04\x64\x25\x3a\x87\xf5\xf2\x7b\x2f\x81\xd1\x1a\xfa\x57\x87\x48\x34\x97\x8b\x32\x60\xad\xc4\xc7\xc1\xc4\xbe\x6b\xbe\xba\x23\xd4\xca\xf7\x6d\xd9\xa9\x5e\xbf\x9f\xa8\x5d\xd0\xd6\xf5\xe1\x6d\x0e\x4a\xd7\x2c\x6e\x2b\xdb\x7d\x99\xbe\xe1\x71\x61\x2f\xec\xae\x83\x13\x1d\xdb\x6c\x17\xb6\x88\x65\x28\xa9\xbb\xf2\x9b\xdf\xbf\xd1\x72\x51\xf9\xc2\x41\xa7\x61\xd2\x13\x9b\x8a\x35\xbd\x76\x40\xa5\x35\xf3\x4f\x00\x00\x00\xff\xff\xb8\xac\xc5\x1e\x10\x0b\x00\x00") func manifests02DeploymentYamlBytes() ([]byte, error) { return bindataRead( @@ -611,8 +611,8 @@ func manifests02DeploymentYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "manifests/02-deployment.yaml", size: 2809, mode: os.FileMode(420), modTime: time.Unix(1, 0)} - a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x1f, 0x3, 0x35, 0xf1, 0xb8, 0xea, 0x53, 0xd3, 0xd4, 0xe0, 0x1b, 0x8b, 0xe4, 0x2d, 0xbd, 0x67, 0x5c, 0x4f, 0xda, 0xed, 0xe1, 0x2e, 0x9, 0x72, 0x30, 0xa5, 0xa7, 0x3d, 0x4b, 0x2f, 0x62, 0xf8}} + info := bindataFileInfo{name: "manifests/02-deployment.yaml", size: 2832, mode: os.FileMode(420), modTime: time.Unix(1, 0)} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xe2, 0x2b, 0xf1, 0xb8, 0x2b, 0x55, 0xdf, 0xa3, 0x95, 0x9b, 0x31, 0x93, 0xb8, 0x75, 0xae, 0x9f, 0x4d, 0x96, 0xcd, 0x9c, 0x14, 0xa6, 0xdf, 0x43, 0x6, 0x94, 0xd0, 0xa6, 0xb8, 0xda, 0xb, 0x32}} return a, nil } diff --git a/pkg/operator/config/config.go b/pkg/operator/config/config.go index 27be9d7674..b40dc1ed25 100644 --- a/pkg/operator/config/config.go +++ b/pkg/operator/config/config.go @@ -12,7 +12,7 @@ type Config struct { // IngressControllerImage is the ingress controller image to manage. IngressControllerImage string - // TrustedCABundle is the fully qualified path of the operator's trusted - // CA certificate bundle. + // TrustedCABundle is the fully qualified path of the trusted CA certificate + // bundle that is watched. TrustedCABundle string } diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 479d3d2ec6..7c744ff7d7 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -16,6 +16,7 @@ import ( dnscontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/dns" ingresscontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/ingress" statuscontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/status" + "github.com/openshift/cluster-ingress-operator/pkg/operator/watcher" operatorutil "github.com/openshift/cluster-ingress-operator/pkg/util" operatorv1 "github.com/openshift/api/operator/v1" @@ -50,11 +51,9 @@ type Operator struct { manager manager.Manager - dnsProvider dns.Provider + caWatcher *watcher.FileWatcher namespace string - - releaseVersion string } // New creates (but does not start) a new operator from configuration. @@ -117,10 +116,15 @@ func New(config operatorconfig.Config, dnsProvider dns.Provider, kubeConfig *res return nil, fmt.Errorf("failed to create dns controller: %v", err) } + // Set up trusted ca bundle watcher + watcher, err := watcher.New(config.TrustedCABundle) + if err != nil { + return nil, fmt.Errorf("failed to create trusted ca watcher: %v", err) + } + return &Operator{ - manager: mgr, - dnsProvider: dnsProvider, - releaseVersion: config.OperatorReleaseVersion, + manager: mgr, + caWatcher: watcher, // TODO: These are only needed for the default ingress controller stuff, which // should be refactored away. client: mgr.GetClient(), @@ -130,7 +134,7 @@ func New(config operatorconfig.Config, dnsProvider dns.Provider, kubeConfig *res // Start creates the default IngressController and then starts the operator // and trusted ca bundle watcher synchronously until a message is received -// on the stop channel. +// on the stop or reload channels. // TODO: Move the default IngressController logic elsewhere. func (o *Operator) Start(stop <-chan struct{}) error { // Periodicaly ensure the default controller exists. @@ -149,14 +153,18 @@ func (o *Operator) Start(stop <-chan struct{}) error { go func() { errChan <- o.manager.Start(stop) }() + + reloadChan := make(chan struct{}) go func() { - errChan <- o.dnsProvider.StartWatcher(o.releaseVersion, stop) + errChan <- o.caWatcher.Start(stop, reloadChan) }() // Wait for the manager or watcher to exit or an explicit stop. select { case <-stop: return nil + case <-reloadChan: + return nil case err := <-errChan: return err } diff --git a/pkg/operator/watcher/filewatcher.go b/pkg/operator/watcher/filewatcher.go index 58ff6f5db7..03cf2d64fa 100644 --- a/pkg/operator/watcher/filewatcher.go +++ b/pkg/operator/watcher/filewatcher.go @@ -15,9 +15,8 @@ var log = logf.Logger.WithName("filewatcher") // FileWatcher watches a file for changes. type FileWatcher struct { sync.Mutex - fileName string - currentData []byte - watcher *fsnotify.Watcher + fileName string + watcher *fsnotify.Watcher } // New returns a new FileWatcher watching the given file. @@ -40,13 +39,6 @@ func New(file string) (*FileWatcher, error) { return fw, nil } -// GetFileData fetches the data of the currently watched file. -func (fw *FileWatcher) GetFileData() []byte { - fw.Lock() - defer fw.Unlock() - return fw.currentData -} - // Start starts the FileWatcher. func (fw *FileWatcher) Start(stopCh <-chan struct{}, reloadCh chan struct{}) error { if err := fw.watcher.Add(fw.fileName); err != nil { @@ -62,7 +54,7 @@ func (fw *FileWatcher) Start(stopCh <-chan struct{}, reloadCh chan struct{}) err } // Watch reads events from the watcher's channel and reacts to changes. -func (fw *FileWatcher) Watch(reload chan struct{}) { +func (fw *FileWatcher) Watch(reloadCh chan struct{}) { for { select { case event, ok := <-fw.watcher.Events: @@ -70,7 +62,7 @@ func (fw *FileWatcher) Watch(reload chan struct{}) { log.Info("file watch events channel closed") return } - fw.handleEvent(event, reload) + fw.handleEvent(event, reloadCh) case err, ok := <-fw.watcher.Errors: if !ok { @@ -92,32 +84,18 @@ func (fw *FileWatcher) ReadFile() error { case len(data) == 0: return fmt.Errorf("file %s contains no data", fw.fileName) } - fw.Lock() - fw.currentData = data - fw.Unlock() - log.Info("reloaded file watcher current data") return nil } -// handleEvent filters events, re-adds and re-reads the watched file -// if removed. -func (fw *FileWatcher) handleEvent(event fsnotify.Event, reload chan struct{}) { +// handleEvent filters events and gracefully terminates the operator +// if a write, remove or create event is received for the watched file. +func (fw *FileWatcher) handleEvent(event fsnotify.Event, reloadCh chan struct{}) { if !(isWrite(event) || isRemove(event) || isCreate(event)) { return } - log.Info("watched file change", "event", event) - - if isRemove(event) { - if err := fw.watcher.Add(event.Name); err != nil { - log.Error(err, "error re-watching file %s", fw.fileName) - } - } - if err := fw.ReadFile(); err != nil { - log.Error(err, "error re-reading watched file %s", fw.fileName) - } else { - reload <- struct{}{} - } + log.Info("watched file changed", "event", event) + reloadCh <- struct{}{} } func isWrite(event fsnotify.Event) bool { From 1dc3d75b72eae20828aa3de58cf95343dc140d0e Mon Sep 17 00:00:00 2001 From: Daneyon Hansen Date: Wed, 11 Dec 2019 22:08:10 -0800 Subject: [PATCH 6/6] Moves watcher to start cmd --- cmd/ingress-operator/start.go | 61 ++++++++++++++- pkg/operator/config/config.go | 4 - pkg/operator/operator.go | 24 +----- pkg/operator/watcher/filewatcher.go | 111 ---------------------------- 4 files changed, 62 insertions(+), 138 deletions(-) delete mode 100644 pkg/operator/watcher/filewatcher.go diff --git a/cmd/ingress-operator/start.go b/cmd/ingress-operator/start.go index fb4ea9da57..5d56be5b26 100644 --- a/cmd/ingress-operator/start.go +++ b/cmd/ingress-operator/start.go @@ -1,12 +1,15 @@ package main import ( + "bytes" "context" "fmt" + "io/ioutil" "os" "github.com/ghodss/yaml" "github.com/spf13/cobra" + "gopkg.in/fsnotify.v1" "github.com/openshift/cluster-ingress-operator/pkg/dns" awsdns "github.com/openshift/cluster-ingress-operator/pkg/dns/aws" @@ -112,7 +115,6 @@ func start() error { OperatorReleaseVersion: releaseVersion, Namespace: operatorNamespace, IngressControllerImage: ingressControllerImage, - TrustedCABundle: defaultTrustedCABundle, } // Set up the DNS manager. @@ -122,12 +124,67 @@ func start() error { os.Exit(1) } + // Set up the channels for the watcher and operator. + stop := make(chan struct{}) + signal := signals.SetupSignalHandler() + + // Set up and start the file watcher. + watcher, err := fsnotify.NewWatcher() + if err != nil { + log.Error(err, "failed to create watcher") + os.Exit(1) + } + defer watcher.Close() + if err := watcher.Add(defaultTrustedCABundle); err != nil { + log.Error(err, "failed to add file to watcher", "filename", defaultTrustedCABundle) + os.Exit(1) + } + log.Info("watching file", "filename", defaultTrustedCABundle) + orig, err := ioutil.ReadFile(defaultTrustedCABundle) + if err != nil { + log.Error(err, "failed to read watched file", "filename", defaultTrustedCABundle) + os.Exit(1) + } + go func() { + for { + select { + case <-signal: + close(stop) + return + case _, ok := <-watcher.Events: + if !ok { + log.Info("file watch events channel closed") + close(stop) + return + } + latest, err := ioutil.ReadFile(defaultTrustedCABundle) + if err != nil { + log.Error(err, "failed to read watched file", "filename", defaultTrustedCABundle) + close(stop) + return + } + if !bytes.Equal(orig, latest) { + log.Info("watched file changed, stopping operator", "filename", defaultTrustedCABundle) + close(stop) + return + } + case err, ok := <-watcher.Errors: + if !ok { + log.Info("file watch error channel closed") + close(stop) + return + } + log.Error(err, "file watch error") + } + } + }() + // Set up and start the operator. op, err := operator.New(operatorConfig, dnsProvider, kubeConfig) if err != nil { return fmt.Errorf("failed to create operator: %v", err) } - return op.Start(signals.SetupSignalHandler()) + return op.Start(stop) } // createDNSManager creates a DNS manager compatible with the given cluster diff --git a/pkg/operator/config/config.go b/pkg/operator/config/config.go index b40dc1ed25..38aa69e37d 100644 --- a/pkg/operator/config/config.go +++ b/pkg/operator/config/config.go @@ -11,8 +11,4 @@ type Config struct { // IngressControllerImage is the ingress controller image to manage. IngressControllerImage string - - // TrustedCABundle is the fully qualified path of the trusted CA certificate - // bundle that is watched. - TrustedCABundle string } diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 7c744ff7d7..f949114aa6 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -16,7 +16,6 @@ import ( dnscontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/dns" ingresscontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/ingress" statuscontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/status" - "github.com/openshift/cluster-ingress-operator/pkg/operator/watcher" operatorutil "github.com/openshift/cluster-ingress-operator/pkg/util" operatorv1 "github.com/openshift/api/operator/v1" @@ -51,8 +50,6 @@ type Operator struct { manager manager.Manager - caWatcher *watcher.FileWatcher - namespace string } @@ -116,15 +113,8 @@ func New(config operatorconfig.Config, dnsProvider dns.Provider, kubeConfig *res return nil, fmt.Errorf("failed to create dns controller: %v", err) } - // Set up trusted ca bundle watcher - watcher, err := watcher.New(config.TrustedCABundle) - if err != nil { - return nil, fmt.Errorf("failed to create trusted ca watcher: %v", err) - } - return &Operator{ - manager: mgr, - caWatcher: watcher, + manager: mgr, // TODO: These are only needed for the default ingress controller stuff, which // should be refactored away. client: mgr.GetClient(), @@ -133,8 +123,7 @@ func New(config operatorconfig.Config, dnsProvider dns.Provider, kubeConfig *res } // Start creates the default IngressController and then starts the operator -// and trusted ca bundle watcher synchronously until a message is received -// on the stop or reload channels. +// synchronously until a message is received on the stop channel. // TODO: Move the default IngressController logic elsewhere. func (o *Operator) Start(stop <-chan struct{}) error { // Periodicaly ensure the default controller exists. @@ -154,17 +143,10 @@ func (o *Operator) Start(stop <-chan struct{}) error { errChan <- o.manager.Start(stop) }() - reloadChan := make(chan struct{}) - go func() { - errChan <- o.caWatcher.Start(stop, reloadChan) - }() - - // Wait for the manager or watcher to exit or an explicit stop. + // Wait for the manager to exit or an explicit stop. select { case <-stop: return nil - case <-reloadChan: - return nil case err := <-errChan: return err } diff --git a/pkg/operator/watcher/filewatcher.go b/pkg/operator/watcher/filewatcher.go deleted file mode 100644 index 03cf2d64fa..0000000000 --- a/pkg/operator/watcher/filewatcher.go +++ /dev/null @@ -1,111 +0,0 @@ -package watcher - -import ( - "fmt" - "io/ioutil" - "sync" - - "gopkg.in/fsnotify.v1" - - logf "github.com/openshift/cluster-ingress-operator/pkg/log" -) - -var log = logf.Logger.WithName("filewatcher") - -// FileWatcher watches a file for changes. -type FileWatcher struct { - sync.Mutex - fileName string - watcher *fsnotify.Watcher -} - -// New returns a new FileWatcher watching the given file. -func New(file string) (*FileWatcher, error) { - fw := &FileWatcher{ - fileName: file, - } - - // Initial read of file. - if err := fw.ReadFile(); err != nil { - return nil, err - } - - if watcher, err := fsnotify.NewWatcher(); err != nil { - return nil, err - } else { - fw.watcher = watcher - } - - return fw, nil -} - -// Start starts the FileWatcher. -func (fw *FileWatcher) Start(stopCh <-chan struct{}, reloadCh chan struct{}) error { - if err := fw.watcher.Add(fw.fileName); err != nil { - return err - } - - go fw.Watch(reloadCh) - log.Info("starting file watcher") - // Block until the stop channel is closed. - <-stopCh - - return fw.watcher.Close() -} - -// Watch reads events from the watcher's channel and reacts to changes. -func (fw *FileWatcher) Watch(reloadCh chan struct{}) { - for { - select { - case event, ok := <-fw.watcher.Events: - if !ok { - log.Info("file watch events channel closed") - return - } - fw.handleEvent(event, reloadCh) - - case err, ok := <-fw.watcher.Errors: - if !ok { - log.Info("file watch error channel closed") - return - } - log.Error(err, "file watch error") - } - } -} - -// ReadFile reads the watched file from disk, parses the file, -// and updates FileWatcher current data. -func (fw *FileWatcher) ReadFile() error { - data, err := ioutil.ReadFile(fw.fileName) - switch { - case err != nil: - return fmt.Errorf("failed to read file %s: %v", fw.fileName, err) - case len(data) == 0: - return fmt.Errorf("file %s contains no data", fw.fileName) - } - - return nil -} - -// handleEvent filters events and gracefully terminates the operator -// if a write, remove or create event is received for the watched file. -func (fw *FileWatcher) handleEvent(event fsnotify.Event, reloadCh chan struct{}) { - if !(isWrite(event) || isRemove(event) || isCreate(event)) { - return - } - log.Info("watched file changed", "event", event) - reloadCh <- struct{}{} -} - -func isWrite(event fsnotify.Event) bool { - return event.Op&fsnotify.Write == fsnotify.Write -} - -func isCreate(event fsnotify.Event) bool { - return event.Op&fsnotify.Create == fsnotify.Create -} - -func isRemove(event fsnotify.Event) bool { - return event.Op&fsnotify.Remove == fsnotify.Remove -}