diff --git a/cmd/ingress-operator/start.go b/cmd/ingress-operator/start.go index 8524da8bd9..288f9bcf55 100644 --- a/cmd/ingress-operator/start.go +++ b/cmd/ingress-operator/start.go @@ -12,10 +12,10 @@ import ( "github.com/openshift/cluster-ingress-operator/pkg/manifests" "github.com/openshift/cluster-ingress-operator/pkg/operator" operatorconfig "github.com/openshift/cluster-ingress-operator/pkg/operator/config" + canarycontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/canary" statuscontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/status" "sigs.k8s.io/controller-runtime/pkg/client/config" - "sigs.k8s.io/controller-runtime/pkg/metrics" "sigs.k8s.io/controller-runtime/pkg/runtime/signals" ) @@ -77,7 +77,6 @@ func NewStartCommand() *cobra.Command { } func start(opts *StartOptions) error { - metrics.DefaultBindAddress = opts.MetricsListenAddr kubeConfig, err := config.GetConfig() if err != nil { @@ -90,16 +89,20 @@ func start(opts *StartOptions) error { log.Info("Warning: no release version is specified", "release version", statuscontroller.UnknownVersionValue) } + // Set up the channels for the watcher, operator, and metrics. + stop := make(chan struct{}) + signal := signals.SetupSignalHandler() + operatorConfig := operatorconfig.Config{ OperatorReleaseVersion: opts.ReleaseVersion, Namespace: opts.OperatorNamespace, IngressControllerImage: opts.IngressControllerImage, CanaryImage: opts.CanaryImage, + Stop: stop, } - // Set up the channels for the watcher and operator. - stop := make(chan struct{}) - signal := signals.SetupSignalHandler() + // Start operator metrics. + go canarycontroller.StartMetricsListener(opts.MetricsListenAddr, stop) // Set up and start the file watcher. watcher, err := fsnotify.NewWatcher() diff --git a/go.mod b/go.mod index 0b43561e9b..9b79a59c4c 100644 --- a/go.mod +++ b/go.mod @@ -21,8 +21,10 @@ require ( github.com/openshift/api v0.0.0-20201130121019-19e3831bc513 github.com/openshift/library-go v0.0.0-20200423123937-d1360419413d github.com/pkg/errors v0.9.1 + github.com/prometheus/client_golang v1.7.1 github.com/spf13/cobra v1.0.0 github.com/stretchr/testify v1.5.1 + github.com/tcnksm/go-httpstat v0.2.1-0.20191008022543-e866bb274419 go.uber.org/zap v1.15.0 golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect gomodules.xyz/jsonpatch/v2 v2.1.0 // indirect diff --git a/go.sum b/go.sum index 18a4d58440..5c8b1fc041 100644 --- a/go.sum +++ b/go.sum @@ -489,6 +489,8 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/tcnksm/go-httpstat v0.2.1-0.20191008022543-e866bb274419 h1:elOIj31UL4RZWgLfLV4pWZA0j5QqGO95/Dll2WIwOZU= +github.com/tcnksm/go-httpstat v0.2.1-0.20191008022543-e866bb274419/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= diff --git a/manifests/02-deployment.yaml b/manifests/02-deployment.yaml index 5a2df7a89b..fa4b949d81 100644 --- a/manifests/02-deployment.yaml +++ b/manifests/02-deployment.yaml @@ -63,7 +63,7 @@ spec: - name: IMAGE value: openshift/origin-haproxy-router:v4.0 - name: CANARY_IMAGE - value: openshift/hello-openshift:latest + value: quay.io/openshift/origin-hello-openshift:latest resources: requests: cpu: 10m diff --git a/manifests/image-references b/manifests/image-references index c6d3ffb3e5..0e4758bfbe 100644 --- a/manifests/image-references +++ b/manifests/image-references @@ -13,8 +13,8 @@ spec: - name: kube-rbac-proxy from: kind: DockerImage - name: quay.io/openshift/origin-kube-rbac-proxy:latest + name: "quay.io/openshift/origin-kube-rbac-proxy:latest" - name: hello-openshift from: kind: DockerImage - name: "openshift/hello-openshift:latest" + name: "quay.io/openshift/origin-hello-openshift:latest" diff --git a/pkg/manifests/bindata.go b/pkg/manifests/bindata.go index 2ae11d9d02..67dfe8cd9e 100644 --- a/pkg/manifests/bindata.go +++ b/pkg/manifests/bindata.go @@ -30,9 +30,9 @@ // manifests/01-service-account.yaml (405B) // manifests/01-service.yaml (538B) // manifests/01-trusted-ca-configmap.yaml (517B) -// manifests/02-deployment.yaml (3.839kB) +// manifests/02-deployment.yaml (3.854kB) // manifests/03-cluster-operator.yaml (566B) -// manifests/image-references (540B) +// manifests/image-references (557B) package manifests @@ -701,7 +701,7 @@ func manifests01TrustedCaConfigmapYaml() (*asset, error) { return a, nil } -var _manifests02DeploymentYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x57\x4f\x6f\xe3\xb6\x13\xbd\xe7\x53\x0c\xfc\xfb\x1d\x5a\x60\x69\x39\xd9\xed\xb6\x2b\x20\x07\xd7\xf1\x6e\x02\xc4\x49\x10\x07\x1b\xf4\x64\x8c\xa9\xb1\xc5\x9a\x22\xb5\xe4\xd0\x8d\xbe\x7d\x41\xf9\xbf\xac\x38\x29\xd0\xea\x64\x93\xf3\xde\x0c\x67\x86\x8f\x24\x96\xea\x3b\x39\xaf\xac\x49\x01\xcb\xd2\x27\xcb\xf3\xb3\x85\x32\x59\x0a\x57\x54\x6a\x5b\x15\x64\xf8\xac\x20\xc6\x0c\x19\xd3\x33\x00\x83\x05\xa5\xa0\xcc\xdc\x91\xf7\xc2\x96\xe4\x90\xad\x5b\x4f\xf8\x12\x25\xa5\x60\x4b\x32\x3e\x57\x33\x16\x2d\x76\x68\x8c\x65\x64\x65\x8d\x8f\x7c\x00\xd2\x9a\x99\x9a\x77\xb7\xa0\xae\xb2\x89\x32\x7f\x92\x64\x51\x3a\xfb\x52\xb5\x7a\x03\x50\x46\xea\x90\x51\xd7\x91\x26\xf4\xd4\xc0\x4f\x0b\x21\xb5\x0d\x99\x28\xd0\xe0\x9c\xb2\x14\x3a\xec\x02\x75\xde\x86\x7a\xd2\xb3\x0d\x4a\xe4\x6a\x9e\x0b\x5c\xa2\xd2\x38\x55\x5a\x71\xf5\x0f\x78\x94\x99\x6b\x12\xc6\x66\x24\x32\x5a\x92\x8e\xd1\x6f\xe1\xbe\x24\x19\x97\xef\xa8\xd4\x4a\xa2\x4f\xe1\xfc\x0c\xc0\xb3\x43\xa6\x79\xb5\x4a\x0c\x57\x25\xa5\xf0\x48\xd2\x11\x32\xc5\x69\xd2\x24\xd9\xba\xd5\x74\x81\x2c\xf3\x5b\x9c\x92\x5e\x27\xf2\x44\x71\x98\x8a\x52\x23\xd3\x1a\xb9\x57\xcf\xf8\xe9\x03\x92\x13\x34\x00\x9b\xb8\x6b\x33\x9b\xd1\xf8\x20\xa4\xf8\x2d\xc2\x94\x9c\x21\x26\x1f\xb3\x60\x7d\x0a\x5a\x99\xf0\xb2\x23\x8f\x09\x71\x56\x53\xf7\xd0\xb2\x40\xcf\x75\x82\x3a\x6b\x53\xb6\x3a\x3a\xde\x35\x0a\x80\x80\x05\xc5\x12\x9c\xe6\xe8\x6c\x7d\x6d\x42\x4f\xa1\x33\x7c\x51\x9e\xfd\x6e\x8a\x66\x33\x92\x9c\x42\xe7\xce\x8e\x65\x4e\x59\xd0\xd4\x69\xf1\xd2\x70\x10\x8c\x23\x94\x39\x4e\x77\xd6\xef\xf5\x32\x7c\x21\x19\x78\x0f\xb6\x5b\xdf\x98\xa4\x35\x59\xec\x81\x8b\xde\xdb\x31\x18\xcb\xc2\x11\x66\xd5\x7f\x1b\x81\x27\xb7\x54\x92\xfa\x52\xda\x60\xf8\xee\xf5\x96\x00\x28\x9d\xb2\x4e\x71\x35\xd0\xe8\xfd\xca\xd2\x57\x9e\x29\xee\xc1\x10\x2b\x22\xa4\x53\xac\x24\xea\x35\x40\x5a\xc3\xa8\x0c\xb9\xbd\xa6\x13\xa7\xda\x6e\x1d\x2f\xb9\x42\x99\x3a\xe0\x11\x79\x8f\x73\x7a\xb0\x5a\xc9\x2a\x85\xaf\xa8\xf5\x14\xe5\xe2\xc9\xde\xda\xb9\xbf\x37\x43\xe7\x0e\x90\xaa\x88\xc6\x41\xeb\x0d\xe0\x66\x76\x67\xf9\xc1\x91\x8f\xfa\xd6\xb0\xdb\x13\xb0\xc4\x3a\x35\x57\x66\xbb\x8e\x66\x70\x69\xdc\x54\x7e\x9f\x41\xda\xa2\x40\x93\xa5\x7b\x43\xe2\xd4\x9a\x04\x78\x46\xc7\x07\x23\x42\x6c\xc5\xf4\x60\xbc\xf3\xff\x9f\x9e\xfb\x4f\x83\xeb\xc9\x5d\x7f\x34\x1c\x3f\xf4\x07\xc3\x9f\x3b\x0d\x60\xbd\x80\x26\xe8\x66\xd4\xff\x76\x6c\x2a\xd1\xa0\xab\xda\x11\x83\xfe\x5d\xff\xf1\x8f\x49\x3b\x70\x2d\x77\x62\xb9\x3a\x38\x9a\xd8\xc7\xe1\xed\xb0\x3f\x1e\x4e\xbe\x0f\x1f\xc7\x37\xf7\x77\x07\x70\x32\xcb\xfd\xcc\xec\xaa\xde\x00\x1d\xd8\x00\x2c\x51\x07\x4a\xa1\xd3\xeb\xf6\xba\xe7\xc2\x1b\x2c\x7d\x6e\xb9\xd3\xca\xd4\xc8\x50\x1b\xd3\x57\x67\x8b\xb4\x31\x01\x30\x53\xa4\xb3\x47\x9a\x1d\xcf\xac\xe7\x1e\x90\xf3\x74\x2b\x9e\xdd\xb6\x1a\xed\xc2\xa8\x53\xd7\xbe\x8c\xa3\xee\xca\xb1\x3e\xe8\x84\xb3\x21\x4a\xe0\xf2\x53\xb7\xd7\xca\xb9\x5f\x95\xb7\xa8\x73\xd2\xda\x8a\xed\xff\xe3\x46\x75\xe4\x6d\x70\x92\xfc\xe1\x7a\x1d\xfd\x08\xe4\xd9\x37\xb3\x20\xcb\x90\xc2\x79\xaf\xd8\x1b\x5e\x5a\x1d\x0a\x1a\x45\x79\xf0\x87\xfd\xbe\x0a\x97\x5d\xdc\x34\x99\x90\x78\xc0\x55\x44\xc0\x2a\x97\x09\xb1\x4c\xca\x85\x4a\x24\x8a\xda\x3a\xa1\x17\x76\x28\x99\xb2\xa4\xa4\xa2\x11\x18\x66\xf7\x46\x57\x35\x2f\xb5\xb8\x9b\xda\x60\x32\xe1\x51\xb0\x5d\x90\x79\xd5\xe5\x12\x5d\xe2\x82\x49\x7c\x3c\x57\xd9\x27\xbb\x94\xad\x15\x0f\x57\x8a\xf7\x1e\xe7\x1b\xd7\x51\xa0\x85\x9b\xa2\x5c\xdd\x58\x8e\x05\xe5\x47\xc0\xaa\x3e\x0e\x9b\xa5\x6f\x20\x8f\xcb\x84\x6e\xde\x48\xae\x10\xda\xce\xd9\x7a\xce\xc8\xb9\xc6\x8c\x27\x19\x1c\x09\xad\x3c\x93\x11\x98\x65\x51\x76\x2e\xd3\x2f\x1f\xbf\x7c\x6c\x58\xb2\xf6\x42\xaa\x32\x27\x27\x7c\x50\x4c\xfe\xf2\xe9\x76\x3c\x19\x0e\xae\xae\x87\x93\xc7\x71\x7f\xf2\x7c\xf3\x74\x3d\xe9\x0f\xc7\x93\xf3\x8b\xdf\x26\xdf\x06\xa3\xc9\xf8\xba\x7f\xf1\xcb\xe7\x0f\x3b\xab\xe1\xe0\xea\x0d\xbb\x23\x9e\xc1\xef\x83\x77\xf1\xb4\xda\x9d\x60\x6b\xac\x2d\x94\x9e\x1d\x61\x71\x99\x33\x97\x69\x92\x9c\x5f\xfc\xda\xad\xa5\x23\xfd\xdc\xeb\xf5\x7a\x49\x5b\x2a\xc8\xb1\x98\x29\x4d\x97\x75\x4f\xb2\xf6\x49\xe9\xd4\x12\x99\xe2\xef\xae\x3c\x52\xe8\x08\x5a\x5b\x88\x05\x55\x27\xb0\x0b\xda\x6f\x88\xd2\xba\xe6\x6e\xd9\x9e\x86\x0f\xd6\x71\x0a\x8d\x62\x6d\xae\x63\x05\xb1\x53\xd2\xff\xab\x3b\x18\xea\xcb\x60\x61\x5d\x95\xc2\xa7\xde\x48\xbd\x6b\x73\x37\xf7\xef\xde\x7a\x5f\x0f\x3b\xe6\xeb\xed\x1d\xb5\x72\xba\x77\xdd\x7b\x9d\x63\xb5\x7d\xf7\x23\x5b\x8d\xdc\xbd\x82\x38\x21\x4a\xab\x07\xc8\x08\xcb\x7d\xb6\x13\x12\xa6\x98\x0a\xdf\x3c\xc6\xea\xeb\x9a\x44\x31\x0d\x26\xd3\xd4\x68\x98\xf8\x95\x75\xc6\xea\x5e\xdb\x5a\xed\x34\xee\x7f\xf0\x94\x2b\xbf\xb9\x74\xc1\x5a\x83\xa0\x96\x32\x90\x68\x60\x4a\x10\x3c\x65\xc0\x16\x4a\x67\x97\x2a\x23\x50\x19\x19\x56\x5c\x81\x0d\xec\xe3\x00\xe7\x04\xeb\x9b\x4a\x77\xcb\xfb\xd5\x3a\xa0\x17\x2c\x4a\x4d\x1f\x80\xa3\x93\x63\xd2\xbf\x14\xe7\xd0\xf7\x3e\x14\xf4\x68\x35\x3d\x2b\xce\x9f\x69\x7a\xb3\xe1\x67\x0b\x18\x38\x8f\xff\x24\x32\xad\xcd\x9f\xc7\x10\xe2\x23\x07\x6e\xfa\x23\xb8\xbf\xb9\x1a\x6c\x02\x73\x80\x26\x83\xf1\xd3\xb8\xdb\xc8\xfd\x2b\x0a\x5d\x3a\x1b\x5f\x7b\x74\x70\x69\x6a\x69\x6d\xd1\xb8\x93\x3e\x45\x96\x66\x97\xaf\xd3\x7c\x74\x04\x00\x60\xc8\x14\x99\x83\xf7\xe9\xd9\xdf\x01\x00\x00\xff\xff\x76\x21\xd3\xcd\xff\x0e\x00\x00") +var _manifests02DeploymentYaml = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xb4\x57\x4f\x6f\xe3\xb6\x13\xbd\xe7\x53\x0c\xfc\xfb\x1d\x5a\x60\x69\x3b\xd9\xed\xb6\x2b\x20\x07\xd7\xf1\x6e\x02\xc4\x49\x10\x07\x1b\xf4\x64\x8c\xa9\xb1\xc5\x9a\x22\xb5\xe4\xd0\x8d\xbe\x7d\x41\xf9\x9f\x2c\x2b\x4e\x0a\xb4\x3e\x25\xe4\xbc\x37\xc3\x99\xe1\xe3\x08\x0b\xf5\x9d\x9c\x57\xd6\x24\x80\x45\xe1\x7b\xab\xf3\xb3\xa5\x32\x69\x02\x57\x54\x68\x5b\xe6\x64\xf8\x2c\x27\xc6\x14\x19\x93\x33\x00\x83\x39\x25\xa0\xcc\xc2\x91\xf7\xc2\x16\xe4\x90\xad\xdb\x6c\xf8\x02\x25\x25\x60\x0b\x32\x3e\x53\x73\x16\x2d\x76\x68\x8c\x65\x64\x65\x8d\x8f\x7c\x00\xd2\x9a\xb9\x5a\x74\x77\xa0\xae\xb2\x3d\x65\xfe\x24\xc9\xa2\x70\xf6\xa5\x6c\xf5\x06\xa0\x8c\xd4\x21\xa5\xae\x23\x4d\xe8\xa9\x81\x9f\xe5\x42\x6a\x1b\x52\x91\xa3\xc1\x05\xa5\x09\x74\xd8\x05\xea\xbc\x0d\xf5\xa4\xe7\x5b\x94\xc8\xd4\x22\x13\xb8\x42\xa5\x71\xa6\xb4\xe2\xf2\x1f\xf0\x28\xb3\xd0\x24\x8c\x4d\x49\xa4\xb4\x22\x1d\xa3\xdf\xc1\x7d\x41\x32\x1e\xdf\x51\xa1\x95\x44\x9f\xc0\xf9\x19\x80\x67\x87\x4c\x8b\x72\x9d\x18\x2e\x0b\x4a\xe0\x91\xa4\x23\x64\x8a\xdb\xa4\x49\xb2\x75\xeb\xed\x1c\x59\x66\xb7\x38\x23\xbd\x49\xe4\x89\xe2\x30\xe5\x85\x46\xa6\x0d\xb2\x56\xcf\xf8\xd3\x07\x24\x27\x68\x00\xb6\x71\x57\x66\x36\xa5\xc9\x41\x48\xf1\xb7\x0c\x33\x72\x86\x98\x7c\xcc\x82\xf5\x09\x68\x65\xc2\xcb\x9e\x3c\x26\xc4\x59\x4d\xdd\x43\xcb\x1c\x3d\x57\x09\xea\x6c\x4c\xd9\xea\xe8\x78\xdf\x28\x00\x02\x96\x14\x4b\x70\x9a\xa3\xb3\xf3\xb5\x0d\x3d\x81\xce\xe8\x45\x79\xf6\xfb\x2d\x9a\xcf\x49\x72\x02\x9d\x3b\x3b\x91\x19\xa5\x41\x53\xa7\xc5\x4b\xc3\x41\x30\x8e\x50\x66\x38\xdb\x5b\xbf\xd7\xcb\xe8\x85\x64\xe0\x1a\x6c\x7f\xbe\x09\x49\x6b\xd2\xd8\x03\x17\xfd\xb7\x63\x30\x96\x85\x23\x4c\xcb\xff\x36\x02\x4f\x6e\xa5\x24\x0d\xa4\xb4\xc1\xf0\xdd\xeb\x2d\x01\x50\x38\x65\x9d\xe2\x72\xa8\xd1\xfb\xb5\xa5\x2f\x3d\x53\xbc\x83\x21\x56\x44\x48\xa7\x58\x49\xd4\x1b\x80\xb4\x86\x51\x19\x72\xb5\xa6\x13\xa7\xda\x6e\x13\x2f\xb9\x5c\x99\x2a\xe0\x31\x79\x8f\x0b\x7a\xb0\x5a\xc9\x32\x81\xaf\xa8\xf5\x0c\xe5\xf2\xc9\xde\xda\x85\xbf\x37\x23\xe7\x0e\x90\x2a\x8f\xc6\x41\xeb\x2d\xe0\x66\x7e\x67\xf9\xc1\x91\x8f\xfa\xd6\xb0\xab\x09\x58\xcf\x3a\xb5\x50\x66\x77\x8e\x66\x70\x49\xbc\x54\xbe\xce\x20\x6d\x9e\xa3\x49\x93\xda\x92\x38\x75\x26\x01\x9e\xd1\xf1\xc1\x8a\x10\x3b\x31\x3d\x58\xef\xfc\xff\xa7\xe7\xc1\xd3\xf0\x7a\x7a\x37\x18\x8f\x26\x0f\x83\xe1\xe8\xe7\x4e\x03\x58\x1d\xa0\x09\xba\x19\x0f\xbe\x1d\x9b\x4a\x34\xe8\xca\x76\xc4\x70\x70\x37\x78\xfc\x63\xda\x0e\xdc\xc8\x9d\x58\xad\x1f\x8e\x26\xf6\x71\x74\x3b\x1a\x4c\x46\xd3\xef\xa3\xc7\xc9\xcd\xfd\xdd\x01\x9c\xcc\xaa\x9e\x99\x7d\xd5\x1b\xa0\x03\x1b\x80\x15\xea\x40\x09\x74\xfa\xdd\x7e\xf7\x5c\x78\x83\x85\xcf\x2c\x77\x5a\x99\x1a\x19\x6a\x63\xfa\xea\x6c\x9e\x34\x36\x00\xe6\x8a\x74\xfa\x48\xf3\xe3\x9d\xcd\xde\x03\x72\x96\xec\xc4\xb3\xdb\x56\xa3\x7d\x18\x55\xea\xda\x8f\x71\xd4\x5d\x19\x56\x0f\x9d\x70\x36\x44\x09\x5c\x7d\xea\xf6\x5b\x39\xeb\x55\x69\xa7\xfe\x11\xb0\xac\x64\xf7\xc8\x05\x69\x6d\xc5\x6e\xf9\xb8\x6f\x1d\x79\x1b\x9c\x24\x7f\x78\x7c\x47\x3f\x02\x79\xf6\xcd\xa4\xc8\x22\x24\x70\xde\xcf\x6b\xcb\x2b\xab\x43\x4e\xe3\xa8\x16\xfe\xb0\xfd\xd7\xd1\xb3\x8b\x77\x28\x15\x12\x0f\xb8\xf2\x08\x58\xa7\xb6\x47\x2c\x7b\xc5\x52\xf5\x24\x8a\xca\xba\x47\x2f\xec\x50\x32\xa5\xbd\x82\xf2\x46\x60\x98\xde\x1b\x5d\x56\xbc\xd4\xe2\x6e\x66\x83\x49\x85\x47\xc1\x76\x49\xe6\x55\x97\x2b\x74\x3d\x17\x4c\xcf\xc7\x67\x96\x7d\x2d\x73\x1b\x01\xc4\xb5\x00\xbe\xc7\xf9\xd6\x75\xd4\x6b\xe1\x66\x28\xd7\x03\xcc\xb1\xbe\xbc\x5a\xa6\x06\xf2\xb8\x4c\xe8\x16\x8d\xe4\x0a\xa1\xed\x82\xad\xe7\x94\x9c\x6b\xec\x78\x92\xc1\x91\xd0\xca\x33\x19\x81\x69\x1a\x55\xe8\x32\xf9\xf2\xf1\xcb\xc7\x86\x25\x6b\x2f\xa4\x2a\x32\x72\xc2\x07\xc5\xe4\x2f\x9f\x6e\x27\xd3\xd1\xf0\xea\x7a\x34\x7d\x9c\x0c\xa6\xcf\x37\x4f\xd7\xd3\xc1\x68\x32\x3d\xbf\xf8\x6d\xfa\x6d\x38\x9e\x4e\xae\x07\x17\xbf\x7c\xfe\xb0\xb7\x1a\x0d\xaf\xde\xb0\x3b\xe2\x19\xfe\x3e\x7c\x17\x4f\xab\xdd\x09\xb6\xc6\xd9\x42\xe1\xd9\x11\xe6\x97\x19\x73\x91\xf4\x7a\xe7\x17\xbf\x76\x2b\x25\x49\x3e\xf7\xfb\xfd\x7e\xaf\x2d\x15\xe4\x58\xcc\x95\xa6\xcb\xaa\x27\x59\xfb\x5e\xe1\xd4\x0a\x99\xe2\xdf\x5d\x79\x24\xd8\x11\xb4\xb1\x10\x4b\x2a\x4f\x60\x97\x54\x6f\x88\xc2\xba\xe6\x6d\xd9\x3d\x8e\x0f\xd6\x71\x02\x8d\x62\x6d\xa7\xb3\x9c\xd8\x29\xe9\xff\xd5\x1b\x0c\xd5\x6c\x98\x5b\x57\x26\xf0\xa9\x3f\x56\xef\xba\xdc\xcd\xfb\x5b\x3b\xef\xeb\x61\xc7\x7c\xbd\x7d\xa3\xd6\x4e\x6b\xd3\xdf\xeb\x1c\xeb\xeb\x5b\x8f\x6c\xbd\x72\xf7\x0a\xe2\x84\x28\xad\xbf\x47\xc6\x58\xd4\xd9\x4e\x48\x98\x62\xca\x7d\xf3\x55\xab\xa6\x37\x89\x62\x16\x4c\xaa\xa9\xd1\x30\xf1\x57\x54\x19\xab\x7a\x6d\x67\xb5\xd7\xb8\xff\xc1\x53\xa6\xfc\x76\x06\x83\x8d\x06\x41\x25\x65\x20\xd1\xc0\x8c\x20\x78\x4a\x81\x2d\x14\xce\xae\x54\x4a\xa0\x52\x32\xac\xb8\x04\x1b\xd8\xc7\x05\xce\x08\x36\x83\x4b\x77\xc7\xfb\xd5\x3a\xa0\x17\xcc\x0b\x4d\x1f\x80\xa3\x93\x63\xd2\xbf\x14\x67\x30\xf0\x3e\xe4\xf4\x68\x35\x3d\x2b\xce\x9e\x69\x76\xb3\xe5\x67\x0b\x18\x38\x8b\xff\x49\x64\xda\x98\x3f\x4f\x20\xc4\x6f\x1e\xb8\x19\x8c\xe1\xfe\xe6\x6a\xb8\x0d\xcc\x01\x9a\x14\x26\x4f\x93\x6e\x23\xf7\xaf\x28\x74\xe1\x6c\xfc\xf8\xa3\x83\x19\xaa\xa5\xb5\x45\x63\x44\x7d\x8a\x2c\xcd\x2e\xdf\xa4\xf9\xe8\x09\x00\xc0\x90\x2a\x32\x07\x9f\xab\x67\x7f\x07\x00\x00\xff\xff\x8f\xdd\x4f\x8f\x0e\x0f\x00\x00") func manifests02DeploymentYamlBytes() ([]byte, error) { return bindataRead( @@ -716,8 +716,8 @@ func manifests02DeploymentYaml() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "manifests/02-deployment.yaml", size: 3839, mode: os.FileMode(420), modTime: time.Unix(1, 0)} - a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x4e, 0xa2, 0xa2, 0xe8, 0x31, 0x11, 0xf8, 0x9a, 0xba, 0xd5, 0x7d, 0xc2, 0xee, 0x17, 0x2d, 0x2f, 0xab, 0xf2, 0x7d, 0xaf, 0x92, 0xfb, 0x1a, 0x3d, 0x79, 0x93, 0x8e, 0xd8, 0xca, 0xa7, 0x77, 0x23}} + info := bindataFileInfo{name: "manifests/02-deployment.yaml", size: 3854, mode: os.FileMode(420), modTime: time.Unix(1, 0)} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x96, 0x63, 0x59, 0xbc, 0x4a, 0x9f, 0x56, 0xe, 0x4a, 0x51, 0x4f, 0x64, 0x96, 0x4a, 0x94, 0x4, 0x27, 0x19, 0xa3, 0x4c, 0x57, 0xa0, 0x76, 0xd0, 0xa, 0x5, 0x67, 0x84, 0xee, 0x9a, 0x9e, 0x71}} return a, nil } @@ -741,7 +741,7 @@ func manifests03ClusterOperatorYaml() (*asset, error) { return a, nil } -var _manifestsImageReferences = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x9c\xd0\xbf\x4e\xc3\x40\x0c\x06\xf0\x3d\x4f\x61\x65\xbf\x14\x24\xa6\x9b\x59\x98\x91\xd8\xdd\xe0\xa6\x56\x92\xf3\x61\xfb\x2a\xfa\xf6\x28\x69\x15\x5d\x8b\x3a\xc0\x94\xe8\xfe\x7c\xbf\xfb\x3c\x72\xfa\x8c\xf0\x36\xe3\x40\xef\xae\x84\x73\x83\x99\x3f\x48\x8d\x25\x45\xe0\x65\xbd\x93\x4c\xc9\x8e\x7c\xf0\x8e\x65\x77\x7a\x6e\x2c\x53\x1f\x1b\x00\xc7\xc1\x96\x6f\x80\x84\x33\x45\xe8\xa7\x62\x4e\x1a\x38\x0d\x4a\x66\x41\x32\x29\xba\x68\x03\x00\x70\x50\x99\x23\xac\xbf\x00\x17\xb5\x7d\x95\x7e\x24\x5d\xf1\xf6\xba\x73\x49\x6a\x37\x72\x27\xca\x03\xa7\xf0\x28\x3b\x4e\xe8\x64\xde\x56\xcf\x38\x62\x56\xf9\x3e\x07\x95\xe2\x54\xe1\xff\xb6\x6f\x03\xe3\xe9\xa5\x7b\xaa\xbd\xb1\xec\x29\xe8\x1e\xfb\xb0\x1e\x7b\x00\x56\xde\x0d\xf7\x55\xf0\xbc\xcc\xf5\x97\x7a\x17\x7b\x2d\x5a\xf7\xa4\x69\x92\xb0\xdd\xfb\xa3\x5b\xd5\xbc\x0b\xda\x46\xfa\x13\x00\x00\xff\xff\x09\x34\x44\xf5\x1c\x02\x00\x00") +var _manifestsImageReferences = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\xd0\xbd\x4e\xc3\x40\x0c\xc0\xf1\x3d\x4f\x61\x65\x77\x0a\x12\xd3\xcd\x2c\xcc\x48\xec\x6e\x70\x53\x2b\xc9\xf9\xb0\x9d\x8a\xbe\x3d\x4a\x8a\xa2\xb4\xd0\x05\x31\x25\xba\x8f\xff\xef\xee\x7a\xc9\xef\x09\x5e\x46\xea\xf8\x35\x8c\x69\xac\xa8\xc8\x1b\x9b\x8b\xe6\x04\x32\x8f\x37\x5a\x38\xfb\x51\x0e\xd1\x88\xee\x4e\x8f\x95\x17\x6e\x53\x05\x10\xd4\xf9\xfc\x45\xc8\x34\x72\x82\x76\x98\x3c\xd8\x50\x72\x67\xec\x8e\x5a\xd8\x28\xd4\x2a\x00\x80\x83\xe9\x98\x60\xf9\x05\xb8\xa8\xf5\xb3\xb6\x3d\xdb\x82\xd7\xdf\x33\x97\x52\xbd\x92\x3b\x35\xe9\x24\xe3\xbd\x76\x1a\x28\xd8\xa3\xde\x1c\xe3\x48\xc5\xf4\xf3\x8c\xa6\x53\xf0\x06\xff\xb3\x7d\x1d\x4c\xa7\xa7\xe6\x61\xeb\xf5\xd3\x9e\xd1\xf6\xd4\xe2\xb2\xec\x0e\xb8\xf1\xae\xb9\x8f\x89\xce\xf3\xc3\xfe\x60\x6f\xba\xbf\xdd\x94\x87\x41\x71\xdd\xf8\x6f\xf2\x4d\x77\x95\xbf\x02\x00\x00\xff\xff\x14\xe5\x02\x83\x2d\x02\x00\x00") func manifestsImageReferencesBytes() ([]byte, error) { return bindataRead( @@ -756,8 +756,8 @@ func manifestsImageReferences() (*asset, error) { return nil, err } - info := bindataFileInfo{name: "manifests/image-references", size: 540, mode: os.FileMode(420), modTime: time.Unix(1, 0)} - a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x36, 0xea, 0xd2, 0x29, 0x4d, 0x25, 0xdf, 0xe7, 0x17, 0x2, 0xf8, 0xe8, 0x7f, 0xbb, 0x8f, 0xfc, 0x1b, 0x7c, 0x44, 0xf6, 0xbc, 0xf3, 0xdd, 0x81, 0xe8, 0x11, 0x41, 0x59, 0x49, 0xfd, 0x8e, 0x2b}} + info := bindataFileInfo{name: "manifests/image-references", size: 557, mode: os.FileMode(420), modTime: time.Unix(1, 0)} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x2c, 0x1a, 0x1b, 0x9f, 0x7e, 0x71, 0xc2, 0x5b, 0xf, 0x67, 0x32, 0x59, 0x96, 0x77, 0x92, 0x8f, 0x79, 0x7, 0x78, 0xfa, 0x3d, 0xf5, 0xa3, 0x84, 0x97, 0x1d, 0x76, 0x2e, 0x88, 0x97, 0x87, 0xe5}} return a, nil } diff --git a/pkg/operator/config/config.go b/pkg/operator/config/config.go index e482f8f899..1724f8ac5f 100644 --- a/pkg/operator/config/config.go +++ b/pkg/operator/config/config.go @@ -14,4 +14,6 @@ type Config struct { // CanaryImage is the ingress canary image to manage. CanaryImage string + + Stop chan struct{} } diff --git a/pkg/operator/controller/canary/controller.go b/pkg/operator/controller/canary/controller.go index 1ef19f5923..cd20508ee5 100644 --- a/pkg/operator/controller/canary/controller.go +++ b/pkg/operator/controller/canary/controller.go @@ -1,23 +1,32 @@ package canary import ( + "context" "fmt" + "sync" + "time" logf "github.com/openshift/cluster-ingress-operator/pkg/log" "github.com/openshift/cluster-ingress-operator/pkg/manifests" "github.com/google/go-cmp/cmp" + operatorcontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller" + ingresscontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/ingress" + operatorv1 "github.com/openshift/api/operator/v1" routev1 "github.com/openshift/api/route/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/predicate" @@ -27,10 +36,19 @@ import ( const ( canaryControllerName = "canary_controller" + // canaryCheckFrequency is how long to wait in between canary checks. + canaryCheckFrequency = 1 * time.Minute + // canaryCheckCycleCount is how many successful canary checks should be observed + // before rotating the canary endpoint. + canaryCheckCycleCount = 5 + // canaryCheckFailureCount is how many successive failing canary checks should + // be observed before the default ingress controller goes degraded. + canaryCheckFailureCount = 5 ) var ( - log = logf.Logger.WithName(canaryControllerName) + log = logf.Logger.WithName(canaryControllerName) + routeProbeRunner sync.Once ) // New creates the canary controller. @@ -47,7 +65,7 @@ func New(mgr manager.Manager, config Config) (controller.Controller, error) { return nil, err } - // Only trigger a reconcile request for the canary controller via events for the default ingress controller. + // trigger reconcile requests for the canary controller via events for the default ingress controller. defaultIcPredicate := predicate.NewPredicateFuncs(func(meta metav1.Object, object runtime.Object) bool { return meta.GetName() == manifests.DefaultIngressControllerName }) @@ -56,9 +74,50 @@ func New(mgr manager.Manager, config Config) (controller.Controller, error) { return nil, err } + // trigger reconcile requests for the canary controller via events for the canary route. + canaryRoutePredicate := predicate.NewPredicateFuncs(func(meta metav1.Object, object runtime.Object) bool { + return meta.GetName() == operatorcontroller.CanaryRouteName().Name + }) + + // filter out canary route updates where the canary controller changes the canary route's Spec.Port, + // so that the controller isn't immediately reverting its own changes. + updateFilter := predicate.Funcs{ + UpdateFunc: func(e event.UpdateEvent) bool { + oldRoute, ok := e.ObjectOld.(*routev1.Route) + if !ok { + return false + } + newRoute, ok := e.ObjectNew.(*routev1.Route) + if !ok { + return false + } + // if Spec.Port has changed, do not trigger a reconcile + return cmp.Equal(oldRoute.Spec.Port, newRoute.Spec.Port) + }, + } + + if err := c.Watch(&source.Kind{Type: &routev1.Route{}}, enqueueRequestForDefaultIngressController(config.Namespace), canaryRoutePredicate, updateFilter); err != nil { + return nil, err + } + return c, nil } +func enqueueRequestForDefaultIngressController(namespace string) handler.EventHandler { + return &handler.EnqueueRequestsFromMapFunc{ + ToRequests: handler.ToRequestsFunc(func(a handler.MapObject) []reconcile.Request { + return []reconcile.Request{ + { + NamespacedName: types.NamespacedName{ + Namespace: namespace, + Name: manifests.DefaultIngressControllerName, + }, + }, + } + }), + } +} + // Reconcile ensures that the canary controller's resources // are in the desired state. func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, error) { @@ -94,19 +153,35 @@ func (r *reconciler) Reconcile(request reconcile.Request) (reconcile.Result, err errors = append(errors, fmt.Errorf("failed to get canary service: %v", err)) } - if haveRoute, _, err := r.ensureCanaryRoute(service); err != nil { + haveRoute, route, err := r.ensureCanaryRoute(service) + if err != nil { errors = append(errors, fmt.Errorf("failed to ensure canary route: %v", err)) } else if !haveRoute { errors = append(errors, fmt.Errorf("failed to get canary route: %v", err)) } - return result, utilerrors.NewAggregate(errors) + // If errors have been encountered during a reconciliation, + // return before starting canary probes. + if len(errors) > 0 { + return result, utilerrors.NewAggregate(errors) + } + + // Start probing the canary route once the canary route + // has been admitted. + if checkRouteAdmitted(route) { + routeProbeRunner.Do(func() { + r.startCanaryRoutePolling(r.Config.Stop) + }) + } + + return result, nil } // Config holds all the things necessary for the controller to run. type Config struct { Namespace string CanaryImage string + Stop chan struct{} } // reconciler handles the actual canary reconciliation logic in response to @@ -117,9 +192,118 @@ type reconciler struct { client client.Client } -// TODO: Canary Controller Phase 2 -// Add callers for these 2 functions -// +func (r *reconciler) startCanaryRoutePolling(stop <-chan struct{}) error { + // Keep track of how many canary checks have passed + // so the route endpoint can be periodically cycled. + checkCount := 0 + + // Keep track of successive canary check failures + // for status reporting. + successiveFail := 0 + + go wait.Until(func() { + // Get the current canary route every iteration in case it has been modified + haveRoute, route, err := r.currentCanaryRoute() + if err != nil { + log.Error(err, "failed to get current canary route for canary check") + return + } else if !haveRoute { + log.Info("canary check route does not exist") + return + } + // Periodically rotate the canary route endpoint. + if checkCount > canaryCheckCycleCount { + haveService, service, err := r.currentCanaryService() + if err != nil { + log.Error(err, "failed to get canary service") + return + } else if !haveService { + log.Info("canary check service does not exist") + return + } + route, err = r.rotateRouteEndpoint(service, route) + if err != nil { + log.Error(err, "failed to rotate canary route endpoint") + return + } + checkCount = 0 + // Give the router time to reload by returning here. + return + } + + err = probeRouteEndpoint(route) + if err != nil { + log.Error(err, "error performing canary route check") + SetCanaryRouteReachableMetric(route.Spec.Host, false) + successiveFail += 1 + // Mark the default ingress controller degraded after 5 successive canary check failures + if successiveFail >= canaryCheckFailureCount { + if err := r.setCanaryFailingStatusCondition(); err != nil { + log.Error(err, "error updating canary status condition") + } + } + return + } + + SetCanaryRouteReachableMetric(route.Spec.Host, true) + if err := r.setCanaryPassingStatusCondition(); err != nil { + log.Error(err, "error updating canary status condition") + } + successiveFail = 0 + checkCount++ + }, canaryCheckFrequency, stop) + + return nil +} + +func (r *reconciler) setCanaryFailingStatusCondition() error { + cond := operatorv1.OperatorCondition{ + Type: ingresscontroller.IngressControllerCanaryCheckSuccessConditionType, + Status: operatorv1.ConditionFalse, + Reason: "CanaryChecksRepetitiveFailures", + Message: "Canary route checks for the default ingress controller are failing", + } + + return r.setCanaryStatusCondition(cond) +} + +func (r *reconciler) setCanaryPassingStatusCondition() error { + cond := operatorv1.OperatorCondition{ + Type: ingresscontroller.IngressControllerCanaryCheckSuccessConditionType, + Status: operatorv1.ConditionTrue, + Reason: "CanaryChecksSucceeding", + Message: "Canary route checks for the default ingress controller are successful", + } + + return r.setCanaryStatusCondition(cond) +} + +// setCanaryStatusCondition applies the given condition to the default ingress controller. +// The assumption here is that cond is a condition that does not overlap with any of the status +// conditions set by the ingress controller in pkg/operator/controller/ingress/status.go. +func (r *reconciler) setCanaryStatusCondition(cond operatorv1.OperatorCondition) error { + ic := &operatorv1.IngressController{ + ObjectMeta: metav1.ObjectMeta{ + Name: manifests.DefaultIngressControllerName, + Namespace: r.Config.Namespace, + }, + } + if err := r.client.Get(context.TODO(), types.NamespacedName{Namespace: ic.Namespace, Name: ic.Name}, ic); err != nil { + return fmt.Errorf("failed to get ingress controller %s: %v", ic.Name, err) + } + + updated := ic.DeepCopy() + updated.Status.Conditions = ingresscontroller.MergeConditions(updated.Status.Conditions, cond) + + if !ingresscontroller.IngressStatusesEqual(updated.Status, ic.Status) { + if err := r.client.Status().Update(context.TODO(), updated); err != nil { + return fmt.Errorf("failed to update ingresscontroller %s status: %v", ic.Name, err) + } + } + + return nil +} + // Switch the current RoutePort that the route points to. // Use this function to periodically update the canary route endpoint // to verify if the router has wedged. @@ -129,9 +313,10 @@ func (r *reconciler) rotateRouteEndpoint(service *corev1.Service, current *route return nil, fmt.Errorf("failed to rotate route port: %v", err) } - _, err = r.updateCanaryRoute(current, updated) - if err != nil { + if changed, err := r.updateCanaryRoute(current, updated); err != nil { return current, err + } else if !changed { + return current, fmt.Errorf("expected canary route to be updated: No relevant changes detected") } return updated, nil @@ -157,7 +342,7 @@ func cycleServicePort(service *corev1.Service, route *routev1.Route) (*routev1.R updated := route.DeepCopy() currentIndex := 0 - // Find the current port index in the service ports slice + // Find the current port index in the service ports slice. for i, port := range servicePorts { if cmp.Equal(port.TargetPort, currentPort.TargetPort) { currentIndex = i diff --git a/pkg/operator/controller/canary/controller_test.go b/pkg/operator/controller/canary/controller_test.go index ed3ff7ae7b..1d229ab6b1 100644 --- a/pkg/operator/controller/canary/controller_test.go +++ b/pkg/operator/controller/canary/controller_test.go @@ -42,6 +42,18 @@ func TestCycleServicePort(t *testing.T) { }, success: false, }, + { + description: "route with no ports", + route: &routev1.Route{ + Spec: routev1.RouteSpec{}, + }, + service: &corev1.Service{ + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{}, + }, + }, + success: false, + }, { description: "service has one port", route: &routev1.Route{ diff --git a/pkg/operator/controller/canary/http.go b/pkg/operator/controller/canary/http.go new file mode 100644 index 0000000000..55ef6cfd31 --- /dev/null +++ b/pkg/operator/controller/canary/http.go @@ -0,0 +1,117 @@ +package canary + +import ( + "errors" + "fmt" + "io/ioutil" + "net" + "net/http" + "os" + "strings" + "time" + + routev1 "github.com/openshift/api/route/v1" + + "github.com/tcnksm/go-httpstat" +) + +const ( + echoServerPortAckHeader = "x-request-port" +) + +// probeRouteEndpoint probes the given route's host +// and returns an error when applicable. +func probeRouteEndpoint(route *routev1.Route) error { + if len(route.Spec.Host) == 0 { + return fmt.Errorf("route.Spec.Host is empty, cannot test route") + } + + // Create HTTP request + request, err := http.NewRequest("GET", "http://"+route.Spec.Host, nil) + if err != nil { + return fmt.Errorf("error creating canary HTTP request %v: %v", request, err) + } + + // Create HTTP result + // for request stats tracking. + result := &httpstat.Result{} + + // Get request context + ctx := httpstat.WithHTTPStat(request.Context(), result) + request = request.WithContext(ctx) + + // Send the HTTP request + timeout, _ := time.ParseDuration("10s") + client := &http.Client{Timeout: timeout} + response, err := client.Do(request) + + if err != nil { + // Check if err is a DNS error + dnsErr := &net.DNSError{} + if errors.As(err, &dnsErr) { + // Handle DNS error + CanaryRouteDNSError.WithLabelValues(route.Spec.Host, dnsErr.Server).Inc() + return fmt.Errorf("error sending canary HTTP request: DNS error: %v", err) + } + // Check if err is a timeout error + if os.IsTimeout(err) { + // Handle timeout error + return fmt.Errorf("error sending canary HTTP Request: Timeout: %v", err) + } + return fmt.Errorf("error sending canary HTTP request to %q: %v", route.Spec.Host, err) + } + + // Close response body even if read fails + defer response.Body.Close() + + // Read response body + bodyBytes, err := ioutil.ReadAll(response.Body) + if err != nil { + return fmt.Errorf("error reading canary response body: %v", err) + } + body := string(bodyBytes) + t := time.Now() + // Mark request as finished + result.End(t) + totalTime := result.Total(t) + + // Verify body contents + if len(body) == 0 { + return fmt.Errorf("expected canary response body to not be empty") + } + + expectedBodyContents := "Hello OpenShift!" + if !strings.Contains(body, expectedBodyContents) { + return fmt.Errorf("expected canary request body to contain %q", expectedBodyContents) + } + + // Verify that the request was received on the correct port + recPort := response.Header.Get(echoServerPortAckHeader) + if len(recPort) == 0 { + return fmt.Errorf("expected %q header in canary response to have a nonempty value", echoServerPortAckHeader) + } + routePortStr := route.Spec.Port.TargetPort.String() + if routePortStr != recPort { + // router wedged, register in metrics counter + CanaryEndpointWrongPortEcho.Inc() + return fmt.Errorf("canary request received on port %s, but route specifies %v", recPort, routePortStr) + } + + // Check status code + switch status := response.StatusCode; status { + case 200: + // Register total time in metrics (use milliseconds) + CanaryRequestTime.WithLabelValues(route.Spec.Host).Observe(float64(totalTime.Milliseconds())) + case 408: + return fmt.Errorf("status code %d: request timed out", status) + case 503: + return fmt.Errorf("status code %d: Canary route not available via router", status) + // TODO (sgreene): + // Add more specific status code checks, if any are missing. + // Also, use HTTP status code constants, if available. + default: + return fmt.Errorf("unexpected status code: %d", status) + } + + return nil +} diff --git a/pkg/operator/controller/canary/metrics.go b/pkg/operator/controller/canary/metrics.go new file mode 100644 index 0000000000..c5bf34738a --- /dev/null +++ b/pkg/operator/controller/canary/metrics.go @@ -0,0 +1,107 @@ +package canary + +import ( + "context" + "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + ctrlruntimemetrics "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +var ( + CanaryRequestTime = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "ingress_canary_check_duration", + Help: "Canary endpoint request time in ms", + Buckets: []float64{25, 50, 100, 200, 400, 800, 1600}, + }, []string{"host"}) + + CanaryEndpointWrongPortEcho = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "ingress_canary_endpoint_wrong_port_echo", + Help: "The ingress canary application received a test request on an incorrect port which may indicate that the router is \"wedged\"", + }) + + CanaryRouteReachable = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "ingress_canary_route_reachable", + Help: "A gauge set to 0 or 1 to signify whether or not the canary application is reachable via a route", + }, []string{"host"}) + + CanaryRouteDNSError = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "ingress_canary_route_DNS_error", + Help: "A counter tracking canary route DNS lookup errors", + }, []string{"host", "dnsServer"}) + + // Populate prometheus collector. + // Individual metrics are stored as public variables + // so that metrics can be globally controlled. + metricsList = []prometheus.Collector{ + CanaryRequestTime, + CanaryEndpointWrongPortEcho, + CanaryRouteReachable, + CanaryRouteDNSError, + } +) + +// SetCanaryRouteMetric is a wrapper function to +// mark the canary route as either online or offline. +func SetCanaryRouteReachableMetric(host string, status bool) { + if status { + CanaryRouteReachable.WithLabelValues(host).Set(1) + } else { + CanaryRouteReachable.WithLabelValues(host).Set(0) + } +} + +// registerCanaryMetrics calls prometheus.Register +// on each metric in metricsList, and returns on errors. +func registerCanaryMetrics() error { + for _, metric := range metricsList { + err := prometheus.Register(metric) + if err != nil { + return err + } + } + return nil +} + +// StartMetricsListener starts the metrics listener on addr. +func StartMetricsListener(addr string, stopCh chan struct{}) { + // These metrics get registered in controller-runtime's registry via an init in the internal/controller/metrics package. + // Unregister the controller-runtime metrics, so that we can combine the controller-runtime metric's registry + // with that of the ingress-operator. This shouldn't have any side effects, as long as no 2 metrics across + // controller runtime or the ingress operator share the same name (which is unlikely). See + // https://github.com/kubernetes/test-infra/blob/master/prow/metrics/metrics.go for additional context. + ctrlruntimemetrics.Registry.Unregister(prometheus.NewGoCollector()) + ctrlruntimemetrics.Registry.Unregister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{})) + + // Create prometheus handler by combining the ingress-operator registry + // with the ingress-operator's controller runtime metrics registry. + handler := promhttp.HandlerFor( + prometheus.Gatherers{prometheus.DefaultGatherer, ctrlruntimemetrics.Registry}, + promhttp.HandlerOpts{}, + ) + + log.Info("registering Prometheus metrics") + if err := registerCanaryMetrics(); err != nil { + log.Error(err, "unable to register metrics") + } + + log.Info("starting metrics listener on ", "addr", addr) + mux := http.NewServeMux() + mux.Handle("/metrics", handler) + s := http.Server{Addr: addr, Handler: mux} + + go func() { + if err := s.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Error(err, "metrics listener exited") + } + }() + <-stopCh + if err := s.Shutdown(context.Background()); err != http.ErrServerClosed { + log.Error(err, "error stopping metrics listener") + } +} diff --git a/pkg/operator/controller/canary/route.go b/pkg/operator/controller/canary/route.go index 450d4fe846..1d887730c0 100644 --- a/pkg/operator/controller/canary/route.go +++ b/pkg/operator/controller/canary/route.go @@ -139,3 +139,21 @@ func desiredCanaryRoute(service *corev1.Service) *routev1.Route { return route } + +// checkRouteAdmitted returns true if a given route has been admitted +// by the default Ingress Controller. +func checkRouteAdmitted(route *routev1.Route) bool { + for _, routeIngress := range route.Status.Ingress { + if routeIngress.RouterName != manifests.DefaultIngressControllerName { + continue + } + conditions := routeIngress.Conditions + for _, cond := range conditions { + if cond.Type == routev1.RouteAdmitted && cond.Status == corev1.ConditionTrue { + return true + } + } + } + + return false +} diff --git a/pkg/operator/controller/ingress/controller.go b/pkg/operator/controller/ingress/controller.go index ee9e6001b1..061bf97c9c 100644 --- a/pkg/operator/controller/ingress/controller.go +++ b/pkg/operator/controller/ingress/controller.go @@ -46,6 +46,7 @@ const ( IngressControllerDeploymentAvailableConditionType = "DeploymentAvailable" IngressControllerDeploymentReplicasMinAvailableConditionType = "DeploymentReplicasMinAvailable" IngressControllerDeploymentReplicasAllAvailableConditionType = "DeploymentReplicasAllAvailable" + IngressControllerCanaryCheckSuccessConditionType = "CanaryChecksSucceeding" ) var ( @@ -258,14 +259,14 @@ func (r *reconciler) admit(current *operatorv1.IngressController, ingressConfig if err := r.validate(updated); err != nil { switch err := err.(type) { case *admissionRejection: - updated.Status.Conditions = mergeConditions(updated.Status.Conditions, operatorv1.OperatorCondition{ + updated.Status.Conditions = MergeConditions(updated.Status.Conditions, operatorv1.OperatorCondition{ Type: IngressControllerAdmittedConditionType, Status: operatorv1.ConditionFalse, Reason: "Invalid", Message: err.Reason, }) updated.Status.ObservedGeneration = updated.Generation - if !ingressStatusesEqual(current.Status, updated.Status) { + if !IngressStatusesEqual(current.Status, updated.Status) { if err := r.client.Status().Update(context.TODO(), updated); err != nil { return fmt.Errorf("failed to update status: %v", err) } @@ -274,13 +275,13 @@ func (r *reconciler) admit(current *operatorv1.IngressController, ingressConfig return err } - updated.Status.Conditions = mergeConditions(updated.Status.Conditions, operatorv1.OperatorCondition{ + updated.Status.Conditions = MergeConditions(updated.Status.Conditions, operatorv1.OperatorCondition{ Type: IngressControllerAdmittedConditionType, Status: operatorv1.ConditionTrue, Reason: "Valid", }) updated.Status.ObservedGeneration = updated.Generation - if !ingressStatusesEqual(current.Status, updated.Status) { + if !IngressStatusesEqual(current.Status, updated.Status) { if err := r.client.Status().Update(context.TODO(), updated); err != nil { return fmt.Errorf("failed to update status: %v", err) } diff --git a/pkg/operator/controller/ingress/status.go b/pkg/operator/controller/ingress/status.go index 56d450e878..d0df3d874e 100644 --- a/pkg/operator/controller/ingress/status.go +++ b/pkg/operator/controller/ingress/status.go @@ -12,6 +12,7 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" iov1 "github.com/openshift/api/operatoringress/v1" + "github.com/openshift/cluster-ingress-operator/pkg/manifests" "github.com/openshift/cluster-ingress-operator/pkg/util/retryableerror" configv1 "github.com/openshift/api/config/v1" @@ -44,18 +45,18 @@ func (r *reconciler) syncIngressControllerStatus(ic *operatorv1.IngressControlle updated.Status.AvailableReplicas = deployment.Status.AvailableReplicas updated.Status.Selector = selector.String() updated.Status.TLSProfile = computeIngressTLSProfile(ic.Status.TLSProfile, deployment) - updated.Status.Conditions = mergeConditions(updated.Status.Conditions, computeDeploymentPodsScheduledCondition(deployment, pods)) - updated.Status.Conditions = mergeConditions(updated.Status.Conditions, computeIngressAvailableCondition(deployment)) - updated.Status.Conditions = mergeConditions(updated.Status.Conditions, computeDeploymentAvailableCondition(deployment)) - updated.Status.Conditions = mergeConditions(updated.Status.Conditions, computeDeploymentReplicasMinAvailableCondition(deployment)) - updated.Status.Conditions = mergeConditions(updated.Status.Conditions, computeDeploymentReplicasAllAvailableCondition(deployment)) - updated.Status.Conditions = mergeConditions(updated.Status.Conditions, computeLoadBalancerStatus(ic, service, operandEvents)...) - updated.Status.Conditions = mergeConditions(updated.Status.Conditions, computeDNSStatus(ic, wildcardRecord, dnsConfig)...) - degradedCondition, err := computeIngressDegradedCondition(updated.Status.Conditions) + updated.Status.Conditions = MergeConditions(updated.Status.Conditions, computeDeploymentPodsScheduledCondition(deployment, pods)) + updated.Status.Conditions = MergeConditions(updated.Status.Conditions, computeIngressAvailableCondition(deployment)) + updated.Status.Conditions = MergeConditions(updated.Status.Conditions, computeDeploymentAvailableCondition(deployment)) + updated.Status.Conditions = MergeConditions(updated.Status.Conditions, computeDeploymentReplicasMinAvailableCondition(deployment)) + updated.Status.Conditions = MergeConditions(updated.Status.Conditions, computeDeploymentReplicasAllAvailableCondition(deployment)) + updated.Status.Conditions = MergeConditions(updated.Status.Conditions, computeLoadBalancerStatus(ic, service, operandEvents)...) + updated.Status.Conditions = MergeConditions(updated.Status.Conditions, computeDNSStatus(ic, wildcardRecord, dnsConfig)...) + degradedCondition, err := computeIngressDegradedCondition(updated.Status.Conditions, updated.Name) errs = append(errs, err) - updated.Status.Conditions = mergeConditions(updated.Status.Conditions, degradedCondition) + updated.Status.Conditions = MergeConditions(updated.Status.Conditions, degradedCondition) - if !ingressStatusesEqual(updated.Status, ic.Status) { + if !IngressStatusesEqual(updated.Status, ic.Status) { if err := r.client.Status().Update(context.TODO(), updated); err != nil { errs = append(errs, fmt.Errorf("failed to update ingresscontroller status: %v", err)) } @@ -64,10 +65,10 @@ func (r *reconciler) syncIngressControllerStatus(ic *operatorv1.IngressControlle return retryableerror.NewMaybeRetryableAggregate(errs) } -// mergeConditions adds or updates matching conditions, and updates +// MergeConditions adds or updates matching conditions, and updates // the transition time if details of a condition have changed. Returns // the updated condition array. -func mergeConditions(conditions []operatorv1.OperatorCondition, updates ...operatorv1.OperatorCondition) []operatorv1.OperatorCondition { +func MergeConditions(conditions []operatorv1.OperatorCondition, updates ...operatorv1.OperatorCondition) []operatorv1.OperatorCondition { now := metav1.NewTime(clock.Now()) var additions []operatorv1.OperatorCondition for i, update := range updates { @@ -337,7 +338,7 @@ func computeDeploymentReplicasAllAvailableCondition(deployment *appsv1.Deploymen // duration value that indicates, if it is non-zero, that the operator should // reconcile the ingresscontroller again after that period to update its status // conditions. -func computeIngressDegradedCondition(conditions []operatorv1.OperatorCondition) (operatorv1.OperatorCondition, error) { +func computeIngressDegradedCondition(conditions []operatorv1.OperatorCondition, icName string) (operatorv1.OperatorCondition, error) { var requeueAfter time.Duration conditionsMap := make(map[string]*operatorv1.OperatorCondition) for i := range conditions { @@ -392,6 +393,23 @@ func computeIngressDegradedCondition(conditions []operatorv1.OperatorCondition) }, } + // Only check the default ingress controller for the canary + // success status condition. + if icName == manifests.DefaultIngressControllerName { + canaryCond := struct { + condition string + status operatorv1.ConditionStatus + ifConditionsTrue []string + gracePeriod time.Duration + }{ + condition: IngressControllerCanaryCheckSuccessConditionType, + status: operatorv1.ConditionTrue, + gracePeriod: time.Second * 60, + } + + expectedConditions = append(expectedConditions, canaryCond) + } + var graceConditions, degradedConditions []*operatorv1.OperatorCondition now := clock.Now() for _, expected := range expectedConditions { @@ -469,10 +487,10 @@ func computeIngressDegradedCondition(conditions []operatorv1.OperatorCondition) return condition, err } -// ingressStatusesEqual compares two IngressControllerStatus values. Returns true +// IngressStatusesEqual compares two IngressControllerStatus values. Returns true // if the provided values should be considered equal for the purpose of determining // whether an update is necessary, false otherwise. -func ingressStatusesEqual(a, b operatorv1.IngressControllerStatus) bool { +func IngressStatusesEqual(a, b operatorv1.IngressControllerStatus) bool { if a.ObservedGeneration != b.ObservedGeneration { return false } diff --git a/pkg/operator/controller/ingress/status_test.go b/pkg/operator/controller/ingress/status_test.go index 6ffc442887..326de73b12 100644 --- a/pkg/operator/controller/ingress/status_test.go +++ b/pkg/operator/controller/ingress/status_test.go @@ -238,6 +238,7 @@ func TestComputePodsScheduledCondition(t *testing.T) { func TestComputeIngressDegradedCondition(t *testing.T) { tests := []struct { name string + icName string conditions []operatorv1.OperatorCondition expectIngressDegradedStatus operatorv1.ConditionStatus expectRequeue bool @@ -407,10 +408,28 @@ func TestComputeIngressDegradedCondition(t *testing.T) { expectIngressDegradedStatus: operatorv1.ConditionFalse, expectRequeue: false, }, + { + name: "default ingress controller, canary check failing", + conditions: []operatorv1.OperatorCondition{ + cond(IngressControllerCanaryCheckSuccessConditionType, operatorv1.ConditionFalse, "", clock.Now().Add(time.Second*-61)), + }, + expectIngressDegradedStatus: operatorv1.ConditionTrue, + expectRequeue: true, + icName: "default", + }, + { + name: "default ingress controller, canary check passing", + conditions: []operatorv1.OperatorCondition{ + cond(IngressControllerCanaryCheckSuccessConditionType, operatorv1.ConditionTrue, "", clock.Now().Add(time.Minute*-1)), + }, + expectIngressDegradedStatus: operatorv1.ConditionFalse, + expectRequeue: false, + icName: "default", + }, } for _, test := range tests { - actual, err := computeIngressDegradedCondition(test.conditions) + actual, err := computeIngressDegradedCondition(test.conditions, test.icName) switch err.(type) { case retryable.Error: if !test.expectRequeue { @@ -990,7 +1009,7 @@ func TestIngressStatusesEqual(t *testing.T) { } for _, tc := range testCases { - if actual := ingressStatusesEqual(tc.a, tc.b); actual != tc.expected { + if actual := IngressStatusesEqual(tc.a, tc.b); actual != tc.expected { t.Fatalf("%q: expected %v, got %v", tc.description, tc.expected, actual) } } @@ -1039,7 +1058,7 @@ func TestMergeConditions(t *testing.T) { for name, test := range tests { t.Logf("test: %s", name) - actual := mergeConditions(test.conditions, test.updates...) + actual := MergeConditions(test.conditions, test.updates...) if !conditionsEqual(test.expected, actual) { t.Errorf("expected:\n%v\nactual:\n%v", toYaml(test.expected), toYaml(actual)) } diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index d771916ec7..2332ce670a 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -120,6 +120,7 @@ func New(config operatorconfig.Config, kubeConfig *rest.Config) (*Operator, erro if _, err := canarycontroller.New(mgr, canarycontroller.Config{ Namespace: config.Namespace, CanaryImage: config.CanaryImage, + Stop: config.Stop, }); err != nil { return nil, fmt.Errorf("failed to create canary controller: %v", err) } diff --git a/test/e2e/canary_test.go b/test/e2e/canary_test.go new file mode 100644 index 0000000000..f6aaf61fba --- /dev/null +++ b/test/e2e/canary_test.go @@ -0,0 +1,177 @@ +// +build e2e + +package e2e + +import ( + "bufio" + "context" + "strings" + "testing" + "time" + + operatorv1 "github.com/openshift/api/operator/v1" + routev1 "github.com/openshift/api/route/v1" + + operatorclient "github.com/openshift/cluster-ingress-operator/pkg/operator/client" + "github.com/openshift/cluster-ingress-operator/pkg/operator/controller" + ingresscontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/ingress" + + "sigs.k8s.io/controller-runtime/pkg/client/config" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" +) + +// TestCanaryRoute tests the ingress canary route +// and checks that the hello-openshift echo server +// works as expected. +func TestCanaryRoute(t *testing.T) { + kubeConfig, err := config.GetConfig() + if err != nil { + t.Fatalf("failed to get kube config: %v", err) + } + kubeClient, err := operatorclient.NewClient(kubeConfig) + if err != nil { + t.Fatalf("failed to create kube client: %s\n", err) + } + + client, err := kubernetes.NewForConfig(kubeConfig) + if err != nil { + t.Fatalf("failed to create kube client: %v", err) + } + + // check that the default ingress controller is ready + def := &operatorv1.IngressController{} + if err := waitForIngressControllerCondition(t, kubeClient, 5*time.Minute, defaultName, defaultAvailableConditions...); err != nil { + t.Fatalf("failed to observe expected conditions: %v", err) + } + + if err := kubeClient.Get(context.TODO(), defaultName, def); err != nil { + t.Fatalf("failed to get default ingresscontroller: %v", err) + } + + // Get default ingress controller deployment + deployment := &appsv1.Deployment{} + if err := kubeClient.Get(context.TODO(), controller.RouterDeploymentName(def), deployment); err != nil { + t.Fatalf("failed to get ingresscontroller deployment: %v", err) + } + + // Get canary route + canaryRoute := &routev1.Route{} + name := controller.CanaryRouteName() + err = wait.PollImmediate(1*time.Second, 1*time.Minute, func() (bool, error) { + if err := kubeClient.Get(context.TODO(), name, canaryRoute); err != nil { + t.Logf("failed to get canary route %s: %v", name, err) + return false, nil + } + + return true, nil + }) + + if err != nil { + t.Fatalf("failed to observe canary route: %v", err) + } + + image := deployment.Spec.Template.Spec.Containers[0].Image + clientPod := buildCanaryCurlPod("canary-route-check", canaryRoute.Namespace, image, canaryRoute.Spec.Host) + if err := kubeClient.Create(context.TODO(), clientPod); err != nil { + t.Fatalf("failed to create pod %s/%s: %v", clientPod.Namespace, clientPod.Name, err) + } + defer func() { + if err := kubeClient.Delete(context.TODO(), clientPod); err != nil { + t.Errorf("failed to delete pod %s/%s: %v", clientPod.Namespace, clientPod.Name, err) + } + }() + + // Test canary route and verify that the hello-openshift echo pod is running properly. + err = wait.PollImmediate(1*time.Second, 5*time.Minute, func() (bool, error) { + readCloser, err := client.CoreV1().Pods(clientPod.Namespace).GetLogs(clientPod.Name, &corev1.PodLogOptions{ + Container: "curl", + Follow: false, + }).Stream(context.TODO()) + if err != nil { + return false, nil + } + scanner := bufio.NewScanner(readCloser) + defer func() { + if err := readCloser.Close(); err != nil { + t.Errorf("failed to close reader for pod %s: %v", clientPod.Name, err) + } + }() + foundBody := false + foundRequestPortHeader := false + for scanner.Scan() { + line := scanner.Text() + if strings.Contains(line, "Hello OpenShift!") { + foundBody = true + } + if strings.Contains(strings.ToLower(line), "x-request-port:") { + foundRequestPortHeader = true + } + if foundBody && foundRequestPortHeader { + return true, nil + } + } + return false, nil + }) + if err != nil { + t.Fatalf("failed to observe the expected canary response body: %v", err) + } +} + +// TestCanaryStatusCondition ensures that the default +// ingress controller has the canary check succeeding +// status condition, which is an indication that canary +// checks are functioning as intended. +func TestCanaryStatusCondition(t *testing.T) { + kubeConfig, err := config.GetConfig() + if err != nil { + t.Fatalf("failed to get kube config: %v", err) + } + kubeClient, err := operatorclient.NewClient(kubeConfig) + if err != nil { + t.Fatalf("failed to create kube client: %s\n", err) + } + + // check that the default ingress controller has the canary success condition set to true + conditions := []operatorv1.OperatorCondition{ + { + Type: ingresscontroller.IngressControllerCanaryCheckSuccessConditionType, + Status: operatorv1.ConditionTrue, + }, + } + + if err := waitForIngressControllerCondition(t, kubeClient, 5*time.Minute, defaultName, conditions...); err != nil { + t.Fatalf("failed to observe expected canary conditions: %v", err) + } +} + +// buildCanaryCurlPod returns a pod definition for a pod with the given name and image +// and in the given namespace that curls the specified route via the route's hostname. +func buildCanaryCurlPod(name, namespace, image, host string) *corev1.Pod { + curlArgs := []string{ + "-s", "-v", + "--retry", "300", "--retry-delay", "1", "--max-time", "2", + } + curlArgs = append(curlArgs, "http://"+host) + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "curl", + Image: image, + Command: []string{"/bin/curl"}, + Args: curlArgs, + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + } +} diff --git a/test/e2e/operator_test.go b/test/e2e/operator_test.go index 3538e76089..d2191cc02e 100644 --- a/test/e2e/operator_test.go +++ b/test/e2e/operator_test.go @@ -106,6 +106,47 @@ func TestOperatorSteadyConditions(t *testing.T) { } } +func TestClusterOperatorStatusRelatedObjects(t *testing.T) { + expected := []configv1.ObjectReference{ + { + Resource: "namespaces", + Name: operatorNamespace, + }, + { + Group: operatorv1.GroupName, + Resource: "IngressController", + Namespace: operatorNamespace, + }, + { + Group: iov1.GroupVersion.Group, + Resource: "DNSRecord", + Namespace: operatorNamespace, + }, + { + Resource: "namespaces", + Name: "openshift-ingress", + }, + { + Resource: "namespaces", + Name: "openshift-ingress-canary", + }, + } + + coName := controller.IngressClusterOperatorName() + err := wait.PollImmediate(1*time.Second, 5*time.Minute, func() (bool, error) { + co := &configv1.ClusterOperator{} + if err := kclient.Get(context.TODO(), coName, co); err != nil { + t.Logf("failed to get ingress cluster operator %s: %v", coName, err) + return false, nil + } + + return reflect.DeepEqual(expected, co.Status.RelatedObjects), nil + }) + if err != nil { + t.Errorf("did not get expected status related objects: %v", err) + } +} + func TestDefaultIngressControllerSteadyConditions(t *testing.T) { if err := waitForIngressControllerCondition(t, kclient, 10*time.Second, defaultName, defaultAvailableConditions...); err != nil { t.Errorf("did not get expected conditions: %v", err) diff --git a/vendor/github.com/tcnksm/go-httpstat/.travis.yml b/vendor/github.com/tcnksm/go-httpstat/.travis.yml new file mode 100644 index 0000000000..4678caa4d0 --- /dev/null +++ b/vendor/github.com/tcnksm/go-httpstat/.travis.yml @@ -0,0 +1,19 @@ +language: go + +go: + - 1.7.5 + - 1.8 + - 1.8.1 + - tip + +os: + - linux + - osx + +sudo: false + +install: + - echo "skipping travis default" + +script: + - make test-all diff --git a/vendor/github.com/tcnksm/go-httpstat/LICENSE b/vendor/github.com/tcnksm/go-httpstat/LICENSE new file mode 100644 index 0000000000..91045c4931 --- /dev/null +++ b/vendor/github.com/tcnksm/go-httpstat/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2016 Taichi Nakashima + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/tcnksm/go-httpstat/Makefile b/vendor/github.com/tcnksm/go-httpstat/Makefile new file mode 100644 index 0000000000..7c48fce498 --- /dev/null +++ b/vendor/github.com/tcnksm/go-httpstat/Makefile @@ -0,0 +1,26 @@ +PACKAGES = $(shell go list ./... | grep -v '/vendor/') + +default: test + +test-all: vet lint test + +test: + go test -v -parallel=4 ${PACKAGES} + +test-race: + go test -v -race ${PACKAGES} + +vet: + go vet ${PACKAGES} + +lint: + @go get github.com/golang/lint/golint + go list ./... | grep -v vendor | xargs -n1 golint + +cover: + @go get golang.org/x/tools/cmd/cover + go test -coverprofile=cover.out + go tool cover -html cover.out + rm cover.out + +.PHONY: test test-race vet lint cover diff --git a/vendor/github.com/tcnksm/go-httpstat/README.md b/vendor/github.com/tcnksm/go-httpstat/README.md new file mode 100644 index 0000000000..70da89138e --- /dev/null +++ b/vendor/github.com/tcnksm/go-httpstat/README.md @@ -0,0 +1,23 @@ +# go-httpstat [![Go Documentation](http://img.shields.io/badge/go-documentation-blue.svg?style=flat-square)][godocs] [![Build Status](http://img.shields.io/travis/tcnksm/go-httpstat.svg?style=flat-square)][travis] [![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat-square)][license] + +[godocs]: http://godoc.org/github.com/tcnksm/go-httpstat +[travis]: https://travis-ci.org/tcnksm/go-httpstat +[license]: /LICENSE + +`go-httpstat` is a golang package to trace golang HTTP request latency (DNSLookup, TCP Connection and so on). Because it uses [`httptrace`](https://golang.org/pkg/net/http/httptrace/) internally, just creating `go-httpstat` powered `context` and giving it your `http.Request` kicks tracing (no big code modification is required). The original idea came from [`httpstat`](https://github.com/reorx/httpstat) command ( and Dave Cheney's [golang implementation](https://github.com/davecheney/httpstat)) 👏. This package now traces same latency infomation as them. + +See usage and example on [GoDoc][godocs]. + +*NOTE*: Since [`httptrace`](https://golang.org/pkg/net/http/httptrace/) was introduced after go1.7, this package may not work with old HTTP client. Especially, if you don't use `net.DialContext` it can not trace DNS and connection. + +## Install + +Use `go get`, + +```bash +$ go get github.com/tcnksm/go-httpstat +``` + +## Author + +[Taichi Nakashima](https://github.com/tcnksm) diff --git a/vendor/github.com/tcnksm/go-httpstat/go18.go b/vendor/github.com/tcnksm/go-httpstat/go18.go new file mode 100644 index 0000000000..3b5df3c3bf --- /dev/null +++ b/vendor/github.com/tcnksm/go-httpstat/go18.go @@ -0,0 +1,134 @@ +// +build go1.8 + +package httpstat + +import ( + "context" + "crypto/tls" + "net/http/httptrace" + "time" +) + +// End sets the time when reading response is done. +// This must be called after reading response body. +func (r *Result) End(t time.Time) { + r.transferDone = t + + // This means result is empty (it does nothing). + // Skip setting value(contentTransfer and total will be zero). + if r.dnsStart.IsZero() { + return + } + + r.contentTransfer = r.transferDone.Sub(r.transferStart) + r.total = r.transferDone.Sub(r.dnsStart) +} + +// ContentTransfer returns the duration of content transfer time. +// It is from first response byte to the given time. The time must +// be time after read body (go-httpstat can not detect that time). +func (r *Result) ContentTransfer(t time.Time) time.Duration { + return t.Sub(r.serverDone) +} + +// Total returns the duration of total http request. +// It is from dns lookup start time to the given time. The +// time must be time after read body (go-httpstat can not detect that time). +func (r *Result) Total(t time.Time) time.Duration { + return t.Sub(r.dnsStart) +} + +func withClientTrace(ctx context.Context, r *Result) context.Context { + return httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{ + DNSStart: func(i httptrace.DNSStartInfo) { + r.dnsStart = time.Now() + }, + + DNSDone: func(i httptrace.DNSDoneInfo) { + r.dnsDone = time.Now() + + r.DNSLookup = r.dnsDone.Sub(r.dnsStart) + r.NameLookup = r.dnsDone.Sub(r.dnsStart) + }, + + ConnectStart: func(_, _ string) { + r.tcpStart = time.Now() + + // When connecting to IP (When no DNS lookup) + if r.dnsStart.IsZero() { + r.dnsStart = r.tcpStart + r.dnsDone = r.tcpStart + } + }, + + ConnectDone: func(network, addr string, err error) { + r.tcpDone = time.Now() + + r.TCPConnection = r.tcpDone.Sub(r.tcpStart) + r.Connect = r.tcpDone.Sub(r.dnsStart) + }, + + TLSHandshakeStart: func() { + r.isTLS = true + r.tlsStart = time.Now() + }, + + TLSHandshakeDone: func(_ tls.ConnectionState, _ error) { + r.tlsDone = time.Now() + + r.TLSHandshake = r.tlsDone.Sub(r.tlsStart) + r.Pretransfer = r.tlsDone.Sub(r.dnsStart) + }, + + GotConn: func(i httptrace.GotConnInfo) { + // Handle when keep alive is used and connection is reused. + // DNSStart(Done) and ConnectStart(Done) is skipped + if i.Reused { + r.isReused = true + } + }, + + WroteRequest: func(info httptrace.WroteRequestInfo) { + r.serverStart = time.Now() + + // When client doesn't use DialContext or using old (before go1.7) `net` + // pakcage, DNS/TCP/TLS hook is not called. + if r.dnsStart.IsZero() && r.tcpStart.IsZero() { + now := r.serverStart + + r.dnsStart = now + r.dnsDone = now + r.tcpStart = now + r.tcpDone = now + } + + // When connection is re-used, DNS/TCP/TLS hook is not called. + if r.isReused { + now := r.serverStart + + r.dnsStart = now + r.dnsDone = now + r.tcpStart = now + r.tcpDone = now + r.tlsStart = now + r.tlsDone = now + } + + if r.isTLS { + return + } + + r.TLSHandshake = r.tcpDone.Sub(r.tcpDone) + r.Pretransfer = r.Connect + }, + + GotFirstResponseByte: func() { + r.serverDone = time.Now() + + r.ServerProcessing = r.serverDone.Sub(r.serverStart) + r.StartTransfer = r.serverDone.Sub(r.dnsStart) + + r.transferStart = r.serverDone + }, + }) +} diff --git a/vendor/github.com/tcnksm/go-httpstat/httpstat.go b/vendor/github.com/tcnksm/go-httpstat/httpstat.go new file mode 100644 index 0000000000..15ce78cafc --- /dev/null +++ b/vendor/github.com/tcnksm/go-httpstat/httpstat.go @@ -0,0 +1,133 @@ +// Package httpstat traces HTTP latency infomation (DNSLookup, TCP Connection and so on) on any golang HTTP request. +// It uses `httptrace` package. Just create `go-httpstat` powered `context.Context` and give it your `http.Request` (no big code modification is required). +package httpstat + +import ( + "bytes" + "context" + "fmt" + "io" + "strings" + "time" +) + +// Result stores httpstat info. +type Result struct { + // The following are duration for each phase + DNSLookup time.Duration + TCPConnection time.Duration + TLSHandshake time.Duration + ServerProcessing time.Duration + contentTransfer time.Duration + + // The followings are timeline of request + NameLookup time.Duration + Connect time.Duration + Pretransfer time.Duration + StartTransfer time.Duration + total time.Duration + + t0 time.Time + t1 time.Time + t2 time.Time + t3 time.Time + t4 time.Time + t5 time.Time // need to be provided from outside + + dnsStart time.Time + dnsDone time.Time + tcpStart time.Time + tcpDone time.Time + tlsStart time.Time + tlsDone time.Time + serverStart time.Time + serverDone time.Time + transferStart time.Time + transferDone time.Time // need to be provided from outside + + // isTLS is true when connection seems to use TLS + isTLS bool + + // isReused is true when connection is reused (keep-alive) + isReused bool +} + +func (r *Result) durations() map[string]time.Duration { + return map[string]time.Duration{ + "DNSLookup": r.DNSLookup, + "TCPConnection": r.TCPConnection, + "TLSHandshake": r.TLSHandshake, + "ServerProcessing": r.ServerProcessing, + "ContentTransfer": r.contentTransfer, + + "NameLookup": r.NameLookup, + "Connect": r.Connect, + "Pretransfer": r.Connect, + "StartTransfer": r.StartTransfer, + "Total": r.total, + } +} + +// Format formats stats result. +func (r Result) Format(s fmt.State, verb rune) { + switch verb { + case 'v': + if s.Flag('+') { + var buf bytes.Buffer + fmt.Fprintf(&buf, "DNS lookup: %4d ms\n", + int(r.DNSLookup/time.Millisecond)) + fmt.Fprintf(&buf, "TCP connection: %4d ms\n", + int(r.TCPConnection/time.Millisecond)) + fmt.Fprintf(&buf, "TLS handshake: %4d ms\n", + int(r.TLSHandshake/time.Millisecond)) + fmt.Fprintf(&buf, "Server processing: %4d ms\n", + int(r.ServerProcessing/time.Millisecond)) + + if r.total > 0 { + fmt.Fprintf(&buf, "Content transfer: %4d ms\n\n", + int(r.contentTransfer/time.Millisecond)) + } else { + fmt.Fprintf(&buf, "Content transfer: %4s ms\n\n", "-") + } + + fmt.Fprintf(&buf, "Name Lookup: %4d ms\n", + int(r.NameLookup/time.Millisecond)) + fmt.Fprintf(&buf, "Connect: %4d ms\n", + int(r.Connect/time.Millisecond)) + fmt.Fprintf(&buf, "Pre Transfer: %4d ms\n", + int(r.Pretransfer/time.Millisecond)) + fmt.Fprintf(&buf, "Start Transfer: %4d ms\n", + int(r.StartTransfer/time.Millisecond)) + + if r.total > 0 { + fmt.Fprintf(&buf, "Total: %4d ms\n", + int(r.total/time.Millisecond)) + } else { + fmt.Fprintf(&buf, "Total: %4s ms\n", "-") + } + io.WriteString(s, buf.String()) + return + } + + fallthrough + case 's', 'q': + d := r.durations() + list := make([]string, 0, len(d)) + for k, v := range d { + // Handle when End function is not called + if (k == "ContentTransfer" || k == "Total") && r.t5.IsZero() { + list = append(list, fmt.Sprintf("%s: - ms", k)) + continue + } + list = append(list, fmt.Sprintf("%s: %d ms", k, v/time.Millisecond)) + } + io.WriteString(s, strings.Join(list, ", ")) + } + +} + +// WithHTTPStat is a wrapper of httptrace.WithClientTrace. It records the +// time of each httptrace hooks. +func WithHTTPStat(ctx context.Context, r *Result) context.Context { + return withClientTrace(ctx, r) +} diff --git a/vendor/github.com/tcnksm/go-httpstat/pre_go18.go b/vendor/github.com/tcnksm/go-httpstat/pre_go18.go new file mode 100644 index 0000000000..92eaff3a47 --- /dev/null +++ b/vendor/github.com/tcnksm/go-httpstat/pre_go18.go @@ -0,0 +1,128 @@ +// +build !go1.8 + +package httpstat + +import ( + "context" + "net" + "net/http/httptrace" + "time" +) + +// End sets the time when reading response is done. +// This must be called after reading response body. +func (r *Result) End(t time.Time) { + r.t5 = t + + // This means result is empty (it does nothing). + // Skip setting value(contentTransfer and total will be zero). + if r.t0.IsZero() { + return + } + + r.contentTransfer = r.t5.Sub(r.t4) + r.total = r.t5.Sub(r.t0) +} + +// ContentTransfer returns the duration of content transfer time. +// It is from first response byte to the given time. The time must +// be time after read body (go-httpstat can not detect that time). +func (r *Result) ContentTransfer(t time.Time) time.Duration { + return t.Sub(r.t4) +} + +// Total returns the duration of total http request. +// It is from dns lookup start time to the given time. The +// time must be time after read body (go-httpstat can not detect that time). +func (r *Result) Total(t time.Time) time.Duration { + return t.Sub(r.t0) +} + +func withClientTrace(ctx context.Context, r *Result) context.Context { + return httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{ + GetConn: func(hostPort string) { + _, port, err := net.SplitHostPort(hostPort) + if err != nil { + return + } + + // Heuristic way to detect + if port == "443" { + r.isTLS = true + } + }, + + DNSStart: func(i httptrace.DNSStartInfo) { + r.t0 = time.Now() + }, + DNSDone: func(i httptrace.DNSDoneInfo) { + r.t1 = time.Now() + r.DNSLookup = r.t1.Sub(r.t0) + r.NameLookup = r.t1.Sub(r.t0) + }, + + ConnectStart: func(_, _ string) { + // When connecting to IP + if r.t0.IsZero() { + r.t0 = time.Now() + r.t1 = r.t0 + } + }, + + ConnectDone: func(network, addr string, err error) { + r.t2 = time.Now() + if r.isTLS { + r.TCPConnection = r.t2.Sub(r.t1) + r.Connect = r.t2.Sub(r.t0) + } + }, + + GotConn: func(i httptrace.GotConnInfo) { + // Handle when keep alive is enabled and connection is reused. + // DNSStart(Done) and ConnectStart(Done) is skipped + if i.Reused { + r.t0 = time.Now() + r.t1 = r.t0 + r.t2 = r.t0 + + r.isReused = true + } + }, + + WroteRequest: func(info httptrace.WroteRequestInfo) { + r.t3 = time.Now() + + // This means DNSStart, Done and ConnectStart is not + // called. This happens if client doesn't use DialContext + // or using net package before go1.7. + if r.t0.IsZero() && r.t1.IsZero() && r.t2.IsZero() { + r.t0 = time.Now() + r.t1 = r.t0 + r.t2 = r.t0 + r.t3 = r.t0 + } + + // When connection is reused, TLS handshake is skipped. + if r.isReused { + r.t3 = r.t0 + } + + if r.isTLS { + r.TLSHandshake = r.t3.Sub(r.t2) + r.Pretransfer = r.t3.Sub(r.t0) + return + } + + r.TCPConnection = r.t3.Sub(r.t1) + r.Connect = r.t3.Sub(r.t0) + + r.TLSHandshake = r.t3.Sub(r.t3) + r.Pretransfer = r.Connect + }, + GotFirstResponseByte: func() { + r.t4 = time.Now() + r.ServerProcessing = r.t4.Sub(r.t3) + r.StartTransfer = r.t4.Sub(r.t0) + }, + }) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 12e49b4cca..af5bcb661e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -170,6 +170,8 @@ github.com/spf13/cobra github.com/spf13/pflag # github.com/stretchr/testify v1.5.1 github.com/stretchr/testify/assert +# github.com/tcnksm/go-httpstat v0.2.1-0.20191008022543-e866bb274419 +github.com/tcnksm/go-httpstat # go.opencensus.io v0.22.2 go.opencensus.io go.opencensus.io/internal