NE-199 Phase 2: Add periodic canary route HTTP checks w/ metrics & basic status reporting

cmd/ingress-operator/start.go

@@ -12,10 +12,10 @@ import (
 	"github.com/openshift/cluster-ingress-operator/pkg/manifests"
 	"github.com/openshift/cluster-ingress-operator/pkg/operator"
 	operatorconfig "github.com/openshift/cluster-ingress-operator/pkg/operator/config"
+	canarycontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/canary"
 	statuscontroller "github.com/openshift/cluster-ingress-operator/pkg/operator/controller/status"
 
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
-	"sigs.k8s.io/controller-runtime/pkg/metrics"
 	"sigs.k8s.io/controller-runtime/pkg/runtime/signals"
 )
 
@@ -77,7 +77,6 @@ func NewStartCommand() *cobra.Command {
 }
 
 func start(opts *StartOptions) error {
-	metrics.DefaultBindAddress = opts.MetricsListenAddr
 
 	kubeConfig, err := config.GetConfig()
 	if err != nil {
@@ -90,16 +89,20 @@ func start(opts *StartOptions) error {
 		log.Info("Warning: no release version is specified", "release version", statuscontroller.UnknownVersionValue)
 	}
 
+	// Set up the channels for the watcher, operator, and metrics.
+	stop := make(chan struct{})
+	signal := signals.SetupSignalHandler()
+
 	operatorConfig := operatorconfig.Config{
 		OperatorReleaseVersion: opts.ReleaseVersion,
 		Namespace:              opts.OperatorNamespace,
 		IngressControllerImage: opts.IngressControllerImage,
 		CanaryImage:            opts.CanaryImage,
+		Stop:                   stop,
 	}
 
-	// Set up the channels for the watcher and operator.
-	stop := make(chan struct{})
-	signal := signals.SetupSignalHandler()
+	// Start operator metrics.
+	go canarycontroller.StartMetricsListener(opts.MetricsListenAddr, stop)
 
 	// Set up and start the file watcher.
 	watcher, err := fsnotify.NewWatcher()

go.mod

@@ -21,8 +21,10 @@ require (
 	github.com/openshift/api v0.0.0-20201130121019-19e3831bc513
 	github.com/openshift/library-go v0.0.0-20200423123937-d1360419413d
 	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_golang v1.7.1
 	github.com/spf13/cobra v1.0.0
 	github.com/stretchr/testify v1.5.1
+	github.com/tcnksm/go-httpstat v0.2.1-0.20191008022543-e866bb274419
 	go.uber.org/zap v1.15.0
 	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
 	gomodules.xyz/jsonpatch/v2 v2.1.0 // indirect

go.sum

@@ -489,6 +489,8 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/tcnksm/go-httpstat v0.2.1-0.20191008022543-e866bb274419 h1:elOIj31UL4RZWgLfLV4pWZA0j5QqGO95/Dll2WIwOZU=
+github.com/tcnksm/go-httpstat v0.2.1-0.20191008022543-e866bb274419/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=

manifests/02-deployment.yaml

@@ -63,7 +63,7 @@ spec:
             - name: IMAGE
               value: openshift/origin-haproxy-router:v4.0
             - name: CANARY_IMAGE
-              value: openshift/hello-openshift:latest
+              value: quay.io/openshift/origin-hello-openshift:latest
           resources:
             requests:
               cpu: 10m

manifests/image-references

@@ -13,8 +13,8 @@ spec:
   - name: kube-rbac-proxy
     from:
       kind: DockerImage
-      name: quay.io/openshift/origin-kube-rbac-proxy:latest
+      name: "quay.io/openshift/origin-kube-rbac-proxy:latest"
   - name: hello-openshift
     from:
       kind: DockerImage
-      name: "openshift/hello-openshift:latest"
+      name: "quay.io/openshift/origin-hello-openshift:latest"

pkg/operator/config/config.go

@@ -14,4 +14,6 @@ type Config struct {
 
 	// CanaryImage is the ingress canary image to manage.
 	CanaryImage string
+
+	Stop chan struct{}
 }