Split ClusterVersion into spec and status, write end-user condition messages, report metrics

cmd/main.go

@@ -8,7 +8,7 @@ import (
 )
 
 const (
-	componentName      = "cluster-version-operator"
+	componentName      = "version"
 	componentNamespace = "openshift-cluster-version"
 )
 

cmd/start.go

@@ -3,6 +3,7 @@ package main
 import (
 	"flag"
 	"math/rand"
+	"net/http"
 	"os"
 	"time"
 
@@ -13,6 +14,7 @@ import (
 	clientset "github.com/openshift/cluster-version-operator/pkg/generated/clientset/versioned"
 	informers "github.com/openshift/cluster-version-operator/pkg/generated/informers/externalversions"
 	"github.com/openshift/cluster-version-operator/pkg/version"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/spf13/cobra"
 	"k8s.io/api/core/v1"
 	apiext "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
@@ -29,7 +31,7 @@ import (
 )
 
 const (
-	minResyncPeriod = 10 * time.Second
+	minResyncPeriod = 2 * time.Minute
 
 	leaseDuration = 90 * time.Second
 	renewDeadline = 45 * time.Second
@@ -47,13 +49,15 @@ var (
 	startOpts struct {
 		kubeconfig string
 		nodeName   string
+		listenAddr string
 
 		enableAutoUpdate bool
 	}
 )
 
 func init() {
 	rootCmd.AddCommand(startCmd)
+	startCmd.PersistentFlags().StringVar(&startOpts.listenAddr, "listen", "0.0.0.0:11345", "Address to listen on for metrics")
 	startCmd.PersistentFlags().StringVar(&startOpts.kubeconfig, "kubeconfig", "", "Kubeconfig file to access a remote cluster (testing only)")
 	startCmd.PersistentFlags().StringVar(&startOpts.nodeName, "node-name", "", "kubernetes node name CVO is scheduled on.")
 	startCmd.PersistentFlags().BoolVar(&startOpts.enableAutoUpdate, "enable-auto-update", true, "Enables the autoupdate controller.")
@@ -78,6 +82,16 @@ func runStartCmd(cmd *cobra.Command, args []string) {
 		glog.Fatalf("missing --release-image flag, it is required")
 	}
 
+	if len(startOpts.listenAddr) > 0 {
+		mux := http.NewServeMux()
+		mux.Handle("/metrics", promhttp.Handler())
+		go func() {
+			if err := http.ListenAndServe(startOpts.listenAddr, mux); err != nil {
+				glog.Fatalf("Unable to start metrics server: %v", err)
+			}
+		}()
+	}
+
 	cb, err := newClientBuilder(startOpts.kubeconfig)
 	if err != nil {
 		glog.Fatalf("error creating clients: %v", err)
@@ -227,14 +241,19 @@ func createControllerContext(cb *clientBuilder, stop <-chan struct{}) *controlle
 }
 
 func startControllers(ctx *controllerContext) error {
+	overrideDirectory := os.Getenv("PAYLOAD_OVERRIDE")
+	if len(overrideDirectory) > 0 {
+		glog.Warningf("Using an override payload directory for testing only: %s", overrideDirectory)
+	}
+
 	go cvo.New(
 		startOpts.nodeName,
 		componentNamespace, componentName,
 		rootOpts.releaseImage,
-		ctx.InformerFactory.Clusterversion().V1().CVOConfigs(),
+		overrideDirectory,
+		ctx.ResyncPeriod(),
+		ctx.InformerFactory.Config().V1().ClusterVersions(),
 		ctx.InformerFactory.Operatorstatus().V1().ClusterOperators(),
-		ctx.APIExtInformerFactory.Apiextensions().V1beta1().CustomResourceDefinitions(),
-		ctx.KubeInformerFactory.Apps().V1().Deployments(),
 		ctx.ClientBuilder.RestConfig(),
 		ctx.ClientBuilder.ClientOrDie(componentName),
 		ctx.ClientBuilder.KubeClientOrDie(componentName),
@@ -244,7 +263,7 @@ func startControllers(ctx *controllerContext) error {
 	if startOpts.enableAutoUpdate {
 		go autoupdate.New(
 			componentNamespace, componentName,
-			ctx.InformerFactory.Clusterversion().V1().CVOConfigs(),
+			ctx.InformerFactory.Config().V1().ClusterVersions(),
 			ctx.InformerFactory.Operatorstatus().V1().ClusterOperators(),
 			ctx.ClientBuilder.ClientOrDie(componentName),
 			ctx.ClientBuilder.KubeClientOrDie(componentName),

docs/dev/metrics.md

@@ -0,0 +1,21 @@
+# CVO Metrics
+
+The Cluster Version Operator reports the following metrics:
+
+```
+# HELP cluster_operator_conditions Report the conditions for active cluster operators. 0 is False and 1 is True.
+# TYPE cluster_operator_conditions gauge
+cluster_operator_conditions{condition="Available",name="version",namespace="openshift-cluster-version"} 1
+cluster_operator_conditions{condition="Failing",name="version",namespace="openshift-cluster-version"} 0
+cluster_operator_conditions{condition="Progressing",name="version",namespace="openshift-cluster-version"} 0
+cluster_operator_conditions{condition="RetrievedUpdates",name="version",namespace="openshift-cluster-version"} 0
+# HELP cluster_operator_up Reports key highlights of the active cluster operators.
+# TYPE cluster_operator_up gauge
+cluster_operator_up{name="version",namespace="openshift-cluster-version",version="4.0.1"} 1
+# HELP cluster_version Reports the version of the cluster.
+# TYPE cluster_version gauge
+cluster_version{payload="test/image:1",type="current",version="4.0.2"} 1
+# HELP cluster_version_available_updates Report the count of available versions for an upstream and channel.
+# TYPE cluster_version_available_updates gauge
+cluster_version_available_updates{channel="fast",upstream="http://localhost:8080/graph"} 0
+```

hack/update-codegen.sh

@@ -6,5 +6,5 @@ ${PROJECT_ROOT}/vendor/k8s.io/code-generator/generate-groups.sh \
 	all \
 	github.com/openshift/cluster-version-operator/pkg/generated \
 	github.com/openshift/cluster-version-operator/pkg/apis \
-	"clusterversion.openshift.io:v1 operatorstatus.openshift.io:v1" \
+	"config.openshift.io:v1 operatorstatus.openshift.io:v1" \
 	$@ \

install/0000_00_cluster-version-operator_01_clusteroperator.crd.yaml

@@ -0,0 +1,99 @@
+kind: CustomResourceDefinition
+apiVersion: apiextensions.k8s.io/v1beta1
+metadata:
+  name: clusteroperators.config.openshift.io
+spec:
+  additionalPrinterColumns:
+  - JSONPath: .status.version
+    description: The version the operator is at.
+    name: Version
+    type: string
+  - JSONPath: .status.conditions[?(@.type=="Available")].status
+    description: Whether the operator is running and stable.
+    name: Available
+    type: string
+  - JSONPath: .status.conditions[?(@.type=="Progressing")].status
+    description: Whether the operator is processing changes.
+    name: Progressing
+    type: string
+  - JSONPath: .status.conditions[?(@.type=="Available")].lastTransitionTime
+    description: The time the operator's Available status last changed.
+    name: Since
+    type: date
+  group: config.openshift.io
+  names:
+    kind: ClusterOperator
+    listKind: ClusterOperatorList
+    plural: clusteroperators
+    singular: clusteroperator
+  scope: Namespaced
+  subresources:
+    status: {}
+  version: v1
+  versions:
+  - name: v1
+    served: true
+    storage: true
+---
+# XXX: This CRD will be removed before 4.0
+kind: CustomResourceDefinition
+apiVersion: apiextensions.k8s.io/v1beta1
+metadata:
+  name: clusteroperators.operatorstatus.openshift.io
+spec:
+  additionalPrinterColumns:
+  - JSONPath: .status.version
+    description: The version the operator is at.
+    name: Version
+    type: string
+  - JSONPath: .status.conditions[?(@.type=="Available")].status
+    description: Whether the operator is running and stable.
+    name: Available
+    type: string
+  - JSONPath: .status.conditions[?(@.type=="Progressing")].status
+    description: Whether the operator is processing changes.
+    name: Progressing
+    type: string
+  - JSONPath: .status.conditions[?(@.type=="Available")].lastTransitionTime
+    description: The time the operator's Available status last changed.
+    name: Since
+    type: date
+  group: operatorstatus.openshift.io
+  names:
+    kind: ClusterOperator
+    listKind: ClusterOperatorList
+    plural: clusteroperators
+    singular: clusteroperator
+  scope: Namespaced
+  subresources:
+    status: {}
+  version: v1
+  versions:
+  - name: v1
+    served: true
+    storage: true
+---
+# XXX: This CRD will be removed before 4.0
+kind: CustomResourceDefinition
+apiVersion: apiextensions.k8s.io/v1beta1
+metadata:
+  name: operatorstatuses.operatorstatus.openshift.io
+spec:
+  additionalPrinterColumns:
+  - JSONPath: .metadata.creationTimestamp
+    description: CreationTimestamp is a timestamp representing the server time when
+      this object was created.
+    name: Age
+    type: date
+  group: operatorstatus.openshift.io
+  names:
+    kind: OperatorStatus
+    listKind: OperatorStatusList
+    plural: operatorstatuses
+    singular: operatorstatus
+  scope: Namespaced
+  version: v1
+  versions:
+  - name: v1
+    served: true
+    storage: true

install/0000_00_cluster-version-operator_01_clusterversion.crd.yaml

@@ -41,3 +41,32 @@ spec:
   - name: Status
     type: string
     JSONPath: .status.conditions[?(@.type=="Progressing")].message
+---
+# XXX: This CRD will be removed before 4.0, use the version above
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  # name must match the spec fields below, and be in the form: <plural>.<group>
+  name: cvoconfigs.clusterversion.openshift.io
+spec:
+  # group name to use for REST API: /apis/<group>/<version>
+  group: clusterversion.openshift.io
+  # list of versions supported by this CustomResourceDefinition
+  versions:
+    - name: v1
+      # Each version can be enabled/disabled by Served flag.
+      served: true
+      # One and only one version must be marked as the storage version.
+      storage: true
+  # either Namespaced or Cluster
+  scope: Namespaced
+  subresources:
+    # enable spec/status
+    status: {}
+  names:
+    # plural name to be used in the URL: /apis/<group>/<version>/<plural>
+    plural: cvoconfigs
+    # singular name to be used as an alias on the CLI and for display
+    singular: cvoconfig
+    # kind is normally the CamelCased singular type. Your resource manifests use this.
+    kind: CVOConfig