helm/metrics: add a new gauge metric to monitor helm releases

go.mod

@@ -14,15 +14,17 @@ require (
 	github.com/openshift/client-go v0.0.0-20210112160336-8889f8b15bd6
 	github.com/openshift/library-go v0.0.0-20210330121117-68dd4a4c9d9e
 	github.com/pkg/profile v1.4.0 // indirect
-	github.com/spf13/cobra v1.1.1
+	github.com/spf13/cobra v1.1.3
 	github.com/spf13/pflag v1.0.5
 	gopkg.in/yaml.v2 v2.4.0
+	helm.sh/helm/v3 v3.6.3
 	k8s.io/api v0.21.1
-	k8s.io/apiextensions-apiserver v0.21.0 // indirect
 	k8s.io/apimachinery v0.21.1
+	k8s.io/cli-runtime v0.21.0
 	k8s.io/client-go v0.21.0
 	k8s.io/component-base v0.21.0
 	k8s.io/klog/v2 v2.8.0
+	rsc.io/letsencrypt v0.0.3 // indirect
 )
 
 // points to temporary-watch-reduction-patch-1.21 to pick up k/k/pull/101102 - please remove it once the pr merges and a new Z release is cut

manifests/03-rbac-role-cluster.yaml

@@ -80,6 +80,9 @@ rules:
   - create
   - update
   - delete
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get", "watch", "list"]
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1

pkg/console/operator/sync_v400.go

@@ -30,6 +30,8 @@ import (
 	"github.com/openshift/library-go/pkg/route/routeapihelpers"
 
 	// operator
+	helmmetrics "github.com/openshift/console-operator/pkg/helm/metrics"
+
 	customerrors "github.com/openshift/console-operator/pkg/console/errors"
 	"github.com/openshift/console-operator/pkg/console/metrics"
 	"github.com/openshift/console-operator/pkg/console/status"
@@ -218,6 +220,7 @@ func (co *consoleOperator) GetActiveRouteInfo(ctx context.Context, activeRouteNa
 func (co *consoleOperator) SyncConsoleConfig(ctx context.Context, consoleConfig *configv1.Console, consoleURL string) (*configv1.Console, error) {
 	oldURL := consoleConfig.Status.ConsoleURL
 	metrics.HandleConsoleURL(oldURL, consoleURL)
+	helmmetrics.HandleHelmChartReleaseHealthStatus()
 	if oldURL != consoleURL {
 		klog.V(4).Infof("updating console.config.openshift.io with url: %v", consoleURL)
 		updated := consoleConfig.DeepCopy()

pkg/helm/metrics/metrics.go

@@ -0,0 +1,83 @@
+package metrics
+
+import (
+	"log"
+	"os"
+
+	"helm.sh/helm/v3/pkg/action"
+	"k8s.io/cli-runtime/pkg/genericclioptions"
+	"k8s.io/client-go/rest"
+	k8smetrics "k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+	"k8s.io/klog/v2"
+)
+
+var (
+	helmChartReleaseHealthStatus = k8smetrics.NewGaugeVec(
+		&k8smetrics.GaugeOpts{
+			Name: "helm_chart_release_health_status",
+			Help: "Health of the Helm release",
+		},
+		[]string{"releaseName", "chartName", "chartVersion"},
+	)
+)
+
+func init() {
+	legacyregistry.MustRegister(helmChartReleaseHealthStatus)
+}
+
+func HandleHelmChartReleaseHealthStatus() {
+	defer recoverMetricPanic()
+
+	if actionConfig, err := getActionConfig(); err != nil {
+		klog.Errorf("metric helm_chart_release_health_status unhandled: %v", err)
+		return
+	} else {
+		listAction := action.NewList(actionConfig)
+		releases, err := listAction.Run()
+		if err != nil {
+			klog.Errorf("metric helm_chart_release_health_status unhandled: %v", err)
+			return
+		}
+
+		for _, release := range releases {
+			if status := release.Info.Status.String(); status == "deployed" {
+				klog.V(4).Infof("metric helm_chart_release_health_status 1: %s %s %s", release.Name, release.Chart.Metadata.Name, release.Chart.Metadata.Version)
+				helmChartReleaseHealthStatus.WithLabelValues(release.Name, release.Chart.Metadata.Name, release.Chart.Metadata.Version).Set(1)
+			} else {
+				klog.V(4).Infof("metric helm_chart_release_health_status 0: %s %s %s", release.Name, release.Chart.Metadata.Name, release.Chart.Metadata.Version)
+				helmChartReleaseHealthStatus.WithLabelValues(release.Name, release.Chart.Metadata.Name, release.Chart.Metadata.Version).Set(0)
+			}
+		}
+	}
+}
+
+// Reference: https://github.com/helm/helm/issues/7430#issuecomment-620489002
+func getActionConfig() (*action.Configuration, error) {
+	actionConfig := new(action.Configuration)
+	var kubeConfig *genericclioptions.ConfigFlags
+	// Create the rest config instance with ServiceAccount values loaded in them
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		return nil, err
+	}
+	// Create the ConfigFlags struct instance with initialized values from ServiceAccount
+	kubeConfig = genericclioptions.NewConfigFlags(false)
+	kubeConfig.APIServer = &config.Host
+	kubeConfig.BearerToken = &config.BearerToken
+	kubeConfig.CAFile = &config.CAFile
+	// Empty string for all namespaces
+	if err := actionConfig.Init(kubeConfig, "", os.Getenv("HELM_DRIVER"), log.Printf); err != nil {
+		return nil, err
+	}
+	return actionConfig, nil
+}
+
+// We will never want to panic our operator because of metric saving.
+// Therefore, we will recover our panics here and error log them
+// for later diagnosis but will never fail the operator.
+func recoverMetricPanic() {
+	if r := recover(); r != nil {
+		klog.Errorf("Recovering from metric function - %v", r)
+	}
+}