Change to single metric with success/failure

Author: stuartleeks

api/v1alpha1/zz_generated.deepcopy.go

@@ -16,7 +16,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// Code generated by controller-gen. DO NOT EDIT.
+// autogenerated by controller-gen object, do not modify manually
 
 package v1alpha1
 

controllers/metrics.go

@@ -16,6 +16,8 @@ limitations under the License.
 package controllers
 
 import (
+	"time"
+
 	"github.com/prometheus/client_golang/prometheus"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
 )
@@ -28,45 +30,34 @@ const (
 var databricksRequestHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{
 	Name: "databricks_request_duration_seconds",
 	Help: "Duration of upstream calls to Databricks REST service endpoints",
-}, []string{"object_type", "action"})
-
-var databricksRequestCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
-	Name: "databricks_request_total",
-	Help: "Counter of upstream calls to Databricks REST service endpoints",
 }, []string{"object_type", "action", "outcome"})
 
 func init() {
 	// Register custom metrics with the global prometheus registry
-	metrics.Registry.MustRegister(databricksRequestHistogram, databricksRequestCounter)
+	metrics.Registry.MustRegister(databricksRequestHistogram)
 }
 
 // NewExecution creates an Execution instance and starts the timer
 func NewExecution(objectType string, action string) Execution {
-	labels := prometheus.Labels{"object_type": objectType, "action": action}
-	observer := databricksRequestHistogram.With(labels)
-	timer := prometheus.NewTimer(observer)
-
 	return Execution{
-		timer:  *timer,
-		labels: labels,
+		begin:  time.Now(),
+		labels: prometheus.Labels{"object_type": objectType, "action": action},
 	}
 }
 
 // Execution tracks state for an API execution for emitting metrics
 type Execution struct {
-	timer  prometheus.Timer
+	begin  time.Time
 	labels prometheus.Labels
 }
 
 // Finish is used to log duration and success/failure
 func (e *Execution) Finish(err error) {
-	e.timer.ObserveDuration()
-
 	if err == nil {
 		e.labels["outcome"] = successMetric
 	} else {
 		e.labels["outcome"] = failureMetric
 	}
-
-	databricksRequestCounter.With(e.labels).Inc()
+	duration := time.Since(e.begin)
+	databricksRequestHistogram.With(e.labels).Observe(duration.Seconds())
 }

controllers/metrics_test.go

@@ -65,13 +65,6 @@ In addition to the standard metrics that kubebuilder provides, the following cus
 
 The `databricks_request_duration_seconds` histogram provides metrics on the duration of calls via the databricks SDK and has the following labels:
 
-|Name|Description|
-|-|-|
-|`object_type`|The type of object that the call relatest to, e.g. `dcluster`|
-|`action`| The action being performed, e.g. `get`, `create`|
-
-The `databricks_request_total` counter provides the total number of calls via the databricks SDK and can be used in `rate` queries to see the rate of successful/failed calls. The labels for this metric are 
-
 |Name|Description|
 |-|-|
 |`object_type`|The type of object that the call relatest to, e.g. `dcluster`|