fix(metrics): remove changing values like 'limit' from metric labels to prevent Prometheus crash

cmd/scheduler/metrics.go

@@ -259,13 +259,21 @@ func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) {
 	quotaUsedDesc := prometheus.NewDesc(
 		"hami_resource_quota_used",
 		"resourcequota usage for a certain device",
-		[]string{"namespace", "quota_name", "limit"}, nil,
+		[]string{"namespace", "quota_name"}, nil,
+	)
+	quotaLimitDesc := prometheus.NewDesc(
+		"hami_resource_quota_limit",
+		"Configured hard limit for a resource quota",
+		[]string{"namespace", "quota_name"}, nil,
 	)
 	for ns, val := range cc.metricsProvider.GetQuotaManager().GetResourceQuota() {
 		for quotaname, q := range *val {
-			if err := sendMetric(ch, quotaUsedDesc, prometheus.GaugeValue, float64(q.Used), ns, quotaname, fmt.Sprint(q.Limit)); err != nil {
+			if err := sendMetric(ch, quotaUsedDesc, prometheus.GaugeValue, float64(q.Used), ns, quotaname); err != nil {
 				klog.V(4).Infof("Failed to send quotaUsedDesc metric: %v", err)
 			}
+			if err := sendMetric(ch, quotaLimitDesc, prometheus.GaugeValue, float64(q.Limit), ns, quotaname); err != nil {
+				klog.V(4).Infof("Failed to send quotaLimitDesc metric: %v", err)
+			}
 			if legacy {
 				sendLegacyMetric(ch, legacyQuotaUsed, prometheus.GaugeValue, float64(q.Used), ns, quotaname, fmt.Sprint(q.Limit))
 			}

cmd/scheduler/metrics_test.go

@@ -215,3 +215,42 @@ nodeGPUMemoryPercentage{deviceidx="2",deviceuuid="normal-memory",nodeid="node-1"
 		t.Fatalf("unexpected collecting result:\n%s", err)
 	}
 }
+
+func TestQuotaMetricHasNoLimitLabel(t *testing.T) {
+	qm := device.NewQuotaManager()
+	qm.Quotas["test-namespace"] = &device.DeviceQuota{
+		"test-quota": &device.Quota{
+			Limit: 10,
+			Used:  3,
+		},
+	}
+
+	collector := ClusterManagerCollector{
+		ClusterManager: &ClusterManager{
+			LegacyMetrics: false,
+		},
+		metricsProvider: &fakeSchedulerMetricsProvider{
+			nodeUsage:    map[string]*schedulerpkg.NodeUsage{},
+			quotaManager: qm,
+			podManager:   device.NewPodManager(),
+		},
+	}
+
+	want := `
+# HELP hami_resource_quota_limit Configured hard limit for a resource quota
+# TYPE hami_resource_quota_limit gauge
+hami_resource_quota_limit{namespace="test-namespace",quota_name="test-quota"} 10
+# HELP hami_resource_quota_used resourcequota usage for a certain device
+# TYPE hami_resource_quota_used gauge
+hami_resource_quota_used{namespace="test-namespace",quota_name="test-quota"} 3
+`
+
+	if err := promtestutil.CollectAndCompare(
+		collector,
+		strings.NewReader(want),
+		"hami_resource_quota_limit",
+		"hami_resource_quota_used",
+	); err != nil {
+		t.Fatalf("unexpected collecting result:\n%s", err)
+	}
+}

cmd/vGPUmonitor/metrics_test.go

@@ -17,6 +17,7 @@
 package main
 
 import (
+	"strings"
 	"testing"
 
 	"github.com/prometheus/client_golang/prometheus"