From 9e15d2f463a4acb892663d42baa7054132fa066c Mon Sep 17 00:00:00 2001 From: SK8-infi Date: Tue, 18 Aug 2026 17:19:09 +0530 Subject: [PATCH 1/2] feat(scheduler): export hami_resource_quota_limit gauge metric Signed-off-by: SK8-infi --- cmd/scheduler/metrics.go | 10 +++++++++- cmd/scheduler/metrics_test.go | 10 ++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/cmd/scheduler/metrics.go b/cmd/scheduler/metrics.go index e8ddfb6ae9..c5b34a17b6 100644 --- a/cmd/scheduler/metrics.go +++ b/cmd/scheduler/metrics.go @@ -281,13 +281,18 @@ func (cc ClusterManagerCollector) collectNodeMetrics(ch chan<- prometheus.Metric } } -// collectQuotaMetrics emits per-namespace resource quota usage. +// collectQuotaMetrics emits per-namespace resource quota usage and limits. func (cc ClusterManagerCollector) collectQuotaMetrics(ch chan<- prometheus.Metric, legacy bool) { quotaUsedDesc := prometheus.NewDesc( "hami_resource_quota_used", "resourcequota usage for a certain device", []string{"namespace", "quota_name", "limit"}, nil, ) + quotaLimitDesc := prometheus.NewDesc( + "hami_resource_quota_limit", + "resourcequota limit for a certain device", + []string{"namespace", "quota_name"}, nil, + ) var legacyQuotaUsed *prometheus.Desc if legacy { legacyQuotaUsed = prometheus.NewDesc( @@ -301,6 +306,9 @@ func (cc ClusterManagerCollector) collectQuotaMetrics(ch chan<- prometheus.Metri if err := sendMetric(ch, quotaUsedDesc, prometheus.GaugeValue, float64(q.Used), ns, quotaname, fmt.Sprint(q.Limit)); err != nil { klog.V(4).Infof("Failed to send quotaUsedDesc metric: %v", err) } + if err := sendMetric(ch, quotaLimitDesc, prometheus.GaugeValue, float64(q.Limit), ns, quotaname); err != nil { + klog.V(4).Infof("Failed to send quotaLimitDesc metric: %v", err) + } if legacy { sendLegacyMetric(ch, legacyQuotaUsed, prometheus.GaugeValue, float64(q.Used), ns, quotaname, fmt.Sprint(q.Limit)) } diff --git a/cmd/scheduler/metrics_test.go b/cmd/scheduler/metrics_test.go index 7707ab2fe1..57e4be5482 100644 --- a/cmd/scheduler/metrics_test.go +++ b/cmd/scheduler/metrics_test.go @@ -438,6 +438,10 @@ func TestClusterManagerCollectorQuotaMetrics(t *testing.T) { metricsProvider: provider, } want := ` +# HELP hami_resource_quota_limit resourcequota limit for a certain device +# TYPE hami_resource_quota_limit gauge +hami_resource_quota_limit{namespace="team-a",quota_name="nvidia.com/gpucore"} 100 +hami_resource_quota_limit{namespace="team-a",quota_name="nvidia.com/gpumem"} 8192 # HELP hami_resource_quota_used resourcequota usage for a certain device # TYPE hami_resource_quota_used gauge hami_resource_quota_used{limit="100",namespace="team-a",quota_name="nvidia.com/gpucore"} 50 @@ -447,6 +451,7 @@ hami_resource_quota_used{limit="8192",namespace="team-a",quota_name="nvidia.com/ collector, strings.NewReader(want), "hami_resource_quota_used", + "hami_resource_quota_limit", ); err != nil { t.Fatalf("unexpected non-legacy collecting result:\n%s", err) } @@ -462,6 +467,10 @@ hami_resource_quota_used{limit="8192",namespace="team-a",quota_name="nvidia.com/ # TYPE QuotaUsed gauge QuotaUsed{limit="100",quotaName="nvidia.com/gpucore",quotanamespace="team-a"} 50 QuotaUsed{limit="8192",quotaName="nvidia.com/gpumem",quotanamespace="team-a"} 4096 +# HELP hami_resource_quota_limit resourcequota limit for a certain device +# TYPE hami_resource_quota_limit gauge +hami_resource_quota_limit{namespace="team-a",quota_name="nvidia.com/gpucore"} 100 +hami_resource_quota_limit{namespace="team-a",quota_name="nvidia.com/gpumem"} 8192 # HELP hami_resource_quota_used resourcequota usage for a certain device # TYPE hami_resource_quota_used gauge hami_resource_quota_used{limit="100",namespace="team-a",quota_name="nvidia.com/gpucore"} 50 @@ -471,6 +480,7 @@ hami_resource_quota_used{limit="8192",namespace="team-a",quota_name="nvidia.com/ collector, strings.NewReader(want), "hami_resource_quota_used", + "hami_resource_quota_limit", "QuotaUsed", ); err != nil { t.Fatalf("unexpected legacy collecting result:\n%s", err) From adf311365ec29b9c1814b59398a2ddedb6981541 Mon Sep 17 00:00:00 2001 From: SK8-infi Date: Wed, 19 Aug 2026 03:00:56 +0530 Subject: [PATCH 2/2] fix(scheduler): omit hami_resource_quota_limit when quota limit is unconfigured (LimitSet == false) Signed-off-by: SK8-infi --- cmd/scheduler/metrics.go | 6 ++++-- cmd/scheduler/metrics_test.go | 36 +++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/cmd/scheduler/metrics.go b/cmd/scheduler/metrics.go index c5b34a17b6..f23b0e10d7 100644 --- a/cmd/scheduler/metrics.go +++ b/cmd/scheduler/metrics.go @@ -306,8 +306,10 @@ func (cc ClusterManagerCollector) collectQuotaMetrics(ch chan<- prometheus.Metri if err := sendMetric(ch, quotaUsedDesc, prometheus.GaugeValue, float64(q.Used), ns, quotaname, fmt.Sprint(q.Limit)); err != nil { klog.V(4).Infof("Failed to send quotaUsedDesc metric: %v", err) } - if err := sendMetric(ch, quotaLimitDesc, prometheus.GaugeValue, float64(q.Limit), ns, quotaname); err != nil { - klog.V(4).Infof("Failed to send quotaLimitDesc metric: %v", err) + if q.LimitSet { + if err := sendMetric(ch, quotaLimitDesc, prometheus.GaugeValue, float64(q.Limit), ns, quotaname); err != nil { + klog.V(4).Infof("Failed to send quotaLimitDesc metric: %v", err) + } } if legacy { sendLegacyMetric(ch, legacyQuotaUsed, prometheus.GaugeValue, float64(q.Used), ns, quotaname, fmt.Sprint(q.Limit)) diff --git a/cmd/scheduler/metrics_test.go b/cmd/scheduler/metrics_test.go index 57e4be5482..0456e0a5e1 100644 --- a/cmd/scheduler/metrics_test.go +++ b/cmd/scheduler/metrics_test.go @@ -487,3 +487,39 @@ hami_resource_quota_used{limit="8192",namespace="team-a",quota_name="nvidia.com/ } }) } + +func TestClusterManagerCollectorQuotaUnconfiguredLimit(t *testing.T) { + const ( + ns = "team-b" + memName = "nvidia.com/gpumem" + ) + + unconfQm := device.NewQuotaManager() + unconfQm.Quotas[ns] = &device.DeviceQuota{ + memName: &device.Quota{Used: 1024, Limit: 0, LimitSet: false}, + } + t.Cleanup(func() { delete(unconfQm.Quotas, ns) }) + + unconfProvider := &fakeMetricsProvider{ + nodeUsage: map[string]*schedulerpkg.NodeUsage{}, + quotaManager: unconfQm, + podManager: device.NewPodManager(), + } + collector := ClusterManagerCollector{ + ClusterManager: &ClusterManager{LegacyMetrics: false}, + metricsProvider: unconfProvider, + } + want := ` +# HELP hami_resource_quota_used resourcequota usage for a certain device +# TYPE hami_resource_quota_used gauge +hami_resource_quota_used{limit="0",namespace="team-b",quota_name="nvidia.com/gpumem"} 1024 +` + if err := promtestutil.CollectAndCompare( + collector, + strings.NewReader(want), + "hami_resource_quota_used", + "hami_resource_quota_limit", + ); err != nil { + t.Fatalf("unexpected unconfigured limit collecting result:\n%s", err) + } +}