diff --git a/cmd/scheduler/metrics.go b/cmd/scheduler/metrics.go index e8ddfb6ae9..f23b0e10d7 100644 --- a/cmd/scheduler/metrics.go +++ b/cmd/scheduler/metrics.go @@ -281,13 +281,18 @@ func (cc ClusterManagerCollector) collectNodeMetrics(ch chan<- prometheus.Metric } } -// collectQuotaMetrics emits per-namespace resource quota usage. +// collectQuotaMetrics emits per-namespace resource quota usage and limits. func (cc ClusterManagerCollector) collectQuotaMetrics(ch chan<- prometheus.Metric, legacy bool) { quotaUsedDesc := prometheus.NewDesc( "hami_resource_quota_used", "resourcequota usage for a certain device", []string{"namespace", "quota_name", "limit"}, nil, ) + quotaLimitDesc := prometheus.NewDesc( + "hami_resource_quota_limit", + "resourcequota limit for a certain device", + []string{"namespace", "quota_name"}, nil, + ) var legacyQuotaUsed *prometheus.Desc if legacy { legacyQuotaUsed = prometheus.NewDesc( @@ -301,6 +306,11 @@ func (cc ClusterManagerCollector) collectQuotaMetrics(ch chan<- prometheus.Metri if err := sendMetric(ch, quotaUsedDesc, prometheus.GaugeValue, float64(q.Used), ns, quotaname, fmt.Sprint(q.Limit)); err != nil { klog.V(4).Infof("Failed to send quotaUsedDesc metric: %v", err) } + if q.LimitSet { + if err := sendMetric(ch, quotaLimitDesc, prometheus.GaugeValue, float64(q.Limit), ns, quotaname); err != nil { + klog.V(4).Infof("Failed to send quotaLimitDesc metric: %v", err) + } + } if legacy { sendLegacyMetric(ch, legacyQuotaUsed, prometheus.GaugeValue, float64(q.Used), ns, quotaname, fmt.Sprint(q.Limit)) } diff --git a/cmd/scheduler/metrics_test.go b/cmd/scheduler/metrics_test.go index 7707ab2fe1..0456e0a5e1 100644 --- a/cmd/scheduler/metrics_test.go +++ b/cmd/scheduler/metrics_test.go @@ -438,6 +438,10 @@ func TestClusterManagerCollectorQuotaMetrics(t *testing.T) { metricsProvider: provider, } want := ` +# HELP hami_resource_quota_limit resourcequota limit for a certain device +# TYPE hami_resource_quota_limit gauge +hami_resource_quota_limit{namespace="team-a",quota_name="nvidia.com/gpucore"} 100 +hami_resource_quota_limit{namespace="team-a",quota_name="nvidia.com/gpumem"} 8192 # HELP hami_resource_quota_used resourcequota usage for a certain device # TYPE hami_resource_quota_used gauge hami_resource_quota_used{limit="100",namespace="team-a",quota_name="nvidia.com/gpucore"} 50 @@ -447,6 +451,7 @@ hami_resource_quota_used{limit="8192",namespace="team-a",quota_name="nvidia.com/ collector, strings.NewReader(want), "hami_resource_quota_used", + "hami_resource_quota_limit", ); err != nil { t.Fatalf("unexpected non-legacy collecting result:\n%s", err) } @@ -462,6 +467,10 @@ hami_resource_quota_used{limit="8192",namespace="team-a",quota_name="nvidia.com/ # TYPE QuotaUsed gauge QuotaUsed{limit="100",quotaName="nvidia.com/gpucore",quotanamespace="team-a"} 50 QuotaUsed{limit="8192",quotaName="nvidia.com/gpumem",quotanamespace="team-a"} 4096 +# HELP hami_resource_quota_limit resourcequota limit for a certain device +# TYPE hami_resource_quota_limit gauge +hami_resource_quota_limit{namespace="team-a",quota_name="nvidia.com/gpucore"} 100 +hami_resource_quota_limit{namespace="team-a",quota_name="nvidia.com/gpumem"} 8192 # HELP hami_resource_quota_used resourcequota usage for a certain device # TYPE hami_resource_quota_used gauge hami_resource_quota_used{limit="100",namespace="team-a",quota_name="nvidia.com/gpucore"} 50 @@ -471,9 +480,46 @@ hami_resource_quota_used{limit="8192",namespace="team-a",quota_name="nvidia.com/ collector, strings.NewReader(want), "hami_resource_quota_used", + "hami_resource_quota_limit", "QuotaUsed", ); err != nil { t.Fatalf("unexpected legacy collecting result:\n%s", err) } }) } + +func TestClusterManagerCollectorQuotaUnconfiguredLimit(t *testing.T) { + const ( + ns = "team-b" + memName = "nvidia.com/gpumem" + ) + + unconfQm := device.NewQuotaManager() + unconfQm.Quotas[ns] = &device.DeviceQuota{ + memName: &device.Quota{Used: 1024, Limit: 0, LimitSet: false}, + } + t.Cleanup(func() { delete(unconfQm.Quotas, ns) }) + + unconfProvider := &fakeMetricsProvider{ + nodeUsage: map[string]*schedulerpkg.NodeUsage{}, + quotaManager: unconfQm, + podManager: device.NewPodManager(), + } + collector := ClusterManagerCollector{ + ClusterManager: &ClusterManager{LegacyMetrics: false}, + metricsProvider: unconfProvider, + } + want := ` +# HELP hami_resource_quota_used resourcequota usage for a certain device +# TYPE hami_resource_quota_used gauge +hami_resource_quota_used{limit="0",namespace="team-b",quota_name="nvidia.com/gpumem"} 1024 +` + if err := promtestutil.CollectAndCompare( + collector, + strings.NewReader(want), + "hami_resource_quota_used", + "hami_resource_quota_limit", + ); err != nil { + t.Fatalf("unexpected unconfigured limit collecting result:\n%s", err) + } +}