feat(scheduler): add filter_duration_seconds and filter_denials_total…

cmd/scheduler/metrics.go

@@ -31,6 +31,7 @@ import (
 	"github.com/Project-HAMi/HAMi/pkg/device"
 	versionmetrics "github.com/Project-HAMi/HAMi/pkg/metrics"
 	schedulerpkg "github.com/Project-HAMi/HAMi/pkg/scheduler"
+	"github.com/Project-HAMi/HAMi/pkg/scheduler/routes"
 )
 
 type ClusterManager struct {
@@ -342,6 +343,7 @@ func initMetrics(bindAddress string, metricsProvider schedulerMetricsProvider, l
 	klog.Info("Initializing metrics for scheduler")
 	reg := prometheus.NewRegistry()
 	reg.MustRegister(versionmetrics.NewBuildInfoCollector())
+	routes.RegisterMetrics(reg)
 
 	NewClusterManager("vGPU", reg, metricsProvider, legacyMetrics)
 

pkg/scheduler/routes/metrics.go

@@ -0,0 +1,49 @@
+/*
+Copyright 2024 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package routes
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	// FilterDuration is a histogram to measure the time taken by the filter phase of the scheduler.
+	FilterDuration = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name: "hami_scheduler_filter_duration_seconds",
+			Help: "Duration of the filter phase in the scheduler extender",
+			// Buckets from 10ms to 10s
+			Buckets: prometheus.ExponentialBucketsRange(0.01, 10, 10),
+		},
+		[]string{"node", "result"}, // result: allowed, denied
+	)
+
+	// FilterDenials is a counter to track the number of pod denials during the filter phase.
+	FilterDenials = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "hami_scheduler_filter_denials_total",
+			Help: "Total number of scheduling filter denials",
+		},
+		[]string{"node", "reason"}, // reason why it was denied
+	)
+)
+
+// RegisterMetrics registers the prometheus metrics defined in this package.
+func RegisterMetrics(registry prometheus.Registerer) {
+	registry.MustRegister(FilterDuration)
+	registry.MustRegister(FilterDenials)
+}

pkg/scheduler/routes/route.go

@@ -22,6 +22,8 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"strings"
+	"time"
 
 	"github.com/julienschmidt/httprouter"
 	"k8s.io/klog/v2"
@@ -71,11 +73,35 @@ func PredicateRoute(s *scheduler.Scheduler) httprouter.Handle {
 					Error: err.Error(),
 				}
 			} else {
-				extenderFilterResult, err = s.Filter(extenderArgs)
-				if err != nil {
-					klog.ErrorS(err, "Filter error for pod", "pod", extenderArgs.Pod.Name)
+				startTime := time.Now()
+				var filterErr error
+				extenderFilterResult, filterErr = s.Filter(extenderArgs)
+				duration := time.Since(startTime).Seconds()
+
+				if filterErr != nil {
+					klog.ErrorS(filterErr, "Filter error for pod", "pod", extenderArgs.Pod.Name)
 					extenderFilterResult = &extenderv1.ExtenderFilterResult{
-						Error: err.Error(),
+						Error: filterErr.Error(),
+					}
+				}
+
+				if extenderFilterResult != nil && extenderFilterResult.FailedNodes != nil {
+					for failedNode, rawReason := range extenderFilterResult.FailedNodes {
+						reasonCode := "hami.io/unknown"
+						if strings.Contains(rawReason, "Insufficient") {
+							reasonCode = "hami.io/insufficient-resources"
+						} else if strings.Contains(rawReason, "node(s) didn't match") {
+							reasonCode = "hami.io/node-affinity"
+						} else if strings.Contains(rawReason, "device") || strings.Contains(rawReason, "vgpu") || strings.Contains(rawReason, "nvidia") {
+							reasonCode = "hami.io/device-unavailable"
+						}
+						FilterDenials.WithLabelValues(failedNode, reasonCode).Inc()
+						FilterDuration.WithLabelValues(failedNode, "denied").Observe(duration)
+					}
+				}
+				if extenderFilterResult != nil && extenderFilterResult.Nodes != nil {
+					for _, allowedNode := range extenderFilterResult.Nodes.Items {
+						FilterDuration.WithLabelValues(allowedNode.Name, "allowed").Observe(duration)
 					}
 				}
 			}