diff --git a/cmd/scheduler/metrics.go b/cmd/scheduler/metrics.go index d652582253..c7955ebbd1 100644 --- a/cmd/scheduler/metrics.go +++ b/cmd/scheduler/metrics.go @@ -31,6 +31,7 @@ import ( "github.com/Project-HAMi/HAMi/pkg/device" versionmetrics "github.com/Project-HAMi/HAMi/pkg/metrics" schedulerpkg "github.com/Project-HAMi/HAMi/pkg/scheduler" + "github.com/Project-HAMi/HAMi/pkg/scheduler/routes" ) type ClusterManager struct { @@ -342,6 +343,7 @@ func initMetrics(bindAddress string, metricsProvider schedulerMetricsProvider, l klog.Info("Initializing metrics for scheduler") reg := prometheus.NewRegistry() reg.MustRegister(versionmetrics.NewBuildInfoCollector()) + routes.RegisterMetrics(reg) NewClusterManager("vGPU", reg, metricsProvider, legacyMetrics) diff --git a/pkg/scheduler/routes/metrics.go b/pkg/scheduler/routes/metrics.go new file mode 100644 index 0000000000..7df15e4f64 --- /dev/null +++ b/pkg/scheduler/routes/metrics.go @@ -0,0 +1,49 @@ +/* +Copyright 2024 The HAMi Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package routes + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +var ( + // FilterDuration is a histogram to measure the time taken by the filter phase of the scheduler. + FilterDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "hami_scheduler_filter_duration_seconds", + Help: "Duration of the filter phase in the scheduler extender", + // Buckets from 10ms to 10s + Buckets: prometheus.ExponentialBucketsRange(0.01, 10, 10), + }, + []string{"node", "result"}, // result: allowed, denied + ) + + // FilterDenials is a counter to track the number of pod denials during the filter phase. + FilterDenials = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "hami_scheduler_filter_denials_total", + Help: "Total number of scheduling filter denials", + }, + []string{"node", "reason"}, // reason why it was denied + ) +) + +// RegisterMetrics registers the prometheus metrics defined in this package. +func RegisterMetrics(registry prometheus.Registerer) { + registry.MustRegister(FilterDuration) + registry.MustRegister(FilterDenials) +} diff --git a/pkg/scheduler/routes/route.go b/pkg/scheduler/routes/route.go index 8c5d29410e..87d6042034 100644 --- a/pkg/scheduler/routes/route.go +++ b/pkg/scheduler/routes/route.go @@ -22,6 +22,8 @@ import ( "fmt" "io" "net/http" + "strings" + "time" "github.com/julienschmidt/httprouter" "k8s.io/klog/v2" @@ -71,11 +73,35 @@ func PredicateRoute(s *scheduler.Scheduler) httprouter.Handle { Error: err.Error(), } } else { - extenderFilterResult, err = s.Filter(extenderArgs) - if err != nil { - klog.ErrorS(err, "Filter error for pod", "pod", extenderArgs.Pod.Name) + startTime := time.Now() + var filterErr error + extenderFilterResult, filterErr = s.Filter(extenderArgs) + duration := time.Since(startTime).Seconds() + + if filterErr != nil { + klog.ErrorS(filterErr, "Filter error for pod", "pod", extenderArgs.Pod.Name) extenderFilterResult = &extenderv1.ExtenderFilterResult{ - Error: err.Error(), + Error: filterErr.Error(), + } + } + + if extenderFilterResult != nil && extenderFilterResult.FailedNodes != nil { + for failedNode, rawReason := range extenderFilterResult.FailedNodes { + reasonCode := "hami.io/unknown" + if strings.Contains(rawReason, "Insufficient") { + reasonCode = "hami.io/insufficient-resources" + } else if strings.Contains(rawReason, "node(s) didn't match") { + reasonCode = "hami.io/node-affinity" + } else if strings.Contains(rawReason, "device") || strings.Contains(rawReason, "vgpu") || strings.Contains(rawReason, "nvidia") { + reasonCode = "hami.io/device-unavailable" + } + FilterDenials.WithLabelValues(failedNode, reasonCode).Inc() + FilterDuration.WithLabelValues(failedNode, "denied").Observe(duration) + } + } + if extenderFilterResult != nil && extenderFilterResult.Nodes != nil { + for _, allowedNode := range extenderFilterResult.Nodes.Items { + FilterDuration.WithLabelValues(allowedNode.Name, "allowed").Observe(duration) } } }