Skip to content

Commit e7198f8

Browse files
fsaminfsamin
authored
feat(engine, api): metrics through REST API (#5089)
1 parent b779efb commit e7198f8

File tree

6 files changed

+153
-14
lines changed

6 files changed

+153
-14
lines changed

‎engine/api/api_routes.go

+2
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package api
33
import (
44
"net/http"
55

6+
"github.com/ovh/cds/engine/api/observability"
67
"github.com/ovh/cds/engine/service"
78
"github.com/ovh/cds/sdk"
89
"github.com/ovh/cds/sdk/log"
@@ -136,6 +137,7 @@ func (api *API) InitRouter() {
136137
r.Handle("/mon/db/migrate", ScopeNone(), r.GET(api.getMonDBStatusMigrateHandler, NeedAdmin(true)))
137138
r.Handle("/mon/metrics", ScopeNone(), r.GET(service.GetPrometheustMetricsHandler(api), Auth(false)))
138139
r.Handle("/mon/metrics/all", ScopeNone(), r.GET(service.GetMetricsHandler, Auth(false)))
140+
r.HandlePrefix("/mon/metrics/detail/", ScopeNone(), r.GET(service.GetMetricHandler(observability.StatsHTTPExporter(), "/mon/metrics/detail/"), Auth(false)))
139141
r.Handle("/mon/errors/{uuid}", ScopeNone(), r.GET(api.getErrorHandler, NeedAdmin(true)))
140142
r.Handle("/mon/panic/{uuid}", ScopeNone(), r.GET(api.getPanicDumpHandler, Auth(false)))
141143

‎engine/api/observability/http_exporter.go

+65
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
package observability
2+
3+
import (
4+
"reflect"
5+
"time"
6+
7+
"go.opencensus.io/stats/view"
8+
)
9+
10+
type HTTPExporter struct {
11+
Views []HTTPExporterView `json:"views"`
12+
}
13+
14+
type HTTPExporterView struct {
15+
Name string `json:"name"`
16+
Tags map[string]string `json:"tags"`
17+
Value float64 `json:"value"`
18+
Date time.Time `json:"date"`
19+
}
20+
21+
func (e *HTTPExporter) GetView(name string, tags map[string]string) *HTTPExporterView {
22+
for i := range e.Views {
23+
if e.Views[i].Name == name && reflect.DeepEqual(e.Views[i].Tags, tags) {
24+
return &e.Views[i]
25+
}
26+
}
27+
return nil
28+
}
29+
30+
func (e *HTTPExporter) NewView(name string, tags map[string]string) *HTTPExporterView {
31+
v := HTTPExporterView{
32+
Name: name,
33+
Tags: tags,
34+
}
35+
e.Views = append(e.Views, v)
36+
return &v
37+
}
38+
39+
func (e *HTTPExporter) ExportView(vd *view.Data) {
40+
for _, row := range vd.Rows {
41+
tags := make(map[string]string)
42+
for _, t := range row.Tags {
43+
tags[t.Key.Name()] = t.Value
44+
}
45+
view := e.GetView(vd.View.Name, tags)
46+
if view == nil {
47+
view = e.NewView(vd.View.Name, tags)
48+
}
49+
view.Record(row.Data)
50+
}
51+
}
52+
53+
func (v *HTTPExporterView) Record(data view.AggregationData) {
54+
v.Date = time.Now()
55+
switch x := data.(type) {
56+
case *view.DistributionData:
57+
v.Value = x.Mean
58+
case *view.CountData:
59+
v.Value = float64(x.Value)
60+
case *view.SumData:
61+
v.Value = x.Value
62+
case *view.LastValueData:
63+
v.Value = x.Value
64+
}
65+
}

‎engine/api/observability/init.go

+11-2
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,9 @@ import (
1414
)
1515

1616
var (
17-
traceExporter trace.Exporter
18-
statsExporter *prometheus.Exporter
17+
traceExporter trace.Exporter
18+
statsExporter *prometheus.Exporter
19+
statsHTTPExporter *HTTPExporter
1920
)
2021

2122
type service interface {
@@ -31,6 +32,10 @@ func StatsExporter() *prometheus.Exporter {
3132
return statsExporter
3233
}
3334

35+
func StatsHTTPExporter() *HTTPExporter {
36+
return statsHTTPExporter
37+
}
38+
3439
// Init the opencensus exporter
3540
func Init(ctx context.Context, cfg Configuration, s service) (context.Context, error) {
3641
ctx = ContextWithTag(ctx,
@@ -70,6 +75,10 @@ func Init(ctx context.Context, cfg Configuration, s service) (context.Context, e
7075
}
7176
view.RegisterExporter(e)
7277
statsExporter = e
78+
79+
he := new(HTTPExporter)
80+
view.RegisterExporter(he)
81+
statsHTTPExporter = he
7382
}
7483

7584
return ctx, nil

‎engine/api/router.go

+23-12
Original file line numberDiff line numberDiff line change
@@ -71,9 +71,9 @@ type HandlerConfigParam func(*service.HandlerConfig)
7171
type HandlerConfigFunc func(service.Handler, ...HandlerConfigParam) *service.HandlerConfig
7272

7373
func (r *Router) pprofLabel(config map[string]*service.HandlerConfig, fn http.HandlerFunc) http.HandlerFunc {
74-
return func(w http.ResponseWriter, r *http.Request) {
74+
return func(w http.ResponseWriter, req *http.Request) {
7575
var name = sdk.RandomString(12)
76-
rc := config[r.Method]
76+
rc := config[req.Method]
7777
if rc != nil && rc.Handler != nil {
7878
name = runtime.FuncForPC(reflect.ValueOf(rc.Handler).Pointer()).Name()
7979
name = strings.Replace(name, ".func1", "", 1)
@@ -82,14 +82,14 @@ func (r *Router) pprofLabel(config map[string]*service.HandlerConfig, fn http.Ha
8282
id := fmt.Sprintf("%d", sdk.GoroutineID())
8383

8484
labels := pprof.Labels(
85-
"http-path", r.URL.Path,
85+
"http-path", req.URL.Path,
8686
"goroutine-id", id,
8787
"goroutine-name", name+"-"+id,
8888
)
89-
ctx := pprof.WithLabels(r.Context(), labels)
89+
ctx := pprof.WithLabels(req.Context(), labels)
9090
pprof.SetGoroutineLabels(ctx)
91-
r = r.WithContext(ctx)
92-
fn(w, r)
91+
req = req.WithContext(ctx)
92+
fn(w, req)
9393
}
9494
}
9595

@@ -112,10 +112,10 @@ func (r *Router) recoverWrap(h http.HandlerFunc) http.HandlerFunc {
112112
switch t := re.(type) {
113113
case string:
114114
err = errors.New(t)
115-
case error:
116-
err = re.(error)
117115
case sdk.Error:
118116
err = re.(sdk.Error)
117+
case error:
118+
err = re.(error)
119119
default:
120120
err = sdk.ErrUnknownError
121121
}
@@ -250,6 +250,18 @@ func (r *Router) computeScopeDetails() {
250250
// Handle adds all handler for their specific verb in gorilla router for given uri
251251
func (r *Router) Handle(uri string, scope HandlerScope, handlers ...*service.HandlerConfig) {
252252
uri = r.Prefix + uri
253+
config, f := r.handle(uri, scope, handlers...)
254+
r.Mux.Handle(uri, r.pprofLabel(config, r.compress(r.recoverWrap(f))))
255+
}
256+
257+
func (r *Router) HandlePrefix(uri string, scope HandlerScope, handlers ...*service.HandlerConfig) {
258+
uri = r.Prefix + uri
259+
config, f := r.handle(uri, scope, handlers...)
260+
r.Mux.PathPrefix(uri).HandlerFunc(r.pprofLabel(config, r.compress(r.recoverWrap(f))))
261+
}
262+
263+
// Handle adds all handler for their specific verb in gorilla router for given uri
264+
func (r *Router) handle(uri string, scope HandlerScope, handlers ...*service.HandlerConfig) (map[string]*service.HandlerConfig, http.HandlerFunc) {
253265
cfg := &service.RouterConfig{
254266
Config: map[string]*service.HandlerConfig{},
255267
}
@@ -385,7 +397,7 @@ func (r *Router) Handle(uri string, scope HandlerScope, handlers ...*service.Han
385397
"route": cleanURL,
386398
"request_uri": req.RequestURI,
387399
"deprecated": rc.IsDeprecated,
388-
}, "[%d] | %s | END | %s [%s]", responseWriter.statusCode, req.Method, req.URL, rc.Name)
400+
}, "%s | END | %s [%s] | [%d]", req.Method, req.URL, rc.Name, responseWriter.statusCode)
389401

390402
observability.RecordFloat64(ctx, ServerLatency, float64(latency)/float64(time.Millisecond))
391403
observability.Record(ctx, ServerRequestBytes, responseWriter.reqSize)
@@ -429,8 +441,7 @@ func (r *Router) Handle(uri string, scope HandlerScope, handlers ...*service.Han
429441
deferFunc(ctx)
430442
}
431443

432-
// The chain is http -> mux -> f -> recover -> wrap -> pprof -> opencensus -> http
433-
r.Mux.Handle(uri, r.pprofLabel(cfg.Config, r.compress(r.recoverWrap(f))))
444+
return cfg.Config, f
434445
}
435446

436447
type asynchronousRequest struct {
@@ -627,7 +638,7 @@ func EnableTracing() HandlerConfigParam {
627638

628639
// NotFoundHandler is called by default by Mux is any matching handler has been found
629640
func NotFoundHandler(w http.ResponseWriter, req *http.Request) {
630-
service.WriteError(context.Background(), w, req, sdk.WithStack(sdk.ErrNotFound))
641+
service.WriteError(context.Background(), w, req, sdk.NewError(sdk.ErrNotFound, fmt.Errorf("%s not found", req.URL.Path)))
631642
}
632643

633644
// StatusPanic returns router status. If nbPanic > 30 -> Alert, if nbPanic > 0 -> Warn

‎engine/service/metrics.go

+51
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,13 @@ package service
22

33
import (
44
"context"
5+
"encoding/json"
56
"fmt"
67
"net/http"
78
"os"
89
"runtime"
910
"strconv"
11+
"strings"
1012
"sync"
1113
"time"
1214

@@ -198,3 +200,52 @@ func RegisterCommonMetricsView(ctx context.Context) {
198200
})
199201
})
200202
}
203+
204+
func writeJSON(w http.ResponseWriter, i interface{}, statusCode int) error {
205+
btes, _ := json.Marshal(i)
206+
w.Header().Add("Content-Type", "application/json")
207+
w.Header().Add("Content-Length", fmt.Sprintf("%d", len(btes)))
208+
w.WriteHeader(statusCode)
209+
_, err := w.Write(btes)
210+
return sdk.WithStack(err)
211+
}
212+
213+
func GetMetricHandler(exporter *observability.HTTPExporter, prefix string) func() Handler {
214+
return func() Handler {
215+
return func(ctx context.Context, w http.ResponseWriter, r *http.Request) error {
216+
view := strings.TrimPrefix(r.URL.Path, prefix)
217+
formValues := r.URL.Query()
218+
tags := make(map[string]string)
219+
threshold := formValues.Get("threshold")
220+
for k := range formValues {
221+
if k != "threshold" {
222+
tags[k] = formValues.Get(k)
223+
}
224+
}
225+
log.Debug("GetMetricHandler> path: %s - tags: %v", view, tags)
226+
227+
if view == "" {
228+
return writeJSON(w, exporter, http.StatusOK)
229+
}
230+
231+
metricsView := exporter.GetView(view, tags)
232+
if metricsView == nil {
233+
return sdk.WithStack(sdk.ErrNotFound)
234+
}
235+
236+
statusCode := http.StatusOK
237+
if threshold != "" {
238+
thresholdF, err := strconv.ParseFloat(threshold, 64)
239+
if err != nil {
240+
return sdk.WithStack(sdk.ErrWrongRequest)
241+
}
242+
if metricsView.Value >= thresholdF {
243+
log.Error(context.Background(), "GetMetricHandler> %s threshold (%s) reached or exceeded : %v", metricsView.Name, threshold, metricsView.Value)
244+
statusCode = 509 // Bandwidth Limit Exceeded
245+
}
246+
}
247+
248+
return writeJSON(w, metricsView, statusCode)
249+
}
250+
}
251+
}

‎go.sum

+1
Original file line numberDiff line numberDiff line change
@@ -465,6 +465,7 @@ github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDf
465465
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
466466
github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8=
467467
github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
468+
github.com/prometheus/client_golang v1.5.1 h1:bdHYieyGlH+6OLEk2YQha8THib30KP0/yD0YH9m6xcA=
468469
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
469470
github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
470471
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=

0 commit comments

Comments
 (0)