diff --git a/api/go.mod b/api/go.mod index e15e46b45651c..5f15a30407446 100644 --- a/api/go.mod +++ b/api/go.mod @@ -11,6 +11,7 @@ require ( github.com/google/uuid v1.6.0 github.com/gravitational/trace v1.4.0 github.com/jonboulle/clockwork v0.4.0 + github.com/prometheus/client_golang v1.19.0 github.com/russellhaering/gosaml2 v0.9.1 github.com/stretchr/testify v1.9.0 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.50.0 @@ -34,7 +35,9 @@ require ( require ( github.com/beevik/etree v1.3.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.1 // indirect @@ -44,6 +47,9 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/russellhaering/goxmldsig v1.4.0 // indirect golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/api/go.sum b/api/go.sum index 634dc74fddb5d..6cf1eb94f9465 100644 --- a/api/go.sum +++ b/api/go.sum @@ -611,6 +611,8 @@ github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2 github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= github.com/beevik/etree v1.3.0 h1:hQTc+pylzIKDb23yYprodCWWTt+ojFfUZyzU09a/hmU= github.com/beevik/etree v1.3.0/go.mod h1:aiPf89g/1k3AShMVAzriilpcE4R/Vuor90y83zVZWFc= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= @@ -620,6 +622,7 @@ github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -850,9 +853,17 @@ github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= +github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= diff --git a/api/utils/transportlogger/common/result.go b/api/utils/transportlogger/common/result.go new file mode 100644 index 0000000000000..c6018e8573dee --- /dev/null +++ b/api/utils/transportlogger/common/result.go @@ -0,0 +1,39 @@ +/* +Copyright 2021 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import "time" + +// Result is the result of the HTTP API call. +type Result struct { + // Information obtained from metricsInfoKey + Name string + // URL is the URL of the request. + URL string + // Host is the hostname of the request. + Host string + // Method is the HTTP method of the request. + Method string + // Err is the error that occurred during the call. + Err error + // HttpStatusCode is the HTTP status code of the response. + HttpStatusCode int + // Duration is the duration of the call. + Duration time.Duration + // Service is the service name. + Service string +} diff --git a/api/utils/transportlogger/logger.go b/api/utils/transportlogger/logger.go new file mode 100644 index 0000000000000..ba3cafb92eee2 --- /dev/null +++ b/api/utils/transportlogger/logger.go @@ -0,0 +1,90 @@ +/* +Copyright 2024 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package transportlogger + +import ( + "context" + "net/http" + + "github.com/jonboulle/clockwork" + + "github.com/gravitational/teleport/api/utils/transportlogger/common" + "github.com/gravitational/teleport/api/utils/transportlogger/providers/prometheus" +) + +type options struct { + roundTripper http.RoundTripper + loggers []LoggerFunc + clock clockwork.Clock + serviceName string +} + +func (o *options) checkAndSetDefaults() error { + if o.roundTripper == nil { + o.roundTripper = http.DefaultTransport + } + if o.clock == nil { + o.clock = clockwork.NewRealClock() + } + if len(o.loggers) == 0 { + o.loggers = append(o.loggers, prometheus.Logger) + } + return nil +} + +type OptionFunc func(o *options) + +// WithRoundTripper sets the underlying http.RoundTripper to use for making requests. +func WithRoundTripper(rt http.RoundTripper) OptionFunc { + return func(o *options) { + o.roundTripper = rt + } +} + +// WithClock sets the underlying http.RoundTripper to use for making requests. +func WithClock(clock clockwork.Clock) OptionFunc { + return func(o *options) { + o.clock = clock + } +} + +// WithServiceName sets the service name to use for making requests. +func WithServiceName(name string) OptionFunc { + return func(o *options) { + o.serviceName = name + } +} + +// WithPrometheusLogger adds a Prometheus logger to the transport. +// This logger will report the result of the API call as Prometheus metrics. +func WithPrometheusLogger() OptionFunc { + return func(o *options) { + o.loggers = append(o.loggers, prometheus.Logger) + } +} + +// LoggerFunc is a function that logs the result of a call. +type LoggerFunc func(ctx context.Context, result *common.Result) + +func (t *Transport) report(ctx context.Context, r *common.Result) { + for _, logFn := range t.loggers { + if logFn == nil { + continue + } + logFn(ctx, r) + } +} diff --git a/api/utils/transportlogger/logger_test.go b/api/utils/transportlogger/logger_test.go new file mode 100644 index 0000000000000..8c68e7518892a --- /dev/null +++ b/api/utils/transportlogger/logger_test.go @@ -0,0 +1,82 @@ +/* +Copyright 2024 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package transportlogger + +import ( + "bytes" + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" + + "github.com/gravitational/teleport/api/utils/transportlogger/providers/prometheus" +) + +func TestLogger(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/ok" { + w.WriteHeader(http.StatusOK) + } else if r.URL.Path == "/not_found" { + w.WriteHeader(http.StatusNotFound) + } + })) + defer srv.Close() + + transport, err := NewTransport( + WithRoundTripper(srv.Client().Transport), + WithServiceName("test_api"), + ) + require.NoError(t, err) + + httpClient := &http.Client{ + Transport: transport, + } + ctx := context.Background() + req, err := http.NewRequest(http.MethodPost, srv.URL+"/ok", nil) + require.NoError(t, err) + if false { + resp, err := httpClient.Do(req.WithContext(WithMetricInfo(ctx, MetricsInfo{CallName: "ok_endpoint"}))) + require.NoError(t, err) + defer resp.Body.Close() + + req, err = http.NewRequest(http.MethodGet, srv.URL+"/not_found", nil) + require.NoError(t, err) + resp, err = httpClient.Do(req.WithContext(WithMetricInfo(ctx, MetricsInfo{CallName: "not_found_endpoint"}))) + require.NoError(t, err) + defer resp.Body.Close() + + want := ` +# HELP teleport_api_call_status Track calls to 3th party API +# TYPE teleport_api_call_status counter +teleport_api_call_status{endpoint="not_found_endpoint",http_method="GET",http_status="404",service="test_api"} 1 +teleport_api_call_status{endpoint="ok_endpoint",http_method="POST",http_status="200",service="test_api"} 1 +` + err = testutil.CollectAndCompare(prometheus.ExternalAPICallMetric, bytes.NewBufferString(want)) + require.NoError(t, err) + + prometheus.ExternalAPICallMetric.Reset() + } + req, err = http.NewRequest(http.MethodGet, srv.URL+"/not_found", nil) + require.NoError(t, err) + resp, err := httpClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + +} diff --git a/api/utils/transportlogger/metrics_info.go b/api/utils/transportlogger/metrics_info.go new file mode 100644 index 0000000000000..45408d8ebbdb3 --- /dev/null +++ b/api/utils/transportlogger/metrics_info.go @@ -0,0 +1,34 @@ +/* +Copyright 2024 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package transportlogger + +import "context" + +var metricsInfoKey struct{} + +// MetricsInfo is a struct that contains information about the call that is being made. +type MetricsInfo struct { + // CallName is the name of the call that is being made. + // Many REST endpoint uses uuid in the URL, so it's for metrics entropy. + // CallName allow to set limited values for the REST API call that will be used for metrics. + CallName string +} + +// WithMetricInfo sets the metrics information for the call. +func WithMetricInfo(ctx context.Context, info MetricsInfo) context.Context { + return context.WithValue(ctx, metricsInfoKey, info) +} diff --git a/api/utils/transportlogger/providers/prometheus/http_internals.go b/api/utils/transportlogger/providers/prometheus/http_internals.go new file mode 100644 index 0000000000000..eb55391617541 --- /dev/null +++ b/api/utils/transportlogger/providers/prometheus/http_internals.go @@ -0,0 +1,34 @@ +/* +Copyright 2024 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package prometheus + +import ( + _ "net/http" + _ "unsafe" +) + +// Link local error variables to unexported errors from http package by using +// go:linkname directive. + +//go:linkname errTimeout net/http.errTimeout +var errTimeout error + +//go:linkname errRequestCanceled net/http.errRequestCanceled +var errRequestCanceled error + +//go:linkname errRequestCanceledConn net/http.errRequestCanceledConn +var errRequestCanceledConn error diff --git a/api/utils/transportlogger/providers/prometheus/metrics.go b/api/utils/transportlogger/providers/prometheus/metrics.go new file mode 100644 index 0000000000000..4579c3f28c7f0 --- /dev/null +++ b/api/utils/transportlogger/providers/prometheus/metrics.go @@ -0,0 +1,57 @@ +/* +Copyright 2024 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package prometheus + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const ( + endpointLabel = "endpoint" + hostLabel = "endpoint" + httpStatusLabel = "http_status" + httpMethodLabel = "http_method" + serviceLabel = "service" +) + +const ( + // NamespaceTeleport is a namespace for all teleport metrics. + NamespaceTeleport = "teleport" + // APICallStatusName is a label for API call status. + APICallStatusName = "api_call_status" + // APICallTimeName is a label for API call time. + APICallTimeName = "api_call_time" +) + +var ( + ExternalAPICallMetric = prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: NamespaceTeleport, + Name: APICallStatusName, + Help: "Track calls to 3th party API", + }, []string{endpointLabel, httpStatusLabel, httpMethodLabel, serviceLabel}) + + ExternalApiCallTimeMetric = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: NamespaceTeleport, + Name: APICallTimeName, + Help: "API call time in seconds", + }, []string{endpointLabel}) +) + +func init() { + prometheus.MustRegister(ExternalAPICallMetric) + prometheus.MustRegister(ExternalApiCallTimeMetric) +} diff --git a/api/utils/transportlogger/providers/prometheus/prometheus.go b/api/utils/transportlogger/providers/prometheus/prometheus.go new file mode 100644 index 0000000000000..22ca05f726f72 --- /dev/null +++ b/api/utils/transportlogger/providers/prometheus/prometheus.go @@ -0,0 +1,68 @@ +/* +Copyright 2024 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package prometheus + +import ( + "context" + "errors" + "fmt" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/gravitational/teleport/api/utils/transportlogger/common" +) + +// Extended HTTP status by custom context errors +const ( + HTTPErrUnknownStatusLabelValue = "unknown_status" + HTTPErrDeadlineExceededLabelValue = "deadline_exceeded" + HTTPErrContextCanceledLabelValue = "context_canceled" + HTTPErrRequestTimeoutLabelValue = "request_timeout" +) + +func Logger(ctx context.Context, result *common.Result) { + if result == nil { + return + } + ExternalAPICallMetric.With(prometheus.Labels{ + endpointLabel: result.Name, + httpStatusLabel: statusFromResult(result), + httpMethodLabel: result.Method, + serviceLabel: result.Service, + }).Inc() + ExternalApiCallTimeMetric.With(prometheus.Labels{ + endpointLabel: result.Name, + }).Observe(result.Duration.Seconds()) +} + +func statusFromResult(result *common.Result) string { + if result.Err != nil { + switch { + case errors.Is(result.Err, context.DeadlineExceeded): + return HTTPErrDeadlineExceededLabelValue + case errors.Is(result.Err, context.Canceled): + return HTTPErrContextCanceledLabelValue + case errors.Is(result.Err, errTimeout), + errors.Is(result.Err, errRequestCanceled), + errors.Is(result.Err, errRequestCanceledConn): + return HTTPErrRequestTimeoutLabelValue + default: + return HTTPErrUnknownStatusLabelValue + } + } + return fmt.Sprintf("%d", result.HttpStatusCode) +} diff --git a/api/utils/transportlogger/transport.go b/api/utils/transportlogger/transport.go new file mode 100644 index 0000000000000..03bf232bf1fff --- /dev/null +++ b/api/utils/transportlogger/transport.go @@ -0,0 +1,80 @@ +/* +Copyright 2024 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package transportlogger + +import ( + "net/http" + "time" + + "github.com/gravitational/trace" + "github.com/jonboulle/clockwork" + + "github.com/gravitational/teleport/api/utils/transportlogger/common" +) + +// NewTransport creates a new Transport. +func NewTransport(optionsFunc ...OptionFunc) (*Transport, error) { + var settings options + for _, option := range optionsFunc { + option(&settings) + } + if err := settings.checkAndSetDefaults(); err != nil { + return nil, trace.Wrap(err) + } + return &Transport{ + roundTripper: settings.roundTripper, + loggers: settings.loggers, + clock: settings.clock, + serviceName: settings.serviceName, + }, nil +} + +// Transport is transport that logs the result of the API call. +type Transport struct { + roundTripper http.RoundTripper + loggers []LoggerFunc + clock clockwork.Clock + serviceName string +} + +// RoundTrip executes a single HTTP transaction and logs the result of the API call. +func (t *Transport) RoundTrip(r *http.Request) (*http.Response, error) { + result := &common.Result{ + Name: r.URL.Path, + URL: r.URL.String(), + Host: r.URL.Hostname(), + Method: r.Method, + Service: t.serviceName, + } + + if mi, ok := r.Context().Value(metricsInfoKey).(MetricsInfo); ok { + result.Name = mi.CallName + } + + start := time.Now() + resp, err := t.roundTripper.RoundTrip(r) + if err == nil { + result.HttpStatusCode = resp.StatusCode + } + result.Err = err + result.Duration = time.Since(start) + + // report the API call result. + t.report(r.Context(), result) + + return resp, trace.Wrap(err) +}