diff --git a/pkg/tempo/servicegraphprocessor/factory.go b/pkg/tempo/servicegraphprocessor/factory.go index 2e96b4d9481f..1c752cb47526 100644 --- a/pkg/tempo/servicegraphprocessor/factory.go +++ b/pkg/tempo/servicegraphprocessor/factory.go @@ -26,6 +26,13 @@ type Config struct { Wait time.Duration `mapstructure:"wait"` MaxItems int `mapstructure:"max_items"` + + SuccessCodes *successCodes `mapstructure:"success_codes"` +} + +type successCodes struct { + http []int64 `mapstructure:"http"` + grpc []int64 `mapstructure:"grpc"` } // NewFactory returns a new factory for the Prometheus service graph processor. diff --git a/pkg/tempo/servicegraphprocessor/processor.go b/pkg/tempo/servicegraphprocessor/processor.go index 964e77e48662..4403e7e598d6 100644 --- a/pkg/tempo/servicegraphprocessor/processor.go +++ b/pkg/tempo/servicegraphprocessor/processor.go @@ -18,6 +18,7 @@ import ( "go.opentelemetry.io/collector/consumer" "go.opentelemetry.io/collector/model/pdata" "go.opentelemetry.io/collector/translator/conventions" + "google.golang.org/grpc/codes" ) var ( @@ -29,6 +30,8 @@ type edgeRequest struct { serverService, clientService string serverLatency, clientLatency time.Duration + // If either the client or the server spans have status code error, + // the request will be considered as failed. failed bool } @@ -55,6 +58,9 @@ type processor struct { serviceGraphUnpairedSpansTotal *prometheus.CounterVec serviceGraphUntaggedSpansTotal *prometheus.CounterVec + httpSuccessCode map[int]struct{} + grpcSuccessCode map[int]struct{} + logger log.Logger } @@ -68,6 +74,19 @@ func newProcessor(nextConsumer consumer.Traces, cfg *Config) *processor { cfg.MaxItems = DefaultMaxItems } + var ( + httpSuccessCode = make(map[int]struct{}) + grpcSuccessCode = make(map[int]struct{}) + ) + if cfg.SuccessCodes != nil { + for _, sc := range cfg.SuccessCodes.http { + httpSuccessCode[int(sc)] = struct{}{} + } + for _, sc := range cfg.SuccessCodes.grpc { + grpcSuccessCode[int(sc)] = struct{}{} + } + } + // TODO(mapno): Add support for an external cache (e.g. memcached) p := &processor{ nextConsumer: nextConsumer, @@ -238,6 +257,7 @@ func (p *processor) consume(trace pdata.Traces) error { } r.clientService = svc.StringVal() r.clientLatency = spanDuration(span) + r.failed = p.spanFailed(span) p.store.SetDefault(k, r) case pdata.SpanKindServer: @@ -250,6 +270,7 @@ func (p *processor) consume(trace pdata.Traces) error { r.serverService = svc.StringVal() r.serverLatency = spanDuration(span) + r.failed = p.spanFailed(span) p.store.SetDefault(k, r) default: @@ -261,6 +282,26 @@ func (p *processor) consume(trace pdata.Traces) error { return nil } +func (p *processor) spanFailed(span pdata.Span) bool { + // Request considered failed if status is not 2XX or added as a successful status code + if statusCode, ok := span.Attributes().Get("http.status_code"); ok { + sc := int(statusCode.IntVal()) + if _, ok := p.httpSuccessCode[sc]; !ok || sc/100 != 2 { + return true + } + } + + // Request considered failed if status is not OK or added as a successful status code + if statusCode, ok := span.Attributes().Get("grpc.status_code"); ok { + sc := int(statusCode.IntVal()) + if _, ok := p.grpcSuccessCode[sc]; !ok || sc != int(codes.OK) { + return true + } + } + + return span.Status().Code() == pdata.StatusCodeError +} + func spanDuration(span pdata.Span) time.Duration { return span.EndTimestamp().AsTime().Sub(span.StartTimestamp().AsTime()) } diff --git a/pkg/tempo/servicegraphprocessor/processor_test.go b/pkg/tempo/servicegraphprocessor/processor_test.go index 4948ad667884..dd20e931f7ab 100644 --- a/pkg/tempo/servicegraphprocessor/processor_test.go +++ b/pkg/tempo/servicegraphprocessor/processor_test.go @@ -151,6 +151,10 @@ const ( tempo_service_graph_request_client_seconds_bucket{client="lb",server="app",le="+Inf"} 3 tempo_service_graph_request_client_seconds_sum{client="lb",server="app"} 7.8 tempo_service_graph_request_client_seconds_count{client="lb",server="app"} 3 + # HELP tempo_service_graph_request_failed_total Total count of failed requests between two nodes + # TYPE tempo_service_graph_request_failed_total counter + tempo_service_graph_request_failed_total{client="app",server="db"} 3 + tempo_service_graph_request_failed_total{client="lb",server="app"} 3 # HELP tempo_service_graph_request_server_seconds Time for a request between two nodes as seen from the server # TYPE tempo_service_graph_request_server_seconds histogram tempo_service_graph_request_server_seconds_bucket{client="app",server="db",le="0.01"} 0 diff --git a/pkg/tempo/servicegraphprocessor/testdata/trace-sample.json b/pkg/tempo/servicegraphprocessor/testdata/trace-sample.json index e4bc282d71c9..763a70658fc1 100644 --- a/pkg/tempo/servicegraphprocessor/testdata/trace-sample.json +++ b/pkg/tempo/servicegraphprocessor/testdata/trace-sample.json @@ -284,7 +284,8 @@ } ], "status":{ - + "deprecatedCode":"DEPRECATED_STATUS_CODE_UNKNOWN_ERROR", + "code":"STATUS_CODE_ERROR" } }, {