Check span error

pkg/tempo/servicegraphprocessor/factory.go

@@ -26,6 +26,13 @@ type Config struct {
 
 	Wait     time.Duration `mapstructure:"wait"`
 	MaxItems int           `mapstructure:"max_items"`
+
+	SuccessCodes *successCodes `mapstructure:"success_codes"`
+}
+
+type successCodes struct {
+	http []int64 `mapstructure:"http"`
+	grpc []int64 `mapstructure:"grpc"`
 }
 
 // NewFactory returns a new factory for the Prometheus service graph processor.

pkg/tempo/servicegraphprocessor/processor.go

@@ -18,6 +18,7 @@ import (
 	"go.opentelemetry.io/collector/consumer"
 	"go.opentelemetry.io/collector/model/pdata"
 	"go.opentelemetry.io/collector/translator/conventions"
+	"google.golang.org/grpc/codes"
 )
 
 var (
@@ -29,6 +30,8 @@ type edgeRequest struct {
 	serverService, clientService string
 	serverLatency, clientLatency time.Duration
 
+	// If either the client or the server spans have status code error,
+	// the request will be considered as failed.
 	failed bool
 }
 
@@ -55,6 +58,9 @@ type processor struct {
 	serviceGraphUnpairedSpansTotal     *prometheus.CounterVec
 	serviceGraphUntaggedSpansTotal     *prometheus.CounterVec
 
+	httpSuccessCode map[int]struct{}
+	grpcSuccessCode map[int]struct{}
+
 	logger log.Logger
 }
 
@@ -68,6 +74,19 @@ func newProcessor(nextConsumer consumer.Traces, cfg *Config) *processor {
 		cfg.MaxItems = DefaultMaxItems
 	}
 
+	var (
+		httpSuccessCode = make(map[int]struct{})
+		grpcSuccessCode = make(map[int]struct{})
+	)
+	if cfg.SuccessCodes != nil {
+		for _, sc := range cfg.SuccessCodes.http {
+			httpSuccessCode[int(sc)] = struct{}{}
+		}
+		for _, sc := range cfg.SuccessCodes.grpc {
+			grpcSuccessCode[int(sc)] = struct{}{}
+		}
+	}
+
 	// TODO(mapno): Add support for an external cache (e.g. memcached)
 	p := &processor{
 		nextConsumer: nextConsumer,
@@ -238,6 +257,7 @@ func (p *processor) consume(trace pdata.Traces) error {
 					}
 					r.clientService = svc.StringVal()
 					r.clientLatency = spanDuration(span)
+					r.failed = p.spanFailed(span)
 					p.store.SetDefault(k, r)
 
 				case pdata.SpanKindServer:
@@ -250,6 +270,7 @@ func (p *processor) consume(trace pdata.Traces) error {
 
 					r.serverService = svc.StringVal()
 					r.serverLatency = spanDuration(span)
+					r.failed = p.spanFailed(span)
 					p.store.SetDefault(k, r)
 
 				default:
@@ -261,6 +282,26 @@ func (p *processor) consume(trace pdata.Traces) error {
 	return nil
 }
 
+func (p *processor) spanFailed(span pdata.Span) bool {
+	// Request considered failed if status is not 2XX or added as a successful status code
+	if statusCode, ok := span.Attributes().Get("http.status_code"); ok {
+		sc := int(statusCode.IntVal())
+		if _, ok := p.httpSuccessCode[sc]; !ok || sc/100 != 2 {
+			return true
+		}
+	}
+
+	// Request considered failed if status is not OK or added as a successful status code
+	if statusCode, ok := span.Attributes().Get("grpc.status_code"); ok {
+		sc := int(statusCode.IntVal())
+		if _, ok := p.grpcSuccessCode[sc]; !ok || sc != int(codes.OK) {
+			return true
+		}
+	}
+
+	return span.Status().Code() == pdata.StatusCodeError
+}
+
 func spanDuration(span pdata.Span) time.Duration {
 	return span.EndTimestamp().AsTime().Sub(span.StartTimestamp().AsTime())
 }

pkg/tempo/servicegraphprocessor/processor_test.go

@@ -151,6 +151,10 @@ const (
 		tempo_service_graph_request_client_seconds_bucket{client="lb",server="app",le="+Inf"} 3
 		tempo_service_graph_request_client_seconds_sum{client="lb",server="app"} 7.8
 		tempo_service_graph_request_client_seconds_count{client="lb",server="app"} 3
+		# HELP tempo_service_graph_request_failed_total Total count of failed requests between two nodes
+		# TYPE tempo_service_graph_request_failed_total counter
+		tempo_service_graph_request_failed_total{client="app",server="db"} 3
+		tempo_service_graph_request_failed_total{client="lb",server="app"} 3
 		# HELP tempo_service_graph_request_server_seconds Time for a request between two nodes as seen from the server
 		# TYPE tempo_service_graph_request_server_seconds histogram
 		tempo_service_graph_request_server_seconds_bucket{client="app",server="db",le="0.01"} 0

pkg/tempo/servicegraphprocessor/testdata/trace-sample.json

@@ -284,7 +284,8 @@
                 }
               ],
               "status":{
-
+                "deprecatedCode":"DEPRECATED_STATUS_CODE_UNKNOWN_ERROR",
+                "code":"STATUS_CODE_ERROR"
               }
             },
             {