From 091a7bead9c5c9df36907b0ba2bd2f4d2987c115 Mon Sep 17 00:00:00 2001 From: Mario Date: Thu, 2 Sep 2021 18:12:15 +0200 Subject: [PATCH 1/4] Service graphs processor (#756) * Bare implementation of the service graph processor * Comment fixes * Collect unpaired spans metric * Add unpaired metric * Implementation improvements * Some more improvements * No need to close * Fix CI * Improve test stability * Some documentation * Fix tests * Truly fix it --- CHANGELOG.md | 2 + docs/configuration/tempo-config.md | 29 +- .../docker-compose/agent/config/agent.yaml | 6 +- go.mod | 4 + go.sum | 50 +- pkg/tempo/config.go | 28 +- pkg/tempo/config_test.go | 39 + pkg/tempo/contextkeys/keys.go | 7 +- pkg/tempo/instance.go | 26 +- pkg/tempo/remotewriteexporter/exporter.go | 4 +- pkg/tempo/remotewriteexporter/factory.go | 4 +- pkg/tempo/servicegraphprocessor/factory.go | 55 + pkg/tempo/servicegraphprocessor/processor.go | 270 +++ .../servicegraphprocessor/processor_test.go | 194 ++ .../testdata/trace-sample.json | 1915 +++++++++++++++++ .../testdata/unpaired-trace-sample.json | 1900 ++++++++++++++++ pkg/tempo/tempo.go | 7 +- pkg/tempo/tempo_test.go | 58 +- 18 files changed, 4545 insertions(+), 53 deletions(-) create mode 100644 pkg/tempo/servicegraphprocessor/factory.go create mode 100644 pkg/tempo/servicegraphprocessor/processor.go create mode 100644 pkg/tempo/servicegraphprocessor/processor_test.go create mode 100644 pkg/tempo/servicegraphprocessor/testdata/trace-sample.json create mode 100644 pkg/tempo/servicegraphprocessor/testdata/unpaired-trace-sample.json diff --git a/CHANGELOG.md b/CHANGELOG.md index a8d2972a41f0..94f68af32b85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ for specific instructions. - [FEATURE] Add `operator-detach` command to agentctl to allow zero-downtime upgrades when removing an Operator CRD. (@rfratto) +- [FEATURE] Service graphs processor (@mapno) + - [ENHANCEMENT] The Grafana Agent Operator will now default to deploying the matching release version of the Grafana Agent instead of v0.14.0. (@rfratto) diff --git a/docs/configuration/tempo-config.md b/docs/configuration/tempo-config.md index 61b01f5c909d..1041fe1bfa74 100644 --- a/docs/configuration/tempo-config.md +++ b/docs/configuration/tempo-config.md @@ -182,7 +182,7 @@ spanmetrics: # a complete trace. This is achieved by waiting a given time for all the spans # before evaluating the trace. # -# Tail sampling also supports multiple agent deployments, allowing to group all +# Tail sampling also supports multi agent deployments, allowing to group all # spans of a trace in the same agent by load balancing the spans by trace ID # between the instances. # * To make use of this feature, check load_balancing below * @@ -242,6 +242,33 @@ load_balancing: [ username: ] [ password: ] [ password_file: ] + +# service_graphs configures processing of traces for building service graphs in +# the form of prometheus metrics. The generated metrics represent edges between +# nodes in the graph. Nodes are represented by `client` and `server` labels. +# +# e.g. tempo_service_graph_request_total{client="app", server="db"} 20 +# +# Service graphs works by inspecting spans and looking for the tag `span.kind`. +# If it finds the span kind to be client or server, it stores the request in a +# local in-memory store. +# +# That request waits until its corresponding client or server pair span is +# processed or until the maximum waiting time has passed. +# When either of those conditions is reached, the request is processed and +# removed from the local store. If the request is complete by that time, it'll +# be recorded as an edge in the graph. +# +# Service graphs supports multi agent deployments, allowing to group all spans +# of a trace in the same agent by load balancing the spans by trace ID between +# the instances. +# * To make use of this feature, check load_balancing above * +service_graphs: + [ enabled: | default = false ] + + [ wait: | default = "10s"] + + [ max_items: | default = 10_000 ] ``` > **Note:** More information on the following types can be found on the diff --git a/example/docker-compose/agent/config/agent.yaml b/example/docker-compose/agent/config/agent.yaml index 766898cdcb2a..c73d12b97a15 100644 --- a/example/docker-compose/agent/config/agent.yaml +++ b/example/docker-compose/agent/config/agent.yaml @@ -23,7 +23,7 @@ prometheus: - job_name: local_scrape static_configs: - - targets: ['127.0.0.1:12345'] + - targets: ['127.0.0.1:12345', '0.0.0.0:8889'] labels: cluster: 'docker_compose' container: 'agent' @@ -97,4 +97,6 @@ tempo: processes: true roots: true spanmetrics: - prom_instance: test + handler_endpoint: 0.0.0.0:8889 + service_graphs: + enabled: true diff --git a/go.mod b/go.mod index 40fa7119c83c..2c0056ed65b1 100644 --- a/go.mod +++ b/go.mod @@ -23,9 +23,11 @@ require ( github.com/google/go-jsonnet v0.17.0 github.com/gorilla/mux v1.8.0 github.com/grafana/loki v1.6.2-0.20210429132126-d88f3996eaa2 + github.com/grafana/tempo v1.0.1 github.com/hashicorp/consul/api v1.8.1 github.com/hashicorp/go-cleanhttp v0.5.2 github.com/hashicorp/go-getter v1.5.3 + github.com/hashicorp/go-multierror v1.1.0 github.com/imdario/mergo v0.3.12 // indirect github.com/infinityworks/github-exporter v0.0.0-20201016091012-831b72461034 github.com/jsternberg/zap-logfmt v1.2.0 @@ -38,11 +40,13 @@ require ( github.com/olekukonko/tablewriter v0.0.2 github.com/oliver006/redis_exporter v1.15.0 github.com/open-telemetry/opentelemetry-collector-contrib/exporter/loadbalancingexporter v0.30.0 + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/batchpersignal v0.0.0-00010101000000-000000000000 github.com/open-telemetry/opentelemetry-collector-contrib/processor/spanmetricsprocessor v0.30.0 github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor v0.30.0 github.com/opentracing-contrib/go-grpc v0.0.0-20210225150812-73cb765af46e github.com/opentracing-contrib/go-stdlib v1.0.0 github.com/opentracing/opentracing-go v1.2.0 + github.com/patrickmn/go-cache v0.0.0-20180527043350-9f6ff22cfff8 github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pkg/errors v0.9.1 github.com/prometheus-community/postgres_exporter v0.0.0-00010101000000-000000000000 diff --git a/go.sum b/go.sum index c04d5ff2eca2..e10997caa8c6 100644 --- a/go.sum +++ b/go.sum @@ -45,15 +45,18 @@ cloud.google.com/go/storage v1.3.0/go.mod h1:9IAwXhoyBJ7z9LcAwkj0/7NnPzYaPeZxxVp cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.15.0 h1:Ljj+ZXVEhCr/1+4ZhvtteN1ND7UUsNTlduGclLh8GO0= +cloud.google.com/go/storage v1.15.0/go.mod h1:mjjQMoxxyGH7Jr8K5qrx6N2O0AHsczI61sMNn03GIZI= code.cloudfoundry.org/clock v1.0.0/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8= collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= +contrib.go.opencensus.io/exporter/prometheus v0.2.0/go.mod h1:TYmVAyE8Tn1lyPcltF5IYYfWp2KHu7lQGIZnj8iZMys= contrib.go.opencensus.io/exporter/prometheus v0.3.0 h1:08FMdJYpItzsknogU6PiiNo7XQZg/25GjH236+YCwD0= contrib.go.opencensus.io/exporter/prometheus v0.3.0/go.mod h1:rpCPVQKhiyH8oomWgm34ZmgIdZa8OVYO5WAIygPbBBE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AlekSi/pointer v1.1.0 h1:SSDMPcXD9jSl8FPy9cRzoRaMJtm9g9ggGTxecRUbQoI= github.com/AlekSi/pointer v1.1.0/go.mod h1:y7BvfRI3wXPWKXEBhU71nbnIEEZX0QTSB2Bj48UJIZE= +github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/Azure/azure-amqp-common-go/v3 v3.0.0/go.mod h1:SY08giD/XbhTz07tJdpw1SoxQXHPN30+DI3Z04SYqyg= github.com/Azure/azure-event-hubs-go/v3 v3.2.0/go.mod h1:BPIIJNH/l/fVHYq3Rm6eg4clbrULrQ3q7+icmqHyyLc= github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= @@ -119,7 +122,9 @@ github.com/DATA-DOG/go-sqlmock v1.4.1 h1:ThlnYciV1iM/V0OSF/dtkqWb6xo5qITT1TJBG1M github.com/DATA-DOG/go-sqlmock v1.4.1/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/datadog-go v0.0.0-20160329135253-cc2f4770f4d6/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/HdrHistogram/hdrhistogram-go v0.9.0/go.mod h1:nxrse8/Tzg2tg3DZcZjm6qEclQKK70g0KxO61gFFZD4= github.com/HdrHistogram/hdrhistogram-go v1.0.1 h1:GX8GAYDuhlFQnI2fRDHQhTlkHMz8bEn0jTI6LJU0mpw= github.com/HdrHistogram/hdrhistogram-go v1.0.1/go.mod h1:BWJ+nMSHY3L41Zj7CA3uXnloDp7xxV0YvstAE7nKTaM= @@ -171,6 +176,7 @@ github.com/SAP/go-hdb v0.12.0/go.mod h1:etBT+FAi1t5k3K3tf5vQTnosgYmhDkRi8jEnQqCn github.com/SermoDigital/jose v0.0.0-20180104203859-803625baeddc/go.mod h1:ARgCUhI1MHQH+ONky/PAtmVHQrP5JlGY0F3poXOp/fA= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/sarama v1.22.2-0.20190604114437-cd910a683f9f/go.mod h1:XLH1GYJnLVE0XCr6KdJGVJRTwY30moWNJ4sERjXX6fs= github.com/Shopify/sarama v1.27.1/go.mod h1:g5s5osgELxgM+Md9Qni9rzo7Rbt+vvFQI4bt/Mc93II= github.com/Shopify/sarama v1.27.2/go.mod h1:g5s5osgELxgM+Md9Qni9rzo7Rbt+vvFQI4bt/Mc93II= github.com/Shopify/sarama v1.29.1 h1:wBAacXbYVLmWieEA/0X/JagDdCZ8NVFOfS6l6+2u5S0= @@ -189,6 +195,7 @@ github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE= +github.com/alecthomas/kong v0.2.11/go.mod h1:kQOmtJgV+Lb4aj+I2LEn40cbtawdWJ9Y8QLq+lElKxE= github.com/alecthomas/kong v0.2.17/go.mod h1:ka3VZ8GZNPXv9Ov+j4YNLkI8mTuhXyr/0ktSlqIydQQ= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -496,7 +503,9 @@ github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4/go.mod h1:zA github.com/denverdino/aliyungo v0.0.0-20170926055100-d3308649c661/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= +github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= github.com/dgraph-io/badger/v3 v3.2103.0/go.mod h1:GHMCYxuDWyzbHkh4k3yyg4PM61tJPFfEGSMbE3Vd5QE= +github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.4-0.20210309073149-3836124cdc5a/go.mod h1:MIonLggsKgZLUSt414ExgwNtlOL5MuEoAJP514mwGe8= github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= @@ -613,6 +622,7 @@ github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= +github.com/frankban/quicktest v1.7.3/go.mod h1:V1d2J5pfxYH6EjBAgSK7YNXcXlTWxUHdE1sVDXkjnig= github.com/frankban/quicktest v1.10.2/go.mod h1:K+q6oSqb0W0Ininfk863uOk1lMy69l/P6txr3mVT54s= github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= @@ -724,6 +734,7 @@ github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcs github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/spec v0.19.5/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= github.com/go-openapi/spec v0.19.6/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= +github.com/go-openapi/spec v0.19.7/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= github.com/go-openapi/spec v0.19.8/go.mod h1:Hm2Jr4jv8G1ciIAo+frC/Ft+rR2kQDh8JHKHb3gWUSk= github.com/go-openapi/spec v0.19.15/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU= github.com/go-openapi/spec v0.20.0/go.mod h1:+81FIL1JwC5P3/Iuuozq3pPE9dXdIEGxFutcFKaVbmU= @@ -970,12 +981,14 @@ github.com/gopherjs/gopherjs v0.0.0-20191106031601-ce3c9ade29de h1:F7WD09S8QB4Lr github.com/gopherjs/gopherjs v0.0.0-20191106031601-ce3c9ade29de/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= @@ -1002,6 +1015,8 @@ github.com/grafana/prometheus v1.8.2-0.20210608193638-7b78de4ccffc h1:brKRMe6V7k github.com/grafana/prometheus v1.8.2-0.20210608193638-7b78de4ccffc/go.mod h1:yUzDYX0hIYu5YVHmpj/JXLOclB6QcLNDgmagD3FUnSU= github.com/grafana/tail v0.0.0-20201004203643-7aa4e4a91f03 h1:fGgFrAraMB0BaPfYumu+iulfDXwHm+GFyHA4xEtBqI8= github.com/grafana/tail v0.0.0-20201004203643-7aa4e4a91f03/go.mod h1:GIMXMPB/lRAllP5rVDvcGif87ryO2hgD7tCtHMdHrho= +github.com/grafana/tempo v1.0.1 h1:8M4u1D/noVASoYmFlDNY9LNerMbVZ+8zPep7Qs0ldfA= +github.com/grafana/tempo v1.0.1/go.mod h1:hd+8igGfN6WE4uIuJxXuPeyZDslpy6Gt0LRAsUWPrKg= github.com/grafana/windows_exporter v0.15.1-0.20210325142439-9e8f66d53433 h1:3WaMH1VOp8T2gCwjM5iHtH2SW3BpN8pAmRhzy9TYVAA= github.com/grafana/windows_exporter v0.15.1-0.20210325142439-9e8f66d53433/go.mod h1:FZy59lGh3jUzgaaS6jYzzTfk1NTNrH5+UK3NCeQF4Ao= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= @@ -1010,6 +1025,7 @@ github.com/grobie/gomemcache v0.0.0-20201204163352-08d7c80fcac6/go.mod h1:L69/dB github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= +github.com/grpc-ecosystem/go-grpc-middleware v1.2.0/go.mod h1:mJzapYve32yjrKlk9GbyCZHuPgZsrbyIbyKhSzOpg6s= github.com/grpc-ecosystem/go-grpc-middleware v1.2.2/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= @@ -1023,6 +1039,7 @@ github.com/grpc-ecosystem/grpc-gateway v1.4.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpg github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= +github.com/grpc-ecosystem/grpc-gateway v1.14.5/go.mod h1:UJ0EZAp832vCd54Wev9N1BMKEyvcZ5+IM0AwDrnlkEc= github.com/grpc-ecosystem/grpc-gateway v1.14.6/go.mod h1:zdiPV4Yse/1gnckTHtghG4GkDEdKCRJduHpTxT3/jcw= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= @@ -1059,6 +1076,7 @@ github.com/hashicorp/go-hclog v0.0.0-20180402200405-69ff559dc25f/go.mod h1:9bjs9 github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v0.14.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v0.14.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v0.16.1 h1:IVQwpTGNRRIHafnTs2dQLIk4ENtneRIEEJWOVDqz99o= github.com/hashicorp/go-hclog v0.16.1/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= @@ -1076,6 +1094,7 @@ github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-plugin v0.0.0-20180331002553-e8d22c780116/go.mod h1:JSqWYsict+jzcj0+xElxyrBQRPNoiWQuddnxArJ7XHQ= github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= +github.com/hashicorp/go-plugin v1.3.0/go.mod h1:F9eH4LrE/ZsRdbwhfjs9k9HoDUwAHnYtXdgmf1AVNs0= github.com/hashicorp/go-plugin v1.4.2/go.mod h1:5fGEH17QVwTTcR0zV7yhDPLLmFX9YSZ38b18Udy6vYQ= github.com/hashicorp/go-retryablehttp v0.0.0-20180531211321-3b087ef2d313/go.mod h1:fXcdFsQoipQa7mwORhKad5jmDCeSy/RCGzWA08PO0lM= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= @@ -1181,6 +1200,7 @@ github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6t github.com/jackc/fake v0.0.0-20150926172116-812a484cc733/go.mod h1:WrMFNQdiFJ80sQsxDoMokWK1W5TQtxBFNpzWTD84ibQ= github.com/jackc/pgx v3.2.0+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= github.com/jackc/pgx v3.6.0+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= +github.com/jaegertracing/jaeger v1.21.0/go.mod h1:PCTGGFohQBPQMR4j333V5lt6If7tj8aWJ+pQNgvZ+wU= github.com/jaegertracing/jaeger v1.24.0 h1:wbzvajFSsV3j5843nIlyUa70+uQevKsT3l7MV29jlxU= github.com/jaegertracing/jaeger v1.24.0/go.mod h1:mqdtFDA447va5j0UewDaAWyNlGreGQyhGxXVhbF58gQ= github.com/jarcoal/httpmock v0.0.0-20180424175123-9c70cfe4a1da/go.mod h1:ks+b9deReOc7jgqp+e7LuFiCBH6Rm5hL32cLcEAArb4= @@ -1188,6 +1208,7 @@ github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFK github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= +github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= @@ -1264,6 +1285,7 @@ github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdY github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.2/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.12/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.1 h1:wXr2uRxZTJXHLly6qhJabee5JqIhTRoLBhDOA74hDEQ= @@ -1425,6 +1447,7 @@ github.com/mitchellh/hashstructure v1.1.0/go.mod h1:xUDAozZz0Wmdiufv0uyhnHkUTN6/ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.3.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -1507,6 +1530,7 @@ github.com/olekukonko/tablewriter v0.0.2 h1:sq53g+DWf0J6/ceFUHpQ0nAEb6WgM++fq16M github.com/olekukonko/tablewriter v0.0.2/go.mod h1:rSAaSIOAGT9odnlyGlUfAJaoc5w2fSBUmeGDbRWPxyQ= github.com/oliver006/redis_exporter v1.15.0 h1:6MkRqNiHF8bh/4THvI6GT+wukJu3plBXOW1Y7QlqeFA= github.com/oliver006/redis_exporter v1.15.0/go.mod h1:VWMvGKpOhg71Y7tR9JDwkqNhVAQNqrpqjm2OTg74WHQ= +github.com/olivere/elastic v6.2.27+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8= github.com/olivere/elastic v6.2.35+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8= github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -1593,12 +1617,14 @@ github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIw github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/patrickmn/go-cache v0.0.0-20180527043350-9f6ff22cfff8 h1:BR6MM54q4W9pn0SySwg6yctZtBKlTdUq6a+b0kArBnE= github.com/patrickmn/go-cache v0.0.0-20180527043350-9f6ff22cfff8/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= +github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= @@ -1611,13 +1637,16 @@ github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pierrec/lz4 v0.0.0-20190327172049-315a67e90e41/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.1/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.3/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -1716,6 +1745,7 @@ github.com/prometheus/statsd_exporter v0.18.1-0.20201124082027-8b2b4c1a2b49/go.m github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1/go.mod h1:JaY6n2sDr+z2WTsXkOmNRUfDy6FN0L6Nk7x06ndm4tY= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= @@ -1851,7 +1881,9 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= +github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= @@ -1919,8 +1951,10 @@ github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJ github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= github.com/tonistiigi/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:Q5IRRDY+cjIaiOjTAnXN5LKQV5MPqVx5ofQn85Jy5Yw= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/uber-go/atomic v1.4.0/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/uber/jaeger-client-go v2.20.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-client-go v2.22.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= +github.com/uber/jaeger-client-go v2.23.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-client-go v2.24.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-client-go v2.28.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= @@ -1956,8 +1990,10 @@ github.com/wavefronthq/wavefront-sdk-go v0.9.2/go.mod h1:hQI6y8M9OtTCtc0xdwh+dCE github.com/weaveworks/promrus v1.2.0 h1:jOLf6pe6/vss4qGHjXmGz4oDJQA+AOCqEL3FvvZGz7M= github.com/weaveworks/promrus v1.2.0/go.mod h1:SaE82+OJ91yqjrE1rsvBWVzNZKcHYFtMUyS1+Ogs/KA= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= +github.com/willf/bloom v2.0.3+incompatible/go.mod h1:MmAltL9pDMNTrvUkxdg0k0q5I0suxmuwp3KbyrZLOZ8= github.com/wvanbergen/kafka v0.0.0-20171203153745-e2edea948ddf/go.mod h1:nxx7XRXbR9ykhnC8lXqQyJS0rfvJGxKyKw/sT1YOttg= github.com/wvanbergen/kazoo-go v0.0.0-20180202103751-f72d8611297a/go.mod h1:vQQATAGxVK20DC1rRubTJbZDDhhpA4QfU02pMdPxGO4= github.com/xanzy/go-gitlab v0.15.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs= @@ -2037,6 +2073,7 @@ go.mongodb.org/mongo-driver v1.1.0/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qL go.mongodb.org/mongo-driver v1.1.1/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.1.2/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= +go.mongodb.org/mongo-driver v1.3.2/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= @@ -2086,6 +2123,7 @@ go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.8.0 h1:CUhrE4N1rqSE6FM9ecihEjRkLQu8cDfgDyoOs83mEY4= go.uber.org/atomic v1.8.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/automaxprocs v1.2.0/go.mod h1:YfO3fm683kQpzETxlTGZhGIVmXAhaw3gxeBADbpZtnU= +go.uber.org/automaxprocs v1.3.0/go.mod h1:9CWT6lKIep8U41DDaPiH6eFscnTyjfTANNQNx6LrIcA= go.uber.org/automaxprocs v1.4.0/go.mod h1:/mTEdr7LvHhs0v7mjdxDreTz1OG5zdZGqgOnhWiR/+Q= go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= @@ -2113,6 +2151,7 @@ golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -2286,6 +2325,7 @@ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210323180902-22b0adad7558/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c h1:SgVl/sCtkicsS7psKkje4H9YtjdEl3xsYh7N+5TDHqY= golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -2339,6 +2379,7 @@ golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2424,6 +2465,7 @@ golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210412220455-f1c623a9e750/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210503080704-8803ae5d1324/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210503173754-0981d6026fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -2527,6 +2569,7 @@ golang.org/x/tools v0.0.0-20200509030707-2212a7e161a5/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200513201620-d5fe73897c97/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200603131246-cc40288be839/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= @@ -2588,6 +2631,7 @@ google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBz google.golang.org/api v0.42.0/go.mod h1:+Oj4s6ch2SEGtPjGqfUfZonBH0GjQH89gTeKKAEGZKI= google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.44.0/go.mod h1:EBOGZqzyhtvMDoxwS97ctnh0zUmYY6CxqXsc1AvkYD8= +google.golang.org/api v0.45.0/go.mod h1:ISLIJCedJolbZvDfAk+Ctuq5hf+aJ33WgtUsfyFoLXA= google.golang.org/api v0.46.0 h1:jkDWHOBIoNSD0OQpq4rtBVu+Rh325MPjXG1rakAp8JU= google.golang.org/api v0.46.0/go.mod h1:ceL4oozhkAiTID8XMmJBsIxID/9wMXJVVFXPg4ylg3I= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -2659,6 +2703,8 @@ google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210312152112-fc591d9ea70f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210413151531-c14fb6ef47c3/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210420162539-3c870d7478d2/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210429181445-86c259c2b4ab/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08 h1:pc16UedxnxXXtGxHCSUhafAoVHQZ0yXl8ZelMH4EETc= @@ -2744,6 +2790,7 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.52.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.61.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= @@ -2751,6 +2798,7 @@ gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= +gopkg.in/jcmturner/gokrb5.v7 v7.2.3/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= gopkg.in/ldap.v3 v3.1.0/go.mod h1:dQjCc0R0kfyFjIlWNMH1DORwUASZyDxo2Ry1B51dXaQ= diff --git a/pkg/tempo/config.go b/pkg/tempo/config.go index 3b20fb5bd63c..e67ca247a49f 100644 --- a/pkg/tempo/config.go +++ b/pkg/tempo/config.go @@ -14,6 +14,7 @@ import ( "github.com/grafana/agent/pkg/tempo/noopreceiver" "github.com/grafana/agent/pkg/tempo/promsdprocessor" "github.com/grafana/agent/pkg/tempo/remotewriteexporter" + "github.com/grafana/agent/pkg/tempo/servicegraphprocessor" "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/loadbalancingexporter" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/spanmetricsprocessor" "github.com/open-telemetry/opentelemetry-collector-contrib/processor/tailsamplingprocessor" @@ -120,10 +121,13 @@ type InstanceConfig struct { AutomaticLogging *automaticloggingprocessor.AutomaticLoggingConfig `yaml:"automatic_logging,omitempty"` // TailSampling defines a sampling strategy for the pipeline - TailSampling *tailSamplingConfig `yaml:"tail_sampling"` + TailSampling *tailSamplingConfig `yaml:"tail_sampling,omitempty"` // LoadBalancing is used to distribute spans of the same trace to the same agent instance LoadBalancing *loadBalancingConfig `yaml:"load_balancing"` + + // ServiceGraphs + ServiceGraphs *serviceGraphsConfig `yaml:"service_graphs,omitempty"` } const ( @@ -241,6 +245,12 @@ type exporterConfig struct { BasicAuth *prom_config.BasicAuth `yaml:"basic_auth,omitempty"` } +type serviceGraphsConfig struct { + Enabled bool `yaml:"enabled,omitempty"` + Wait time.Duration `yaml:"wait,omitempty"` + MaxItems int `yaml:"max_items,omitempty"` +} + // exporter builds an OTel exporter from RemoteWriteConfig func exporter(rwCfg RemoteWriteConfig) (map[string]interface{}, error) { if len(rwCfg.Endpoint) == 0 { @@ -556,6 +566,14 @@ func (c *InstanceConfig) otelConfig() (*config.Config, error) { } } + if c.ServiceGraphs != nil && c.ServiceGraphs.Enabled { + processors[servicegraphprocessor.TypeStr] = map[string]interface{}{ + "wait": c.ServiceGraphs.Wait, + "max_items": c.ServiceGraphs.MaxItems, + } + processorNames = append(processorNames, servicegraphprocessor.TypeStr) + } + // Build Pipelines splitPipeline := c.TailSampling != nil && c.LoadBalancing != nil orderedSplitProcessors := orderProcessors(processorNames, splitPipeline) @@ -647,6 +665,7 @@ func tracingFactories() (component.Factories, error) { spanmetricsprocessor.NewFactory(), automaticloggingprocessor.NewFactory(), tailsamplingprocessor.NewFactory(), + servicegraphprocessor.NewFactory(), ) if err != nil { return component.Factories{}, err @@ -667,9 +686,10 @@ func orderProcessors(processors []string, splitPipelines bool) [][]string { order := map[string]int{ "attributes": 0, "spanmetrics": 1, - "tail_sampling": 2, - "automatic_logging": 3, - "batch": 4, + "service_graphs": 2, + "tail_sampling": 3, + "automatic_logging": 4, + "batch": 5, } sort.Slice(processors, func(i, j int) bool { diff --git a/pkg/tempo/config_test.go b/pkg/tempo/config_test.go index 6000e2c108a4..03b9e09a894d 100644 --- a/pkg/tempo/config_test.go +++ b/pkg/tempo/config_test.go @@ -773,6 +773,39 @@ service: exporters: ["otlphttp/0", "otlp/1"] processors: [] receivers: ["jaeger"] +`, + }, + { + name: "service graphs", + cfg: ` +receivers: + jaeger: + protocols: + grpc: +remote_write: + - endpoint: example.com:12345 +service_graphs: + enabled: true +`, + expectedConfig: ` +receivers: + jaeger: + protocols: + grpc: +exporters: + otlp/0: + endpoint: example.com:12345 + compression: gzip + retry_on_failure: + max_elapsed_time: 60s +processors: + service_graphs: +service: + pipelines: + traces: + exporters: ["otlp/0"] + processors: ["service_graphs"] + receivers: ["jaeger"] `, }, } @@ -871,11 +904,14 @@ tail_sampling: values: - value1 - value2 +service_graphs: + enabled: true `, expectedProcessors: map[string][]config.ComponentID{ "traces": { config.NewID("attributes"), config.NewID("spanmetrics"), + config.NewID("service_graphs"), config.NewID("tail_sampling"), config.NewID("automatic_logging"), config.NewID("batch"), @@ -926,11 +962,14 @@ load_balancing: dns: hostname: agent port: 4318 +service_graphs: + enabled: true `, expectedProcessors: map[string][]config.ComponentID{ "traces/0": { config.NewID("attributes"), config.NewID("spanmetrics"), + config.NewID("service_graphs"), }, "traces/1": { config.NewID("tail_sampling"), diff --git a/pkg/tempo/contextkeys/keys.go b/pkg/tempo/contextkeys/keys.go index 1939db0f1a61..e61a3c081f32 100644 --- a/pkg/tempo/contextkeys/keys.go +++ b/pkg/tempo/contextkeys/keys.go @@ -6,6 +6,9 @@ const ( // Logs is used to pass *logs.Logs through the context Logs key = iota - // Prometheus is used to pass instance.Manager through the context - Prometheus + // InstanceManager is used to pass instance.Manager through the context + InstanceManager + + // PrometheusRegisterer is used to pass prometheus.Registerer through the context + PrometheusRegisterer ) diff --git a/pkg/tempo/instance.go b/pkg/tempo/instance.go index 96a97d2618a8..05b1a664a74e 100644 --- a/pkg/tempo/instance.go +++ b/pkg/tempo/instance.go @@ -9,6 +9,7 @@ import ( "github.com/grafana/agent/pkg/build" "github.com/grafana/agent/pkg/logs" "github.com/grafana/agent/pkg/metrics/instance" + "github.com/grafana/agent/pkg/tempo/automaticloggingprocessor" "github.com/grafana/agent/pkg/tempo/contextkeys" "github.com/grafana/agent/pkg/util" "github.com/prometheus/client_golang/prometheus" @@ -43,14 +44,14 @@ func NewInstance(logsSubsystem *logs.Logs, reg prometheus.Registerer, cfg Instan return nil, fmt.Errorf("failed to create metric views: %w", err) } - if err := instance.ApplyConfig(logsSubsystem, promInstanceManager, cfg); err != nil { + if err := instance.ApplyConfig(logsSubsystem, promInstanceManager, reg, cfg); err != nil { return nil, err } return instance, nil } // ApplyConfig updates the configuration of the Instance. -func (i *Instance) ApplyConfig(logsSubsystem *logs.Logs, promInstanceManager instance.Manager, cfg InstanceConfig) error { +func (i *Instance) ApplyConfig(logsSubsystem *logs.Logs, promInstanceManager instance.Manager, reg prometheus.Registerer, cfg InstanceConfig) error { i.mut.Lock() defer i.mut.Unlock() @@ -63,8 +64,7 @@ func (i *Instance) ApplyConfig(logsSubsystem *logs.Logs, promInstanceManager ins // Shut down any existing pipeline i.stop() - createCtx := context.WithValue(context.Background(), contextkeys.Logs, logsSubsystem) - err := i.buildAndStartPipeline(createCtx, cfg, promInstanceManager) + err := i.buildAndStartPipeline(context.Background(), cfg, logsSubsystem, promInstanceManager, reg) if err != nil { return fmt.Errorf("failed to create pipeline: %w", err) } @@ -130,7 +130,7 @@ func (i *Instance) stop() { i.exporter = nil } -func (i *Instance) buildAndStartPipeline(ctx context.Context, cfg InstanceConfig, promManager instance.Manager) error { +func (i *Instance) buildAndStartPipeline(ctx context.Context, cfg InstanceConfig, logs *logs.Logs, instManager instance.Manager, reg prometheus.Registerer) error { // create component factories otelConfig, err := cfg.otelConfig() if err != nil { @@ -149,12 +149,20 @@ func (i *Instance) buildAndStartPipeline(ctx context.Context, cfg InstanceConfig } if cfg.SpanMetrics != nil && len(cfg.SpanMetrics.PromInstance) != 0 { - ctx = context.WithValue(ctx, contextkeys.Prometheus, promManager) + ctx = context.WithValue(ctx, contextkeys.InstanceManager, instManager) } - if cfg.TailSampling != nil && cfg.LoadBalancing == nil { - i.logger.Warn("Configuring tail_sampling without load_balance." + - "Load balancing is required for tail sampling to properly work in multi instance deployments") + if cfg.LoadBalancing == nil && (cfg.TailSampling != nil || cfg.ServiceGraphs != nil) { + i.logger.Warn("Configuring tail_sampling and/or service_graphs without load_balance." + + "Load balancing is required for those features to properly work in multi agent deployments") + } + + if cfg.AutomaticLogging != nil && cfg.AutomaticLogging.Backend != automaticloggingprocessor.BackendStdout { + ctx = context.WithValue(ctx, contextkeys.Logs, logs) + } + + if cfg.ServiceGraphs != nil { + ctx = context.WithValue(ctx, contextkeys.PrometheusRegisterer, reg) } factories, err := tracingFactories() diff --git a/pkg/tempo/remotewriteexporter/exporter.go b/pkg/tempo/remotewriteexporter/exporter.go index 50ac087f655b..15252b250158 100644 --- a/pkg/tempo/remotewriteexporter/exporter.go +++ b/pkg/tempo/remotewriteexporter/exporter.go @@ -60,9 +60,9 @@ func newRemoteWriteExporter(cfg *Config) (component.MetricsExporter, error) { } func (e *remoteWriteExporter) Start(ctx context.Context, _ component.Host) error { - manager, ok := ctx.Value(contextkeys.Prometheus).(instance.Manager) + manager, ok := ctx.Value(contextkeys.InstanceManager).(instance.Manager) if !ok || manager == nil { - return fmt.Errorf("key does not contain a Prometheus instance") + return fmt.Errorf("key does not contain a InstanceManager instance") } e.manager = manager return nil diff --git a/pkg/tempo/remotewriteexporter/factory.go b/pkg/tempo/remotewriteexporter/factory.go index 9d0c28984513..c4f2d5b848f3 100644 --- a/pkg/tempo/remotewriteexporter/factory.go +++ b/pkg/tempo/remotewriteexporter/factory.go @@ -14,7 +14,7 @@ const ( TypeStr = "remote_write" ) -// Config holds the configuration for the Prometheus SD processor. +// Config holds the configuration for the Prometheus remote write processor. type Config struct { config.ProcessorSettings `mapstructure:",squash"` @@ -23,7 +23,7 @@ type Config struct { PromInstance string `mapstructure:"prom_instance"` } -// NewFactory returns a new factory for the Attributes processor. +// NewFactory returns a new factory for the Prometheus remote write processor. func NewFactory() component.ExporterFactory { return exporterhelper.NewFactory( TypeStr, diff --git a/pkg/tempo/servicegraphprocessor/factory.go b/pkg/tempo/servicegraphprocessor/factory.go new file mode 100644 index 000000000000..2e96b4d9481f --- /dev/null +++ b/pkg/tempo/servicegraphprocessor/factory.go @@ -0,0 +1,55 @@ +package servicegraphprocessor + +import ( + "context" + "time" + + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/processor/processorhelper" +) + +const ( + // TypeStr is the unique identifier for the Prometheus service graph exporter. + TypeStr = "service_graphs" + + // DefaultWait is the default value to wait for an edgeRequest to be completed + DefaultWait = time.Second * 10 + // DefaultMaxItems is the default amount of edges that will be stored in the store + DefaultMaxItems = 10_000 +) + +// Config holds the configuration for the Prometheus service graph processor. +type Config struct { + config.ProcessorSettings `mapstructure:",squash"` + + Wait time.Duration `mapstructure:"wait"` + MaxItems int `mapstructure:"max_items"` +} + +// NewFactory returns a new factory for the Prometheus service graph processor. +func NewFactory() component.ProcessorFactory { + return processorhelper.NewFactory( + TypeStr, + createDefaultConfig, + processorhelper.WithTraces(createTracesProcessor), + ) +} + +func createDefaultConfig() config.Processor { + return &Config{ + ProcessorSettings: config.NewProcessorSettings(config.NewIDWithName(TypeStr, TypeStr)), + } +} + +func createTracesProcessor( + _ context.Context, + _ component.ProcessorCreateSettings, + cfg config.Processor, + nextConsumer consumer.Traces, +) (component.TracesProcessor, error) { + eCfg := cfg.(*Config) + + return newProcessor(nextConsumer, eCfg), nil +} diff --git a/pkg/tempo/servicegraphprocessor/processor.go b/pkg/tempo/servicegraphprocessor/processor.go new file mode 100644 index 000000000000..964e77e48662 --- /dev/null +++ b/pkg/tempo/servicegraphprocessor/processor.go @@ -0,0 +1,270 @@ +package servicegraphprocessor + +import ( + "context" + "errors" + "fmt" + "time" + + util "github.com/cortexproject/cortex/pkg/util/log" + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/grafana/agent/pkg/tempo/contextkeys" + "github.com/hashicorp/go-multierror" + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/batchpersignal" + "github.com/patrickmn/go-cache" + "github.com/prometheus/client_golang/prometheus" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/model/pdata" + "go.opentelemetry.io/collector/translator/conventions" +) + +var ( + errTooManyItems = errors.New("too many items in store") +) + +// edgeRequest is a request between two nodes in the graph +type edgeRequest struct { + serverService, clientService string + serverLatency, clientLatency time.Duration + + failed bool +} + +// complete returns true if the corresponding client and server +// pair spans have been processed for the given request +func (e *edgeRequest) complete() bool { + return len(e.clientService) != 0 && len(e.serverService) != 0 +} + +var _ component.TracesProcessor = (*processor)(nil) + +type processor struct { + nextConsumer consumer.Traces + reg prometheus.Registerer + + // store is a local storage for request between graphs nodes + store *cache.Cache + maxItems int + + serviceGraphRequestTotal *prometheus.CounterVec + serviceGraphRequestFailedTotal *prometheus.CounterVec + serviceGraphRequestServerHistogram *prometheus.HistogramVec + serviceGraphRequestClientHistogram *prometheus.HistogramVec + serviceGraphUnpairedSpansTotal *prometheus.CounterVec + serviceGraphUntaggedSpansTotal *prometheus.CounterVec + + logger log.Logger +} + +func newProcessor(nextConsumer consumer.Traces, cfg *Config) *processor { + logger := log.With(util.Logger, "component", "tempo service graphs") + + if cfg.Wait == 0 { + cfg.Wait = DefaultWait + } + if cfg.MaxItems == 0 { + cfg.MaxItems = DefaultMaxItems + } + + // TODO(mapno): Add support for an external cache (e.g. memcached) + p := &processor{ + nextConsumer: nextConsumer, + // Cleanup period is hardcoded to twice the waiting time for simplicity + // Most likely not ideal in every scenario + store: cache.New(cfg.Wait, cfg.Wait*2), + maxItems: cfg.MaxItems, + logger: logger, + } + + return p +} + +func (p *processor) Start(ctx context.Context, _ component.Host) error { + reg, ok := ctx.Value(contextkeys.PrometheusRegisterer).(prometheus.Registerer) + if !ok || reg == nil { + return fmt.Errorf("key does not contain a prometheus registerer") + } + p.reg = reg + return p.registerMetrics() +} + +func (p *processor) registerMetrics() error { + p.serviceGraphRequestTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "tempo_service_graph_request_total", + Help: "Total count of requests between two nodes", + }, []string{"client", "server"}) + p.serviceGraphRequestFailedTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "tempo_service_graph_request_failed_total", + Help: "Total count of failed requests between two nodes", + }, []string{"client", "server"}) + p.serviceGraphRequestServerHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "tempo_service_graph_request_server_seconds", + Help: "Time for a request between two nodes as seen from the server", + Buckets: prometheus.ExponentialBuckets(0.01, 2, 12), + }, []string{"client", "server"}) + p.serviceGraphRequestClientHistogram = prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "tempo_service_graph_request_client_seconds", + Help: "Time for a request between two nodes as seen from the client", + Buckets: prometheus.ExponentialBuckets(0.01, 2, 12), + }, []string{"client", "server"}) + p.serviceGraphUnpairedSpansTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "tempo_service_graph_unpaired_spans_total", + Help: "Total count of requests between two nodes", + }, []string{"client", "server"}) + p.serviceGraphUntaggedSpansTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "tempo_service_graph_untagged_spans_total", + Help: "Total count of spans processed that were not tagged with span.kind", + }, []string{"span_kind"}) + + cs := []prometheus.Collector{ + p.serviceGraphRequestTotal, + p.serviceGraphRequestFailedTotal, + p.serviceGraphRequestServerHistogram, + p.serviceGraphRequestClientHistogram, + p.serviceGraphUnpairedSpansTotal, + p.serviceGraphUntaggedSpansTotal, + } + + for _, c := range cs { + if err := p.reg.Register(c); err != nil { + return err + } + } + + // Collect unpaired spans when evicting items from the store during + // periodic cleanup + p.store.OnEvicted(func(s string, i interface{}) { + e := i.(edgeRequest) + if !e.complete() { + p.serviceGraphUnpairedSpansTotal.WithLabelValues(e.clientService, e.serverService).Inc() + } + }) + + return nil +} + +func (p *processor) Shutdown(context.Context) error { + p.unregisterMetrics() + p.store.Flush() + return nil +} + +func (p *processor) unregisterMetrics() { + cs := []prometheus.Collector{ + p.serviceGraphRequestTotal, + p.serviceGraphRequestFailedTotal, + p.serviceGraphRequestServerHistogram, + p.serviceGraphRequestClientHistogram, + p.serviceGraphUnpairedSpansTotal, + p.serviceGraphUntaggedSpansTotal, + } + + for _, c := range cs { + p.reg.Unregister(c) + } +} + +func (p *processor) Capabilities() consumer.Capabilities { + return consumer.Capabilities{} +} + +func (p *processor) ConsumeTraces(ctx context.Context, td pdata.Traces) error { + level.Debug(p.logger).Log("msg", "consuming traces") + + var errs error + for _, trace := range batchpersignal.SplitTraces(td) { + if err := p.consume(trace); err != nil { + if errors.Is(err, errTooManyItems) { + level.Warn(p.logger).Log("msg", "skipped processing of spans", "maxItems", p.maxItems, "err", errTooManyItems) + break + } + errs = multierror.Append(errs, err) + } + } + if errs != nil { + level.Error(p.logger).Log("msg", "failed consuming traces", "err", errs) + } + + p.collectMetrics() + + return p.nextConsumer.ConsumeTraces(ctx, td) +} + +func (p *processor) collectMetrics() { + for k, v := range p.store.Items() { + e := v.Object.(edgeRequest) + if e.complete() { + p.serviceGraphRequestTotal.WithLabelValues(e.clientService, e.serverService).Inc() + if e.failed { + p.serviceGraphRequestFailedTotal.WithLabelValues(e.clientService, e.serverService).Inc() + } + p.serviceGraphRequestServerHistogram.WithLabelValues(e.clientService, e.serverService).Observe(e.serverLatency.Seconds()) + p.serviceGraphRequestClientHistogram.WithLabelValues(e.clientService, e.serverService).Observe(e.clientLatency.Seconds()) + p.store.Delete(k) + } + } +} + +func (p *processor) consume(trace pdata.Traces) error { + rSpansSlice := trace.ResourceSpans() + for i := 0; i < rSpansSlice.Len(); i++ { + rSpan := rSpansSlice.At(i) + + svc, ok := rSpan.Resource().Attributes().Get(conventions.AttributeServiceName) + if !ok { + continue + } + + ilsSlice := rSpan.InstrumentationLibrarySpans() + for j := 0; j < ilsSlice.Len(); j++ { + ils := ilsSlice.At(j) + + for k := 0; k < ils.Spans().Len(); k++ { + if p.store.ItemCount() >= p.maxItems { + return errTooManyItems + } + + span := ils.Spans().At(k) + + switch span.Kind() { + case pdata.SpanKindClient: + k := key(span.TraceID().HexString(), span.SpanID().HexString()) + + var r edgeRequest + if v, ok := p.store.Get(k); ok { + r = v.(edgeRequest) + } + r.clientService = svc.StringVal() + r.clientLatency = spanDuration(span) + p.store.SetDefault(k, r) + + case pdata.SpanKindServer: + k := key(span.TraceID().HexString(), span.ParentSpanID().HexString()) + + var r edgeRequest + if v, ok := p.store.Get(k); ok { + r = v.(edgeRequest) + } + + r.serverService = svc.StringVal() + r.serverLatency = spanDuration(span) + p.store.SetDefault(k, r) + + default: + p.serviceGraphUntaggedSpansTotal.WithLabelValues(span.Kind().String()).Inc() + } + } + } + } + return nil +} + +func spanDuration(span pdata.Span) time.Duration { + return span.EndTimestamp().AsTime().Sub(span.StartTimestamp().AsTime()) +} + +func key(k1, k2 string) string { + return fmt.Sprintf("%s-%s", k1, k2) +} diff --git a/pkg/tempo/servicegraphprocessor/processor_test.go b/pkg/tempo/servicegraphprocessor/processor_test.go new file mode 100644 index 000000000000..4948ad667884 --- /dev/null +++ b/pkg/tempo/servicegraphprocessor/processor_test.go @@ -0,0 +1,194 @@ +package servicegraphprocessor + +import ( + "bytes" + "context" + "os" + "testing" + "time" + + "github.com/gogo/protobuf/jsonpb" + "github.com/grafana/agent/pkg/tempo/contextkeys" + "github.com/grafana/tempo/pkg/tempopb" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/model/otlp" + "go.opentelemetry.io/collector/model/pdata" +) + +const ( + traceSamplePath = "testdata/trace-sample.json" + unpairedTraceSamplePath = "testdata/unpaired-trace-sample.json" +) + +func TestConsumeMetrics(t *testing.T) { + for _, tc := range []struct { + name string + sampleDataPath string + cfg *Config + expectedMetrics string + }{ + { + name: "happy case", + sampleDataPath: traceSamplePath, + cfg: &Config{}, + expectedMetrics: happyCaseExpectedMetrics, + }, + { + name: "unpaired spans", + sampleDataPath: unpairedTraceSamplePath, + cfg: &Config{ + Wait: time.Millisecond, + }, + expectedMetrics: ` + # HELP tempo_service_graph_unpaired_spans_total Total count of requests between two nodes + # TYPE tempo_service_graph_unpaired_spans_total counter + tempo_service_graph_unpaired_spans_total{client="",server="db"} 2 + tempo_service_graph_unpaired_spans_total{client="app",server=""} 3 + tempo_service_graph_unpaired_spans_total{client="lb",server=""} 3 + # HELP tempo_service_graph_untagged_spans_total Total count of spans processed that were not tagged with span.kind + # TYPE tempo_service_graph_untagged_spans_total counter + tempo_service_graph_untagged_spans_total{span_kind="SPAN_KIND_UNSPECIFIED"} 5 +`, + }, + { + name: "max items in store is reached", + sampleDataPath: traceSamplePath, + cfg: &Config{ + Wait: time.Millisecond, + MaxItems: 1, // Configure max number of items in store to 1. Only one edgeRequest will be processed. + }, + expectedMetrics: ` + # HELP tempo_service_graph_unpaired_spans_total Total count of requests between two nodes + # TYPE tempo_service_graph_unpaired_spans_total counter + tempo_service_graph_unpaired_spans_total{client="lb",server=""} 1 + # HELP tempo_service_graph_untagged_spans_total Total count of spans processed that were not tagged with span.kind + # TYPE tempo_service_graph_untagged_spans_total counter + tempo_service_graph_untagged_spans_total{span_kind="SPAN_KIND_UNSPECIFIED"} 1 +`, + }, + } { + t.Run(tc.name, func(t *testing.T) { + p := newProcessor(&mockConsumer{}, tc.cfg) + + reg := prometheus.NewRegistry() + ctx := context.WithValue(context.Background(), contextkeys.PrometheusRegisterer, reg) + + err := p.Start(ctx, nil) + require.NoError(t, err) + + traces := traceSamples(t, tc.sampleDataPath) + err = p.ConsumeTraces(context.Background(), traces) + require.NoError(t, err) + + assert.Eventually(t, func() bool { + return testutil.GatherAndCompare(reg, bytes.NewBufferString(tc.expectedMetrics)) == nil + }, time.Second, time.Millisecond*100) + err = testutil.GatherAndCompare(reg, bytes.NewBufferString(tc.expectedMetrics)) + require.NoError(t, err) + }) + } + +} + +func traceSamples(t *testing.T, path string) pdata.Traces { + f, err := os.Open(path) + require.NoError(t, err) + + r := &tempopb.Trace{} + err = jsonpb.Unmarshal(f, r) + require.NoError(t, err) + + b, err := r.Marshal() + require.NoError(t, err) + + traces, err := otlp.NewProtobufTracesUnmarshaler().UnmarshalTraces(b) + require.NoError(t, err) + + return traces +} + +type mockConsumer struct{} + +func (m *mockConsumer) Capabilities() consumer.Capabilities { return consumer.Capabilities{} } + +func (m *mockConsumer) ConsumeTraces(context.Context, pdata.Traces) error { return nil } + +const ( + happyCaseExpectedMetrics = ` + # HELP tempo_service_graph_request_client_seconds Time for a request between two nodes as seen from the client + # TYPE tempo_service_graph_request_client_seconds histogram + tempo_service_graph_request_client_seconds_bucket{client="app",server="db",le="0.01"} 0 + tempo_service_graph_request_client_seconds_bucket{client="app",server="db",le="0.02"} 0 + tempo_service_graph_request_client_seconds_bucket{client="app",server="db",le="0.04"} 0 + tempo_service_graph_request_client_seconds_bucket{client="app",server="db",le="0.08"} 0 + tempo_service_graph_request_client_seconds_bucket{client="app",server="db",le="0.16"} 0 + tempo_service_graph_request_client_seconds_bucket{client="app",server="db",le="0.32"} 0 + tempo_service_graph_request_client_seconds_bucket{client="app",server="db",le="0.64"} 0 + tempo_service_graph_request_client_seconds_bucket{client="app",server="db",le="1.28"} 2 + tempo_service_graph_request_client_seconds_bucket{client="app",server="db",le="2.56"} 3 + tempo_service_graph_request_client_seconds_bucket{client="app",server="db",le="5.12"} 3 + tempo_service_graph_request_client_seconds_bucket{client="app",server="db",le="10.24"} 3 + tempo_service_graph_request_client_seconds_bucket{client="app",server="db",le="20.48"} 3 + tempo_service_graph_request_client_seconds_bucket{client="app",server="db",le="+Inf"} 3 + tempo_service_graph_request_client_seconds_sum{client="app",server="db"} 4.4 + tempo_service_graph_request_client_seconds_count{client="app",server="db"} 3 + tempo_service_graph_request_client_seconds_bucket{client="lb",server="app",le="0.01"} 0 + tempo_service_graph_request_client_seconds_bucket{client="lb",server="app",le="0.02"} 0 + tempo_service_graph_request_client_seconds_bucket{client="lb",server="app",le="0.04"} 0 + tempo_service_graph_request_client_seconds_bucket{client="lb",server="app",le="0.08"} 0 + tempo_service_graph_request_client_seconds_bucket{client="lb",server="app",le="0.16"} 0 + tempo_service_graph_request_client_seconds_bucket{client="lb",server="app",le="0.32"} 0 + tempo_service_graph_request_client_seconds_bucket{client="lb",server="app",le="0.64"} 0 + tempo_service_graph_request_client_seconds_bucket{client="lb",server="app",le="1.28"} 0 + tempo_service_graph_request_client_seconds_bucket{client="lb",server="app",le="2.56"} 2 + tempo_service_graph_request_client_seconds_bucket{client="lb",server="app",le="5.12"} 3 + tempo_service_graph_request_client_seconds_bucket{client="lb",server="app",le="10.24"} 3 + tempo_service_graph_request_client_seconds_bucket{client="lb",server="app",le="20.48"} 3 + tempo_service_graph_request_client_seconds_bucket{client="lb",server="app",le="+Inf"} 3 + tempo_service_graph_request_client_seconds_sum{client="lb",server="app"} 7.8 + tempo_service_graph_request_client_seconds_count{client="lb",server="app"} 3 + # HELP tempo_service_graph_request_server_seconds Time for a request between two nodes as seen from the server + # TYPE tempo_service_graph_request_server_seconds histogram + tempo_service_graph_request_server_seconds_bucket{client="app",server="db",le="0.01"} 0 + tempo_service_graph_request_server_seconds_bucket{client="app",server="db",le="0.02"} 0 + tempo_service_graph_request_server_seconds_bucket{client="app",server="db",le="0.04"} 0 + tempo_service_graph_request_server_seconds_bucket{client="app",server="db",le="0.08"} 0 + tempo_service_graph_request_server_seconds_bucket{client="app",server="db",le="0.16"} 0 + tempo_service_graph_request_server_seconds_bucket{client="app",server="db",le="0.32"} 0 + tempo_service_graph_request_server_seconds_bucket{client="app",server="db",le="0.64"} 0 + tempo_service_graph_request_server_seconds_bucket{client="app",server="db",le="1.28"} 1 + tempo_service_graph_request_server_seconds_bucket{client="app",server="db",le="2.56"} 3 + tempo_service_graph_request_server_seconds_bucket{client="app",server="db",le="5.12"} 3 + tempo_service_graph_request_server_seconds_bucket{client="app",server="db",le="10.24"} 3 + tempo_service_graph_request_server_seconds_bucket{client="app",server="db",le="20.48"} 3 + tempo_service_graph_request_server_seconds_bucket{client="app",server="db",le="+Inf"} 3 + tempo_service_graph_request_server_seconds_sum{client="app",server="db"} 5 + tempo_service_graph_request_server_seconds_count{client="app",server="db"} 3 + tempo_service_graph_request_server_seconds_bucket{client="lb",server="app",le="0.01"} 0 + tempo_service_graph_request_server_seconds_bucket{client="lb",server="app",le="0.02"} 0 + tempo_service_graph_request_server_seconds_bucket{client="lb",server="app",le="0.04"} 0 + tempo_service_graph_request_server_seconds_bucket{client="lb",server="app",le="0.08"} 0 + tempo_service_graph_request_server_seconds_bucket{client="lb",server="app",le="0.16"} 0 + tempo_service_graph_request_server_seconds_bucket{client="lb",server="app",le="0.32"} 0 + tempo_service_graph_request_server_seconds_bucket{client="lb",server="app",le="0.64"} 0 + tempo_service_graph_request_server_seconds_bucket{client="lb",server="app",le="1.28"} 1 + tempo_service_graph_request_server_seconds_bucket{client="lb",server="app",le="2.56"} 2 + tempo_service_graph_request_server_seconds_bucket{client="lb",server="app",le="5.12"} 3 + tempo_service_graph_request_server_seconds_bucket{client="lb",server="app",le="10.24"} 3 + tempo_service_graph_request_server_seconds_bucket{client="lb",server="app",le="20.48"} 3 + tempo_service_graph_request_server_seconds_bucket{client="lb",server="app",le="+Inf"} 3 + tempo_service_graph_request_server_seconds_sum{client="lb",server="app"} 6.2 + tempo_service_graph_request_server_seconds_count{client="lb",server="app"} 3 + # HELP tempo_service_graph_request_total Total count of requests between two nodes + # TYPE tempo_service_graph_request_total counter + tempo_service_graph_request_total{client="app",server="db"} 3 + tempo_service_graph_request_total{client="lb",server="app"} 3 + # HELP tempo_service_graph_untagged_spans_total Total count of spans processed that were not tagged with span.kind + # TYPE tempo_service_graph_untagged_spans_total counter + tempo_service_graph_untagged_spans_total{span_kind="SPAN_KIND_UNSPECIFIED"} 5 +` +) diff --git a/pkg/tempo/servicegraphprocessor/testdata/trace-sample.json b/pkg/tempo/servicegraphprocessor/testdata/trace-sample.json new file mode 100644 index 000000000000..e4bc282d71c9 --- /dev/null +++ b/pkg/tempo/servicegraphprocessor/testdata/trace-sample.json @@ -0,0 +1,1915 @@ +{ + "batches":[ + { + "resource":{ + "attributes":[ + { + "key":"service.name", + "value":{ + "stringValue":"lb" + } + }, + { + "key":"cluster", + "value":{ + "stringValue":"tns-demo" + } + }, + { + "key":"namespace", + "value":{ + "stringValue":"tns-demo" + } + }, + { + "key":"opencensus.exporterversion", + "value":{ + "stringValue":"Jaeger-Go-2.22.1" + } + }, + { + "key":"host.name", + "value":{ + "stringValue":"loadgen-6b59dff4c-jdkdk" + } + }, + { + "key":"ip", + "value":{ + "stringValue":"10.136.11.153" + } + }, + { + "key":"client-uuid", + "value":{ + "stringValue":"653fad9a76c115ac" + } + }, + { + "key":"container", + "value":{ + "stringValue":"loadgen" + } + }, + { + "key":"pod", + "value":{ + "stringValue":"loadgen-6b59dff4c-jdkdk" + } + } + ] + }, + "instrumentationLibrarySpans":[ + { + "instrumentationLibrary":{ + + }, + "spans":[ + { + "traceId":"AAAAAAAAAABbdZuWJO/uiw==", + "spanId":"W3WbliTv7os=", + "name":"HTTP Client", + "startTimeUnixNano":"1626717505784699000", + "endTimeUnixNano":"1626717505833874000", + "attributes":[ + { + "key":"sampler.type", + "value":{ + "stringValue":"const" + } + }, + { + "key":"sampler.param", + "value":{ + "boolValue":true + } + } + ], + "status":{ + + } + }, + { + "traceId":"AAAAAAAAAABbdZuWJO/uiw==", + "spanId":"d5ZXpoG4mUc=", + "parentSpanId":"W3WbliTv7os=", + "name":"HTTP POST", + "kind":"SPAN_KIND_CLIENT", + "startTimeUnixNano":"1626717503329568000", + "endTimeUnixNano":"1626717505829568000", + "attributes":[ + { + "key":"http.status_code", + "value":{ + "intValue":"302" + } + }, + { + "key":"component", + "value":{ + "stringValue":"net/http" + } + }, + { + "key":"http.method", + "value":{ + "stringValue":"POST" + } + }, + { + "key":"http.url", + "value":{ + "stringValue":"app:80" + } + }, + { + "key":"net/http.reused", + "value":{ + "boolValue":false + } + }, + { + "key":"net/http.was_idle", + "value":{ + "boolValue":false + } + } + ], + "events":[ + { + "timeUnixNano":"1626717505784725000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GetConn" + } + } + ] + }, + { + "timeUnixNano":"1626717505784771000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"DNSStart" + } + }, + { + "key":"host", + "value":{ + "stringValue":"app" + } + } + ] + }, + { + "timeUnixNano":"1626717505822812000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"DNSDone" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.116.196" + } + } + ] + }, + { + "timeUnixNano":"1626717505822821000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ConnectStart" + } + }, + { + "key":"network", + "value":{ + "stringValue":"tcp" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.116.196:80" + } + } + ] + }, + { + "timeUnixNano":"1626717505822983000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ConnectDone" + } + }, + { + "key":"network", + "value":{ + "stringValue":"tcp" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.116.196:80" + } + } + ] + }, + { + "timeUnixNano":"1626717505823035000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GotConn" + } + } + ] + }, + { + "timeUnixNano":"1626717505823116000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"WroteHeaders" + } + } + ] + }, + { + "timeUnixNano":"1626717505823121000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"WroteRequest" + } + } + ] + }, + { + "timeUnixNano":"1626717505829460000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GotFirstResponseByte" + } + } + ] + }, + { + "timeUnixNano":"1626717505829568000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ClosedBody" + } + } + ] + } + ], + "status":{ + + } + }, + { + "traceId":"AAAAAAAAAABbdZuWJO/uiw==", + "spanId":"T3b9lSy4e6o=", + "parentSpanId":"W3WbliTv7os=", + "name":"HTTP GET", + "kind":"SPAN_KIND_CLIENT", + "startTimeUnixNano":"1626717504533933000", + "endTimeUnixNano":"1626717505833933000", + "attributes":[ + { + "key":"http.status_code", + "value":{ + "intValue":"200" + } + }, + { + "key":"component", + "value":{ + "stringValue":"net/http" + } + }, + { + "key":"http.method", + "value":{ + "stringValue":"GET" + } + }, + { + "key":"http.url", + "value":{ + "stringValue":"app:80" + } + }, + { + "key":"net/http.reused", + "value":{ + "boolValue":false + } + }, + { + "key":"net/http.was_idle", + "value":{ + "boolValue":false + } + } + ], + "events":[ + { + "timeUnixNano":"1626717505829603000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GetConn" + } + } + ] + }, + { + "timeUnixNano":"1626717505829642000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"DNSStart" + } + }, + { + "key":"host", + "value":{ + "stringValue":"app" + } + } + ] + }, + { + "timeUnixNano":"1626717505830180000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"DNSDone" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.116.196" + } + } + ] + }, + { + "timeUnixNano":"1626717505830186000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ConnectStart" + } + }, + { + "key":"network", + "value":{ + "stringValue":"tcp" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.116.196:80" + } + } + ] + }, + { + "timeUnixNano":"1626717505830301000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ConnectDone" + } + }, + { + "key":"network", + "value":{ + "stringValue":"tcp" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.116.196:80" + } + } + ] + }, + { + "timeUnixNano":"1626717505830332000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GotConn" + } + } + ] + }, + { + "timeUnixNano":"1626717505830372000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"WroteHeaders" + } + } + ] + }, + { + "timeUnixNano":"1626717505830373000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"WroteRequest" + } + } + ] + }, + { + "timeUnixNano":"1626717505833806000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GotFirstResponseByte" + } + } + ] + }, + { + "timeUnixNano":"1626717505833933000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ClosedBody" + } + } + ] + } + ], + "status":{ + + } + } + ] + } + ] + }, + { + "resource":{ + "attributes":[ + { + "key":"service.name", + "value":{ + "stringValue":"app" + } + }, + { + "key":"cluster", + "value":{ + "stringValue":"tns-demo" + } + }, + { + "key":"namespace", + "value":{ + "stringValue":"tns-demo" + } + }, + { + "key":"opencensus.exporterversion", + "value":{ + "stringValue":"Jaeger-Go-2.22.1" + } + }, + { + "key":"host.name", + "value":{ + "stringValue":"app-7c474df6bc-xpm6j" + } + }, + { + "key":"ip", + "value":{ + "stringValue":"10.136.11.151" + } + }, + { + "key":"client-uuid", + "value":{ + "stringValue":"264ce1d77c354156" + } + }, + { + "key":"container", + "value":{ + "stringValue":"app" + } + }, + { + "key":"pod", + "value":{ + "stringValue":"app-7c474df6bc-xpm6j" + } + } + ] + }, + "instrumentationLibrarySpans":[ + { + "instrumentationLibrary":{ + + }, + "spans":[ + { + "traceId":"AAAAAAAAAABbdZuWJO/uiw==", + "spanId":"CyLVKnG8lfk=", + "parentSpanId":"d5ZXpoG4mUc=", + "name":"HTTP POST - post", + "kind":"SPAN_KIND_SERVER", + "startTimeUnixNano":"1626717504829303000", + "endTimeUnixNano":"1626717505829303000", + "attributes":[ + { + "key":"http.status_code", + "value":{ + "intValue":"302" + } + }, + { + "key":"http.method", + "value":{ + "stringValue":"POST" + } + }, + { + "key":"http.url", + "value":{ + "stringValue":"/post" + } + }, + { + "key":"component", + "value":{ + "stringValue":"net/http" + } + } + ], + "status":{ + + } + }, + { + "traceId":"AAAAAAAAAABbdZuWJO/uiw==", + "spanId":"bzp4d/duh20=", + "parentSpanId":"CyLVKnG8lfk=", + "name":"HTTP Client", + "startTimeUnixNano":"1626717505823375000", + "endTimeUnixNano":"1626717505829164000", + "status":{ + + } + }, + { + "traceId":"AAAAAAAAAABbdZuWJO/uiw==", + "spanId":"T/rh1/wSL9g=", + "parentSpanId":"bzp4d/duh20=", + "name":"HTTP POST", + "kind":"SPAN_KIND_CLIENT", + "startTimeUnixNano":"1626717504929264000", + "endTimeUnixNano":"1626717505829264000", + "attributes":[ + { + "key":"http.status_code", + "value":{ + "intValue":"208" + } + }, + { + "key":"component", + "value":{ + "stringValue":"net/http" + } + }, + { + "key":"http.method", + "value":{ + "stringValue":"POST" + } + }, + { + "key":"http.url", + "value":{ + "stringValue":"db:80" + } + }, + { + "key":"net/http.reused", + "value":{ + "boolValue":false + } + }, + { + "key":"net/http.was_idle", + "value":{ + "boolValue":false + } + } + ], + "events":[ + { + "timeUnixNano":"1626717505823393000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GetConn" + } + } + ] + }, + { + "timeUnixNano":"1626717505823439000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"DNSStart" + } + }, + { + "key":"host", + "value":{ + "stringValue":"db" + } + } + ] + }, + { + "timeUnixNano":"1626717505827663000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"DNSDone" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.106.8" + } + } + ] + }, + { + "timeUnixNano":"1626717505827673000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ConnectStart" + } + }, + { + "key":"network", + "value":{ + "stringValue":"tcp" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.106.8:80" + } + } + ] + }, + { + "timeUnixNano":"1626717505827797000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ConnectDone" + } + }, + { + "key":"network", + "value":{ + "stringValue":"tcp" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.106.8:80" + } + } + ] + }, + { + "timeUnixNano":"1626717505827824000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GotConn" + } + } + ] + }, + { + "timeUnixNano":"1626717505827896000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"WroteHeaders" + } + } + ] + }, + { + "timeUnixNano":"1626717505827901000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"WroteRequest" + } + } + ] + }, + { + "timeUnixNano":"1626717505829057000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GotFirstResponseByte" + } + } + ] + }, + { + "timeUnixNano":"1626717505829264000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ClosedBody" + } + } + ] + } + ], + "status":{ + + } + }, + { + "traceId":"AAAAAAAAAABbdZuWJO/uiw==", + "spanId":"NvbVfAxyM10=", + "parentSpanId":"T3b9lSy4e6o=", + "name":"HTTP GET - root", + "kind":"SPAN_KIND_SERVER", + "startTimeUnixNano":"1626717503833939000", + "endTimeUnixNano":"1626717505833939000", + "attributes":[ + { + "key":"http.status_code", + "value":{ + "intValue":"200" + } + }, + { + "key":"http.method", + "value":{ + "stringValue":"GET" + } + }, + { + "key":"http.url", + "value":{ + "stringValue":"/" + } + }, + { + "key":"component", + "value":{ + "stringValue":"net/http" + } + } + ], + "status":{ + + } + }, + { + "traceId":"AAAAAAAAAABbdZuWJO/uiw==", + "spanId":"M464LUSGHVU=", + "parentSpanId":"NvbVfAxyM10=", + "name":"HTTP Client", + "startTimeUnixNano":"1626717505830578000", + "endTimeUnixNano":"1626717505833516000", + "status":{ + + } + }, + { + "traceId":"AAAAAAAAAABbdZuWJO/uiw==", + "spanId":"FcfJmhEwNxM=", + "parentSpanId":"M464LUSGHVU=", + "name":"HTTP GET", + "kind":"SPAN_KIND_CLIENT", + "startTimeUnixNano":"1626717504833879000", + "endTimeUnixNano":"1626717505833879000", + "attributes":[ + { + "key":"http.status_code", + "value":{ + "intValue":"200" + } + }, + { + "key":"component", + "value":{ + "stringValue":"net/http" + } + }, + { + "key":"http.method", + "value":{ + "stringValue":"GET" + } + }, + { + "key":"http.url", + "value":{ + "stringValue":"db:80" + } + }, + { + "key":"net/http.reused", + "value":{ + "boolValue":false + } + }, + { + "key":"net/http.was_idle", + "value":{ + "boolValue":false + } + } + ], + "events":[ + { + "timeUnixNano":"1626717505830596000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GetConn" + } + } + ] + }, + { + "timeUnixNano":"1626717505830647000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"DNSStart" + } + }, + { + "key":"host", + "value":{ + "stringValue":"db" + } + } + ] + }, + { + "timeUnixNano":"1626717505832975000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"DNSDone" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.106.8" + } + } + ] + }, + { + "timeUnixNano":"1626717505832983000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ConnectStart" + } + }, + { + "key":"network", + "value":{ + "stringValue":"tcp" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.106.8:80" + } + } + ] + }, + { + "timeUnixNano":"1626717505833096000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ConnectDone" + } + }, + { + "key":"network", + "value":{ + "stringValue":"tcp" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.106.8:80" + } + } + ] + }, + { + "timeUnixNano":"1626717505833116000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GotConn" + } + } + ] + }, + { + "timeUnixNano":"1626717505833153000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"WroteHeaders" + } + } + ] + }, + { + "timeUnixNano":"1626717505833154000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"WroteRequest" + } + } + ] + }, + { + "timeUnixNano":"1626717505833460000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GotFirstResponseByte" + } + } + ] + }, + { + "timeUnixNano":"1626717505833880000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ClosedBody" + } + } + ] + } + ], + "status":{ + + } + } + ] + } + ] + }, + { + "resource":{ + "attributes":[ + { + "key":"service.name", + "value":{ + "stringValue":"db" + } + }, + { + "key":"cluster", + "value":{ + "stringValue":"tns-demo" + } + }, + { + "key":"namespace", + "value":{ + "stringValue":"tns-demo" + } + }, + { + "key":"opencensus.exporterversion", + "value":{ + "stringValue":"Jaeger-Go-2.22.1" + } + }, + { + "key":"host.name", + "value":{ + "stringValue":"db-7488656cb4-m8ljw" + } + }, + { + "key":"ip", + "value":{ + "stringValue":"10.136.11.152" + } + }, + { + "key":"client-uuid", + "value":{ + "stringValue":"392a5faabe967ba3" + } + }, + { + "key":"container", + "value":{ + "stringValue":"db" + } + }, + { + "key":"pod", + "value":{ + "stringValue":"db-7488656cb4-m8ljw" + } + } + ] + }, + "instrumentationLibrarySpans":[ + { + "instrumentationLibrary":{ + + }, + "spans":[ + { + "traceId":"AAAAAAAAAABbdZuWJO/uiw==", + "spanId":"IElW8xeWvqs=", + "parentSpanId":"T/rh1/wSL9g=", + "name":"HTTP POST - post", + "kind":"SPAN_KIND_SERVER", + "startTimeUnixNano":"1626717504328961000", + "endTimeUnixNano":"1626717505828961000", + "attributes":[ + { + "key":"http.status_code", + "value":{ + "intValue":"208" + } + }, + { + "key":"http.method", + "value":{ + "stringValue":"POST" + } + }, + { + "key":"http.url", + "value":{ + "stringValue":"/post" + } + }, + { + "key":"component", + "value":{ + "stringValue":"net/http" + } + } + ], + "status":{ + + } + }, + { + "traceId":"AAAAAAAAAABbdZuWJO/uiw==", + "spanId":"frLAE97IEMg=", + "parentSpanId":"FcfJmhEwNxM=", + "name":"HTTP GET - root", + "kind":"SPAN_KIND_SERVER", + "startTimeUnixNano":"1626717504833394000", + "endTimeUnixNano":"1626717505833394000", + "attributes":[ + { + "key":"http.status_code", + "value":{ + "intValue":"200" + } + }, + { + "key":"http.method", + "value":{ + "stringValue":"GET" + } + }, + { + "key":"http.url", + "value":{ + "stringValue":"/" + } + }, + { + "key":"component", + "value":{ + "stringValue":"net/http" + } + } + ], + "status":{ + + } + } + ] + } + ] + }, + { + "resource":{ + "attributes":[ + { + "key":"service.name", + "value":{ + "stringValue":"lb" + } + }, + { + "key":"cluster", + "value":{ + "stringValue":"tns-demo" + } + }, + { + "key":"namespace", + "value":{ + "stringValue":"tns-demo" + } + }, + { + "key":"opencensus.exporterversion", + "value":{ + "stringValue":"Jaeger-Go-2.22.1" + } + }, + { + "key":"host.name", + "value":{ + "stringValue":"loadgen-6b59dff4c-jdkdk" + } + }, + { + "key":"ip", + "value":{ + "stringValue":"10.136.11.153" + } + }, + { + "key":"client-uuid", + "value":{ + "stringValue":"653fad9a76c115ac" + } + }, + { + "key":"container", + "value":{ + "stringValue":"loadgen" + } + }, + { + "key":"pod", + "value":{ + "stringValue":"loadgen-6b59dff4c-jdkdk" + } + } + ] + }, + "instrumentationLibrarySpans":[ + { + "instrumentationLibrary":{ + + }, + "spans":[ + { + "traceId":"AAAAAAAAAABFFdxqWiBZ9g==", + "spanId":"RRXcalogWfY=", + "name":"HTTP Client", + "startTimeUnixNano":"1626717505125848000", + "endTimeUnixNano":"1626717505130325000", + "attributes":[ + { + "key":"sampler.type", + "value":{ + "stringValue":"const" + } + }, + { + "key":"sampler.param", + "value":{ + "boolValue":true + } + } + ], + "status":{ + + } + }, + { + "traceId":"AAAAAAAAAABFFdxqWiBZ9g==", + "spanId":"Uoyxh+9zsBo=", + "parentSpanId":"RRXcalogWfY=", + "name":"HTTP GET", + "kind":"SPAN_KIND_CLIENT", + "startTimeUnixNano":"1626717501130383000", + "endTimeUnixNano":"1626717505130383000", + "attributes":[ + { + "key":"http.status_code", + "value":{ + "intValue":"200" + } + }, + { + "key":"component", + "value":{ + "stringValue":"net/http" + } + }, + { + "key":"http.method", + "value":{ + "stringValue":"GET" + } + }, + { + "key":"http.url", + "value":{ + "stringValue":"app:80" + } + }, + { + "key":"net/http.reused", + "value":{ + "boolValue":false + } + }, + { + "key":"net/http.was_idle", + "value":{ + "boolValue":false + } + } + ], + "events":[ + { + "timeUnixNano":"1626717505125868000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GetConn" + } + } + ] + }, + { + "timeUnixNano":"1626717505125909000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"DNSStart" + } + }, + { + "key":"host", + "value":{ + "stringValue":"app" + } + } + ] + }, + { + "timeUnixNano":"1626717505126450000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"DNSDone" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.116.196" + } + } + ] + }, + { + "timeUnixNano":"1626717505126455000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ConnectStart" + } + }, + { + "key":"network", + "value":{ + "stringValue":"tcp" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.116.196:80" + } + } + ] + }, + { + "timeUnixNano":"1626717505126534000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ConnectDone" + } + }, + { + "key":"network", + "value":{ + "stringValue":"tcp" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.116.196:80" + } + } + ] + }, + { + "timeUnixNano":"1626717505126556000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GotConn" + } + } + ] + }, + { + "timeUnixNano":"1626717505126585000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"WroteHeaders" + } + } + ] + }, + { + "timeUnixNano":"1626717505126586000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"WroteRequest" + } + } + ] + }, + { + "timeUnixNano":"1626717505130263000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GotFirstResponseByte" + } + } + ] + }, + { + "timeUnixNano":"1626717505130383000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ClosedBody" + } + } + ] + } + ], + "status":{ + + } + } + ] + } + ] + }, + { + "resource":{ + "attributes":[ + { + "key":"service.name", + "value":{ + "stringValue":"app" + } + }, + { + "key":"cluster", + "value":{ + "stringValue":"tns-demo" + } + }, + { + "key":"namespace", + "value":{ + "stringValue":"tns-demo" + } + }, + { + "key":"opencensus.exporterversion", + "value":{ + "stringValue":"Jaeger-Go-2.22.1" + } + }, + { + "key":"host.name", + "value":{ + "stringValue":"app-7c474df6bc-xpm6j" + } + }, + { + "key":"ip", + "value":{ + "stringValue":"10.136.11.151" + } + }, + { + "key":"client-uuid", + "value":{ + "stringValue":"264ce1d77c354156" + } + }, + { + "key":"pod", + "value":{ + "stringValue":"app-7c474df6bc-xpm6j" + } + }, + { + "key":"container", + "value":{ + "stringValue":"app" + } + } + ] + }, + "instrumentationLibrarySpans":[ + { + "instrumentationLibrary":{ + + }, + "spans":[ + { + "traceId":"AAAAAAAAAABFFdxqWiBZ9g==", + "spanId":"UQZfs+d8Sdw=", + "parentSpanId":"Uoyxh+9zsBo=", + "name":"HTTP GET - root", + "kind":"SPAN_KIND_SERVER", + "startTimeUnixNano":"1626717501930340000", + "endTimeUnixNano":"1626717505130340000", + "attributes":[ + { + "key":"http.status_code", + "value":{ + "intValue":"200" + } + }, + { + "key":"http.method", + "value":{ + "stringValue":"GET" + } + }, + { + "key":"http.url", + "value":{ + "stringValue":"/" + } + }, + { + "key":"component", + "value":{ + "stringValue":"net/http" + } + } + ], + "status":{ + + } + }, + { + "traceId":"AAAAAAAAAABFFdxqWiBZ9g==", + "spanId":"H7/n/FTsqf0=", + "parentSpanId":"UQZfs+d8Sdw=", + "name":"HTTP Client", + "startTimeUnixNano":"1626717505126741000", + "endTimeUnixNano":"1626717505130038000", + "status":{ + + } + }, + { + "traceId":"AAAAAAAAAABFFdxqWiBZ9g==", + "spanId":"ERieXkZU3MU=", + "parentSpanId":"H7/n/FTsqf0=", + "name":"HTTP GET", + "kind":"SPAN_KIND_CLIENT", + "startTimeUnixNano":"1626717502630304000", + "endTimeUnixNano":"1626717505130304000", + "attributes":[ + { + "key":"http.status_code", + "value":{ + "intValue":"200" + } + }, + { + "key":"component", + "value":{ + "stringValue":"net/http" + } + }, + { + "key":"http.method", + "value":{ + "stringValue":"GET" + } + }, + { + "key":"http.url", + "value":{ + "stringValue":"db:80" + } + }, + { + "key":"net/http.reused", + "value":{ + "boolValue":false + } + }, + { + "key":"net/http.was_idle", + "value":{ + "boolValue":false + } + } + ], + "events":[ + { + "timeUnixNano":"1626717505126754000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GetConn" + } + } + ] + }, + { + "timeUnixNano":"1626717505126800000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"DNSStart" + } + }, + { + "key":"host", + "value":{ + "stringValue":"db" + } + } + ] + }, + { + "timeUnixNano":"1626717505129605000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"DNSDone" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.106.8" + } + } + ] + }, + { + "timeUnixNano":"1626717505129609000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ConnectStart" + } + }, + { + "key":"network", + "value":{ + "stringValue":"tcp" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.106.8:80" + } + } + ] + }, + { + "timeUnixNano":"1626717505129678000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ConnectDone" + } + }, + { + "key":"network", + "value":{ + "stringValue":"tcp" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.106.8:80" + } + } + ] + }, + { + "timeUnixNano":"1626717505129695000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GotConn" + } + } + ] + }, + { + "timeUnixNano":"1626717505129727000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"WroteHeaders" + } + } + ] + }, + { + "timeUnixNano":"1626717505129727000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"WroteRequest" + } + } + ] + }, + { + "timeUnixNano":"1626717505129965000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GotFirstResponseByte" + } + } + ] + }, + { + "timeUnixNano":"1626717505130303000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ClosedBody" + } + } + ] + } + ], + "status":{ + + } + } + ] + } + ] + }, + { + "resource":{ + "attributes":[ + { + "key":"service.name", + "value":{ + "stringValue":"db" + } + }, + { + "key":"cluster", + "value":{ + "stringValue":"tns-demo" + } + }, + { + "key":"namespace", + "value":{ + "stringValue":"tns-demo" + } + }, + { + "key":"opencensus.exporterversion", + "value":{ + "stringValue":"Jaeger-Go-2.22.1" + } + }, + { + "key":"host.name", + "value":{ + "stringValue":"db-7488656cb4-m8ljw" + } + }, + { + "key":"ip", + "value":{ + "stringValue":"10.136.11.152" + } + }, + { + "key":"client-uuid", + "value":{ + "stringValue":"392a5faabe967ba3" + } + }, + { + "key":"container", + "value":{ + "stringValue":"db" + } + }, + { + "key":"pod", + "value":{ + "stringValue":"db-7488656cb4-m8ljw" + } + } + ] + }, + "instrumentationLibrarySpans":[ + { + "instrumentationLibrary":{ + + }, + "spans":[ + { + "traceId":"AAAAAAAAAABFFdxqWiBZ9g==", + "spanId":"ZCqiD3w+XGc=", + "parentSpanId":"ERieXkZU3MU=", + "name":"HTTP GET - root", + "kind":"SPAN_KIND_SERVER", + "startTimeUnixNano":"1626717502629891000", + "endTimeUnixNano":"1626717505129891000", + "attributes":[ + { + "key":"http.status_code", + "value":{ + "intValue":"200" + } + }, + { + "key":"http.method", + "value":{ + "stringValue":"GET" + } + }, + { + "key":"http.url", + "value":{ + "stringValue":"/" + } + }, + { + "key":"component", + "value":{ + "stringValue":"net/http" + } + } + ], + "status":{ + + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/pkg/tempo/servicegraphprocessor/testdata/unpaired-trace-sample.json b/pkg/tempo/servicegraphprocessor/testdata/unpaired-trace-sample.json new file mode 100644 index 000000000000..ad3a0f6a2f69 --- /dev/null +++ b/pkg/tempo/servicegraphprocessor/testdata/unpaired-trace-sample.json @@ -0,0 +1,1900 @@ +{ + "batches":[ + { + "resource":{ + "attributes":[ + { + "key":"service.name", + "value":{ + "stringValue":"lb" + } + }, + { + "key":"cluster", + "value":{ + "stringValue":"tns-demo" + } + }, + { + "key":"namespace", + "value":{ + "stringValue":"tns-demo" + } + }, + { + "key":"opencensus.exporterversion", + "value":{ + "stringValue":"Jaeger-Go-2.22.1" + } + }, + { + "key":"host.name", + "value":{ + "stringValue":"loadgen-6b59dff4c-jdkdk" + } + }, + { + "key":"ip", + "value":{ + "stringValue":"10.136.11.153" + } + }, + { + "key":"client-uuid", + "value":{ + "stringValue":"653fad9a76c115ac" + } + }, + { + "key":"container", + "value":{ + "stringValue":"loadgen" + } + }, + { + "key":"pod", + "value":{ + "stringValue":"loadgen-6b59dff4c-jdkdk" + } + } + ] + }, + "instrumentationLibrarySpans":[ + { + "instrumentationLibrary":{ + + }, + "spans":[ + { + "traceId":"AAAAAAAAAABbdZuWJO/uiw==", + "spanId":"W3WbliTv7os=", + "name":"HTTP Client", + "startTimeUnixNano":"1626717505784699000", + "endTimeUnixNano":"1626717505833874000", + "attributes":[ + { + "key":"sampler.type", + "value":{ + "stringValue":"const" + } + }, + { + "key":"sampler.param", + "value":{ + "boolValue":true + } + } + ], + "status":{ + + } + }, + { + "traceId":"AAAAAAAAAABbdZuWJO/uiw==", + "spanId":"d5ZXpoG4mUc=", + "name":"HTTP POST", + "kind":"SPAN_KIND_CLIENT", + "startTimeUnixNano":"1626717503329568000", + "endTimeUnixNano":"1626717505829568000", + "attributes":[ + { + "key":"http.status_code", + "value":{ + "intValue":"302" + } + }, + { + "key":"component", + "value":{ + "stringValue":"net/http" + } + }, + { + "key":"http.method", + "value":{ + "stringValue":"POST" + } + }, + { + "key":"http.url", + "value":{ + "stringValue":"app:80" + } + }, + { + "key":"net/http.reused", + "value":{ + "boolValue":false + } + }, + { + "key":"net/http.was_idle", + "value":{ + "boolValue":false + } + } + ], + "events":[ + { + "timeUnixNano":"1626717505784725000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GetConn" + } + } + ] + }, + { + "timeUnixNano":"1626717505784771000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"DNSStart" + } + }, + { + "key":"host", + "value":{ + "stringValue":"app" + } + } + ] + }, + { + "timeUnixNano":"1626717505822812000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"DNSDone" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.116.196" + } + } + ] + }, + { + "timeUnixNano":"1626717505822821000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ConnectStart" + } + }, + { + "key":"network", + "value":{ + "stringValue":"tcp" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.116.196:80" + } + } + ] + }, + { + "timeUnixNano":"1626717505822983000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ConnectDone" + } + }, + { + "key":"network", + "value":{ + "stringValue":"tcp" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.116.196:80" + } + } + ] + }, + { + "timeUnixNano":"1626717505823035000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GotConn" + } + } + ] + }, + { + "timeUnixNano":"1626717505823116000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"WroteHeaders" + } + } + ] + }, + { + "timeUnixNano":"1626717505823121000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"WroteRequest" + } + } + ] + }, + { + "timeUnixNano":"1626717505829460000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GotFirstResponseByte" + } + } + ] + }, + { + "timeUnixNano":"1626717505829568000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ClosedBody" + } + } + ] + } + ], + "status":{ + + } + }, + { + "traceId":"AAAAAAAAAABbdZuWJO/uiw==", + "spanId":"T3b9lSy4e6o=", + "name":"HTTP GET", + "kind":"SPAN_KIND_CLIENT", + "startTimeUnixNano":"1626717504533933000", + "endTimeUnixNano":"1626717505833933000", + "attributes":[ + { + "key":"http.status_code", + "value":{ + "intValue":"200" + } + }, + { + "key":"component", + "value":{ + "stringValue":"net/http" + } + }, + { + "key":"http.method", + "value":{ + "stringValue":"GET" + } + }, + { + "key":"http.url", + "value":{ + "stringValue":"app:80" + } + }, + { + "key":"net/http.reused", + "value":{ + "boolValue":false + } + }, + { + "key":"net/http.was_idle", + "value":{ + "boolValue":false + } + } + ], + "events":[ + { + "timeUnixNano":"1626717505829603000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GetConn" + } + } + ] + }, + { + "timeUnixNano":"1626717505829642000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"DNSStart" + } + }, + { + "key":"host", + "value":{ + "stringValue":"app" + } + } + ] + }, + { + "timeUnixNano":"1626717505830180000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"DNSDone" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.116.196" + } + } + ] + }, + { + "timeUnixNano":"1626717505830186000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ConnectStart" + } + }, + { + "key":"network", + "value":{ + "stringValue":"tcp" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.116.196:80" + } + } + ] + }, + { + "timeUnixNano":"1626717505830301000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ConnectDone" + } + }, + { + "key":"network", + "value":{ + "stringValue":"tcp" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.116.196:80" + } + } + ] + }, + { + "timeUnixNano":"1626717505830332000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GotConn" + } + } + ] + }, + { + "timeUnixNano":"1626717505830372000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"WroteHeaders" + } + } + ] + }, + { + "timeUnixNano":"1626717505830373000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"WroteRequest" + } + } + ] + }, + { + "timeUnixNano":"1626717505833806000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GotFirstResponseByte" + } + } + ] + }, + { + "timeUnixNano":"1626717505833933000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ClosedBody" + } + } + ] + } + ], + "status":{ + + } + } + ] + } + ] + }, + { + "resource":{ + "attributes":[ + { + "key":"service.name", + "value":{ + "stringValue":"app" + } + }, + { + "key":"cluster", + "value":{ + "stringValue":"tns-demo" + } + }, + { + "key":"namespace", + "value":{ + "stringValue":"tns-demo" + } + }, + { + "key":"opencensus.exporterversion", + "value":{ + "stringValue":"Jaeger-Go-2.22.1" + } + }, + { + "key":"host.name", + "value":{ + "stringValue":"app-7c474df6bc-xpm6j" + } + }, + { + "key":"ip", + "value":{ + "stringValue":"10.136.11.151" + } + }, + { + "key":"client-uuid", + "value":{ + "stringValue":"264ce1d77c354156" + } + }, + { + "key":"container", + "value":{ + "stringValue":"app" + } + }, + { + "key":"pod", + "value":{ + "stringValue":"app-7c474df6bc-xpm6j" + } + } + ] + }, + "instrumentationLibrarySpans":[ + { + "instrumentationLibrary":{ + + }, + "spans":[ + { + "traceId":"AAAAAAAAAABbdZuWJO/uiw==", + "spanId":"CyLVKnG8lfk=", + "name":"HTTP POST - post", + "kind":"SPAN_KIND_SERVER", + "startTimeUnixNano":"1626717504829303000", + "endTimeUnixNano":"1626717505829303000", + "attributes":[ + { + "key":"http.status_code", + "value":{ + "intValue":"302" + } + }, + { + "key":"http.method", + "value":{ + "stringValue":"POST" + } + }, + { + "key":"http.url", + "value":{ + "stringValue":"/post" + } + }, + { + "key":"component", + "value":{ + "stringValue":"net/http" + } + } + ], + "status":{ + + } + }, + { + "traceId":"AAAAAAAAAABbdZuWJO/uiw==", + "spanId":"bzp4d/duh20=", + "name":"HTTP Client", + "startTimeUnixNano":"1626717505823375000", + "endTimeUnixNano":"1626717505829164000", + "status":{ + + } + }, + { + "traceId":"AAAAAAAAAABbdZuWJO/uiw==", + "spanId":"T/rh1/wSL9g=", + "name":"HTTP POST", + "kind":"SPAN_KIND_CLIENT", + "startTimeUnixNano":"1626717504929264000", + "endTimeUnixNano":"1626717505829264000", + "attributes":[ + { + "key":"http.status_code", + "value":{ + "intValue":"208" + } + }, + { + "key":"component", + "value":{ + "stringValue":"net/http" + } + }, + { + "key":"http.method", + "value":{ + "stringValue":"POST" + } + }, + { + "key":"http.url", + "value":{ + "stringValue":"db:80" + } + }, + { + "key":"net/http.reused", + "value":{ + "boolValue":false + } + }, + { + "key":"net/http.was_idle", + "value":{ + "boolValue":false + } + } + ], + "events":[ + { + "timeUnixNano":"1626717505823393000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GetConn" + } + } + ] + }, + { + "timeUnixNano":"1626717505823439000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"DNSStart" + } + }, + { + "key":"host", + "value":{ + "stringValue":"db" + } + } + ] + }, + { + "timeUnixNano":"1626717505827663000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"DNSDone" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.106.8" + } + } + ] + }, + { + "timeUnixNano":"1626717505827673000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ConnectStart" + } + }, + { + "key":"network", + "value":{ + "stringValue":"tcp" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.106.8:80" + } + } + ] + }, + { + "timeUnixNano":"1626717505827797000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ConnectDone" + } + }, + { + "key":"network", + "value":{ + "stringValue":"tcp" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.106.8:80" + } + } + ] + }, + { + "timeUnixNano":"1626717505827824000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GotConn" + } + } + ] + }, + { + "timeUnixNano":"1626717505827896000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"WroteHeaders" + } + } + ] + }, + { + "timeUnixNano":"1626717505827901000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"WroteRequest" + } + } + ] + }, + { + "timeUnixNano":"1626717505829057000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GotFirstResponseByte" + } + } + ] + }, + { + "timeUnixNano":"1626717505829264000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ClosedBody" + } + } + ] + } + ], + "status":{ + + } + }, + { + "traceId":"AAAAAAAAAABbdZuWJO/uiw==", + "spanId":"NvbVfAxyM10=", + "name":"HTTP GET - root", + "kind":"SPAN_KIND_SERVER", + "startTimeUnixNano":"1626717503833939000", + "endTimeUnixNano":"1626717505833939000", + "attributes":[ + { + "key":"http.status_code", + "value":{ + "intValue":"200" + } + }, + { + "key":"http.method", + "value":{ + "stringValue":"GET" + } + }, + { + "key":"http.url", + "value":{ + "stringValue":"/" + } + }, + { + "key":"component", + "value":{ + "stringValue":"net/http" + } + } + ], + "status":{ + + } + }, + { + "traceId":"AAAAAAAAAABbdZuWJO/uiw==", + "spanId":"M464LUSGHVU=", + "name":"HTTP Client", + "startTimeUnixNano":"1626717505830578000", + "endTimeUnixNano":"1626717505833516000", + "status":{ + + } + }, + { + "traceId":"AAAAAAAAAABbdZuWJO/uiw==", + "spanId":"FcfJmhEwNxM=", + "name":"HTTP GET", + "kind":"SPAN_KIND_CLIENT", + "startTimeUnixNano":"1626717504833879000", + "endTimeUnixNano":"1626717505833879000", + "attributes":[ + { + "key":"http.status_code", + "value":{ + "intValue":"200" + } + }, + { + "key":"component", + "value":{ + "stringValue":"net/http" + } + }, + { + "key":"http.method", + "value":{ + "stringValue":"GET" + } + }, + { + "key":"http.url", + "value":{ + "stringValue":"db:80" + } + }, + { + "key":"net/http.reused", + "value":{ + "boolValue":false + } + }, + { + "key":"net/http.was_idle", + "value":{ + "boolValue":false + } + } + ], + "events":[ + { + "timeUnixNano":"1626717505830596000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GetConn" + } + } + ] + }, + { + "timeUnixNano":"1626717505830647000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"DNSStart" + } + }, + { + "key":"host", + "value":{ + "stringValue":"db" + } + } + ] + }, + { + "timeUnixNano":"1626717505832975000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"DNSDone" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.106.8" + } + } + ] + }, + { + "timeUnixNano":"1626717505832983000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ConnectStart" + } + }, + { + "key":"network", + "value":{ + "stringValue":"tcp" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.106.8:80" + } + } + ] + }, + { + "timeUnixNano":"1626717505833096000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ConnectDone" + } + }, + { + "key":"network", + "value":{ + "stringValue":"tcp" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.106.8:80" + } + } + ] + }, + { + "timeUnixNano":"1626717505833116000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GotConn" + } + } + ] + }, + { + "timeUnixNano":"1626717505833153000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"WroteHeaders" + } + } + ] + }, + { + "timeUnixNano":"1626717505833154000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"WroteRequest" + } + } + ] + }, + { + "timeUnixNano":"1626717505833460000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GotFirstResponseByte" + } + } + ] + }, + { + "timeUnixNano":"1626717505833880000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ClosedBody" + } + } + ] + } + ], + "status":{ + + } + } + ] + } + ] + }, + { + "resource":{ + "attributes":[ + { + "key":"service.name", + "value":{ + "stringValue":"db" + } + }, + { + "key":"cluster", + "value":{ + "stringValue":"tns-demo" + } + }, + { + "key":"namespace", + "value":{ + "stringValue":"tns-demo" + } + }, + { + "key":"opencensus.exporterversion", + "value":{ + "stringValue":"Jaeger-Go-2.22.1" + } + }, + { + "key":"host.name", + "value":{ + "stringValue":"db-7488656cb4-m8ljw" + } + }, + { + "key":"ip", + "value":{ + "stringValue":"10.136.11.152" + } + }, + { + "key":"client-uuid", + "value":{ + "stringValue":"392a5faabe967ba3" + } + }, + { + "key":"container", + "value":{ + "stringValue":"db" + } + }, + { + "key":"pod", + "value":{ + "stringValue":"db-7488656cb4-m8ljw" + } + } + ] + }, + "instrumentationLibrarySpans":[ + { + "instrumentationLibrary":{ + + }, + "spans":[ + { + "traceId":"AAAAAAAAAABbdZuWJO/uiw==", + "spanId":"IElW8xeWvqs=", + "name":"HTTP POST - post", + "kind":"SPAN_KIND_SERVER", + "startTimeUnixNano":"1626717504328961000", + "endTimeUnixNano":"1626717505828961000", + "attributes":[ + { + "key":"http.status_code", + "value":{ + "intValue":"208" + } + }, + { + "key":"http.method", + "value":{ + "stringValue":"POST" + } + }, + { + "key":"http.url", + "value":{ + "stringValue":"/post" + } + }, + { + "key":"component", + "value":{ + "stringValue":"net/http" + } + } + ], + "status":{ + + } + }, + { + "traceId":"AAAAAAAAAABbdZuWJO/uiw==", + "spanId":"frLAE97IEMg=", + "name":"HTTP GET - root", + "kind":"SPAN_KIND_SERVER", + "startTimeUnixNano":"1626717504833394000", + "endTimeUnixNano":"1626717505833394000", + "attributes":[ + { + "key":"http.status_code", + "value":{ + "intValue":"200" + } + }, + { + "key":"http.method", + "value":{ + "stringValue":"GET" + } + }, + { + "key":"http.url", + "value":{ + "stringValue":"/" + } + }, + { + "key":"component", + "value":{ + "stringValue":"net/http" + } + } + ], + "status":{ + + } + } + ] + } + ] + }, + { + "resource":{ + "attributes":[ + { + "key":"service.name", + "value":{ + "stringValue":"lb" + } + }, + { + "key":"cluster", + "value":{ + "stringValue":"tns-demo" + } + }, + { + "key":"namespace", + "value":{ + "stringValue":"tns-demo" + } + }, + { + "key":"opencensus.exporterversion", + "value":{ + "stringValue":"Jaeger-Go-2.22.1" + } + }, + { + "key":"host.name", + "value":{ + "stringValue":"loadgen-6b59dff4c-jdkdk" + } + }, + { + "key":"ip", + "value":{ + "stringValue":"10.136.11.153" + } + }, + { + "key":"client-uuid", + "value":{ + "stringValue":"653fad9a76c115ac" + } + }, + { + "key":"container", + "value":{ + "stringValue":"loadgen" + } + }, + { + "key":"pod", + "value":{ + "stringValue":"loadgen-6b59dff4c-jdkdk" + } + } + ] + }, + "instrumentationLibrarySpans":[ + { + "instrumentationLibrary":{ + + }, + "spans":[ + { + "traceId":"AAAAAAAAAABFFdxqWiBZ9g==", + "spanId":"RRXcalogWfY=", + "name":"HTTP Client", + "startTimeUnixNano":"1626717505125848000", + "endTimeUnixNano":"1626717505130325000", + "attributes":[ + { + "key":"sampler.type", + "value":{ + "stringValue":"const" + } + }, + { + "key":"sampler.param", + "value":{ + "boolValue":true + } + } + ], + "status":{ + + } + }, + { + "traceId":"AAAAAAAAAABFFdxqWiBZ9g==", + "spanId":"Uoyxh+9zsBo=", + "name":"HTTP GET", + "kind":"SPAN_KIND_CLIENT", + "startTimeUnixNano":"1626717501130383000", + "endTimeUnixNano":"1626717505130383000", + "attributes":[ + { + "key":"http.status_code", + "value":{ + "intValue":"200" + } + }, + { + "key":"component", + "value":{ + "stringValue":"net/http" + } + }, + { + "key":"http.method", + "value":{ + "stringValue":"GET" + } + }, + { + "key":"http.url", + "value":{ + "stringValue":"app:80" + } + }, + { + "key":"net/http.reused", + "value":{ + "boolValue":false + } + }, + { + "key":"net/http.was_idle", + "value":{ + "boolValue":false + } + } + ], + "events":[ + { + "timeUnixNano":"1626717505125868000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GetConn" + } + } + ] + }, + { + "timeUnixNano":"1626717505125909000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"DNSStart" + } + }, + { + "key":"host", + "value":{ + "stringValue":"app" + } + } + ] + }, + { + "timeUnixNano":"1626717505126450000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"DNSDone" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.116.196" + } + } + ] + }, + { + "timeUnixNano":"1626717505126455000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ConnectStart" + } + }, + { + "key":"network", + "value":{ + "stringValue":"tcp" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.116.196:80" + } + } + ] + }, + { + "timeUnixNano":"1626717505126534000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ConnectDone" + } + }, + { + "key":"network", + "value":{ + "stringValue":"tcp" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.116.196:80" + } + } + ] + }, + { + "timeUnixNano":"1626717505126556000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GotConn" + } + } + ] + }, + { + "timeUnixNano":"1626717505126585000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"WroteHeaders" + } + } + ] + }, + { + "timeUnixNano":"1626717505126586000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"WroteRequest" + } + } + ] + }, + { + "timeUnixNano":"1626717505130263000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GotFirstResponseByte" + } + } + ] + }, + { + "timeUnixNano":"1626717505130383000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ClosedBody" + } + } + ] + } + ], + "status":{ + + } + } + ] + } + ] + }, + { + "resource":{ + "attributes":[ + { + "key":"service.name", + "value":{ + "stringValue":"app" + } + }, + { + "key":"cluster", + "value":{ + "stringValue":"tns-demo" + } + }, + { + "key":"namespace", + "value":{ + "stringValue":"tns-demo" + } + }, + { + "key":"opencensus.exporterversion", + "value":{ + "stringValue":"Jaeger-Go-2.22.1" + } + }, + { + "key":"host.name", + "value":{ + "stringValue":"app-7c474df6bc-xpm6j" + } + }, + { + "key":"ip", + "value":{ + "stringValue":"10.136.11.151" + } + }, + { + "key":"client-uuid", + "value":{ + "stringValue":"264ce1d77c354156" + } + }, + { + "key":"pod", + "value":{ + "stringValue":"app-7c474df6bc-xpm6j" + } + }, + { + "key":"container", + "value":{ + "stringValue":"app" + } + } + ] + }, + "instrumentationLibrarySpans":[ + { + "instrumentationLibrary":{ + + }, + "spans":[ + { + "traceId":"AAAAAAAAAABFFdxqWiBZ9g==", + "spanId":"UQZfs+d8Sdw=", + "name":"HTTP GET - root", + "kind":"SPAN_KIND_SERVER", + "startTimeUnixNano":"1626717501930340000", + "endTimeUnixNano":"1626717505130340000", + "attributes":[ + { + "key":"http.status_code", + "value":{ + "intValue":"200" + } + }, + { + "key":"http.method", + "value":{ + "stringValue":"GET" + } + }, + { + "key":"http.url", + "value":{ + "stringValue":"/" + } + }, + { + "key":"component", + "value":{ + "stringValue":"net/http" + } + } + ], + "status":{ + + } + }, + { + "traceId":"AAAAAAAAAABFFdxqWiBZ9g==", + "spanId":"H7/n/FTsqf0=", + "name":"HTTP Client", + "startTimeUnixNano":"1626717505126741000", + "endTimeUnixNano":"1626717505130038000", + "status":{ + + } + }, + { + "traceId":"AAAAAAAAAABFFdxqWiBZ9g==", + "spanId":"ERieXkZU3MU=", + "name":"HTTP GET", + "kind":"SPAN_KIND_CLIENT", + "startTimeUnixNano":"1626717502630304000", + "endTimeUnixNano":"1626717505130304000", + "attributes":[ + { + "key":"http.status_code", + "value":{ + "intValue":"200" + } + }, + { + "key":"component", + "value":{ + "stringValue":"net/http" + } + }, + { + "key":"http.method", + "value":{ + "stringValue":"GET" + } + }, + { + "key":"http.url", + "value":{ + "stringValue":"db:80" + } + }, + { + "key":"net/http.reused", + "value":{ + "boolValue":false + } + }, + { + "key":"net/http.was_idle", + "value":{ + "boolValue":false + } + } + ], + "events":[ + { + "timeUnixNano":"1626717505126754000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GetConn" + } + } + ] + }, + { + "timeUnixNano":"1626717505126800000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"DNSStart" + } + }, + { + "key":"host", + "value":{ + "stringValue":"db" + } + } + ] + }, + { + "timeUnixNano":"1626717505129605000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"DNSDone" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.106.8" + } + } + ] + }, + { + "timeUnixNano":"1626717505129609000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ConnectStart" + } + }, + { + "key":"network", + "value":{ + "stringValue":"tcp" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.106.8:80" + } + } + ] + }, + { + "timeUnixNano":"1626717505129678000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ConnectDone" + } + }, + { + "key":"network", + "value":{ + "stringValue":"tcp" + } + }, + { + "key":"addr", + "value":{ + "stringValue":"10.188.106.8:80" + } + } + ] + }, + { + "timeUnixNano":"1626717505129695000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GotConn" + } + } + ] + }, + { + "timeUnixNano":"1626717505129727000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"WroteHeaders" + } + } + ] + }, + { + "timeUnixNano":"1626717505129727000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"WroteRequest" + } + } + ] + }, + { + "timeUnixNano":"1626717505129965000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"GotFirstResponseByte" + } + } + ] + }, + { + "timeUnixNano":"1626717505130303000", + "attributes":[ + { + "key":"event", + "value":{ + "stringValue":"ClosedBody" + } + } + ] + } + ], + "status":{ + + } + } + ] + } + ] + }, + { + "resource":{ + "attributes":[ + { + "key":"service.name", + "value":{ + "stringValue":"db" + } + }, + { + "key":"cluster", + "value":{ + "stringValue":"tns-demo" + } + }, + { + "key":"namespace", + "value":{ + "stringValue":"tns-demo" + } + }, + { + "key":"opencensus.exporterversion", + "value":{ + "stringValue":"Jaeger-Go-2.22.1" + } + }, + { + "key":"host.name", + "value":{ + "stringValue":"db-7488656cb4-m8ljw" + } + }, + { + "key":"ip", + "value":{ + "stringValue":"10.136.11.152" + } + }, + { + "key":"client-uuid", + "value":{ + "stringValue":"392a5faabe967ba3" + } + }, + { + "key":"container", + "value":{ + "stringValue":"db" + } + }, + { + "key":"pod", + "value":{ + "stringValue":"db-7488656cb4-m8ljw" + } + } + ] + }, + "instrumentationLibrarySpans":[ + { + "instrumentationLibrary":{ + + }, + "spans":[ + { + "traceId":"AAAAAAAAAABFFdxqWiBZ9g==", + "spanId":"ZCqiD3w+XGc=", + "name":"HTTP GET - root", + "kind":"SPAN_KIND_SERVER", + "startTimeUnixNano":"1626717502629891000", + "endTimeUnixNano":"1626717505129891000", + "attributes":[ + { + "key":"http.status_code", + "value":{ + "intValue":"200" + } + }, + { + "key":"http.method", + "value":{ + "stringValue":"GET" + } + }, + { + "key":"http.url", + "value":{ + "stringValue":"/" + } + }, + { + "key":"component", + "value":{ + "stringValue":"net/http" + } + } + ], + "status":{ + + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/pkg/tempo/tempo.go b/pkg/tempo/tempo.go index 5795f88f4e38..6b118e182411 100644 --- a/pkg/tempo/tempo.go +++ b/pkg/tempo/tempo.go @@ -60,9 +60,13 @@ func (t *Tempo) ApplyConfig(logsSubsystem *logs.Logs, promInstanceManager instan newInstances := make(map[string]*Instance, len(cfg.Configs)) for _, c := range cfg.Configs { + var ( + instReg = prom_client.WrapRegistererWith(prom_client.Labels{"tempo_config": c.Name}, t.reg) + ) + // If an old instance exists, update it and move it to the new map. if old, ok := t.instances[c.Name]; ok { - err := old.ApplyConfig(logsSubsystem, promInstanceManager, c) + err := old.ApplyConfig(logsSubsystem, promInstanceManager, instReg, c) if err != nil { return err } @@ -73,7 +77,6 @@ func (t *Tempo) ApplyConfig(logsSubsystem *logs.Logs, promInstanceManager instan var ( instLogger = t.logger.With(zap.String("tempo_config", c.Name)) - instReg = prom_client.WrapRegistererWith(prom_client.Labels{"tempo_config": c.Name}, t.reg) ) inst, err := NewInstance(logsSubsystem, instReg, c, instLogger, t.promInstanceManager) diff --git a/pkg/tempo/tempo_test.go b/pkg/tempo/tempo_test.go index 12a83bb73a0d..810821d70ee0 100644 --- a/pkg/tempo/tempo_test.go +++ b/pkg/tempo/tempo_test.go @@ -28,15 +28,15 @@ func TestTempo(t *testing.T) { configs: - name: default receivers: - jaeger: - protocols: - thrift_compact: - push_config: - endpoint: %s - insecure: true - batch: - timeout: 100ms - send_batch_size: 1 + jaeger: + protocols: + thrift_compact: + remote_write: + - endpoint: %s + insecure: true + batch: + timeout: 100ms + send_batch_size: 1 `, tracesAddr)) var cfg Config @@ -75,16 +75,18 @@ func TestTempo_ApplyConfig(t *testing.T) { configs: - name: default receivers: - jaeger: - protocols: - thrift_compact: - push_config: - endpoint: 127.0.0.1:80 # deliberately the wrong endpoint - insecure: true - batch: - timeout: 100ms - send_batch_size: 1 - `) + jaeger: + protocols: + thrift_compact: + remote_write: + - endpoint: 127.0.0.1:80 # deliberately the wrong endpoint + insecure: true + batch: + timeout: 100ms + send_batch_size: 1 + service_graphs: + enabled: true +`) var cfg Config dec := yaml.NewDecoder(strings.NewReader(tempoCfgText)) @@ -101,15 +103,15 @@ configs: configs: - name: default receivers: - jaeger: - protocols: - thrift_compact: - push_config: - endpoint: %s - insecure: true - batch: - timeout: 100ms - send_batch_size: 1 + jaeger: + protocols: + thrift_compact: + remote_write: + - endpoint: %s + insecure: true + batch: + timeout: 100ms + send_batch_size: 1 `, tracesAddr)) var fixedConfig Config From e3b871779cfdcc8d076ba784f2774cb7bbefabd5 Mon Sep 17 00:00:00 2001 From: Mario Date: Wed, 8 Sep 2021 14:15:42 +0200 Subject: [PATCH 2/4] Move port to load_balancing config as receiver_port (#891) * Move port to load_balancing config as receiver_port When extracting load balancing from tail sampling, only the the load_balancing block was moved into its own block, but the receiver's port was left in tail_sampling. This means that the receiver port for load_balancing could not be configured without tail_sampling. Now its moved to load_balancing as receiver_port. * Load balance without tail sampling * Move port up --- docs/upgrade-guide/_index.md | 5 +++- pkg/tempo/config.go | 13 +++++---- pkg/tempo/config_test.go | 55 ++++++++++++++++++++++++++++++++++-- 3 files changed, 63 insertions(+), 10 deletions(-) diff --git a/docs/upgrade-guide/_index.md b/docs/upgrade-guide/_index.md index fc8efaa023d4..ab65ac753e59 100644 --- a/docs/upgrade-guide/_index.md +++ b/docs/upgrade-guide/_index.md @@ -21,7 +21,8 @@ receiving all spans for a trace in the same agent to be processed, such as service graphs. As a consequence, `tail_sampling.load_balancing` has been deprecated in favor of -a `load_balancing` block. +a `load_balancing` block. Also, `port` has been renamed to `receiver_port` and +moved to the new `load_balancing` block. Example old config: @@ -29,6 +30,7 @@ Example old config: tail_sampling: policies: - always_sample: + port: 4318 load_balancing: exporter: insecure: true @@ -51,6 +53,7 @@ load_balancing: dns: hostname: agent port: 4318 + receiver_port: 4318 ``` ### Operator: Rename of Prometheus to Metrics (Breaking change) diff --git a/pkg/tempo/config.go b/pkg/tempo/config.go index e67ca247a49f..8a590bef894a 100644 --- a/pkg/tempo/config.go +++ b/pkg/tempo/config.go @@ -226,8 +226,6 @@ type tailSamplingConfig struct { Policies []map[string]interface{} `yaml:"policies"` // DecisionWait defines the time to wait for a complete trace before making a decision DecisionWait time.Duration `yaml:"decision_wait,omitempty"` - // Port is the port the instance will use to receive load balanced traces - Port string `yaml:"port"` } // loadBalancingConfig defines the configuration for load balancing spans between agent instances @@ -235,6 +233,8 @@ type tailSamplingConfig struct { type loadBalancingConfig struct { Exporter exporterConfig `yaml:"exporter"` Resolver map[string]interface{} `yaml:"resolver"` + // ReceiverPort is the port the instance will use to receive load balanced traces + ReceiverPort string `yaml:"receiver_port"` } // exporterConfig defined the config for a otlp exporter for load balancing @@ -554,8 +554,8 @@ func (c *InstanceConfig) otelConfig() (*config.Config, error) { exporters["loadbalancing"] = internalExporter receiverPort := defaultLoadBalancingPort - if c.TailSampling.Port != "" { - receiverPort = c.TailSampling.Port + if c.LoadBalancing.ReceiverPort != "" { + receiverPort = c.LoadBalancing.ReceiverPort } c.Receivers["otlp/lb"] = map[string]interface{}{ "protocols": map[string]interface{}{ @@ -575,7 +575,7 @@ func (c *InstanceConfig) otelConfig() (*config.Config, error) { } // Build Pipelines - splitPipeline := c.TailSampling != nil && c.LoadBalancing != nil + splitPipeline := c.LoadBalancing != nil orderedSplitProcessors := orderProcessors(processorNames, splitPipeline) if splitPipeline { // load balancing pipeline @@ -710,7 +710,8 @@ func orderProcessors(processors []string, splitPipelines bool) [][]string { foundAt := len(processors) for i, processor := range processors { if processor == "batch" || - processor == "tail_sampling" { + processor == "tail_sampling" || + processor == "automatic_logging" { foundAt = i break } diff --git a/pkg/tempo/config_test.go b/pkg/tempo/config_test.go index 03b9e09a894d..04eb22946670 100644 --- a/pkg/tempo/config_test.go +++ b/pkg/tempo/config_test.go @@ -607,12 +607,13 @@ tail_sampling: - value1 - value2 load_balancing: + receiver_port: 8080 exporter: insecure: true resolver: dns: hostname: agent - port: 4318 + port: 8080 `, expectedConfig: ` receivers: @@ -622,7 +623,7 @@ receivers: otlp/lb: protocols: grpc: - endpoint: "0.0.0.0:4318" + endpoint: "0.0.0.0:8080" exporters: otlp/0: endpoint: example.com:12345 @@ -639,7 +640,7 @@ exporters: resolver: dns: hostname: agent - port: 4318 + port: 8080 processors: tail_sampling: decision_wait: 5s @@ -979,6 +980,54 @@ service_graphs: "metrics/spanmetrics": nil, }, }, + { + name: "load balancing without tail sampling", + cfg: ` +receivers: + jaeger: + protocols: + grpc: +remote_write: + - endpoint: example.com:12345 + headers: + x-some-header: Some value! +attributes: + actions: + - key: montgomery + value: forever + action: update +spanmetrics: + latency_histogram_buckets: [2ms, 6ms, 10ms, 100ms, 250ms] + dimensions: + - name: http.method + default: GET + - name: http.status_code + prom_instance: tempo +automatic_logging: + spans: true +batch: + timeout: 5s + send_batch_size: 100 +load_balancing: + exporter: + insecure: true + resolver: + dns: + hostname: agent + port: 4318 +`, + expectedProcessors: map[string][]config.ComponentID{ + "traces/0": { + config.NewID("attributes"), + config.NewID("spanmetrics"), + }, + "traces/1": { + config.NewID("automatic_logging"), + config.NewID("batch"), + }, + "metrics/spanmetrics": nil, + }, + }, } for _, tc := range tt { From a008f175624bd22db7ac836db293cb44a66eae8f Mon Sep 17 00:00:00 2001 From: Mario Date: Thu, 23 Sep 2021 15:20:26 +0200 Subject: [PATCH 3/4] Check span error (#901) * Check span's error * Also check http status code * Update tests * Inspect http and grpc status too * Use map for performance * Cleanup --- pkg/tempo/servicegraphprocessor/factory.go | 7 ++++ pkg/tempo/servicegraphprocessor/processor.go | 41 +++++++++++++++++++ .../servicegraphprocessor/processor_test.go | 4 ++ .../testdata/trace-sample.json | 3 +- 4 files changed, 54 insertions(+), 1 deletion(-) diff --git a/pkg/tempo/servicegraphprocessor/factory.go b/pkg/tempo/servicegraphprocessor/factory.go index 2e96b4d9481f..1c752cb47526 100644 --- a/pkg/tempo/servicegraphprocessor/factory.go +++ b/pkg/tempo/servicegraphprocessor/factory.go @@ -26,6 +26,13 @@ type Config struct { Wait time.Duration `mapstructure:"wait"` MaxItems int `mapstructure:"max_items"` + + SuccessCodes *successCodes `mapstructure:"success_codes"` +} + +type successCodes struct { + http []int64 `mapstructure:"http"` + grpc []int64 `mapstructure:"grpc"` } // NewFactory returns a new factory for the Prometheus service graph processor. diff --git a/pkg/tempo/servicegraphprocessor/processor.go b/pkg/tempo/servicegraphprocessor/processor.go index 964e77e48662..4403e7e598d6 100644 --- a/pkg/tempo/servicegraphprocessor/processor.go +++ b/pkg/tempo/servicegraphprocessor/processor.go @@ -18,6 +18,7 @@ import ( "go.opentelemetry.io/collector/consumer" "go.opentelemetry.io/collector/model/pdata" "go.opentelemetry.io/collector/translator/conventions" + "google.golang.org/grpc/codes" ) var ( @@ -29,6 +30,8 @@ type edgeRequest struct { serverService, clientService string serverLatency, clientLatency time.Duration + // If either the client or the server spans have status code error, + // the request will be considered as failed. failed bool } @@ -55,6 +58,9 @@ type processor struct { serviceGraphUnpairedSpansTotal *prometheus.CounterVec serviceGraphUntaggedSpansTotal *prometheus.CounterVec + httpSuccessCode map[int]struct{} + grpcSuccessCode map[int]struct{} + logger log.Logger } @@ -68,6 +74,19 @@ func newProcessor(nextConsumer consumer.Traces, cfg *Config) *processor { cfg.MaxItems = DefaultMaxItems } + var ( + httpSuccessCode = make(map[int]struct{}) + grpcSuccessCode = make(map[int]struct{}) + ) + if cfg.SuccessCodes != nil { + for _, sc := range cfg.SuccessCodes.http { + httpSuccessCode[int(sc)] = struct{}{} + } + for _, sc := range cfg.SuccessCodes.grpc { + grpcSuccessCode[int(sc)] = struct{}{} + } + } + // TODO(mapno): Add support for an external cache (e.g. memcached) p := &processor{ nextConsumer: nextConsumer, @@ -238,6 +257,7 @@ func (p *processor) consume(trace pdata.Traces) error { } r.clientService = svc.StringVal() r.clientLatency = spanDuration(span) + r.failed = p.spanFailed(span) p.store.SetDefault(k, r) case pdata.SpanKindServer: @@ -250,6 +270,7 @@ func (p *processor) consume(trace pdata.Traces) error { r.serverService = svc.StringVal() r.serverLatency = spanDuration(span) + r.failed = p.spanFailed(span) p.store.SetDefault(k, r) default: @@ -261,6 +282,26 @@ func (p *processor) consume(trace pdata.Traces) error { return nil } +func (p *processor) spanFailed(span pdata.Span) bool { + // Request considered failed if status is not 2XX or added as a successful status code + if statusCode, ok := span.Attributes().Get("http.status_code"); ok { + sc := int(statusCode.IntVal()) + if _, ok := p.httpSuccessCode[sc]; !ok || sc/100 != 2 { + return true + } + } + + // Request considered failed if status is not OK or added as a successful status code + if statusCode, ok := span.Attributes().Get("grpc.status_code"); ok { + sc := int(statusCode.IntVal()) + if _, ok := p.grpcSuccessCode[sc]; !ok || sc != int(codes.OK) { + return true + } + } + + return span.Status().Code() == pdata.StatusCodeError +} + func spanDuration(span pdata.Span) time.Duration { return span.EndTimestamp().AsTime().Sub(span.StartTimestamp().AsTime()) } diff --git a/pkg/tempo/servicegraphprocessor/processor_test.go b/pkg/tempo/servicegraphprocessor/processor_test.go index 4948ad667884..dd20e931f7ab 100644 --- a/pkg/tempo/servicegraphprocessor/processor_test.go +++ b/pkg/tempo/servicegraphprocessor/processor_test.go @@ -151,6 +151,10 @@ const ( tempo_service_graph_request_client_seconds_bucket{client="lb",server="app",le="+Inf"} 3 tempo_service_graph_request_client_seconds_sum{client="lb",server="app"} 7.8 tempo_service_graph_request_client_seconds_count{client="lb",server="app"} 3 + # HELP tempo_service_graph_request_failed_total Total count of failed requests between two nodes + # TYPE tempo_service_graph_request_failed_total counter + tempo_service_graph_request_failed_total{client="app",server="db"} 3 + tempo_service_graph_request_failed_total{client="lb",server="app"} 3 # HELP tempo_service_graph_request_server_seconds Time for a request between two nodes as seen from the server # TYPE tempo_service_graph_request_server_seconds histogram tempo_service_graph_request_server_seconds_bucket{client="app",server="db",le="0.01"} 0 diff --git a/pkg/tempo/servicegraphprocessor/testdata/trace-sample.json b/pkg/tempo/servicegraphprocessor/testdata/trace-sample.json index e4bc282d71c9..763a70658fc1 100644 --- a/pkg/tempo/servicegraphprocessor/testdata/trace-sample.json +++ b/pkg/tempo/servicegraphprocessor/testdata/trace-sample.json @@ -284,7 +284,8 @@ } ], "status":{ - + "deprecatedCode":"DEPRECATED_STATUS_CODE_UNKNOWN_ERROR", + "code":"STATUS_CODE_ERROR" } }, { From 4b1ac7ddc338296551df4cefe393aa3eee4bec9e Mon Sep 17 00:00:00 2001 From: Mario Date: Wed, 6 Oct 2021 20:19:25 +0200 Subject: [PATCH 4/4] Count number of dropped spans and remove untagged metric (#967) --- pkg/tempo/servicegraphprocessor/processor.go | 22 ++++++++++--------- .../servicegraphprocessor/processor_test.go | 16 +++++--------- 2 files changed, 17 insertions(+), 21 deletions(-) diff --git a/pkg/tempo/servicegraphprocessor/processor.go b/pkg/tempo/servicegraphprocessor/processor.go index 4403e7e598d6..be7c25620b7c 100644 --- a/pkg/tempo/servicegraphprocessor/processor.go +++ b/pkg/tempo/servicegraphprocessor/processor.go @@ -56,7 +56,7 @@ type processor struct { serviceGraphRequestServerHistogram *prometheus.HistogramVec serviceGraphRequestClientHistogram *prometheus.HistogramVec serviceGraphUnpairedSpansTotal *prometheus.CounterVec - serviceGraphUntaggedSpansTotal *prometheus.CounterVec + serviceGraphDroppedSpansTotal *prometheus.CounterVec httpSuccessCode map[int]struct{} grpcSuccessCode map[int]struct{} @@ -130,12 +130,12 @@ func (p *processor) registerMetrics() error { }, []string{"client", "server"}) p.serviceGraphUnpairedSpansTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ Name: "tempo_service_graph_unpaired_spans_total", - Help: "Total count of requests between two nodes", + Help: "Total count of unpaired spans", }, []string{"client", "server"}) - p.serviceGraphUntaggedSpansTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ - Name: "tempo_service_graph_untagged_spans_total", - Help: "Total count of spans processed that were not tagged with span.kind", - }, []string{"span_kind"}) + p.serviceGraphDroppedSpansTotal = prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "tempo_service_graph_dropped_spans_total", + Help: "Total count of dropped spans", + }, []string{"service"}) cs := []prometheus.Collector{ p.serviceGraphRequestTotal, @@ -143,7 +143,7 @@ func (p *processor) registerMetrics() error { p.serviceGraphRequestServerHistogram, p.serviceGraphRequestClientHistogram, p.serviceGraphUnpairedSpansTotal, - p.serviceGraphUntaggedSpansTotal, + p.serviceGraphDroppedSpansTotal, } for _, c := range cs { @@ -177,7 +177,6 @@ func (p *processor) unregisterMetrics() { p.serviceGraphRequestServerHistogram, p.serviceGraphRequestClientHistogram, p.serviceGraphUnpairedSpansTotal, - p.serviceGraphUntaggedSpansTotal, } for _, c := range cs { @@ -196,7 +195,7 @@ func (p *processor) ConsumeTraces(ctx context.Context, td pdata.Traces) error { for _, trace := range batchpersignal.SplitTraces(td) { if err := p.consume(trace); err != nil { if errors.Is(err, errTooManyItems) { - level.Warn(p.logger).Log("msg", "skipped processing of spans", "maxItems", p.maxItems, "err", errTooManyItems) + level.Info(p.logger).Log("msg", "skipped processing of spans", "maxItems", p.maxItems, "err", errTooManyItems) break } errs = multierror.Append(errs, err) @@ -241,7 +240,11 @@ func (p *processor) consume(trace pdata.Traces) error { ils := ilsSlice.At(j) for k := 0; k < ils.Spans().Len(); k++ { + if p.store.ItemCount() >= p.maxItems { + remainingSpans := float64(ils.Spans().Len() - k) + p.serviceGraphDroppedSpansTotal.WithLabelValues(svc.StringVal()).Add(remainingSpans) + return errTooManyItems } @@ -274,7 +277,6 @@ func (p *processor) consume(trace pdata.Traces) error { p.store.SetDefault(k, r) default: - p.serviceGraphUntaggedSpansTotal.WithLabelValues(span.Kind().String()).Inc() } } } diff --git a/pkg/tempo/servicegraphprocessor/processor_test.go b/pkg/tempo/servicegraphprocessor/processor_test.go index dd20e931f7ab..ee6f9ccf0b85 100644 --- a/pkg/tempo/servicegraphprocessor/processor_test.go +++ b/pkg/tempo/servicegraphprocessor/processor_test.go @@ -44,14 +44,11 @@ func TestConsumeMetrics(t *testing.T) { Wait: time.Millisecond, }, expectedMetrics: ` - # HELP tempo_service_graph_unpaired_spans_total Total count of requests between two nodes + # HELP tempo_service_graph_unpaired_spans_total Total count of unpaired spans # TYPE tempo_service_graph_unpaired_spans_total counter tempo_service_graph_unpaired_spans_total{client="",server="db"} 2 tempo_service_graph_unpaired_spans_total{client="app",server=""} 3 tempo_service_graph_unpaired_spans_total{client="lb",server=""} 3 - # HELP tempo_service_graph_untagged_spans_total Total count of spans processed that were not tagged with span.kind - # TYPE tempo_service_graph_untagged_spans_total counter - tempo_service_graph_untagged_spans_total{span_kind="SPAN_KIND_UNSPECIFIED"} 5 `, }, { @@ -62,12 +59,12 @@ func TestConsumeMetrics(t *testing.T) { MaxItems: 1, // Configure max number of items in store to 1. Only one edgeRequest will be processed. }, expectedMetrics: ` - # HELP tempo_service_graph_unpaired_spans_total Total count of requests between two nodes + # HELP tempo_service_graph_dropped_spans_total Total count of dropped spans + # TYPE tempo_service_graph_dropped_spans_total counter + tempo_service_graph_dropped_spans_total{service="lb"} 1 + # HELP tempo_service_graph_unpaired_spans_total Total count of unpaired spans # TYPE tempo_service_graph_unpaired_spans_total counter tempo_service_graph_unpaired_spans_total{client="lb",server=""} 1 - # HELP tempo_service_graph_untagged_spans_total Total count of spans processed that were not tagged with span.kind - # TYPE tempo_service_graph_untagged_spans_total counter - tempo_service_graph_untagged_spans_total{span_kind="SPAN_KIND_UNSPECIFIED"} 1 `, }, } { @@ -191,8 +188,5 @@ const ( # TYPE tempo_service_graph_request_total counter tempo_service_graph_request_total{client="app",server="db"} 3 tempo_service_graph_request_total{client="lb",server="app"} 3 - # HELP tempo_service_graph_untagged_spans_total Total count of spans processed that were not tagged with span.kind - # TYPE tempo_service_graph_untagged_spans_total counter - tempo_service_graph_untagged_spans_total{span_kind="SPAN_KIND_UNSPECIFIED"} 5 ` )