diff --git a/.changesets/maint_bryn_otel_0_31_migration.md b/.changesets/maint_bryn_otel_0_31_migration.md new file mode 100644 index 0000000000..1f41b41624 --- /dev/null +++ b/.changesets/maint_bryn_otel_0_31_migration.md @@ -0,0 +1,12 @@ +### Update to OpenTelemetry 0.31.0 ([PR #8922](https://github.com/apollographql/router/pull/8922)) + +The Router now uses v0.31.0 of the OpenTelemetry Rust libraries. This update includes many bug fixes and performance improvements from upstream. + +The Router does not guarantee the stability of downstream pre 1.0 APIs so users that directly interact with OpenTelemetry must update their code accordingly. + +As part of this upgrade Zipkin Native exporter has been deprecated as this is being [deprecated upstream](https://opentelemetry.io/blog/2025/deprecating-zipkin-exporters/). Users should switch to OTLP exporter, which Zipkin now supports natively. + +Zipkin Native exporter also no longer supports setting service name, users that need this should switch to OTLP exporter. + + +By [@BrynCooke](https://github.com/BrynCooke) [@goto-bus-stop](https://github.com/goto-bus-stop) [@rohan-b99](https://github.com/rohan-b99) in https://github.com/apollographql/router/pull/8922 diff --git a/Cargo.lock b/Cargo.lock index afd376224d..f8e8b597fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21,7 +21,7 @@ dependencies = [ "http 1.4.0", "serde_json", "tokio", - "tower 0.5.2", + "tower", ] [[package]] @@ -237,7 +237,7 @@ dependencies = [ "nom_locate", "parking_lot", "percent-encoding", - "petgraph 0.8.3", + "petgraph", "pretty_assertions", "regex", "ron", @@ -304,7 +304,7 @@ dependencies = [ "aws-smithy-http-client", "aws-smithy-runtime-api", "aws-types", - "axum 0.8.8", + "axum", "base64 0.22.1", "blake3", "bloomfilter", @@ -418,6 +418,7 @@ dependencies = [ "sha2", "shellexpand", "similar", + "socket2 0.5.10", "static_assertions", "strum 0.27.2", "sys-info", @@ -435,8 +436,8 @@ dependencies = [ "tokio-tungstenite", "tokio-util", "tonic", - "tonic-build", - "tower 0.5.2", + "tonic-prost-build", + "tower", "tower-http", "tower-service", "tower-test", @@ -473,7 +474,7 @@ dependencies = [ "once_cell", "serde_json", "tokio", - "tower 0.5.2", + "tower", ] [[package]] @@ -574,30 +575,7 @@ dependencies = [ "serde_json", "serde_json_bytes", "tokio", - "tower 0.5.2", -] - -[[package]] -name = "async-channel" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" -dependencies = [ - "concurrent-queue", - "event-listener 2.5.3", - "futures-core", -] - -[[package]] -name = "async-channel" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" -dependencies = [ - "concurrent-queue", - "event-listener-strategy", - "futures-core", - "pin-project-lite", + "tower", ] [[package]] @@ -613,35 +591,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "async-executor" -version = "1.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497c00e0fd83a72a79a39fcbd8e3e2f055d6f6c7e025f3b3d91f4f8e76527fb8" -dependencies = [ - "async-task", - "concurrent-queue", - "fastrand", - "futures-lite", - "pin-project-lite", - "slab", -] - -[[package]] -name = "async-global-executor" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" -dependencies = [ - "async-channel 2.5.0", - "async-executor", - "async-io", - "async-lock", - "blocking", - "futures-lite", - "once_cell", -] - [[package]] name = "async-graphql" version = "7.0.17" @@ -682,7 +631,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8725874ecfbf399e071150b8619c4071d7b2b7a2f117e173dddef53c6bdb6bb1" dependencies = [ "async-graphql", - "axum 0.8.8", + "axum", "bytes", "futures-util", "serde_json", @@ -733,98 +682,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "async-io" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19634d6336019ef220f09fd31168ce5c184b295cbf80345437cc36094ef223ca" -dependencies = [ - "async-lock", - "cfg-if", - "concurrent-queue", - "futures-io", - "futures-lite", - "parking", - "polling", - "rustix", - "slab", - "windows-sys 0.60.2", -] - -[[package]] -name = "async-lock" -version = "3.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd03604047cee9b6ce9de9f70c6cd540a0520c813cbd49bae61f33ab80ed1dc" -dependencies = [ - "event-listener 5.4.1", - "event-listener-strategy", - "pin-project-lite", -] - -[[package]] -name = "async-process" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65daa13722ad51e6ab1a1b9c01299142bc75135b337923cfa10e79bbbd669f00" -dependencies = [ - "async-channel 2.5.0", - "async-io", - "async-lock", - "async-signal", - "async-task", - "blocking", - "cfg-if", - "event-listener 5.4.1", - "futures-lite", - "rustix", -] - -[[package]] -name = "async-signal" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f567af260ef69e1d52c2b560ce0ea230763e6fbb9214a85d768760a920e3e3c1" -dependencies = [ - "async-io", - "async-lock", - "atomic-waker", - "cfg-if", - "futures-core", - "futures-io", - "rustix", - "signal-hook-registry", - "slab", - "windows-sys 0.60.2", -] - -[[package]] -name = "async-std" -version = "1.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c8e079a4ab67ae52b7403632e4618815d6db36d2a010cfe41b02c1b1578f93b" -dependencies = [ - "async-channel 1.9.0", - "async-global-executor", - "async-io", - "async-lock", - "async-process", - "crossbeam-utils", - "futures-channel", - "futures-core", - "futures-io", - "futures-lite", - "gloo-timers", - "kv-log-macro", - "log", - "memchr", - "once_cell", - "pin-project-lite", - "pin-utils", - "slab", - "wasm-bindgen-futures", -] - [[package]] name = "async-stream" version = "0.3.6" @@ -847,12 +704,6 @@ dependencies = [ "syn 2.0.106", ] -[[package]] -name = "async-task" -version = "4.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" - [[package]] name = "async-trait" version = "0.1.89" @@ -1034,7 +885,7 @@ dependencies = [ "rustls-pki-types", "tokio", "tokio-rustls", - "tower 0.5.2", + "tower", "tracing", ] @@ -1152,40 +1003,13 @@ dependencies = [ "tracing", ] -[[package]] -name = "axum" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" -dependencies = [ - "async-trait", - "axum-core 0.4.5", - "bytes", - "futures-util", - "http 1.4.0", - "http-body 1.0.1", - "http-body-util", - "itoa", - "matchit 0.7.3", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper", - "tower 0.5.2", - "tower-layer", - "tower-service", -] - [[package]] name = "axum" version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" dependencies = [ - "axum-core 0.5.5", + "axum-core", "base64 0.22.1", "bytes", "form_urlencoded", @@ -1196,7 +1020,7 @@ dependencies = [ "hyper", "hyper-util", "itoa", - "matchit 0.8.4", + "matchit", "memchr", "mime", "percent-encoding", @@ -1209,32 +1033,12 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-tungstenite", - "tower 0.5.2", + "tower", "tower-layer", "tower-service", "tracing", ] -[[package]] -name = "axum-core" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http 1.4.0", - "http-body 1.0.1", - "http-body-util", - "mime", - "pin-project-lite", - "rustversion", - "sync_wrapper", - "tower-layer", - "tower-service", -] - [[package]] name = "axum-core" version = "0.5.5" @@ -1332,11 +1136,11 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.9.3" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34efbcccd345379ca2868b2b2c9d3782e9cc58ba87bc7d79d5b53d9c9ae6f25d" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" dependencies = [ - "serde", + "serde_core", ] [[package]] @@ -1362,19 +1166,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "blocking" -version = "1.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" -dependencies = [ - "async-channel 2.5.0", - "async-task", - "futures-io", - "futures-lite", - "piper", -] - [[package]] name = "bloomfilter" version = "3.0.1" @@ -1504,7 +1295,7 @@ dependencies = [ "http 1.4.0", "serde_json", "tokio", - "tower 0.5.2", + "tower", ] [[package]] @@ -1716,15 +1507,6 @@ dependencies = [ "windows-sys 0.45.0", ] -[[package]] -name = "concurrent-queue" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "console" version = "0.15.11" @@ -1749,6 +1531,18 @@ dependencies = [ "windows-sys 0.61.0", ] +[[package]] +name = "const-hex" +version = "1.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "531185e432bb31db1ecda541e9e7ab21468d4d844ad7505e0546a49b4945d49b" +dependencies = [ + "cfg-if", + "cpufeatures", + "proptest", + "serde_core", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -1809,7 +1603,7 @@ dependencies = [ "apollo-router", "async-trait", "http 1.4.0", - "tower 0.5.2", + "tower", "tracing", ] @@ -1847,7 +1641,7 @@ dependencies = [ "http 1.4.0", "serde_json", "tokio", - "tower 0.5.2", + "tower", ] [[package]] @@ -2538,7 +2332,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -2547,43 +2341,16 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5692dd7b5a1978a5aeb0ce83b7655c58ca8efdcb79d21036ea249da95afec2c6" -[[package]] -name = "event-listener" -version = "2.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" - -[[package]] -name = "event-listener" -version = "5.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" -dependencies = [ - "concurrent-queue", - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener-strategy" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" -dependencies = [ - "event-listener 5.4.1", - "pin-project-lite", -] - [[package]] name = "everything-subgraph" version = "0.1.0" dependencies = [ "async-graphql", "async-graphql-axum", - "axum 0.8.8", + "axum", "env_logger", "tokio", - "tower 0.5.2", + "tower", ] [[package]] @@ -2602,7 +2369,7 @@ dependencies = [ "serde", "serde_json", "tokio", - "tower 0.5.2", + "tower", "tracing", ] @@ -2730,7 +2497,7 @@ dependencies = [ "http 1.4.0", "serde_json", "tokio", - "tower 0.5.2", + "tower", "tracing", ] @@ -2743,7 +2510,7 @@ dependencies = [ "http 1.4.0", "serde_json", "tokio", - "tower 0.5.2", + "tower", ] [[package]] @@ -2898,19 +2665,6 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" -[[package]] -name = "futures-lite" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" -dependencies = [ - "fastrand", - "futures-core", - "futures-io", - "parking", - "pin-project-lite", -] - [[package]] name = "futures-macro" version = "0.3.31" @@ -3101,18 +2855,6 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "gloo-timers" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" -dependencies = [ - "futures-channel", - "futures-core", - "js-sys", - "wasm-bindgen", -] - [[package]] name = "graphql-introspection-query" version = "0.2.0" @@ -3290,7 +3032,7 @@ dependencies = [ "serde", "serde_json", "tokio", - "tower 0.5.2", + "tower", "tracing", ] @@ -3553,7 +3295,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.0", "tokio", "tower-service", "tracing", @@ -3739,7 +3481,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f37dccff2791ab604f9babef0ba14fbe0be30bd368dc541e2b08d07c8aa908f3" dependencies = [ - "bitflags 2.9.3", + "bitflags 2.11.0", "inotify-sys", "libc", ] @@ -3825,7 +3567,7 @@ checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3834,15 +3576,6 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.13.0" @@ -4009,7 +3742,7 @@ dependencies = [ "http 1.4.0", "serde_json", "tokio", - "tower 0.5.2", + "tower", ] [[package]] @@ -4032,15 +3765,6 @@ dependencies = [ "libc", ] -[[package]] -name = "kv-log-macro" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" -dependencies = [ - "log", -] - [[package]] name = "lazy_static" version = "1.5.0" @@ -4084,7 +3808,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "391290121bad3d37fbddad76d8f5d1c1c314cfc646d143d7e07a3086ddff0ce3" dependencies = [ - "bitflags 2.9.3", + "bitflags 2.11.0", "libc", "redox_syscall", ] @@ -4160,9 +3884,6 @@ name = "log" version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" -dependencies = [ - "value-bag", -] [[package]] name = "loom" @@ -4207,12 +3928,6 @@ dependencies = [ "regex-automata", ] -[[package]] -name = "matchit" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" - [[package]] name = "matchit" version = "0.8.4" @@ -4442,7 +4157,7 @@ version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3" dependencies = [ - "bitflags 2.9.3", + "bitflags 2.11.0", "inotify", "kqueue", "libc", @@ -4604,7 +4319,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c10c2894a6fed806ade6027bcd50662746363a9589d3ec9d9bef30a4e4bc166" dependencies = [ - "bitflags 2.9.3", + "bitflags 2.11.0", ] [[package]] @@ -4722,7 +4437,7 @@ dependencies = [ "http 1.4.0", "serde_json", "tokio", - "tower 0.5.2", + "tower", ] [[package]] @@ -4733,42 +4448,39 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "opentelemetry" -version = "0.24.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c365a63eec4f55b7efeceb724f1336f26a9cf3427b70e59e2cd2a5b947fba96" +checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0" dependencies = [ "futures-core", "futures-sink", "js-sys", - "once_cell", "pin-project-lite", - "thiserror 1.0.69", + "thiserror 2.0.17", + "tracing", ] [[package]] name = "opentelemetry-aws" -version = "0.12.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f2e5bd1a2e1d14877086a2defe4ac968f42a6a15cfc5862a0f0ecd0f3530135" +checksum = "09fbe9af6b9403e7fe43c11cc341d320d7cf5e779c6708b41415228af1921045" dependencies = [ - "once_cell", "opentelemetry", "opentelemetry_sdk", + "tracing", ] [[package]] name = "opentelemetry-datadog" -version = "0.12.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e55061f0b4acd624ce67434c4a6d6d1b5c341d62564bf80094bdaef884f1bf5b" +checksum = "9a6b2d4db32343691eb945e6153e5a4bd494dbf9d931d5bf7d1d7f59bee156d0" dependencies = [ "ahash", - "futures-core", "http 1.4.0", "indexmap 2.12.1", - "itertools 0.11.0", "itoa", - "once_cell", "opentelemetry", "opentelemetry-http", "opentelemetry-semantic-conventions", @@ -4776,15 +4488,15 @@ dependencies = [ "reqwest", "rmp", "ryu", - "thiserror 1.0.69", + "thiserror 2.0.17", "url", ] [[package]] name = "opentelemetry-http" -version = "0.13.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad31e9de44ee3538fb9d64fe3376c1362f406162434609e79aea2a41a0af78ab" +checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d" dependencies = [ "async-trait", "bytes", @@ -4795,21 +4507,19 @@ dependencies = [ [[package]] name = "opentelemetry-jaeger-propagator" -version = "0.3.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0a68a13b92fc708d875ad659b08b35d08b8ef2403e01944b39ca21e5b08b17" +checksum = "ba3bbd907f151104a112f749f3b8387ef669b7264e0bb80546ea0700a3b307b7" dependencies = [ "opentelemetry", ] [[package]] name = "opentelemetry-otlp" -version = "0.17.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b925a602ffb916fb7421276b86756027b37ee708f9dce2dbdcc51739f07e727" +checksum = "7a2366db2dca4d2ad033cad11e6ee42844fd727007af5ad04a1730f4cb8163bf" dependencies = [ - "async-trait", - "futures-core", "http 1.4.0", "opentelemetry", "opentelemetry-http", @@ -4817,100 +4527,89 @@ dependencies = [ "opentelemetry_sdk", "prost", "reqwest", - "thiserror 1.0.69", + "thiserror 2.0.17", "tokio", "tonic", ] [[package]] name = "opentelemetry-prometheus" -version = "0.17.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc4191ce34aa274621861a7a9d68dbcf618d5b6c66b10081631b61fd81fbc015" +checksum = "14095eb06b569eb5d538fa4555969f7e8a410ed7910c903bfd295f9e1a50d7ea" dependencies = [ "once_cell", "opentelemetry", "opentelemetry_sdk", "prometheus", - "protobuf", + "tracing", ] [[package]] name = "opentelemetry-proto" -version = "0.7.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30ee9f20bff9c984511a02f082dc8ede839e4a9bf15cc2487c8d6fea5ad850d9" +checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f" dependencies = [ - "hex", + "base64 0.22.1", + "const-hex", "opentelemetry", "opentelemetry_sdk", "prost", "serde", + "serde_json", "tonic", + "tonic-prost", ] [[package]] name = "opentelemetry-semantic-conventions" -version = "0.16.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cefe0543875379e47eb5f1e68ff83f45cc41366a92dfd0d073d513bf68e9a05" +checksum = "e62e29dfe041afb8ed2a6c9737ab57db4907285d999ef8ad3a59092a36bdc846" [[package]] name = "opentelemetry-stdout" -version = "0.5.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d408d4345b8be6129a77c46c3bfc75f0d3476f3091909c7dd99c1f3d78582287" +checksum = "bc8887887e169414f637b18751487cce4e095be787d23fad13c454e2fb1b3811" dependencies = [ - "async-trait", "chrono", - "futures-util", "opentelemetry", "opentelemetry_sdk", - "ordered-float", - "serde", - "serde_json", - "thiserror 1.0.69", ] [[package]] name = "opentelemetry-zipkin" -version = "0.22.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e68336254a44c5c20574989699582175910b933be85a593a13031ee58811d93d" +checksum = "f27fcd074586dab55936b003c6a499acaabd6debbd539c3f36356bca2ef2fce2" dependencies = [ - "async-trait", - "futures-core", "http 1.4.0", "once_cell", "opentelemetry", "opentelemetry-http", - "opentelemetry-semantic-conventions", "opentelemetry_sdk", "reqwest", "serde", "serde_json", - "thiserror 1.0.69", + "thiserror 2.0.17", "typed-builder", ] [[package]] name = "opentelemetry_sdk" -version = "0.24.1" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "692eac490ec80f24a17828d49b40b60f5aeaccdfe6a503f939713afd22bc28df" +checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd" dependencies = [ - "async-std", - "async-trait", "futures-channel", "futures-executor", "futures-util", - "glob", - "once_cell", "opentelemetry", "percent-encoding", - "rand 0.8.5", - "serde_json", - "thiserror 1.0.69", + "rand 0.9.2", + "thiserror 2.0.17", "tokio", "tokio-stream", ] @@ -4921,15 +4620,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" -[[package]] -name = "ordered-float" -version = "4.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" -dependencies = [ - "num-traits", -] - [[package]] name = "outref" version = "0.5.2" @@ -4970,12 +4660,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "parking" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" - [[package]] name = "parking_lot" version = "0.12.5" @@ -5074,16 +4758,6 @@ dependencies = [ "sha2", ] -[[package]] -name = "petgraph" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" -dependencies = [ - "fixedbitset", - "indexmap 2.12.1", -] - [[package]] name = "petgraph" version = "0.8.3" @@ -5129,17 +4803,6 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" -[[package]] -name = "piper" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96c8c490f422ef9a4efd2cb5b42b76c8613d7e7dfc1caf667b8a3350a5acc066" -dependencies = [ - "atomic-waker", - "fastrand", - "futures-io", -] - [[package]] name = "pkcs1" version = "0.7.5" @@ -5195,20 +4858,6 @@ dependencies = [ "plotters-backend", ] -[[package]] -name = "polling" -version = "3.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5bd19146350fe804f7cb2669c851c03d69da628803dab0d98018142aaa5d829" -dependencies = [ - "cfg-if", - "concurrent-queue", - "hermit-abi", - "pin-project-lite", - "rustix", - "windows-sys 0.60.2", -] - [[package]] name = "portable-atomic" version = "1.11.1" @@ -5345,9 +4994,9 @@ dependencies = [ [[package]] name = "prometheus" -version = "0.13.4" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1" +checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a" dependencies = [ "cfg-if", "fnv", @@ -5355,7 +5004,7 @@ dependencies = [ "memchr", "parking_lot", "protobuf", - "thiserror 1.0.69", + "thiserror 2.0.17", ] [[package]] @@ -5370,14 +5019,29 @@ dependencies = [ "serde", "serde_json", "tokio", - "tower 0.5.2", + "tower", +] + +[[package]] +name = "proptest" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37566cb3fdacef14c0737f9546df7cfeadbfbc9fef10991038bf5015d0c80532" +dependencies = [ + "bitflags 2.11.0", + "num-traits", + "rand 0.9.2", + "rand_chacha 0.9.0", + "rand_xorshift", + "regex-syntax", + "unarray", ] [[package]] name = "prost" -version = "0.13.5" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" dependencies = [ "bytes", "prost-derive", @@ -5385,19 +5049,20 @@ dependencies = [ [[package]] name = "prost-build" -version = "0.13.5" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" +checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "itertools 0.14.0", "log", "multimap 0.10.1", - "once_cell", - "petgraph 0.7.1", + "petgraph", "prettyplease", "prost", "prost-types", + "pulldown-cmark", + "pulldown-cmark-to-cmark", "regex", "syn 2.0.106", "tempfile", @@ -5405,9 +5070,9 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.13.5" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", "itertools 0.14.0", @@ -5418,9 +5083,9 @@ dependencies = [ [[package]] name = "prost-types" -version = "0.13.5" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" dependencies = [ "prost", ] @@ -5441,9 +5106,43 @@ dependencies = [ [[package]] name = "protobuf" -version = "2.28.0" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4" +dependencies = [ + "once_cell", + "protobuf-support", + "thiserror 1.0.69", +] + +[[package]] +name = "protobuf-support" +version = "3.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" +checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6" +dependencies = [ + "thiserror 1.0.69", +] + +[[package]] +name = "pulldown-cmark" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83c41efbf8f90ac44de7f3a868f0867851d261b56291732d0cbf7cceaaeb55a6" +dependencies = [ + "bitflags 2.11.0", + "memchr", + "unicase", +] + +[[package]] +name = "pulldown-cmark-to-cmark" +version = "22.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50793def1b900256624a709439404384204a5dc3a6ec580281bfaac35e882e90" +dependencies = [ + "pulldown-cmark", +] [[package]] name = "quinn" @@ -5497,7 +5196,7 @@ dependencies = [ "once_cell", "socket2 0.5.10", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5615,6 +5314,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.3", +] + [[package]] name = "rayon" version = "1.11.0" @@ -5655,7 +5363,7 @@ version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" dependencies = [ - "bitflags 2.9.3", + "bitflags 2.11.0", ] [[package]] @@ -5779,7 +5487,7 @@ dependencies = [ "tokio", "tokio-rustls", "tokio-util", - "tower 0.5.2", + "tower", "tower-http", "tower-service", "url", @@ -5813,7 +5521,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4e35aaaa439a5bda2f8d15251bc375e4edfac75f9865734644782c9701b5709" dependencies = [ "ahash", - "bitflags 2.9.3", + "bitflags 2.11.0", "instant", "no-std-compat", "num-traits", @@ -5834,7 +5542,7 @@ dependencies = [ "http 1.4.0", "serde_json", "tokio", - "tower 0.5.2", + "tower", ] [[package]] @@ -5846,7 +5554,7 @@ dependencies = [ "http 1.4.0", "serde_json", "tokio", - "tower 0.5.2", + "tower", ] [[package]] @@ -5858,7 +5566,7 @@ dependencies = [ "http 1.4.0", "serde_json", "tokio", - "tower 0.5.2", + "tower", ] [[package]] @@ -5870,7 +5578,7 @@ dependencies = [ "http 1.4.0", "serde_json", "tokio", - "tower 0.5.2", + "tower", ] [[package]] @@ -5882,7 +5590,7 @@ dependencies = [ "http 1.4.0", "serde_json", "tokio", - "tower 0.5.2", + "tower", ] [[package]] @@ -5927,7 +5635,7 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd490c5b18261893f14449cbd28cb9c0b637aebf161cd77900bfdedaff21ec32" dependencies = [ - "bitflags 2.9.3", + "bitflags 2.11.0", "once_cell", "serde", "serde_derive", @@ -5955,7 +5663,7 @@ dependencies = [ "serde", "serde_json", "serde_json_bytes", - "tower 0.5.2", + "tower", ] [[package]] @@ -6100,11 +5808,11 @@ version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" dependencies = [ - "bitflags 2.9.3", + "bitflags 2.11.0", "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.0", ] [[package]] @@ -6261,7 +5969,7 @@ version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80fb1d92c5028aa318b4b8bd7302a5bfcf48be96a37fc6fc790f806b0004ee0c" dependencies = [ - "bitflags 2.9.3", + "bitflags 2.11.0", "core-foundation", "core-foundation-sys", "libc", @@ -6700,7 +6408,7 @@ dependencies = [ "apollo-compiler", "apollo-router", "async-trait", - "tower 0.5.2", + "tower", "tracing", ] @@ -6797,7 +6505,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.0", ] [[package]] @@ -6907,7 +6615,7 @@ dependencies = [ "http-body-util", "serde_json", "tokio", - "tower 0.5.2", + "tower", ] [[package]] @@ -7131,13 +6839,12 @@ dependencies = [ [[package]] name = "tonic" -version = "0.12.3" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +checksum = "fec7c61a0695dc1887c1b53952990f3ad2e3a31453e1f49f10e75424943a93ec" dependencies = [ - "async-stream", "async-trait", - "axum 0.7.9", + "axum", "base64 0.22.1", "bytes", "flate2", @@ -7150,14 +6857,13 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "prost", "rustls-native-certs", - "rustls-pemfile", - "socket2 0.5.10", + "socket2 0.6.0", + "sync_wrapper", "tokio", "tokio-rustls", "tokio-stream", - "tower 0.4.13", + "tower", "tower-layer", "tower-service", "tracing", @@ -7165,36 +6871,41 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.12.3" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9557ce109ea773b399c9b9e5dca39294110b74f1f342cb347a80d1fce8c26a11" +checksum = "1882ac3bf5ef12877d7ed57aad87e75154c11931c2ba7e6cde5e22d63522c734" dependencies = [ "prettyplease", "proc-macro2", - "prost-build", - "prost-types", "quote", "syn 2.0.106", ] [[package]] -name = "tower" -version = "0.4.13" +name = "tonic-prost" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +checksum = "a55376a0bbaa4975a3f10d009ad763d8f4108f067c7c2e74f3001fb49778d309" dependencies = [ - "futures-core", - "futures-util", - "indexmap 1.9.3", - "pin-project", - "pin-project-lite", - "rand 0.8.5", - "slab", - "tokio", - "tokio-util", - "tower-layer", - "tower-service", - "tracing", + "bytes", + "prost", + "tonic", +] + +[[package]] +name = "tonic-prost-build" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3144df636917574672e93d0f56d7edec49f90305749c668df5101751bb8f95a" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "prost-types", + "quote", + "syn 2.0.106", + "tempfile", + "tonic-build", ] [[package]] @@ -7225,7 +6936,7 @@ checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ "async-compression", "base64 0.22.1", - "bitflags 2.9.3", + "bitflags 2.11.0", "bytes", "futures-core", "futures-util", @@ -7241,7 +6952,7 @@ dependencies = [ "pin-project-lite", "tokio", "tokio-util", - "tower 0.5.2", + "tower", "tower-layer", "tower-service", "tracing", @@ -7342,15 +7053,16 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.25.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9784ed4da7d921bc8df6963f8c80a0e4ce34ba6ba76668acadd3edbd985ff3b" +checksum = "1e6e5658463dd88089aba75c7791e1d3120633b1bfde22478b28f625a9bb1b8e" dependencies = [ "js-sys", - "once_cell", "opentelemetry", "opentelemetry_sdk", + "rustversion", "smallvec", + "thiserror 2.0.17", "tracing", "tracing-core", "tracing-log", @@ -7479,18 +7191,18 @@ checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a" [[package]] name = "typed-builder" -version = "0.18.2" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77739c880e00693faef3d65ea3aad725f196da38b22fdc7ea6ded6e1ce4d3add" +checksum = "cd9d30e3a08026c78f246b173243cf07b3696d274debd26680773b6773c2afc7" dependencies = [ "typed-builder-macro", ] [[package]] name = "typed-builder-macro" -version = "0.18.2" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f718dfaf347dcb5b983bfc87608144b0bad87970aebcbea5ce44d2a30c08e63" +checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28" dependencies = [ "proc-macro2", "quote", @@ -7548,6 +7260,12 @@ dependencies = [ "libc", ] +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicase" version = "2.8.1" @@ -7669,12 +7387,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" -[[package]] -name = "value-bag" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ba6f5989077681266825251a52748b8c1d8a4ad098cc37e440103d0ea717fc0" - [[package]] name = "version_check" version = "0.9.5" @@ -7879,7 +7591,7 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0978bf7171b3d90bac376700cb56d606feb40f251a475a5d6634613564460b22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] diff --git a/apollo-router/Cargo.toml b/apollo-router/Cargo.toml index 6556b86e21..09f67f9bb0 100644 --- a/apollo-router/Cargo.toml +++ b/apollo-router/Cargo.toml @@ -101,6 +101,9 @@ fred = { version = "10.1.0", features = [ "serde-json", "replicas" ] } +# We don't use this dependency, but the "all" feature is required for fred's +# "tcp-user-timeouts" feature to work: https://github.com/aembke/fred.rs/pull/369 +socket2 = { version = "0.5", features = ["all"] } futures = { version = "0.3.30", features = ["thread-pool"] } graphql_client = "0.14.0" hex.workspace = true @@ -161,20 +164,20 @@ once_cell = "1.19.0" # groups `^tracing` and `^opentelemetry*` dependencies together as of # https://github.com/apollographql/router/pull/1509. A comment which exists # there (and on `tracing` packages below) should be updated should this change. -opentelemetry = { version = "0.24.0", features = ["trace", "metrics"] } -opentelemetry_sdk = { version = "0.24.1", default-features = false, features = [ +opentelemetry = { version = "0.31", features = ["trace", "metrics"] } +opentelemetry_sdk = { version = "0.31", default-features = false, features = [ "rt-tokio", "trace", + "metrics", + "spec_unstable_metrics_views", + "experimental_trace_batch_span_processor_with_async_runtime", + "experimental_metrics_periodicreader_with_async_runtime", ] } -opentelemetry-aws = "0.12.0" -# START TEMP DATADOG Temporarily remove until we upgrade otel to the latest version -# This means including the rmp library -# opentelemetry-datadog = { version = "0.12.0", features = ["reqwest-client"] } +opentelemetry-aws = "0.19" rmp = "0.8" -# END TEMP DATADOG -opentelemetry-http = "0.13.0" -opentelemetry-jaeger-propagator = "0.3.0" -opentelemetry-otlp = { version = "0.17.0", default-features = false, features = [ +opentelemetry-http = "0.31" +opentelemetry-jaeger-propagator = "0.31" +opentelemetry-otlp = { version = "0.31", default-features = false, features = [ "grpc-tonic", "gzip-tonic", "tonic", @@ -184,17 +187,17 @@ opentelemetry-otlp = { version = "0.17.0", default-features = false, features = "reqwest-client", "trace", ] } -opentelemetry-semantic-conventions = "0.16.0" -opentelemetry-zipkin = { version = "0.22.0", default-features = false, features = [ +opentelemetry-semantic-conventions = { version = "0.31", features = ["semconv_experimental"] } +opentelemetry-zipkin = { version = "0.31", default-features = false, features = [ "reqwest-client", "reqwest-rustls", ] } -opentelemetry-prometheus = "0.17.0" +opentelemetry-prometheus = "0.31" paste = "1.0.15" pin-project-lite = "0.2.14" -prometheus = "0.13" -prost = "0.13.0" -prost-types = "0.13.0" +prometheus = "0.14" +prost = "0.14.0" +prost-types = "0.14.0" proteus = "0.5.0" rand = "0.9.0" rhai = { version = "1.23.6", features = ["sync", "serde", "internals"] } @@ -233,10 +236,10 @@ thiserror = "2.0.0" tokio.workspace = true tokio-stream = { version = "0.1.15", features = ["sync", "net", "fs"] } tokio-util = { version = "0.7.11", features = ["net", "codec", "time", "compat"] } -tonic = { version = "0.12.3", features = [ +tonic = { version = "0.14.5", features = [ "transport", - "tls", - "tls-roots", + "tls-ring", + "tls-native-roots", "gzip", ] } tower.workspace = true @@ -319,16 +322,16 @@ memchr = { version = "2.7.4", default-features = false } mockall = "0.14.0" num-traits = "0.2.19" once_cell.workspace = true -opentelemetry-stdout = { version = "0.5.0", features = ["trace"] } -opentelemetry = { version = "0.24.0", features = ["testing"] } -opentelemetry_sdk = { version = "0.24.1", features = ["testing"] } -opentelemetry-proto = { version = "0.7.0", features = [ +opentelemetry-stdout = { version = "0.31", features = ["trace"] } +opentelemetry = { version = "0.31", features = ["testing"] } +opentelemetry_sdk = { version = "0.31", features = ["testing"] } +opentelemetry-proto = { version = "0.31", features = [ "metrics", "trace", "gen-tonic-messages", "with-serde", ] } -opentelemetry-datadog = { version = "0.12.0", features = ["reqwest-client"] } +opentelemetry-datadog = { version = "0.19", features = ["reqwest-client"] } p256 = "0.13.2" pretty_assertions = "1.4.1" reqwest = { version = "0.12.9", default-features = false, features = [ @@ -355,7 +358,7 @@ tracing-subscriber = { version = "0.3.20", default-features = false, features = "env-filter", "fmt", ] } -tracing-opentelemetry = "0.25.0" +tracing-opentelemetry = "0.32" tracing-test = "=0.2.5" tracing-mock = "0.1.0-beta.1" walkdir = "2.5.0" @@ -373,12 +376,12 @@ hyperlocal = { version = "0.9.1", default-features = false, features = [ ] } [build-dependencies] -tonic-build = "0.12.3" +tonic-prost-build = "0.14.0" serde_json.workspace = true [package.metadata.cargo-machete] ignored = [ - "serde_regex", # Referenced only as a string in a macro + "socket2", # Only used to enable a transitive feature pending https://github.com/aembke/fred.rs/pull/369 ] [[test]] diff --git a/apollo-router/build/studio.rs b/apollo-router/build/studio.rs index a5ef7b9fc5..36df7f9b12 100644 --- a/apollo-router/build/studio.rs +++ b/apollo-router/build/studio.rs @@ -26,7 +26,7 @@ pub fn main() -> Result<(), Box> { // Process the proto files - tonic_build::configure() + tonic_prost_build::configure() .field_attribute( "Trace.start_time", "#[serde(serialize_with = \"crate::plugins::telemetry::apollo_exporter::serialize_timestamp\")]", @@ -49,9 +49,8 @@ pub fn main() -> Result<(), Box> { ) .type_attribute(".", "#[derive(serde::Serialize)]") .type_attribute(".", "#[allow(dead_code)]") - .type_attribute("StatsContext", "#[derive(Eq, Hash)]") .emit_rerun_if_changed(false) - .compile_protos(&[reports_out], &[&out_dir])?; + .compile_protos(&[reports_out], &[out_dir])?; Ok(()) } diff --git a/apollo-router/src/axum_factory/metrics.rs b/apollo-router/src/axum_factory/metrics.rs index 9f2d3ced3e..58a522fbde 100644 --- a/apollo-router/src/axum_factory/metrics.rs +++ b/apollo-router/src/axum_factory/metrics.rs @@ -32,7 +32,7 @@ pub(crate) mod jemalloc { tracing::warn!("Failed to read jemalloc {} stats", stringify!($name)); } }) - .init() + .build() }; } diff --git a/apollo-router/src/cache/metrics.rs b/apollo-router/src/cache/metrics.rs index 3e63a7032a..4ea32f26c5 100644 --- a/apollo-router/src/cache/metrics.rs +++ b/apollo-router/src/cache/metrics.rs @@ -1,56 +1,33 @@ use std::sync::Arc; -use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; use std::time::Duration; use fred::interfaces::MetricsInterface; use fred::prelude::Pool as RedisPool; use opentelemetry::KeyValue; +use opentelemetry::metrics::Gauge; use opentelemetry::metrics::MeterProvider; -use opentelemetry::metrics::ObservableGauge; use tokio::task::AbortHandle; use super::redis::ACTIVE_CLIENT_COUNT; +use crate::metrics::FutureMetricsExt; use crate::metrics::meter_provider; -/// Collection of Redis metrics gauges -pub(crate) struct RedisMetricsGauges { - pub(crate) _queue_length: ObservableGauge, - pub(crate) _network_latency: ObservableGauge, - pub(crate) _latency: ObservableGauge, - pub(crate) _request_size: ObservableGauge, - pub(crate) _response_size: ObservableGauge, - _active_client_count: ObservableGauge, -} - /// Weighted sum data for calculating averages -#[derive(Default, Clone)] +#[derive(Default)] struct WeightedSum { weighted_sum: u64, total_samples: u64, } -/// Configuration for metrics collection -struct MetricsConfig { - pool: Arc, - caller: &'static str, - metrics_interval: Duration, - queue_length: Arc, - network_latency_metric: WeightedAverageMetric, - latency_metric: WeightedAverageMetric, - request_size_metric: WeightedAverageMetric, - response_size_metric: WeightedAverageMetric, -} - -/// Configuration for a weighted average metric -#[derive(Clone)] -struct WeightedAverageMetric { - weighted_sum: Arc, - sample_count: Arc, - name: &'static str, - description: &'static str, - unit: &'static str, - unit_conversion: f64, // e.g., 1000.0 for ms->μs conversion +impl WeightedSum { + fn average(&self, unit_conversion: f64) -> f64 { + if self.total_samples > 0 { + (self.weighted_sum as f64) / (self.total_samples as f64) / unit_conversion + } else { + 0.0 + } + } } /// Aggregated metrics collected from Redis clients @@ -65,217 +42,139 @@ struct ClientMetrics { response_size: WeightedSum, } -/// Redis metrics collection functionality -pub(crate) struct RedisMetricsCollector { - // Task handle and gauges - abort_handle: AbortHandle, - _gauges: RedisMetricsGauges, +/// Sync gauges for Redis metrics. +/// These are created once in the background task and recorded to periodically. +struct RedisGauges { + queue_length: Gauge, + network_latency: Gauge, + latency: Gauge, + request_size: Gauge, + response_size: Gauge, + client_count: Gauge, } -impl WeightedAverageMetric { - /// Create a new weighted average metric - fn new( - name: &'static str, - description: &'static str, - unit: &'static str, - unit_conversion: f64, - ) -> Self { +impl RedisGauges { + fn new() -> Self { + let meter = meter_provider().meter("apollo/router"); + Self { - weighted_sum: Arc::new(AtomicU64::new(0)), - sample_count: Arc::new(AtomicU64::new(0)), - name, - description, - unit, - unit_conversion, + queue_length: meter + .u64_gauge("apollo.router.cache.redis.command_queue_length") + .with_description("Number of Redis commands buffered and not yet sent") + .with_unit("{command}") + .build(), + network_latency: meter + .f64_gauge("experimental.apollo.router.cache.redis.network_latency_avg") + .with_description("Average Redis network latency") + .with_unit("s") + .build(), + latency: meter + .f64_gauge("experimental.apollo.router.cache.redis.latency_avg") + .with_description("Average Redis command latency") + .with_unit("s") + .build(), + request_size: meter + .f64_gauge("experimental.apollo.router.cache.redis.request_size_avg") + .with_description("Average Redis request size") + .with_unit("bytes") + .build(), + response_size: meter + .f64_gauge("experimental.apollo.router.cache.redis.response_size_avg") + .with_description("Average Redis response size") + .with_unit("bytes") + .build(), + client_count: meter + .u64_gauge("apollo.router.cache.redis.clients") + .with_description("Number of active Redis clients") + .with_unit("{client}") + .build(), } } - /// Update the atomic counters with new weighted sum data - fn update(&self, weighted_sum: &WeightedSum) { - self.weighted_sum - .store(weighted_sum.weighted_sum, Ordering::Relaxed); - self.sample_count - .store(weighted_sum.total_samples, Ordering::Relaxed); + fn record(&self, metrics: &ClientMetrics, caller: &'static str) { + let attrs = &[KeyValue::new("kind", caller)]; + + self.queue_length.record(metrics.total_queue_len, attrs); + // Fred returns milliseconds, convert to seconds + self.network_latency + .record(metrics.network_latency.average(1000.0), attrs); + self.latency.record(metrics.latency.average(1000.0), attrs); + // Bytes - no conversion needed + self.request_size + .record(metrics.request_size.average(1.0), attrs); + self.response_size + .record(metrics.response_size.average(1.0), attrs); + // Client count has no "kind" attribute + self.client_count + .record(ACTIVE_CLIENT_COUNT.load(Ordering::Relaxed), &[]); } } +/// Redis metrics collection functionality. +/// +/// The background task that polls Redis client metrics is only spawned when +/// `activate()` is called. This ensures all metric instruments are registered +/// with the correct meter provider (after Telemetry.activate() has run). +pub(crate) struct RedisMetricsCollector { + /// None until activate() is called + /// TODO(@goto-bus-stop): actually this should maybe be a Once? + abort_handle: parking_lot::Mutex>, + pool: Arc, + caller: &'static str, + metrics_interval: Duration, +} + impl Drop for RedisMetricsCollector { fn drop(&mut self) { - self.abort_handle.abort(); + if let Some(handle) = self.abort_handle.lock().take() { + handle.abort(); + } } } impl RedisMetricsCollector { - /// Create a new metrics collector and start the collection task + /// Create a new metrics collector. + /// + /// The background task is NOT started until `activate()` is called. pub(crate) fn new( pool: Arc, caller: &'static str, metrics_interval: Duration, ) -> Self { - // Create atomic counters for metrics - let queue_length = Arc::new(AtomicU64::new(0)); - - let network_latency_metric = WeightedAverageMetric::new( - "experimental.apollo.router.cache.redis.network_latency_avg", - "Average Redis network latency", - "s", - 1000.0, // Fred returns milliseconds, convert to seconds for display - ); - let latency_metric = WeightedAverageMetric::new( - "experimental.apollo.router.cache.redis.latency_avg", - "Average Redis command latency", - "s", - 1000.0, // Fred returns milliseconds, convert to seconds for display - ); - let request_size_metric = WeightedAverageMetric::new( - "experimental.apollo.router.cache.redis.request_size_avg", - "Average Redis request size", - "bytes", - 1.0, - ); - let response_size_metric = WeightedAverageMetric::new( - "experimental.apollo.router.cache.redis.response_size_avg", - "Average Redis response size", - "bytes", - 1.0, - ); - - let config = MetricsConfig { - pool: pool.clone(), + Self { + abort_handle: parking_lot::Mutex::new(None), + pool, caller, metrics_interval, - queue_length: queue_length.clone(), - network_latency_metric, - latency_metric, - request_size_metric, - response_size_metric, - }; - - let (abort_handle, gauges) = Self::start_collection_task_for_metrics(config); - - Self { - abort_handle, - _gauges: gauges, } } - /// Start the metrics collection task and create gauges - fn start_collection_task_for_metrics( - config: MetricsConfig, - ) -> (AbortHandle, RedisMetricsGauges) { - let queue_length_gauge = - Self::create_queue_length_gauge(config.queue_length.clone(), config.caller); - let network_latency_gauge = - Self::create_weighted_average_gauge(&config.network_latency_metric, config.caller); - let latency_gauge = - Self::create_weighted_average_gauge(&config.latency_metric, config.caller); - let request_size_gauge = - Self::create_weighted_average_gauge(&config.request_size_metric, config.caller); - let response_size_gauge = - Self::create_weighted_average_gauge(&config.response_size_metric, config.caller); - let client_count_gauge = Self::create_client_count_gauge(); - let metrics_handle = Self::spawn_metrics_collection_task(config); - - let gauges = RedisMetricsGauges { - _queue_length: queue_length_gauge, - _network_latency: network_latency_gauge, - _latency: latency_gauge, - _request_size: request_size_gauge, - _response_size: response_size_gauge, - _active_client_count: client_count_gauge, - }; - - (metrics_handle.abort_handle(), gauges) - } - - /// Create the queue length observable gauge - fn create_queue_length_gauge( - queue_length: Arc, - caller: &'static str, - ) -> ObservableGauge { - let meter = meter_provider().meter("apollo/router"); - let queue_length_for_gauge = queue_length; - - meter - .u64_observable_gauge("apollo.router.cache.redis.command_queue_length") - .with_description("Number of Redis commands buffered and not yet sent") - .with_unit("{command}") - .with_callback(move |gauge| { - gauge.observe( - queue_length_for_gauge.load(Ordering::Relaxed), - &[KeyValue::new("kind", caller)], - ); - }) - .init() - } - - /// Generic method to create a weighted average gauge - fn create_weighted_average_gauge( - metric: &WeightedAverageMetric, - caller: &'static str, - ) -> ObservableGauge { - let meter = meter_provider().meter("apollo/router"); - let weighted_sum_for_gauge = metric.weighted_sum.clone(); - let sample_count_for_gauge = metric.sample_count.clone(); - let unit_conversion = metric.unit_conversion; - - meter - .f64_observable_gauge(metric.name) - .with_description(metric.description) - .with_unit(metric.unit) - .with_callback(move |gauge| { - let total_samples = sample_count_for_gauge.load(Ordering::Relaxed); - let weighted_sum = weighted_sum_for_gauge.load(Ordering::Relaxed); - - let average = if total_samples > 0 { - // Convert from milliseconds to seconds for display - (weighted_sum as f64) / (total_samples as f64) / unit_conversion - } else { - // Emit 0 to show the gauge exists even when no samples are available at scrape time - 0.0 - }; - - gauge.observe(average, &[KeyValue::new("kind", caller)]); - }) - .init() - } - - fn create_client_count_gauge() -> ObservableGauge { - let meter = meter_provider().meter("apollo/router"); - meter - .u64_observable_gauge("apollo.router.cache.redis.clients") - .with_description("Number of active Redis clients") - .with_unit("{client}") - .with_callback(move |gauge| { - gauge.observe(ACTIVE_CLIENT_COUNT.load(Ordering::Relaxed), &[]); - }) - .init() - } - - /// Spawn the metrics collection task - fn spawn_metrics_collection_task(config: MetricsConfig) -> tokio::task::JoinHandle<()> { - tokio::spawn(async move { - let mut interval = tokio::time::interval(config.metrics_interval); - loop { - interval.tick().await; - - let metrics = Self::collect_client_metrics(&config.pool); - - // Update atomic counters for gauges - config - .queue_length - .store(metrics.total_queue_len, Ordering::Relaxed); - config - .network_latency_metric - .update(&metrics.network_latency); - config.latency_metric.update(&metrics.latency); - config.request_size_metric.update(&metrics.request_size); - config.response_size_metric.update(&metrics.response_size); - - // Emit counters - Self::emit_counter_metrics(&metrics, config.caller); + /// Start the metrics collection task. + /// + /// This MUST be called after `Telemetry.activate()` to ensure all metric + /// instruments are registered with the correct meter provider. + pub(crate) fn activate(&self) { + let pool = self.pool.clone(); + let caller = self.caller; + let metrics_interval = self.metrics_interval; + + let handle = tokio::spawn( + async move { + let mut interval = tokio::time::interval(metrics_interval); + let gauges = RedisGauges::new(); + + loop { + interval.tick().await; + + let metrics = Self::collect_client_metrics(&pool); + gauges.record(&metrics, caller); + Self::emit_counter_metrics(&metrics, caller); + } } - }) + .with_current_meter_provider(), + ); + + *self.abort_handle.lock() = Some(handle.abort_handle()); } /// Collect metrics from all Redis clients @@ -283,64 +182,32 @@ impl RedisMetricsCollector { let mut metrics = ClientMetrics::default(); for client in pool.clients() { - // Basic metrics always available let redelivery_count = client.take_redelivery_count(); metrics.total_redelivery_count += redelivery_count as u64; let queue_len = client.command_queue_len(); metrics.total_queue_len += queue_len as u64; - // Collect weighted average metrics directly - Self::update_average_weighted_metric( + Self::update_weighted_sum( client.take_network_latency_metrics(), &mut metrics.network_latency, - 1.0, // Fred returns milliseconds, store as-is for precision - ); - - Self::update_average_weighted_metric( - client.take_latency_metrics(), - &mut metrics.latency, - 1.0, // Fred returns milliseconds, store as-is for precision ); - - Self::update_average_weighted_metric( - client.take_req_size_metrics(), - &mut metrics.request_size, - 1.0, - ); - - Self::update_average_weighted_metric( - client.take_res_size_metrics(), - &mut metrics.response_size, - 1.0, - ); - - // Get commands executed from latency stats (already collected above) - // Note: We use latency samples as a proxy for total commands executed - // since latency stats track all commands that were executed + Self::update_weighted_sum(client.take_latency_metrics(), &mut metrics.latency); + Self::update_weighted_sum(client.take_req_size_metrics(), &mut metrics.request_size); + Self::update_weighted_sum(client.take_res_size_metrics(), &mut metrics.response_size); } - // Set total commands executed based on latency samples metrics.total_commands_executed = metrics.latency.total_samples; - metrics } - /// Generic method to collect weighted metrics - fn update_average_weighted_metric( - stats: fred::types::Stats, - weighted_sum: &mut WeightedSum, - unit_conversion: f64, - ) { + fn update_weighted_sum(stats: fred::types::Stats, weighted_sum: &mut WeightedSum) { if stats.samples > 0 { - // Apply unit conversion (Fred returns milliseconds) - let converted_avg = (stats.avg * unit_conversion) as u64; - weighted_sum.weighted_sum += converted_avg * stats.samples; + weighted_sum.weighted_sum += (stats.avg as u64) * stats.samples; weighted_sum.total_samples += stats.samples; } } - /// Emit counter metrics fn emit_counter_metrics(metrics: &ClientMetrics, caller: &'static str) { u64_counter_with_unit!( "apollo.router.cache.redis.redelivery_count", @@ -362,142 +229,82 @@ impl RedisMetricsCollector { #[cfg(test)] mod tests { - use std::sync::Arc; - - use fred::mocks::SimpleMap; - + use super::*; use crate::cache::redis::RedisCacheStorage; use crate::cache::redis::RedisKey; use crate::cache::redis::RedisValue; - use crate::cache::storage::ValueType; - use crate::metrics::FutureMetricsExt; use crate::metrics::test_utils::MetricType; #[test] - fn test_weighted_sum_calculation() { - let mut weighted_sum = super::WeightedSum::default(); + fn test_weighted_sum_average() { + let mut ws = WeightedSum::default(); + + // Empty case + assert_eq!(ws.average(1.0), 0.0); + + // Add some samples: avg=10ms, 5 samples + ws.weighted_sum = 50; // 10 * 5 + ws.total_samples = 5; + + // No conversion + assert_eq!(ws.average(1.0), 10.0); + + // Convert ms to seconds + assert_eq!(ws.average(1000.0), 0.01); + } + + #[test] + fn test_update_weighted_sum() { + let mut ws = WeightedSum::default(); - // Test adding first stats - super::RedisMetricsCollector::update_average_weighted_metric( + // Test with samples + RedisMetricsCollector::update_weighted_sum( fred::types::Stats { - avg: 10.0, // 10ms (Fred returns milliseconds) + avg: 10.0, samples: 5, max: 15, min: 5, stddev: 2.0, sum: 50, }, - &mut weighted_sum, - 1.0, // Store milliseconds as-is - ); - - assert_eq!(weighted_sum.total_samples, 5); - assert_eq!(weighted_sum.weighted_sum, 50); // 10.0 * 1.0 * 5 = 50 milliseconds - - // Test adding more stats - super::RedisMetricsCollector::update_average_weighted_metric( - fred::types::Stats { - avg: 20.0, // 20ms (Fred returns milliseconds) - samples: 3, - max: 25, - min: 15, - stddev: 3.0, - sum: 60, - }, - &mut weighted_sum, - 1.0, + &mut ws, ); - assert_eq!(weighted_sum.total_samples, 8); // 5 + 3 - assert_eq!(weighted_sum.weighted_sum, 110); // 50 + 60 milliseconds - } - - #[test] - fn test_weighted_sum_with_zero_samples() { - let mut weighted_sum = super::WeightedSum::default(); + assert_eq!(ws.total_samples, 5); + assert_eq!(ws.weighted_sum, 50); // 10 * 5 - // Test that zero samples don't affect the weighted sum - super::RedisMetricsCollector::update_average_weighted_metric( + // Test with zero samples (should not change) + RedisMetricsCollector::update_weighted_sum( fred::types::Stats { - avg: 0.010, // 10ms in seconds + avg: 100.0, samples: 0, max: 0, min: 0, stddev: 0.0, sum: 0, }, - &mut weighted_sum, - 1000000.0, + &mut ws, ); - assert_eq!(weighted_sum.total_samples, 0); - assert_eq!(weighted_sum.weighted_sum, 0); - } - - #[test] - fn test_weighted_sum_with_unit_conversion() { - let mut weighted_sum = super::WeightedSum::default(); - - // Test with different unit conversions (bytes - no conversion) - super::RedisMetricsCollector::update_average_weighted_metric( - fred::types::Stats { - avg: 100.0, - samples: 2, - max: 120, - min: 80, - stddev: 20.0, - sum: 200, - }, - &mut weighted_sum, - 1.0, // No conversion (bytes) - ); - - assert_eq!(weighted_sum.total_samples, 2); - assert_eq!(weighted_sum.weighted_sum, 200); // 100.0 * 1.0 * 2 - } - - #[test] - fn test_latency_metric_conversion_to_seconds() { - // This test demonstrates that latency metrics are correctly converted to seconds - let mut weighted_sum = super::WeightedSum::default(); - - // Simulate Redis latency stats (Fred returns milliseconds) - super::RedisMetricsCollector::update_average_weighted_metric( - fred::types::Stats { - avg: 5.0, // 5ms (Fred returns milliseconds) - samples: 10, - max: 8, - min: 2, - stddev: 1.5, - sum: 50, - }, - &mut weighted_sum, - 1.0, // Store milliseconds as-is - ); - - assert_eq!(weighted_sum.total_samples, 10); - assert_eq!(weighted_sum.weighted_sum, 50); // 5.0 * 1.0 * 10 = 50 milliseconds - - // Verify conversion to seconds for gauge emission - let final_average_seconds = - (weighted_sum.weighted_sum as f64) / (weighted_sum.total_samples as f64) / 1000.0; - assert_eq!(final_average_seconds, 0.005); // Should be 0.005 seconds (5ms converted from ms) + assert_eq!(ws.total_samples, 5); // unchanged + assert_eq!(ws.weighted_sum, 50); // unchanged } #[tokio::test] async fn test_redis_storage_with_mocks() { async { - let simple_map = Arc::new(SimpleMap::new()); + let simple_map = Arc::new(fred::mocks::SimpleMap::new()); let storage = RedisCacheStorage::from_mocks(simple_map.clone()) .await .expect("Failed to create Redis storage with mocks"); + storage.activate(); #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] struct TestValue { data: String, } - impl ValueType for TestValue { + impl crate::cache::storage::ValueType for TestValue { fn estimated_size(&self) -> Option { Some(self.data.len()) } @@ -518,17 +325,17 @@ mod tests { assert!(retrieved.is_ok(), "Should have retrieved value from mock"); assert_eq!(retrieved.unwrap().0.data, "test_value"); + // Pause to ensure that queue length is zero & metrics have been exported + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + // Verify Redis connection metrics are emitted. // Since this metric is based on a global AtomicU64, it's not unique across tests - so // we can only reliably check for metric existence, rather than a specific value. - crate::metrics::collect_metrics().metric_exists::( + assert!(crate::metrics::collect_metrics().metric_exists( "apollo.router.cache.redis.clients", MetricType::Gauge, &[], - ); - - // Pause to ensure that queue length is zero - tokio::time::sleep(std::time::Duration::from_millis(100)).await; + )); // Verify Redis gauge metrics are available (observables are created immediately) assert_gauge!( diff --git a/apollo-router/src/cache/redis.rs b/apollo-router/src/cache/redis.rs index f4abcaaa3e..424b7b7f26 100644 --- a/apollo-router/src/cache/redis.rs +++ b/apollo-router/src/cache/redis.rs @@ -129,8 +129,15 @@ struct DropSafeRedisPool { pool: Arc, caller: &'static str, heartbeat_abort_handle: AbortHandle, - // Metrics collector handles its own abort and gauges - _metrics_collector: RedisMetricsCollector, + // Metrics collector handles its own abort and spawns a background task for gauge updates + metrics_collector: RedisMetricsCollector, +} + +impl DropSafeRedisPool { + /// Signal that the meter provider is ready and metrics gauges can be created. + fn activate(&self) { + self.metrics_collector.activate(); + } } impl Deref for DropSafeRedisPool { @@ -483,7 +490,7 @@ impl RedisCacheStorage { pool: pooled_client_arc, caller, heartbeat_abort_handle: heartbeat_handle.abort_handle(), - _metrics_collector: metrics_collector, + metrics_collector, }), namespace: namespace.map(Arc::new), ttl, @@ -496,6 +503,14 @@ impl RedisCacheStorage { self.ttl } + /// Signal that the meter provider is ready and metrics gauges can be created. + /// + /// This MUST be called after `Telemetry.activate()` to ensure gauges are + /// registered with the correct meter provider. + pub(crate) fn activate(&self) { + self.inner.activate(); + } + /// Helper method to record Redis errors for metrics fn record_query_error(&self, error: &RedisError) { record_redis_error(error, self.inner.caller, "query"); diff --git a/apollo-router/src/cache/storage.rs b/apollo-router/src/cache/storage.rs index bfb8eed3e5..0240e261bd 100644 --- a/apollo-router/src/cache/storage.rs +++ b/apollo-router/src/cache/storage.rs @@ -128,7 +128,7 @@ where ], ) }) - .init() + .build() } fn create_cache_estimated_storage_size_gauge(&self) -> ObservableGauge { @@ -153,7 +153,7 @@ where ) } }) - .init() + .build() } /// `init_from_redis` is called with values newly deserialized from Redis cache @@ -279,11 +279,16 @@ where } pub(crate) fn activate(&self) { - // Gauges MUST be created after the meter provider is initialized - // This means that on reload we need a non-fallible way to recreate the gauges, hence this function. + // Gauges MUST be created after the meter provider is initialized. + // This means that on reload we need a non-fallible way to recreate the gauges. *self.cache_size_gauge.lock() = Some(self.create_cache_size_gauge()); *self.cache_estimated_storage_gauge.lock() = Some(self.create_cache_estimated_storage_size_gauge()); + + // Also activate Redis metrics if present + if let Some(redis) = &self.redis { + redis.activate(); + } } } diff --git a/apollo-router/src/compute_job/mod.rs b/apollo-router/src/compute_job/mod.rs index 51aa0a6860..c442394d9b 100644 --- a/apollo-router/src/compute_job/mod.rs +++ b/apollo-router/src/compute_job/mod.rs @@ -410,7 +410,7 @@ pub(crate) fn create_queue_size_gauge() -> ObservableGauge { "Number of computation jobs (parsing, planning, …) waiting to be scheduled", ) .with_callback(move |m| m.observe(queue().queued_count() as u64, &[])) - .init() + .build() } #[cfg(test)] diff --git a/apollo-router/src/configuration/metrics.rs b/apollo-router/src/configuration/metrics.rs index 8ed4a58851..28e531086c 100644 --- a/apollo-router/src/configuration/metrics.rs +++ b/apollo-router/src/configuration/metrics.rs @@ -591,7 +591,7 @@ impl From for Metrics { .with_callback(move |observer| { observer.observe(value, &attributes); }) - .init() + .build() }) .collect(), } @@ -605,6 +605,7 @@ mod test { use crate::configuration::metrics::InstrumentData; use crate::configuration::metrics::Metrics; + use crate::metrics::FutureMetricsExt; use crate::uplink::license_enforcement::LicenseLimits; use crate::uplink::license_enforcement::LicenseState; @@ -612,20 +613,27 @@ mod test { #[folder = "src/configuration/testdata/metrics"] struct Asset; - #[test] - fn test_metrics() { + #[tokio::test] + async fn test_metrics() { + // Each config file needs to be tested in isolation to avoid callback accumulation + // across iterations (observable gauge callbacks persist until provider shutdown) for file_name in Asset::iter() { - let source = Asset::get(&file_name).expect("test file must exist"); - let input = std::str::from_utf8(&source.data) - .expect("expected utf8") - .to_string(); - let yaml = &serde_yaml::from_str::(&input) - .expect("config must be valid yaml"); - - let mut data = InstrumentData::default(); - data.populate_config_instruments(yaml); - let _metrics: Metrics = data.into(); - assert_non_zero_metrics_snapshot!(file_name); + let file_name_owned = file_name.to_string(); + async { + let source = Asset::get(&file_name_owned).expect("test file must exist"); + let input = std::str::from_utf8(&source.data) + .expect("expected utf8") + .to_string(); + let yaml = &serde_yaml::from_str::(&input) + .expect("config must be valid yaml"); + + let mut data = InstrumentData::default(); + data.populate_config_instruments(yaml); + let _metrics: Metrics = data.into(); + assert_non_zero_metrics_snapshot!(file_name_owned); + } + .with_metrics() + .await; } } diff --git a/apollo-router/src/executable.rs b/apollo-router/src/executable.rs index d6ba909f54..5f25b45d00 100644 --- a/apollo-router/src/executable.rs +++ b/apollo-router/src/executable.rs @@ -466,8 +466,13 @@ impl Executable { if apollo_telemetry_initialized { // We should be good to shutdown OpenTelemetry now as the router should have finished everything. tokio::task::spawn_blocking(move || { - opentelemetry::global::shutdown_tracer_provider(); - meter_provider_internal().shutdown(); + // Setting a new default provider causes the old one to be dropped and shut down + opentelemetry::global::set_tracer_provider( + opentelemetry_sdk::trace::SdkTracerProvider::default(), + ); + if let Err(error) = meter_provider_internal().shutdown() { + tracing::error!(%error, "Failed to shut down OTel meter provider cleanly"); + } }) .await?; } diff --git a/apollo-router/src/logging/mod.rs b/apollo-router/src/logging/mod.rs index 5d0b75879d..0947938c3e 100644 --- a/apollo-router/src/logging/mod.rs +++ b/apollo-router/src/logging/mod.rs @@ -70,7 +70,6 @@ pub(crate) mod test { .lines() .map(|line| { let mut line: serde_json::Value = serde_json::from_str(line).unwrap(); - // move the message field to the top level let fields = line .as_object_mut() .unwrap() @@ -78,6 +77,8 @@ pub(crate) mod test { .unwrap() .as_object_mut() .unwrap(); + + // move the message field to the top level let message = fields.remove("message").unwrap_or_default(); line.as_object_mut() .unwrap() diff --git a/apollo-router/src/metrics/aggregation.rs b/apollo-router/src/metrics/aggregation.rs index 3ff20ff97b..f5d1a276a8 100644 --- a/apollo-router/src/metrics/aggregation.rs +++ b/apollo-router/src/metrics/aggregation.rs @@ -1,37 +1,38 @@ -use std::any::Any; use std::borrow::Cow; use std::collections::HashMap; +use std::collections::HashSet; use std::mem; use std::sync::Arc; +use std::time::Duration; use derive_more::From; -use itertools::Itertools; +use opentelemetry::InstrumentationScope; use opentelemetry::KeyValue; use opentelemetry::metrics::AsyncInstrument; -use opentelemetry::metrics::Callback; -use opentelemetry::metrics::CallbackRegistration; +use opentelemetry::metrics::AsyncInstrumentBuilder; use opentelemetry::metrics::Counter; use opentelemetry::metrics::Gauge; use opentelemetry::metrics::Histogram; +use opentelemetry::metrics::HistogramBuilder; +use opentelemetry::metrics::InstrumentBuilder; use opentelemetry::metrics::InstrumentProvider; use opentelemetry::metrics::Meter; use opentelemetry::metrics::MeterProvider; use opentelemetry::metrics::ObservableCounter; use opentelemetry::metrics::ObservableGauge; use opentelemetry::metrics::ObservableUpDownCounter; -use opentelemetry::metrics::Observer; -use opentelemetry::metrics::SyncCounter; -use opentelemetry::metrics::SyncGauge; -use opentelemetry::metrics::SyncHistogram; -use opentelemetry::metrics::SyncUpDownCounter; +use opentelemetry::metrics::SyncInstrument; use opentelemetry::metrics::UpDownCounter; -use opentelemetry::metrics::noop::NoopMeterProvider; -use opentelemetry_sdk::metrics::SdkMeterProvider; +use opentelemetry_sdk::error::OTelSdkError; +use opentelemetry_sdk::error::OTelSdkResult; use parking_lot::Mutex; use strum::Display; use strum::EnumCount; use strum::EnumIter; +use strum::FromRepr; +use strum::IntoEnumIterator; +use super::NoopInstrumentProvider; use crate::metrics::filter::FilterMeterProvider; // This meter provider enables us to combine multiple meter providers. The reasons we need this are: @@ -42,7 +43,7 @@ use crate::metrics::filter::FilterMeterProvider; // `Meters are identified by name, version, and schema_url fields. When more than one Meter of the same name, version, and schema_url is created, it is unspecified whether or under which conditions the same or different Meter instances are returned. It is a user error to create Meters with different attributes but the same identity.` #[derive( - Hash, Ord, PartialOrd, Eq, PartialEq, Copy, Clone, Debug, EnumCount, EnumIter, Display, + Hash, Ord, PartialOrd, Eq, PartialEq, Copy, Clone, Debug, EnumCount, EnumIter, Display, FromRepr, )] #[repr(u8)] pub(crate) enum MeterProviderType { @@ -59,43 +60,58 @@ pub(crate) struct AggregateMeterProvider { impl Default for AggregateMeterProvider { fn default() -> Self { - let meter_provider = AggregateMeterProvider { + // All providers start as noop to avoid OTel SDK errors. + // Real providers are set via set() during configuration. + AggregateMeterProvider { inner: Arc::new(Mutex::new(Some(Inner::default()))), - }; - - // If the regular global meter provider has been set then the aggregate meter provider will use it. Otherwise it'll default to a no-op. - // For this to work the global meter provider must be set before the aggregate meter provider is created. - // This functionality is not guaranteed to stay like this, so use at your own risk. - meter_provider.set( - MeterProviderType::OtelDefault, - FilterMeterProvider::public(opentelemetry::global::meter_provider()), - ); - - meter_provider + } } } pub(crate) struct Inner { providers: Vec<(FilterMeterProvider, HashMap)>, registered_instruments: Vec, + /// Shared registries for observable instruments - tracks registrations per provider + observable_registries: Arc, } impl Default for Inner { fn default() -> Self { Inner { + // Initialize with noop providers to avoid OTel SDK errors during startup. + // Real providers are set via AggregateMeterProvider::set() during configuration. providers: (0..MeterProviderType::COUNT) - .map(|_| { - ( - FilterMeterProvider::public(SdkMeterProvider::default()), - HashMap::new(), - ) - }) + .map(|_| (FilterMeterProvider::noop(), HashMap::new())) .collect(), registered_instruments: Vec::new(), + observable_registries: Arc::new(SharedObservableRegistries::new( + MeterProviderType::COUNT, + )), } } } +// HACK(@goto-bus-stop): see https://github.com/apollographql/router/pull/8976 +// _in tests_, we store the AggregateMeterProvider in a thread local. +// During Drop, the otel meter providers may log using `tracing`. This also uses a thread local. +// When the thread exits, such locals are dropped in unspecified order: `tracing` may be dropped +// _before_ the AggregateMeterProvider. In that case, otel would try to log and `tracing` may +// panic internally. +// +// This implementation works around that issue by manually dropping the otel meter providers while +// _disabling_ tracing logs altogether. +// +// This assumes that in _normal_ router operation, we always call `shutdown()` explicitly, else we +// would lose trace events. +impl Drop for Inner { + fn drop(&mut self) { + let noop = tracing::subscriber::NoSubscriber::new(); + tracing::subscriber::with_default(noop, || { + self.providers.clear(); + }); + } +} + /// Fields are never used directly but strong references here /// keep weak references elsewhere upgradable. #[derive(From)] @@ -151,6 +167,11 @@ impl AggregateMeterProvider { // Once invalidated all metrics callsites will try to obtain new instruments, but will be blocked on the mutex. inner.invalidate(); + // Clear observable registrations for this provider so new gauges will re-register + inner + .observable_registries + .clear_provider(meter_provider_type); + //Now update the meter provider let mut swap = (meter_provider, HashMap::new()); mem::swap( @@ -177,7 +198,24 @@ impl AggregateMeterProvider { } /// Shutdown MUST be called from a blocking thread. - pub(crate) fn shutdown(&self) { + pub(crate) fn shutdown_with_timeout(&self, timeout: Duration) -> OTelSdkResult { + /// Prefix internal failure error message with "[providername]". + /// Returns the original error if it is not an internal failure or if the index is invalid. + fn tag_error_with_provider_type(err: OTelSdkError, index: usize) -> OTelSdkError { + let OTelSdkError::InternalFailure(message) = &err else { + return err; + }; + + let Ok(ty) = u8::try_from(index) else { + return err; + }; + let Some(ty) = MeterProviderType::from_repr(ty) else { + return err; + }; + + OTelSdkError::InternalFailure(format!("[{ty}] {message}")) + } + // Make sure that we don't deadlock by dropping the mutex guard before actual shutdown happens // This means that if we have any misbehaving code that tries to access the meter provider during shutdown, e.g. for export metrics // then we don't get stuck on the mutex. @@ -187,7 +225,48 @@ impl AggregateMeterProvider { let mut guard = self.inner.lock(); let old = guard.take(); drop(guard); - drop(old); + + let Some(inner) = old else { return Ok(()) }; + + let mut result = Ok(()); + for (index, (provider, _)) in inner.providers.iter().enumerate() { + let Err(err) = provider.shutdown_with_timeout(timeout) else { + continue; + }; + + let err = tag_error_with_provider_type(err, index); + + // Report errors as, in order of precedence: + // - combined internal failures tagged with provider type + // - or timeout if one of them timed out + // - or AlreadyShutdown if one of them was already shut down + // - or nothing + // "Less important" errors get silenced in favour of the "more important" ones. Scare + // quotes due to inherent subjectivity. + result = match (result, err) { + // Report all stacking internal failures + ( + Err(OTelSdkError::InternalFailure(old_error)), + OTelSdkError::InternalFailure(new_error), + ) => Err(OTelSdkError::InternalFailure(format!( + "{old_error}\n{new_error}" + ))), + // If we already had an internal failure, keep that + (result @ Err(OTelSdkError::InternalFailure(_)), _) => result, + // If we now encountered an internal failure, keep the new error + (_, err @ OTelSdkError::InternalFailure(_)) => Err(err), + // If we already had an error, keep that + (result @ Err(_), _) => result, + // If we did not yet have an error, stash the new error + (Ok(_), err) => Err(err), + }; + } + Ok(()) + } + + /// Shutdown MUST be called from a blocking thread. + pub(crate) fn shutdown(&self) -> OTelSdkResult { + self.shutdown_with_timeout(Duration::from_secs(5)) } /// Create a registered instrument. This enables caching at callsites and invalidation at the meter provider via weak reference. @@ -221,23 +300,14 @@ impl Inner { self.registered_instruments.clear() } pub(crate) fn meter(&mut self, name: impl Into>) -> Meter { - self.versioned_meter( - name, - None::>, - None::>, - None, - ) + let scope = InstrumentationScope::builder(name).build(); + self.meter_with_scope(scope) } - pub(crate) fn versioned_meter( - &mut self, - name: impl Into>, - version: Option>>, - schema_url: Option>>, - attributes: Option>, - ) -> Meter { - let name = name.into(); - let version = version.map(|v| v.into()); - let schema_url = schema_url.map(|v| v.into()); + pub(crate) fn meter_with_scope(&mut self, scope: InstrumentationScope) -> Meter { + let name: Cow<'static, str> = Cow::Owned(scope.name().to_string()); + let version: Option> = scope.version().map(|v| Cow::Owned(v.to_string())); + let schema_url: Option> = + scope.schema_url().map(|v| Cow::Owned(v.to_string())); let mut meters = Vec::with_capacity(self.providers.len()); for (provider, existing_meters) in &mut self.providers { @@ -248,19 +318,15 @@ impl Inner { version: version.clone(), schema_url: schema_url.clone(), }) - .or_insert_with(|| { - provider.versioned_meter( - name.clone(), - version.clone(), - schema_url.clone(), - attributes.clone(), - ) - }) + .or_insert_with(|| provider.meter_with_scope(scope.clone())) .clone(), ); } - Meter::new(Arc::new(AggregateInstrumentProvider { meters })) + Meter::new(Arc::new(AggregateInstrumentProvider { + meters, + registries: Arc::clone(&self.observable_registries), + })) } pub(crate) fn create_registered_instrument( @@ -277,61 +343,41 @@ impl Inner { } impl MeterProvider for AggregateMeterProvider { - fn versioned_meter( - &self, - name: impl Into>, - version: Option>>, - schema_url: Option>>, - attributes: Option>, - ) -> Meter { + fn meter_with_scope(&self, scope: InstrumentationScope) -> Meter { let mut inner = self.inner.lock(); if let Some(inner) = inner.as_mut() { - inner.versioned_meter(name, version, schema_url, attributes) + inner.meter_with_scope(scope) } else { // The meter was used after shutdown. Default to Noop since the instrument cannot actually be used - NoopMeterProvider::default().versioned_meter(name, version, schema_url, attributes) + Meter::new(Arc::new(NoopInstrumentProvider)) } } } pub(crate) struct AggregateInstrumentProvider { meters: Vec, + /// Shared registries for observable instruments (owned by Inner) + registries: Arc, } pub(crate) struct AggregateCounter { delegates: Vec>, } -impl SyncCounter for AggregateCounter { - fn add(&self, value: T, attributes: &[KeyValue]) { +impl SyncInstrument for AggregateCounter { + fn measure(&self, value: T, attributes: &[KeyValue]) { for counter in &self.delegates { counter.add(value, attributes) } } } -pub(crate) struct AggregateObservableCounter { - delegates: Vec<(ObservableCounter, Option)>, -} - -impl AsyncInstrument for AggregateObservableCounter { - fn observe(&self, value: T, attributes: &[KeyValue]) { - for (counter, _) in &self.delegates { - counter.observe(value, attributes) - } - } - - fn as_any(&self) -> Arc { - unreachable!() - } -} - pub(crate) struct AggregateHistogram { delegates: Vec>, } -impl SyncHistogram for AggregateHistogram { - fn record(&self, value: T, attributes: &[KeyValue]) { +impl SyncInstrument for AggregateHistogram { + fn measure(&self, value: T, attributes: &[KeyValue]) { for histogram in &self.delegates { histogram.record(value, attributes) } @@ -342,164 +388,350 @@ pub(crate) struct AggregateUpDownCounter { delegates: Vec>, } -impl SyncUpDownCounter for AggregateUpDownCounter { - fn add(&self, value: T, attributes: &[KeyValue]) { +impl SyncInstrument for AggregateUpDownCounter { + fn measure(&self, value: T, attributes: &[KeyValue]) { for counter in &self.delegates { counter.add(value, attributes) } } } -pub(crate) struct AggregateObservableUpDownCounter { - delegates: Vec<(ObservableUpDownCounter, Option)>, +pub(crate) struct AggregateGauge { + delegates: Vec>, } -impl AsyncInstrument for AggregateObservableUpDownCounter { - fn observe(&self, value: T, attributes: &[KeyValue]) { - for (counter, _) in &self.delegates { - counter.observe(value, attributes) +impl SyncInstrument for AggregateGauge { + fn measure(&self, value: T, attributes: &[KeyValue]) { + for gauge in &self.delegates { + gauge.record(value, attributes) } } - - fn as_any(&self) -> Arc { - unreachable!() - } } -pub(crate) struct AggregateGauge { - delegates: Vec>, +/// Type alias for observable callbacks +type ObservableCallback = Arc) + Send + Sync>; + +/// Registry for observable instrument callbacks. +/// +/// In OTel 0.31+, observable instrument types like `ObservableGauge` are just +/// `PhantomData` - dropping them does nothing. Callbacks are registered with +/// the SDK at build time and live until provider shutdown. +/// +/// This registry provides proper lifecycle management: +/// - Multiple callbacks can be registered per instrument_name (e.g., different caches +/// using the same gauge name but different attribute values) +/// - One OTel instrument per (meter_provider_type, instrument_name) is registered lazily +/// - The consolidated callback invokes ALL registered user callbacks for that instrument +/// - When a provider is replaced, its registrations are cleared so new gauges re-register +struct ObservableCallbackRegistry { + /// instrument_name -> list of callbacks + /// Multiple callbacks can exist for the same instrument name when different components + /// (e.g., query planner cache, APQ cache) use the same gauge name with different attributes + callbacks: Mutex>>>, + /// Tracks which (meter_provider_type, instrument_name) pairs have been registered with OTel SDK + registered: Mutex>, } -impl SyncGauge for AggregateGauge { - fn record(&self, value: T, attributes: &[KeyValue]) { - for gauge in &self.delegates { - gauge.record(value, attributes) +impl ObservableCallbackRegistry { + fn new() -> Self { + Self { + callbacks: Mutex::new(HashMap::new()), + registered: Mutex::new(HashSet::new()), + } + } + + /// Register a callback for an instrument name. + /// Multiple callbacks can be registered for the same instrument name. + /// This supports scenarios like multiple caches using the same gauge name + /// but with different attribute values (e.g., `kind="query planner"` vs `kind="apq"`). + fn register_callback(&self, instrument_name: &str, callback: ObservableCallback) { + let mut callbacks = self.callbacks.lock(); + callbacks + .entry(instrument_name.to_string()) + .or_default() + .push(callback); + } + + /// Invoke all callbacks for an instrument name. + fn invoke_callback(&self, instrument_name: &str, observer: &dyn AsyncInstrument) { + let callbacks = self.callbacks.lock(); + if let Some(callback_list) = callbacks.get(instrument_name) { + for callback in callback_list { + callback(observer); + } } } + + /// Register an instrument for a provider if not already registered. + /// Returns `true` if newly registered, `false` if already registered. + fn try_register_for_provider( + &self, + meter_provider_type: MeterProviderType, + instrument_name: String, + ) -> bool { + self.registered + .lock() + .insert((meter_provider_type, instrument_name)) + } + + /// Clear registrations for a specific provider (called when provider is replaced) + fn clear_provider_registrations(&self, meter_provider_type: MeterProviderType) { + let mut registered = self.registered.lock(); + registered.retain(|(provider_type, _)| *provider_type != meter_provider_type); + } + + /// Clear all callbacks (called during reload when services will be recreated) + fn clear_callbacks(&self) { + self.callbacks.lock().clear(); + } } -pub(crate) struct AggregateObservableGauge { - delegates: Vec<(ObservableGauge, Option)>, +/// Shared registries for all observable instrument types. +/// This is stored at the `Inner` level and shared across all meters. +pub(crate) struct SharedObservableRegistries { + u64_gauge: ObservableCallbackRegistry, + i64_gauge: ObservableCallbackRegistry, + f64_gauge: ObservableCallbackRegistry, + u64_counter: ObservableCallbackRegistry, + f64_counter: ObservableCallbackRegistry, + i64_up_down_counter: ObservableCallbackRegistry, + f64_up_down_counter: ObservableCallbackRegistry, } -impl AsyncInstrument for AggregateObservableGauge { - fn observe(&self, measurement: T, attributes: &[KeyValue]) { - for (gauge, _) in &self.delegates { - gauge.observe(measurement, attributes) +impl SharedObservableRegistries { + fn new(_num_providers: usize) -> Self { + Self { + u64_gauge: ObservableCallbackRegistry::new(), + i64_gauge: ObservableCallbackRegistry::new(), + f64_gauge: ObservableCallbackRegistry::new(), + u64_counter: ObservableCallbackRegistry::new(), + f64_counter: ObservableCallbackRegistry::new(), + i64_up_down_counter: ObservableCallbackRegistry::new(), + f64_up_down_counter: ObservableCallbackRegistry::new(), } } - fn as_any(&self) -> Arc { - unreachable!() + /// Clear registrations for a specific provider and all callbacks. + /// + /// Called when a provider is replaced. We clear: + /// 1. Provider registrations - so new gauges will re-register with the new provider + /// 2. All callbacks - because services will be recreated and re-register their callbacks + /// + /// This is safe because when any provider is replaced, the entire service graph is + /// recreated, so all gauges will be recreated and add fresh callbacks. + fn clear_provider(&self, meter_provider_type: MeterProviderType) { + // Clear registrations for this provider so new gauges will register with it + self.u64_gauge + .clear_provider_registrations(meter_provider_type); + self.i64_gauge + .clear_provider_registrations(meter_provider_type); + self.f64_gauge + .clear_provider_registrations(meter_provider_type); + self.u64_counter + .clear_provider_registrations(meter_provider_type); + self.f64_counter + .clear_provider_registrations(meter_provider_type); + self.i64_up_down_counter + .clear_provider_registrations(meter_provider_type); + self.f64_up_down_counter + .clear_provider_registrations(meter_provider_type); + + // Clear all callbacks - services will be recreated and re-register them + self.u64_gauge.clear_callbacks(); + self.i64_gauge.clear_callbacks(); + self.f64_gauge.clear_callbacks(); + self.u64_counter.clear_callbacks(); + self.f64_counter.clear_callbacks(); + self.i64_up_down_counter.clear_callbacks(); + self.f64_up_down_counter.clear_callbacks(); } } -// Observable instruments don't need to have a ton of optimisation because they are only read on demand. -macro_rules! aggregate_observable_instrument_fn { + +// Macro for sync instruments (Counter, UpDownCounter, Gauge) +macro_rules! aggregate_instrument_fn { ($name:ident, $ty:ty, $wrapper:ident, $implementation:ident) => { - fn $name( - &self, - name: Cow<'static, str>, - description: Option>, - unit: Option>, - callback: Vec>, - ) -> opentelemetry::metrics::Result<$wrapper<$ty>> { - let callback: Vec>> = - callback.into_iter().map(|c| Arc::new(c)).collect_vec(); - let delegates = self + fn $name(&self, builder: InstrumentBuilder<'_, $wrapper<$ty>>) -> $wrapper<$ty> { + let delegates: Vec<$wrapper<$ty>> = self .meters .iter() .map(|meter| { - let mut builder = meter.$name(name.clone()); - if let Some(description) = &description { - builder = builder.with_description(description.clone()); + let mut b = meter.$name(builder.name.clone()); + if let Some(description) = &builder.description { + b = b.with_description(description.clone()); } - if let Some(unit) = &unit { - builder = builder.with_unit(unit.clone()); + if let Some(unit) = &builder.unit { + b = b.with_unit(unit.clone()); } - // We must not set callback in the builder as it will leak memory. - // Instead we use callback registration on the meter provider as it allows unregistration - // Also we need to filter out no-op instruments as passing these to the meter provider as these will fail with a cryptic message about different implementations. - // Confusingly the implementation of as_any() on an instrument will return 'other stuff'. In particular no-ops return Arc<()>. This is why we need to check for this. - let delegate: $wrapper<$ty> = builder.try_init()?; - let registration = if delegate.clone().as_any().downcast_ref::<()>().is_some() { - // This is a no-op instrument, so we don't need to register a callback. - None - } else { - let delegate = delegate.clone(); - let callback = callback.clone(); - Some( - meter.register_callback(&[delegate.clone().as_any()], move |_| { - for callback in &callback { - callback(&delegate); - } - })?, - ) - }; - let result: opentelemetry::metrics::Result<_> = - Ok((delegate, registration.map(DroppingUnregister))); - result + b.build() }) - .try_collect()?; - Ok($wrapper::new(Arc::new($implementation { delegates }))) + .collect(); + $wrapper::new(Arc::new($implementation { delegates })) } }; } -struct DroppingUnregister(Box); - -macro_rules! aggregate_instrument_fn { +// Macro for histogram instruments +macro_rules! aggregate_histogram_fn { ($name:ident, $ty:ty, $wrapper:ident, $implementation:ident) => { - fn $name( - &self, - name: Cow<'static, str>, - description: Option>, - unit: Option>, - ) -> opentelemetry::metrics::Result<$wrapper<$ty>> { - let delegates = self + fn $name(&self, builder: HistogramBuilder<'_, $wrapper<$ty>>) -> $wrapper<$ty> { + let delegates: Vec<$wrapper<$ty>> = self .meters .iter() - .map(|p| { - let mut b = p.$name(name.clone()); - if let Some(description) = &description { + .map(|meter| { + let mut b = meter.$name(builder.name.clone()); + if let Some(description) = &builder.description { b = b.with_description(description.clone()); } - if let Some(unit) = &unit { + if let Some(unit) = &builder.unit { b = b.with_unit(unit.clone()); } - b.try_init() + // Copy boundaries if set + if let Some(boundaries) = &builder.boundaries { + b = b.with_boundaries(boundaries.clone()); + } + b.build() }) - .try_collect()?; - Ok($wrapper::new(Arc::new($implementation { delegates }))) + .collect(); + $wrapper::new(Arc::new($implementation { delegates })) } }; } -impl Drop for DroppingUnregister { - fn drop(&mut self) { - if let Err(e) = self.0.unregister() { - ::tracing::error!(error = %e, "failed to unregister callback") + +/// Macro for observable gauge instruments using the registry pattern. +/// +/// In OTel 0.31+, observable instruments work through callbacks registered at build time. +/// The SDK's `ObservableGauge` is just `PhantomData` - dropping it does nothing. +/// +/// This macro implements a registry pattern: +/// 1. User callbacks are stored in a shared registry indexed by gauge name +/// 2. One OTel gauge per (provider, name) is registered lazily with a consolidated callback +/// 3. The consolidated callback invokes all registered user callbacks +/// 4. When a provider is replaced, its registrations are cleared so new gauges re-register +macro_rules! aggregate_observable_gauge_fn { + ($name:ident, $ty:ty, $registry:ident) => { + fn $name( + &self, + builder: AsyncInstrumentBuilder<'_, ObservableGauge<$ty>, $ty>, + ) -> ObservableGauge<$ty> { + let gauge_name = builder.name.to_string(); + let description = builder.description.as_ref().map(|s| s.to_string()); + let unit = builder.unit.as_ref().map(|s| s.to_string()); + + // Wrap callbacks in Arc so they can be shared + let shared_callbacks: Vec> = + builder.callbacks.into_iter().map(Arc::from).collect(); + + // If no callbacks, just return noop (matches OTel behavior) + if shared_callbacks.is_empty() { + return ObservableGauge::new(); + } + + // Register callbacks in the shared registry + for callback in shared_callbacks { + self.registries + .$registry + .register_callback(&gauge_name, callback); + } + + // Register with each delegate meter that hasn't been registered yet + for (meter, meter_provider_type) in self.meters.iter().zip(MeterProviderType::iter()) { + if !self + .registries + .$registry + .try_register_for_provider(meter_provider_type, gauge_name.clone()) + { + continue; + } + + let mut b = meter.$name(gauge_name.clone()); + if let Some(desc) = &description { + b = b.with_description(desc.clone()); + } + if let Some(u) = &unit { + b = b.with_unit(u.clone()); + } + // Consolidated callback that invokes all registered callbacks + let registry = Arc::clone(&self.registries); + let name = gauge_name.clone(); + b = b.with_callback(move |observer| { + registry.$registry.invoke_callback(&name, observer); + }); + // Build registers the callback with OTel SDK + // The returned ObservableGauge is PhantomData, no need to store it + let _ = b.build(); + } + + ObservableGauge::new() } - } + }; +} + +/// Macro for observable counter/up-down-counter instruments using the registry pattern. +macro_rules! aggregate_observable_counter_fn { + ($name:ident, $ty:ty, $wrapper:ident, $registry:ident) => { + fn $name(&self, builder: AsyncInstrumentBuilder<'_, $wrapper<$ty>, $ty>) -> $wrapper<$ty> { + let instrument_name = builder.name.to_string(); + let description = builder.description.as_ref().map(|s| s.to_string()); + let unit = builder.unit.as_ref().map(|s| s.to_string()); + + // Wrap callbacks in Arc so they can be shared + let shared_callbacks: Vec> = + builder.callbacks.into_iter().map(Arc::from).collect(); + + // If no callbacks, just return noop (matches OTel behavior) + if shared_callbacks.is_empty() { + return $wrapper::new(); + } + + // Register callbacks in the shared registry + for callback in shared_callbacks { + self.registries + .$registry + .register_callback(&instrument_name, callback); + } + + // Register with each delegate meter that hasn't been registered yet + for (meter, meter_provider_type) in self.meters.iter().zip(MeterProviderType::iter()) { + if !self + .registries + .$registry + .try_register_for_provider(meter_provider_type, instrument_name.clone()) + { + continue; + } + + let mut b = meter.$name(instrument_name.clone()); + if let Some(desc) = &description { + b = b.with_description(desc.clone()); + } + if let Some(u) = &unit { + b = b.with_unit(u.clone()); + } + // Consolidated callback that invokes all registered callbacks + let registry = Arc::clone(&self.registries); + let name = instrument_name.clone(); + b = b.with_callback(move |observer| { + registry.$registry.invoke_callback(&name, observer); + }); + // Build registers the callback with OTel SDK + // The returned type is PhantomData, no need to store it + let _ = b.build(); + } + + $wrapper::new() + } + }; } impl InstrumentProvider for AggregateInstrumentProvider { aggregate_instrument_fn!(u64_counter, u64, Counter, AggregateCounter); aggregate_instrument_fn!(f64_counter, f64, Counter, AggregateCounter); - aggregate_observable_instrument_fn!( - f64_observable_counter, - f64, - ObservableCounter, - AggregateObservableCounter - ); - aggregate_observable_instrument_fn!( - u64_observable_counter, - u64, - ObservableCounter, - AggregateObservableCounter - ); + aggregate_observable_counter_fn!(f64_observable_counter, f64, ObservableCounter, f64_counter); + aggregate_observable_counter_fn!(u64_observable_counter, u64, ObservableCounter, u64_counter); - aggregate_instrument_fn!(u64_histogram, u64, Histogram, AggregateHistogram); - aggregate_instrument_fn!(f64_histogram, f64, Histogram, AggregateHistogram); + aggregate_histogram_fn!(u64_histogram, u64, Histogram, AggregateHistogram); + aggregate_histogram_fn!(f64_histogram, f64, Histogram, AggregateHistogram); aggregate_instrument_fn!( i64_up_down_counter, @@ -517,46 +749,22 @@ impl InstrumentProvider for AggregateInstrumentProvider { aggregate_instrument_fn!(i64_gauge, i64, Gauge, AggregateGauge); aggregate_instrument_fn!(f64_gauge, f64, Gauge, AggregateGauge); - aggregate_observable_instrument_fn!( + aggregate_observable_counter_fn!( i64_observable_up_down_counter, i64, ObservableUpDownCounter, - AggregateObservableUpDownCounter + i64_up_down_counter ); - aggregate_observable_instrument_fn!( + aggregate_observable_counter_fn!( f64_observable_up_down_counter, f64, ObservableUpDownCounter, - AggregateObservableUpDownCounter + f64_up_down_counter ); - aggregate_observable_instrument_fn!( - f64_observable_gauge, - f64, - ObservableGauge, - AggregateObservableGauge - ); - aggregate_observable_instrument_fn!( - i64_observable_gauge, - i64, - ObservableGauge, - AggregateObservableGauge - ); - aggregate_observable_instrument_fn!( - u64_observable_gauge, - u64, - ObservableGauge, - AggregateObservableGauge - ); - - fn register_callback( - &self, - _instruments: &[Arc], - _callbacks: Box, - ) -> opentelemetry::metrics::Result> { - // We may implement this in future, but for now we don't need it and it's a pain to implement because we need to unwrap the aggregate instruments and pass them to the meter provider that owns them. - unimplemented!("register_callback is not supported on AggregateInstrumentProvider"); - } + aggregate_observable_gauge_fn!(f64_observable_gauge, f64, f64_gauge); + aggregate_observable_gauge_fn!(i64_observable_gauge, i64, i64_gauge); + aggregate_observable_gauge_fn!(u64_observable_gauge, u64, u64_gauge); } #[cfg(test)] @@ -567,23 +775,19 @@ mod test { use std::sync::atomic::AtomicI64; use std::time::Duration; - use async_trait::async_trait; - use opentelemetry::global::GlobalMeterProvider; use opentelemetry::metrics::MeterProvider; - use opentelemetry::metrics::Result; - use opentelemetry_sdk::metrics::Aggregation; + use opentelemetry_sdk::error::OTelSdkResult; use opentelemetry_sdk::metrics::InstrumentKind; use opentelemetry_sdk::metrics::ManualReader; use opentelemetry_sdk::metrics::MeterProviderBuilder; - use opentelemetry_sdk::metrics::PeriodicReader; use opentelemetry_sdk::metrics::Pipeline; - use opentelemetry_sdk::metrics::data::Gauge; + use opentelemetry_sdk::metrics::Temporality; + use opentelemetry_sdk::metrics::data::AggregatedMetrics; + use opentelemetry_sdk::metrics::data::MetricData; use opentelemetry_sdk::metrics::data::ResourceMetrics; - use opentelemetry_sdk::metrics::data::Temporality; - use opentelemetry_sdk::metrics::exporter::PushMetricsExporter; - use opentelemetry_sdk::metrics::reader::AggregationSelector; + use opentelemetry_sdk::metrics::exporter::PushMetricExporter; + use opentelemetry_sdk::metrics::periodic_reader_with_async_runtime::PeriodicReader; use opentelemetry_sdk::metrics::reader::MetricReader; - use opentelemetry_sdk::metrics::reader::TemporalitySelector; use opentelemetry_sdk::runtime; use crate::metrics::aggregation::AggregateMeterProvider; @@ -593,38 +797,33 @@ mod test { #[derive(Clone, Debug)] struct SharedReader(Arc); - impl TemporalitySelector for SharedReader { - fn temporality(&self, kind: InstrumentKind) -> Temporality { - self.0.temporality(kind) - } - } - - impl AggregationSelector for SharedReader { - fn aggregation(&self, kind: InstrumentKind) -> Aggregation { - self.0.aggregation(kind) - } - } - impl MetricReader for SharedReader { fn register_pipeline(&self, pipeline: Weak) { self.0.register_pipeline(pipeline) } - fn collect(&self, rm: &mut ResourceMetrics) -> Result<()> { + fn collect(&self, rm: &mut ResourceMetrics) -> OTelSdkResult { self.0.collect(rm) } - fn force_flush(&self) -> Result<()> { + fn force_flush(&self) -> OTelSdkResult { self.0.force_flush() } - fn shutdown(&self) -> Result<()> { - self.0.shutdown() + fn shutdown_with_timeout(&self, timeout: Duration) -> OTelSdkResult { + self.0.shutdown_with_timeout(timeout) + } + + fn temporality(&self, kind: InstrumentKind) -> Temporality { + self.0.temporality(kind) } } #[test] - fn test_i64_gauge_drop() { + fn test_i64_gauge_callback_invocation() { + // In OTel 0.31+, observable instrument callbacks are registered with the SDK + // and persist until the meter provider is shut down. Dropping the returned + // ObservableGauge marker type does NOT unregister the callback. let reader = SharedReader(Arc::new(ManualReader::builder().build())); let delegate = MeterProviderBuilder::default() @@ -639,19 +838,16 @@ mod test { let observe_counter = Arc::new(AtomicI64::new(0)); let callback_observe_counter = observe_counter.clone(); - let gauge = meter + let _gauge = meter .i64_observable_gauge("test") .with_callback(move |i| { let count = callback_observe_counter.fetch_add(1, std::sync::atomic::Ordering::SeqCst); i.observe(count + 1, &[]) }) - .init(); + .build(); - let mut result = ResourceMetrics { - resource: Default::default(), - scope_metrics: Default::default(), - }; + let mut result = ResourceMetrics::default(); // Fetching twice will call the observer twice reader @@ -661,21 +857,14 @@ mod test { .collect(&mut result) .expect("metrics must be collected"); - assert_eq!(get_gauge_value(&mut result), 2); - - // Dropping the gauge should remove the observer registration - drop(gauge); - - // No further increment will happen - reader - .collect(&mut result) - .expect("metrics must be collected"); - + assert_eq!(get_gauge_value(&result), 2); assert_eq!(observe_counter.load(std::sync::atomic::Ordering::SeqCst), 2); } #[test] - fn test_i64_gauge_lifecycle() { + fn test_i64_gauge_multiple_callbacks() { + // In OTel 0.31+, multiple observable gauges with the same name can coexist + // and their callbacks are all invoked during collection. let reader = SharedReader(Arc::new(ManualReader::builder().build())); let delegate = MeterProviderBuilder::default() @@ -690,86 +879,42 @@ mod test { let observe_counter = Arc::new(AtomicI64::new(0)); let callback_observe_counter1 = observe_counter.clone(); - let callback_observe_counter2 = observe_counter.clone(); - let gauge1 = meter + let _gauge1 = meter .i64_observable_gauge("test") .with_callback(move |i| { let count = callback_observe_counter1.fetch_add(1, std::sync::atomic::Ordering::SeqCst); i.observe(count + 1, &[]) }) - .init(); + .build(); - let mut result = ResourceMetrics { - resource: Default::default(), - scope_metrics: Default::default(), - }; + let mut result = ResourceMetrics::default(); // Fetching metrics will call the observer reader .collect(&mut result) .expect("metrics must be collected"); - assert_eq!(get_gauge_value(&mut result), 1); - drop(gauge1); - - // The first gauge is dropped, let's create a new one - let gauge2 = meter - .i64_observable_gauge("test") - .with_callback(move |i| { - let count = - callback_observe_counter2.fetch_add(1, std::sync::atomic::Ordering::SeqCst); - i.observe(count + 1, &[]) - }) - .init(); - - // Fetching metrics will call the observer ONLY on the remaining gauge - reader - .collect(&mut result) - .expect("metrics must be collected"); + assert_eq!(get_gauge_value(&result), 1); + assert_eq!(observe_counter.load(std::sync::atomic::Ordering::SeqCst), 1); + } - assert_eq!(get_gauge_value(&mut result), 2); - drop(gauge2); - } - - fn get_gauge_value(result: &mut ResourceMetrics) -> i64 { - assert_eq!(result.scope_metrics.len(), 1); - assert_eq!(result.scope_metrics.first().unwrap().metrics.len(), 1); - assert_eq!( - result - .scope_metrics - .first() - .unwrap() - .metrics - .first() - .unwrap() - .data - .as_any() - .downcast_ref::>() - .unwrap() - .data_points - .len(), - 1 - ); - result - .scope_metrics - .first() - .unwrap() - .metrics - .first() - .unwrap() - .data - .as_any() - .downcast_ref::>() - .unwrap() - .data_points - .first() - .unwrap() - .value + fn get_gauge_value(result: &ResourceMetrics) -> i64 { + let scope_metrics: Vec<_> = result.scope_metrics().collect(); + assert_eq!(scope_metrics.len(), 1); + let metrics: Vec<_> = scope_metrics.first().unwrap().metrics().collect(); + assert_eq!(metrics.len(), 1); + let metric = metrics.first().unwrap(); + if let AggregatedMetrics::I64(MetricData::Gauge(gauge)) = metric.data() { + assert_eq!(gauge.data_points().count(), 1); + gauge.data_points().next().unwrap().value() + } else { + panic!("Expected i64 gauge") + } } #[test] - fn test_global_meter_provider() { + fn test_otel_default_meter_provider() { // The global meter provider is populated in AggregateMeterProvider::Default, but we can't test that without interacting with statics. // Setting it explicitly is the next best thing. @@ -782,20 +927,17 @@ mod test { let meter_provider = AggregateMeterProvider::default(); meter_provider.set( MeterProviderType::OtelDefault, - FilterMeterProvider::public(GlobalMeterProvider::new(delegate)), + FilterMeterProvider::public(delegate), ); let counter = meter_provider - .versioned_meter("test", None::, None::, None) + .meter("test") .u64_counter("test.counter") - .init(); + .build(); counter.add(1, &[]); - let mut resource_metrics = ResourceMetrics { - resource: Default::default(), - scope_metrics: vec![], - }; + let mut resource_metrics = ResourceMetrics::default(); reader.collect(&mut resource_metrics).unwrap(); - assert_eq!(1, resource_metrics.scope_metrics.len()); + assert_eq!(1, resource_metrics.scope_metrics().count()); } struct TestExporter { @@ -803,45 +945,39 @@ mod test { shutdown: Arc, } - impl AggregationSelector for TestExporter { - fn aggregation(&self, _kind: InstrumentKind) -> Aggregation { - Aggregation::Default - } - } - - impl TemporalitySelector for TestExporter { - fn temporality(&self, _kind: InstrumentKind) -> Temporality { - Temporality::Cumulative - } - } - - #[async_trait] - impl PushMetricsExporter for TestExporter { - async fn export(&self, _metrics: &mut ResourceMetrics) -> Result<()> { + impl PushMetricExporter for TestExporter { + fn export( + &self, + _metrics: &ResourceMetrics, + ) -> impl std::future::Future + Send { self.count(); - Ok(()) + std::future::ready(Ok(())) } - async fn force_flush(&self) -> Result<()> { + fn force_flush(&self) -> OTelSdkResult { self.count(); Ok(()) } - fn shutdown(&self) -> Result<()> { + fn shutdown_with_timeout(&self, _timeout: Duration) -> OTelSdkResult { self.count(); self.shutdown .store(true, std::sync::atomic::Ordering::SeqCst); Ok(()) } + + fn temporality(&self) -> Temporality { + Temporality::Cumulative + } } impl TestExporter { fn count(&self) { let counter = self .meter_provider - .versioned_meter("test", None::, None::, None) + .meter("test") .u64_counter("test.counter") - .init(); + .build(); counter.add(1, &[]); } } @@ -860,11 +996,11 @@ mod test { meter_provider.set( MeterProviderType::OtelDefault, - FilterMeterProvider::public(GlobalMeterProvider::new(delegate)), + FilterMeterProvider::public(delegate), ); tokio::time::sleep(Duration::from_millis(20)).await; - meter_provider.shutdown(); + meter_provider.shutdown().unwrap(); tokio::time::sleep(Duration::from_millis(20)).await; assert!(shutdown.load(std::sync::atomic::Ordering::SeqCst)); @@ -884,7 +1020,7 @@ mod test { meter_provider.set( MeterProviderType::OtelDefault, - FilterMeterProvider::public(GlobalMeterProvider::new(delegate)), + FilterMeterProvider::public(delegate), ); tokio::time::sleep(Duration::from_millis(20)).await; @@ -898,7 +1034,7 @@ mod test { // Setting the meter provider should not deadlock. meter_provider.set( MeterProviderType::OtelDefault, - FilterMeterProvider::public(GlobalMeterProvider::new(delegate)), + FilterMeterProvider::public(delegate), ); tokio::time::sleep(Duration::from_millis(20)).await; @@ -911,7 +1047,7 @@ mod test { fn reader( meter_provider: &AggregateMeterProvider, shutdown: &Arc, - ) -> PeriodicReader { + ) -> PeriodicReader { PeriodicReader::builder( TestExporter { meter_provider: meter_provider.clone(), @@ -920,7 +1056,6 @@ mod test { runtime::Tokio, ) .with_interval(Duration::from_millis(10)) - .with_timeout(Duration::from_millis(10)) .build() } } diff --git a/apollo-router/src/metrics/filter.rs b/apollo-router/src/metrics/filter.rs index c2a1031081..faadadde6c 100644 --- a/apollo-router/src/metrics/filter.rs +++ b/apollo-router/src/metrics/filter.rs @@ -1,73 +1,48 @@ -use std::any::Any; -use std::borrow::Cow; use std::sync::Arc; +use std::time::Duration; use buildstructor::buildstructor; -use opentelemetry::KeyValue; -use opentelemetry::metrics::Callback; -use opentelemetry::metrics::CallbackRegistration; +use opentelemetry::InstrumentationScope; +use opentelemetry::metrics::AsyncInstrumentBuilder; use opentelemetry::metrics::Counter; use opentelemetry::metrics::Gauge; use opentelemetry::metrics::Histogram; +use opentelemetry::metrics::HistogramBuilder; +use opentelemetry::metrics::InstrumentBuilder; use opentelemetry::metrics::InstrumentProvider; use opentelemetry::metrics::Meter; use opentelemetry::metrics::MeterProvider as OtelMeterProvider; use opentelemetry::metrics::ObservableCounter; use opentelemetry::metrics::ObservableGauge; use opentelemetry::metrics::ObservableUpDownCounter; -use opentelemetry::metrics::Observer; use opentelemetry::metrics::UpDownCounter; -use opentelemetry::metrics::noop::NoopMeterProvider; +use opentelemetry_sdk::error::OTelSdkResult; +use opentelemetry_sdk::metrics::SdkMeterProvider; use regex::Regex; +use super::NoopInstrumentProvider; + +/// Wrapper for different meter provider types #[derive(Clone)] -pub(crate) enum MeterProvider { - Regular(opentelemetry_sdk::metrics::SdkMeterProvider), - Global(opentelemetry::global::GlobalMeterProvider), +enum MeterProviderInner { + Sdk(SdkMeterProvider), + /// Noop provider - returns noop instruments without going through OTel SDK. + /// Used as a placeholder to avoid OTel SDK errors during startup/reconfiguration. + Noop, } -impl MeterProvider { - fn versioned_meter( - &self, - name: impl Into>, - version: Option>>, - schema_url: Option>>, - attributes: Option>, - ) -> Meter { - match &self { - MeterProvider::Regular(provider) => { - provider.versioned_meter(name, version, schema_url, attributes) - } - MeterProvider::Global(provider) => { - provider.versioned_meter(name, version, schema_url, attributes) - } - } - } - - #[cfg(test)] - fn force_flush(&self) -> opentelemetry::metrics::Result<()> { +impl OtelMeterProvider for MeterProviderInner { + fn meter_with_scope(&self, scope: InstrumentationScope) -> Meter { match self { - MeterProvider::Regular(provider) => provider.force_flush(), - MeterProvider::Global(_provider) => Ok(()), + MeterProviderInner::Sdk(p) => p.meter_with_scope(scope), + MeterProviderInner::Noop => Meter::new(Arc::new(NoopInstrumentProvider)), } } } -impl From for MeterProvider { - fn from(provider: opentelemetry_sdk::metrics::SdkMeterProvider) -> Self { - MeterProvider::Regular(provider) - } -} - -impl From for MeterProvider { - fn from(provider: opentelemetry::global::GlobalMeterProvider) -> Self { - MeterProvider::Global(provider) - } -} - #[derive(Clone)] pub(crate) struct FilterMeterProvider { - delegate: MeterProvider, + delegate: MeterProviderInner, deny: Option, allow: Option, } @@ -75,9 +50,9 @@ pub(crate) struct FilterMeterProvider { #[buildstructor] impl FilterMeterProvider { #[builder] - fn new>(delegate: T, deny: Option, allow: Option) -> Self { + fn new(delegate: MeterProviderInner, deny: Option, allow: Option) -> Self { FilterMeterProvider { - delegate: delegate.into(), + delegate, deny, allow, } @@ -88,16 +63,9 @@ impl FilterMeterProvider { .expect("regex should have been valid") } - pub(crate) fn apollo_realtime>(delegate: T) -> Self { - FilterMeterProvider::builder() - .delegate(delegate) - .allow(Self::get_private_realtime_regex().clone()) - .build() - } - - pub(crate) fn apollo>(delegate: T) -> Self { + pub(crate) fn apollo(delegate: SdkMeterProvider) -> Self { FilterMeterProvider::builder() - .delegate(delegate) + .delegate(MeterProviderInner::Sdk(delegate)) .allow( Regex::new( r"apollo\.(graphos\.cloud|router\.(operations?|lifecycle|config|schema|query|query_planning|telemetry|instance|graphql_error))(\..*|$)|apollo_router_uplink_fetch_count_total|apollo_router_uplink_fetch_duration_seconds", @@ -108,9 +76,16 @@ impl FilterMeterProvider { .build() } - pub(crate) fn public>(delegate: T) -> Self { + pub(crate) fn apollo_realtime(delegate: SdkMeterProvider) -> Self { + FilterMeterProvider::builder() + .delegate(MeterProviderInner::Sdk(delegate)) + .allow(Self::get_private_realtime_regex().clone()) + .build() + } + + pub(crate) fn public(delegate: SdkMeterProvider) -> Self { FilterMeterProvider::builder() - .delegate(delegate) + .delegate(MeterProviderInner::Sdk(delegate)) .deny( Regex::new(r"apollo\.router\.(config|entities|instance|operations\.(connectors|fetch|request_size|response_size|error)|schema\.connectors)(\..*|$)") .expect("regex should have been valid"), @@ -119,13 +94,35 @@ impl FilterMeterProvider { } #[cfg(test)] - pub(crate) fn all>(delegate: T) -> Self { - FilterMeterProvider::builder().delegate(delegate).build() + pub(crate) fn all(delegate: SdkMeterProvider) -> Self { + FilterMeterProvider::builder() + .delegate(MeterProviderInner::Sdk(delegate)) + .build() + } + + /// Create a noop filter provider that returns noop instruments. + /// Used as a placeholder to avoid OTel SDK errors during startup/reconfiguration. + pub(crate) fn noop() -> Self { + FilterMeterProvider { + delegate: MeterProviderInner::Noop, + deny: None, + allow: None, + } } #[cfg(test)] - pub(crate) fn force_flush(&self) -> opentelemetry::metrics::Result<()> { - self.delegate.force_flush() + pub(crate) fn force_flush(&self) -> OTelSdkResult { + match &self.delegate { + MeterProviderInner::Sdk(p) => p.force_flush(), + MeterProviderInner::Noop => Ok(()), + } + } + + pub(crate) fn shutdown_with_timeout(&self, timeout: Duration) -> OTelSdkResult { + match &self.delegate { + MeterProviderInner::Sdk(p) => p.shutdown_with_timeout(timeout), + MeterProviderInner::Noop => Ok(()), + } } } @@ -138,56 +135,95 @@ struct FilteredInstrumentProvider { macro_rules! filter_instrument_fn { ($name:ident, $ty:ty, $wrapper:ident) => { - fn $name( - &self, - name: Cow<'static, str>, - description: Option>, - unit: Option>, - ) -> opentelemetry::metrics::Result<$wrapper<$ty>> { - let mut builder = match (&self.deny, &self.allow) { + fn $name(&self, builder: InstrumentBuilder<'_, $wrapper<$ty>>) -> $wrapper<$ty> { + let meter = match (&self.deny, &self.allow) { // Deny match takes precedence over allow match - (Some(deny), _) if deny.is_match(&name) => self.noop.$name(name), - (_, Some(allow)) if !allow.is_match(&name) => self.noop.$name(name), - (_, _) => self.delegate.$name(name), + (Some(deny), _) if deny.is_match(&builder.name) => &self.noop, + (_, Some(allow)) if !allow.is_match(&builder.name) => &self.noop, + (_, _) => &self.delegate, }; - if let Some(description) = &description { - builder = builder.with_description(description.clone()) + let mut b = meter.$name(builder.name.clone()); + if let Some(description) = &builder.description { + b = b.with_description(description.clone()); } - if let Some(unit) = &unit { - builder = builder.with_unit(unit.clone()); + if let Some(unit) = &builder.unit { + b = b.with_unit(unit.clone()); } - builder.try_init() + b.build() } }; } -macro_rules! filter_observable_instrument_fn { +macro_rules! filter_histogram_fn { ($name:ident, $ty:ty, $wrapper:ident) => { - fn $name( - &self, - name: Cow<'static, str>, - description: Option>, - unit: Option>, - callback: Vec>, - ) -> opentelemetry::metrics::Result<$wrapper<$ty>> { - let mut builder = match (&self.deny, &self.allow) { + fn $name(&self, builder: HistogramBuilder<'_, $wrapper<$ty>>) -> $wrapper<$ty> { + let meter = match (&self.deny, &self.allow) { // Deny match takes precedence over allow match - (Some(deny), _) if deny.is_match(&name) => self.noop.$name(name), - (_, Some(allow)) if !allow.is_match(&name) => self.noop.$name(name), - (_, _) => self.delegate.$name(name), + (Some(deny), _) if deny.is_match(&builder.name) => &self.noop, + (_, Some(allow)) if !allow.is_match(&builder.name) => &self.noop, + (_, _) => &self.delegate, }; - if let Some(description) = &description { - builder = builder.with_description(description.clone()); + let mut b = meter.$name(builder.name.clone()); + if let Some(description) = &builder.description { + b = b.with_description(description.clone()); + } + if let Some(unit) = &builder.unit { + b = b.with_unit(unit.clone()); } - if let Some(unit) = &unit { - builder = builder.with_unit(unit.clone()); + if let Some(boundaries) = &builder.boundaries { + b = b.with_boundaries(boundaries.clone()); } + b.build() + } + }; +} - for callback in callback { - builder = builder.with_callback(callback); +macro_rules! filter_observable_instrument_fn { + ($name:ident, $ty:ty, $wrapper:ident) => { + fn $name(&self, builder: AsyncInstrumentBuilder<'_, $wrapper<$ty>, $ty>) -> $wrapper<$ty> { + let is_filtered = match (&self.deny, &self.allow) { + // Deny match takes precedence over allow match + (Some(deny), _) if deny.is_match(&builder.name) => true, + (_, Some(allow)) if !allow.is_match(&builder.name) => true, + (_, _) => false, + }; + + // For filtered observable instruments, route through the noop meter. + // This avoids registering callbacks with the SDK which would log + // errors about views not producing measures in OTel 0.31+. + if is_filtered { + return self.noop.$name(builder.name.clone()).build(); } - builder.try_init() + // Extract builder fields before consuming callbacks + let name = builder.name; + let description = builder.description; + let unit = builder.unit; + + // Wrap callbacks in Arc for sharing + let shared_callbacks: Vec< + std::sync::Arc< + dyn Fn(&dyn opentelemetry::metrics::AsyncInstrument<$ty>) + Send + Sync, + >, + > = builder + .callbacks + .into_iter() + .map(std::sync::Arc::from) + .collect(); + + let mut b = self.delegate.$name(name); + if let Some(desc) = &description { + b = b.with_description(desc.clone()); + } + if let Some(u) = &unit { + b = b.with_unit(u.clone()); + } + // Forward callbacks to the delegate + for callback in shared_callbacks { + let cb = std::sync::Arc::clone(&callback); + b = b.with_callback(move |observer| cb(observer)); + } + b.build() } }; } @@ -203,8 +239,8 @@ impl InstrumentProvider for FilteredInstrumentProvider { filter_observable_instrument_fn!(f64_observable_counter, f64, ObservableCounter); filter_observable_instrument_fn!(u64_observable_counter, u64, ObservableCounter); - filter_instrument_fn!(u64_histogram, u64, Histogram); - filter_instrument_fn!(f64_histogram, f64, Histogram); + filter_histogram_fn!(u64_histogram, u64, Histogram); + filter_histogram_fn!(f64_histogram, f64, Histogram); filter_instrument_fn!(i64_up_down_counter, i64, UpDownCounter); filter_instrument_fn!(f64_up_down_counter, f64, UpDownCounter); @@ -215,29 +251,13 @@ impl InstrumentProvider for FilteredInstrumentProvider { filter_observable_instrument_fn!(f64_observable_gauge, f64, ObservableGauge); filter_observable_instrument_fn!(i64_observable_gauge, i64, ObservableGauge); filter_observable_instrument_fn!(u64_observable_gauge, u64, ObservableGauge); - - fn register_callback( - &self, - instruments: &[Arc], - callbacks: Box, - ) -> opentelemetry::metrics::Result> { - self.delegate.register_callback(instruments, callbacks) - } } -impl opentelemetry::metrics::MeterProvider for FilterMeterProvider { - fn versioned_meter( - &self, - name: impl Into>, - version: Option>>, - schema_url: Option>>, - attributes: Option>, - ) -> Meter { +impl OtelMeterProvider for FilterMeterProvider { + fn meter_with_scope(&self, scope: InstrumentationScope) -> Meter { Meter::new(Arc::new(FilteredInstrumentProvider { - noop: NoopMeterProvider::default().meter(""), - delegate: self - .delegate - .versioned_meter(name, version, schema_url, attributes), + noop: Meter::new(Arc::new(NoopInstrumentProvider)), + delegate: self.delegate.meter_with_scope(scope), deny: self.deny.clone(), allow: self.allow.clone(), })) @@ -246,280 +266,205 @@ impl opentelemetry::metrics::MeterProvider for FilterMeterProvider { #[cfg(test)] mod test { - use opentelemetry::global::GlobalMeterProvider; use opentelemetry::metrics::MeterProvider; + use opentelemetry_sdk::metrics::InMemoryMetricExporter; use opentelemetry_sdk::metrics::MeterProviderBuilder; - use opentelemetry_sdk::metrics::PeriodicReader; + use opentelemetry_sdk::metrics::periodic_reader_with_async_runtime::PeriodicReader; use opentelemetry_sdk::runtime; - use opentelemetry_sdk::testing::metrics::InMemoryMetricsExporter; use crate::metrics::filter::FilterMeterProvider; #[tokio::test(flavor = "multi_thread")] async fn test_private_metrics() { - let exporter = InMemoryMetricsExporter::default(); + let exporter = InMemoryMetricExporter::default(); let meter_provider = FilterMeterProvider::apollo( MeterProviderBuilder::default() .with_reader(PeriodicReader::builder(exporter.clone(), runtime::Tokio).build()) .build(), ); - let filtered = meter_provider.versioned_meter("filtered", "".into(), "".into(), None); + let filtered = meter_provider.meter("filtered"); // Matches allow filtered .u64_counter("apollo.router.operations") - .init() + .build() .add(1, &[]); filtered .u64_counter("apollo.router.operations.test") - .init() + .build() .add(1, &[]); filtered .u64_counter("apollo.graphos.cloud.test") - .init() + .build() .add(1, &[]); filtered .u64_counter("apollo.router.query_planning.test") - .init() + .build() .add(1, &[]); filtered .u64_counter("apollo.router.lifecycle.api_schema") - .init() + .build() .add(1, &[]); filtered .u64_counter("apollo.router.operations.connectors") - .init() + .build() .add(1, &[]); filtered .u64_observable_gauge("apollo.router.schema.connectors") .with_callback(move |observer| observer.observe(1, &[])) - .init(); + .build(); // Mismatches allow filtered .u64_counter("apollo.router.unknown.test") - .init() + .build() .add(1, &[]); // Matches deny filtered .u64_counter("apollo.router.operations.error") - .init() + .build() .add(1, &[]); meter_provider.force_flush().unwrap(); - let metrics: Vec<_> = exporter - .get_finished_metrics() - .unwrap() - .into_iter() - .flat_map(|m| m.scope_metrics.into_iter()) - .flat_map(|m| m.metrics) + let finished = exporter.get_finished_metrics().unwrap(); + let metric_names: Vec<_> = finished + .iter() + .flat_map(|m| m.scope_metrics()) + .flat_map(|m| m.metrics()) + .map(|m| m.name()) .collect(); // Matches allow - assert!( - metrics - .iter() - .any(|m| m.name == "apollo.router.operations.test") - ); + assert!(metric_names.contains(&"apollo.router.operations.test")); - assert!(metrics.iter().any(|m| m.name == "apollo.router.operations")); + assert!(metric_names.contains(&"apollo.router.operations")); - assert!( - metrics - .iter() - .any(|m| m.name == "apollo.graphos.cloud.test") - ); + assert!(metric_names.contains(&"apollo.graphos.cloud.test")); - assert!( - metrics - .iter() - .any(|m| m.name == "apollo.router.lifecycle.api_schema") - ); + assert!(metric_names.contains(&"apollo.router.lifecycle.api_schema")); - assert!( - metrics - .iter() - .any(|m| m.name == "apollo.router.operations.connectors") - ); - assert!( - metrics - .iter() - .any(|m| m.name == "apollo.router.schema.connectors") - ); + assert!(metric_names.contains(&"apollo.router.operations.connectors")); + assert!(metric_names.contains(&"apollo.router.schema.connectors")); // Mismatches allow - assert!( - !metrics - .iter() - .any(|m| m.name == "apollo.router.unknown.test") - ); + assert!(!metric_names.contains(&"apollo.router.unknown.test")); // Matches deny - assert!( - !metrics - .iter() - .any(|m| m.name == "apollo.router.operations.error") - ); + assert!(!metric_names.contains(&"apollo.router.operations.error")); } #[tokio::test(flavor = "multi_thread")] async fn test_description_and_unit() { - let exporter = InMemoryMetricsExporter::default(); + let exporter = InMemoryMetricExporter::default(); let meter_provider = FilterMeterProvider::apollo( MeterProviderBuilder::default() .with_reader(PeriodicReader::builder(exporter.clone(), runtime::Tokio).build()) .build(), ); - let filtered = meter_provider.versioned_meter("filtered", "".into(), "".into(), None); + let filtered = meter_provider.meter("filtered"); filtered .u64_counter("apollo.router.operations") .with_description("desc") .with_unit("ms") - .init() + .build() .add(1, &[]); meter_provider.force_flush().unwrap(); - let metrics: Vec<_> = exporter - .get_finished_metrics() - .unwrap() - .into_iter() - .flat_map(|m| m.scope_metrics.into_iter()) - .flat_map(|m| m.metrics) - .collect(); - assert!(metrics.iter().any(|m| m.name == "apollo.router.operations" - && m.description == "desc" - && m.unit == "ms")); + let finished = exporter.get_finished_metrics().unwrap(); + let found = finished + .iter() + .flat_map(|m| m.scope_metrics()) + .flat_map(|m| m.metrics()) + .any(|m| { + m.name() == "apollo.router.operations" + && m.description() == "desc" + && m.unit() == "ms" + }); + assert!(found); } #[tokio::test(flavor = "multi_thread")] - async fn test_public_metrics_using_meter_provider() { - let exporter = InMemoryMetricsExporter::default(); - test_public_metrics( - exporter.clone(), + async fn test_public_metrics() { + let exporter = InMemoryMetricExporter::default(); + let meter_provider = FilterMeterProvider::public( MeterProviderBuilder::default() .with_reader(PeriodicReader::builder(exporter.clone(), runtime::Tokio).build()) .build(), - ) - .await; - } - - #[tokio::test(flavor = "multi_thread")] - async fn test_public_metrics_using_global_meter_provider() { - let exporter = InMemoryMetricsExporter::default(); - - test_public_metrics( - exporter.clone(), - GlobalMeterProvider::new( - MeterProviderBuilder::default() - .with_reader(PeriodicReader::builder(exporter.clone(), runtime::Tokio).build()) - .build(), - ), - ) - .await; - } - async fn test_public_metrics>( - exporter: InMemoryMetricsExporter, - meter_provider: T, - ) { - let meter_provider = FilterMeterProvider::public(meter_provider); - let filtered = meter_provider.versioned_meter("filtered", "".into(), "".into(), None); + ); + let filtered = meter_provider.meter("filtered"); filtered .u64_counter("apollo.router.config") - .init() + .build() .add(1, &[]); filtered .u64_counter("apollo.router.config.test") - .init() + .build() .add(1, &[]); filtered .u64_counter("apollo.router.entities") - .init() + .build() .add(1, &[]); filtered .u64_counter("apollo.router.entities.test") - .init() + .build() .add(1, &[]); filtered .u64_counter("apollo.router.operations.connectors") - .init() + .build() .add(1, &[]); filtered .u64_observable_gauge("apollo.router.schema.connectors") .with_callback(move |observer| observer.observe(1, &[])) - .init(); + .build(); meter_provider.force_flush().unwrap(); - let metrics: Vec<_> = exporter - .get_finished_metrics() - .unwrap() - .into_iter() - .flat_map(|m| m.scope_metrics.into_iter()) - .flat_map(|m| m.metrics) + let finished = exporter.get_finished_metrics().unwrap(); + let metric_names: Vec<_> = finished + .iter() + .flat_map(|m| m.scope_metrics()) + .flat_map(|m| m.metrics()) + .map(|m| m.name()) .collect(); - assert!(!metrics.iter().any(|m| m.name == "apollo.router.config")); - assert!( - !metrics - .iter() - .any(|m| m.name == "apollo.router.config.test") - ); - assert!(!metrics.iter().any(|m| m.name == "apollo.router.entities")); - assert!( - !metrics - .iter() - .any(|m| m.name == "apollo.router.entities.test") - ); - assert!( - !metrics - .iter() - .any(|m| m.name == "apollo.router.operations.connectors") - ); - assert!( - !metrics - .iter() - .any(|m| m.name == "apollo.router.schema.connectors") - ); + assert!(!metric_names.contains(&"apollo.router.config")); + assert!(!metric_names.contains(&"apollo.router.config.test")); + assert!(!metric_names.contains(&"apollo.router.entities")); + assert!(!metric_names.contains(&"apollo.router.entities.test")); + assert!(!metric_names.contains(&"apollo.router.operations.connectors")); + assert!(!metric_names.contains(&"apollo.router.schema.connectors")); } #[tokio::test(flavor = "multi_thread")] async fn test_private_realtime_metrics() { - let exporter = InMemoryMetricsExporter::default(); + let exporter = InMemoryMetricExporter::default(); let meter_provider = FilterMeterProvider::apollo_realtime( MeterProviderBuilder::default() .with_reader(PeriodicReader::builder(exporter.clone(), runtime::Tokio).build()) .build(), ); - let filtered = meter_provider.versioned_meter("filtered", "".into(), "".into(), None); + let filtered = meter_provider.meter("filtered"); filtered .u64_counter("apollo.router.operations.error") - .init() + .build() .add(1, &[]); filtered .u64_counter("apollo.router.operations.mismatch") - .init() + .build() .add(1, &[]); meter_provider.force_flush().unwrap(); - let metrics: Vec<_> = exporter - .get_finished_metrics() - .unwrap() - .into_iter() - .flat_map(|m| m.scope_metrics.into_iter()) - .flat_map(|m| m.metrics) + let finished = exporter.get_finished_metrics().unwrap(); + let metric_names: Vec<_> = finished + .iter() + .flat_map(|m| m.scope_metrics()) + .flat_map(|m| m.metrics()) + .map(|m| m.name()) .collect(); // Matches - assert!( - metrics - .iter() - .any(|m| m.name == "apollo.router.operations.error") - ); + assert!(metric_names.contains(&"apollo.router.operations.error")); // Mismatches - assert!( - !metrics - .iter() - .any(|m| m.name == "apollo.router.operations.mismatch") - ); + assert!(!metric_names.contains(&"apollo.router.operations.mismatch")); } } diff --git a/apollo-router/src/metrics/mod.rs b/apollo-router/src/metrics/mod.rs index 1f8f0aefe7..c72733d42f 100644 --- a/apollo-router/src/metrics/mod.rs +++ b/apollo-router/src/metrics/mod.rs @@ -76,6 +76,7 @@ use std::sync::OnceLock; #[cfg(test)] use futures::FutureExt; +use opentelemetry::metrics::InstrumentProvider; use crate::metrics::aggregation::AggregateMeterProvider; @@ -154,6 +155,14 @@ impl NoopGuard { } } +/// Noop InstrumentProvider - all methods use the default trait implementations +/// which return noop instruments. +// This can be replaced with NoopMeterProvider once the changes in +// https://github.com/open-telemetry/opentelemetry-rust/pull/3111 +// are released. +struct NoopInstrumentProvider; +impl InstrumentProvider for NoopInstrumentProvider {} + #[cfg(test)] pub(crate) mod test_utils { use std::cmp::Ordering; @@ -171,23 +180,20 @@ pub(crate) mod test_utils { use opentelemetry::KeyValue; use opentelemetry::StringValue; use opentelemetry::Value; - use opentelemetry_sdk::metrics::Aggregation; - use opentelemetry_sdk::metrics::AttributeSet; + use opentelemetry_sdk::error::OTelSdkResult; use opentelemetry_sdk::metrics::InstrumentKind; use opentelemetry_sdk::metrics::ManualReader; use opentelemetry_sdk::metrics::MeterProviderBuilder; use opentelemetry_sdk::metrics::Pipeline; - use opentelemetry_sdk::metrics::data::DataPoint; - use opentelemetry_sdk::metrics::data::Gauge; - use opentelemetry_sdk::metrics::data::Histogram; + use opentelemetry_sdk::metrics::Temporality; + use opentelemetry_sdk::metrics::data::AggregatedMetrics; + use opentelemetry_sdk::metrics::data::GaugeDataPoint; use opentelemetry_sdk::metrics::data::HistogramDataPoint; use opentelemetry_sdk::metrics::data::Metric; + use opentelemetry_sdk::metrics::data::MetricData; use opentelemetry_sdk::metrics::data::ResourceMetrics; - use opentelemetry_sdk::metrics::data::Sum; - use opentelemetry_sdk::metrics::data::Temporality; - use opentelemetry_sdk::metrics::reader::AggregationSelector; + use opentelemetry_sdk::metrics::data::SumDataPoint; use opentelemetry_sdk::metrics::reader::MetricReader; - use opentelemetry_sdk::metrics::reader::TemporalitySelector; use serde::Serialize; use tokio::task_local; @@ -206,32 +212,25 @@ pub(crate) mod test_utils { reader: Arc, } - impl TemporalitySelector for ClonableManualReader { - fn temporality(&self, kind: InstrumentKind) -> Temporality { - self.reader.temporality(kind) - } - } - - impl AggregationSelector for ClonableManualReader { - fn aggregation(&self, kind: InstrumentKind) -> Aggregation { - self.reader.aggregation(kind) - } - } impl MetricReader for ClonableManualReader { fn register_pipeline(&self, pipeline: Weak) { self.reader.register_pipeline(pipeline) } - fn collect(&self, rm: &mut ResourceMetrics) -> opentelemetry::metrics::Result<()> { + fn collect(&self, rm: &mut ResourceMetrics) -> OTelSdkResult { self.reader.collect(rm) } - fn force_flush(&self) -> opentelemetry::metrics::Result<()> { + fn force_flush(&self) -> OTelSdkResult { self.reader.force_flush() } - fn shutdown(&self) -> opentelemetry::metrics::Result<()> { - self.reader.shutdown() + fn shutdown_with_timeout(&self, timeout: std::time::Duration) -> OTelSdkResult { + self.reader.shutdown_with_timeout(timeout) + } + + fn temporality(&self, kind: InstrumentKind) -> Temporality { + self.reader.temporality(kind) } } @@ -265,21 +264,11 @@ pub(crate) mod test_utils { } } + #[derive(Default)] pub(crate) struct Metrics { resource_metrics: ResourceMetrics, } - impl Default for Metrics { - fn default() -> Self { - Metrics { - resource_metrics: ResourceMetrics { - resource: Default::default(), - scope_metrics: vec![], - }, - } - } - } - pub(crate) fn collect_metrics() -> Metrics { let mut metrics = Metrics::default(); let (_, reader) = meter_provider_and_readers(); @@ -292,13 +281,11 @@ pub(crate) mod test_utils { impl Metrics { pub(crate) fn find(&self, name: &str) -> Option<&opentelemetry_sdk::metrics::data::Metric> { self.resource_metrics - .scope_metrics - .iter() + .scope_metrics() .flat_map(|scope_metrics| { scope_metrics - .metrics - .iter() - .filter(|metric| metric.name == name) + .metrics() + .filter(|metric| metric.name() == name) }) .next() } @@ -312,21 +299,20 @@ pub(crate) mod test_utils { count: bool, attributes: &[KeyValue], ) -> bool { - let attributes = AttributeSet::from(attributes); if let Some(value) = value.to_u64() - && self.metric_matches(name, &ty, value, count, &attributes) + && self.metric_matches_u64(name, &ty, value, count, attributes) { return true; } if let Some(value) = value.to_i64() - && self.metric_matches(name, &ty, value, count, &attributes) + && self.metric_matches_i64(name, &ty, value, count, attributes) { return true; } if let Some(value) = value.to_f64() - && self.metric_matches(name, &ty, value, count, &attributes) + && self.metric_matches_f64(name, &ty, value, count, attributes) { return true; } @@ -334,92 +320,175 @@ pub(crate) mod test_utils { false } - pub(crate) fn metric_matches( + fn metric_matches_u64( + &self, + name: &str, + ty: &MetricType, + value: u64, + count: bool, + attributes: &[KeyValue], + ) -> bool { + // Try ALL metrics with this name, not just the first one + for scope_metrics in self.resource_metrics.scope_metrics() { + for metric in scope_metrics.metrics().filter(|m| m.name() == name) { + if let AggregatedMetrics::U64(metric_data) = metric.data() + && Self::check_metric_data(metric_data, ty, value, count, attributes) + { + return true; + } + } + } + false + } + + fn metric_matches_i64( + &self, + name: &str, + ty: &MetricType, + value: i64, + count: bool, + attributes: &[KeyValue], + ) -> bool { + // Try ALL metrics with this name, not just the first one + for scope_metrics in self.resource_metrics.scope_metrics() { + for metric in scope_metrics.metrics().filter(|m| m.name() == name) { + if let AggregatedMetrics::I64(metric_data) = metric.data() + && Self::check_metric_data(metric_data, ty, value, count, attributes) + { + return true; + } + } + } + false + } + + fn metric_matches_f64( &self, name: &str, ty: &MetricType, + value: f64, + count: bool, + attributes: &[KeyValue], + ) -> bool { + // Try ALL metrics with this name across all scopes + // (there can be multiple metrics with the same name but different types) + for scope_metrics in self.resource_metrics.scope_metrics() { + for metric in scope_metrics.metrics().filter(|m| m.name() == name) { + if let AggregatedMetrics::F64(metric_data) = metric.data() + && Self::check_metric_data(metric_data, ty, value, count, attributes) + { + return true; + } + } + } + false + } + + fn check_metric_data( + metric_data: &MetricData, + ty: &MetricType, value: T, count: bool, - attributes: &AttributeSet, + attributes: &[KeyValue], ) -> bool { - if let Some(metric) = self.find(name) { - // Try to downcast the metric to each type of aggregation and assert that the value is correct. - if let Some(gauge) = metric.data.as_any().downcast_ref::>() { - // Find the datapoint with the correct attributes. + match metric_data { + MetricData::Gauge(gauge) => { if matches!(ty, MetricType::Gauge) { - return gauge.data_points.iter().any(|datapoint| { - datapoint.value == value - && Self::equal_attributes(attributes, &datapoint.attributes) + return gauge.data_points().any(|datapoint| { + datapoint.value() == value + && Self::equal_attributes(attributes, datapoint.attributes()) }); } - } else if let Some(sum) = metric.data.as_any().downcast_ref::>() { - // Note that we can't actually tell if the sum is monotonic or not, so we just check if it's a sum. + } + MetricData::Sum(sum) => { if matches!(ty, MetricType::Counter | MetricType::UpDownCounter) { - return sum.data_points.iter().any(|datapoint| { - datapoint.value == value - && Self::equal_attributes(attributes, &datapoint.attributes) + return sum.data_points().any(|datapoint| { + datapoint.value() == value + && Self::equal_attributes(attributes, datapoint.attributes()) }); } - } else if let Some(histogram) = metric.data.as_any().downcast_ref::>() - && matches!(ty, MetricType::Histogram) - { - if count { - return histogram.data_points.iter().any(|datapoint| { - datapoint.count == value.to_u64().unwrap() - && Self::equal_attributes(attributes, &datapoint.attributes) - }); - } else { - return histogram.data_points.iter().any(|datapoint| { - datapoint.sum == value - && Self::equal_attributes(attributes, &datapoint.attributes) - }); + } + MetricData::Histogram(histogram) => { + if matches!(ty, MetricType::Histogram) { + if count { + return histogram.data_points().any(|datapoint| { + datapoint.count() == value.to_u64().unwrap() + && Self::equal_attributes(attributes, datapoint.attributes()) + }); + } else { + return histogram.data_points().any(|datapoint| { + datapoint.sum() == value + && Self::equal_attributes(attributes, datapoint.attributes()) + }); + } } } + MetricData::ExponentialHistogram(_) => {} } false } - pub(crate) fn metric_exists( + #[must_use] + pub(crate) fn metric_exists( &self, name: &str, ty: MetricType, attributes: &[KeyValue], ) -> bool { - let attributes = AttributeSet::from(attributes); if let Some(metric) = self.find(name) { - // Try to downcast the metric to each type of aggregation and assert that the value is correct. - if let Some(gauge) = metric.data.as_any().downcast_ref::>() { - // Find the datapoint with the correct attributes. + match metric.data() { + AggregatedMetrics::U64(metric_data) => { + return Self::check_metric_exists(metric_data, &ty, attributes); + } + AggregatedMetrics::I64(metric_data) => { + return Self::check_metric_exists(metric_data, &ty, attributes); + } + AggregatedMetrics::F64(metric_data) => { + return Self::check_metric_exists(metric_data, &ty, attributes); + } + } + } + false + } + + fn check_metric_exists( + metric_data: &MetricData, + ty: &MetricType, + attributes: &[KeyValue], + ) -> bool { + match metric_data { + MetricData::Gauge(gauge) => { if matches!(ty, MetricType::Gauge) { - return gauge.data_points.iter().any(|datapoint| { - Self::equal_attributes(&attributes, &datapoint.attributes) + return gauge.data_points().any(|datapoint| { + Self::equal_attributes(attributes, datapoint.attributes()) }); } - } else if let Some(sum) = metric.data.as_any().downcast_ref::>() { - // Note that we can't actually tell if the sum is monotonic or not, so we just check if it's a sum. + } + MetricData::Sum(sum) => { if matches!(ty, MetricType::Counter | MetricType::UpDownCounter) { - return sum.data_points.iter().any(|datapoint| { - Self::equal_attributes(&attributes, &datapoint.attributes) + return sum.data_points().any(|datapoint| { + Self::equal_attributes(attributes, datapoint.attributes()) + }); + } + } + MetricData::Histogram(histogram) => { + if matches!(ty, MetricType::Histogram) { + return histogram.data_points().any(|datapoint| { + Self::equal_attributes(attributes, datapoint.attributes()) }); } - } else if let Some(histogram) = metric.data.as_any().downcast_ref::>() - && matches!(ty, MetricType::Histogram) - { - return histogram.data_points.iter().any(|datapoint| { - Self::equal_attributes(&attributes, &datapoint.attributes) - }); } + MetricData::ExponentialHistogram(_) => {} } false } #[allow(dead_code)] - pub(crate) fn all(self) -> Vec { + pub(crate) fn all(&self) -> Vec { self.resource_metrics - .scope_metrics - .into_iter() + .scope_metrics() .flat_map(|scope_metrics| { - scope_metrics.metrics.into_iter().map(|metric| { + scope_metrics.metrics().map(|metric| { let serde_metric: SerdeMetric = metric.into(); serde_metric }) @@ -429,7 +498,7 @@ pub(crate) mod test_utils { } #[allow(dead_code)] - pub(crate) fn non_zero(self) -> Vec { + pub(crate) fn non_zero(&self) -> Vec { self.all() .into_iter() .filter(|m| { @@ -447,18 +516,30 @@ pub(crate) mod test_utils { .collect() } - fn equal_attributes(expected: &AttributeSet, actual: &[KeyValue]) -> bool { + fn equal_attributes<'a>( + expected: &[KeyValue], + actual: impl Iterator, + ) -> bool { + let mut actual_vec: Vec<_> = actual.collect(); // If lengths are different, we can short circuit. This also accounts for a bug where // an empty attributes list would always be considered "equal" due to zip capping at // the shortest iter's length - if expected.iter().count() != actual.len() { + if expected.len() != actual_vec.len() { return false; } - // This works because the attributes are always sorted - expected.iter().zip(actual.iter()).all(|((k, v), kv)| { - kv.key == *k - && (kv.value == *v || kv.value == Value::String(StringValue::from(""))) - }) + // Sort both sides by key for comparison (attributes may not be in same order) + let mut expected_sorted: Vec<_> = expected.iter().collect(); + expected_sorted.sort_by(|a, b| a.key.cmp(&b.key)); + actual_vec.sort_by(|a, b| a.key.cmp(&b.key)); + + expected_sorted + .iter() + .zip(actual_vec.iter()) + .all(|(exp, act)| { + exp.key == act.key + && (exp.value == act.value + || exp.value == Value::String(StringValue::from(""))) + }) } } @@ -516,35 +597,77 @@ pub(crate) mod test_utils { } impl SerdeMetricData { - fn extract_datapoints + Clone + 'static>( - metric_data: &mut SerdeMetricData, - value: &dyn opentelemetry_sdk::metrics::data::Aggregation, - ) { - if let Some(gauge) = value.as_any().downcast_ref::>() { - gauge.data_points.iter().for_each(|datapoint| { - metric_data.datapoints.push(datapoint.into()); - }); + fn extract_datapoints_f64(metric_data: &mut SerdeMetricData, value: &MetricData) { + match value { + MetricData::Gauge(gauge) => { + gauge.data_points().for_each(|datapoint| { + metric_data.datapoints.push(datapoint.into()); + }); + } + MetricData::Sum(sum) => { + sum.data_points().for_each(|datapoint| { + metric_data.datapoints.push(datapoint.into()); + }); + } + MetricData::Histogram(histogram) => { + histogram.data_points().for_each(|datapoint| { + metric_data.datapoints.push(datapoint.into()); + }); + } + MetricData::ExponentialHistogram(_) => {} } - if let Some(sum) = value.as_any().downcast_ref::>() { - sum.data_points.iter().for_each(|datapoint| { - metric_data.datapoints.push(datapoint.into()); - }); + } + + fn extract_datapoints_u64(metric_data: &mut SerdeMetricData, value: &MetricData) { + match value { + MetricData::Gauge(gauge) => { + gauge.data_points().for_each(|datapoint| { + metric_data.datapoints.push(datapoint.into()); + }); + } + MetricData::Sum(sum) => { + sum.data_points().for_each(|datapoint| { + metric_data.datapoints.push(datapoint.into()); + }); + } + MetricData::Histogram(histogram) => { + histogram.data_points().for_each(|datapoint| { + metric_data.datapoints.push(datapoint.into()); + }); + } + MetricData::ExponentialHistogram(_) => {} } - if let Some(histogram) = value.as_any().downcast_ref::>() { - histogram.data_points.iter().for_each(|datapoint| { - metric_data.datapoints.push(datapoint.into()); - }); + } + + fn extract_datapoints_i64(metric_data: &mut SerdeMetricData, value: &MetricData) { + match value { + MetricData::Gauge(gauge) => { + gauge.data_points().for_each(|datapoint| { + metric_data.datapoints.push(datapoint.into()); + }); + } + MetricData::Sum(sum) => { + sum.data_points().for_each(|datapoint| { + metric_data.datapoints.push(datapoint.into()); + }); + } + MetricData::Histogram(histogram) => { + histogram.data_points().for_each(|datapoint| { + metric_data.datapoints.push(datapoint.into()); + }); + } + MetricData::ExponentialHistogram(_) => {} } } } - impl From for SerdeMetric { - fn from(value: Metric) -> Self { + impl From<&Metric> for SerdeMetric { + fn from(value: &Metric) -> Self { let mut serde_metric = SerdeMetric { - name: value.name.into_owned(), - description: value.description.into_owned(), - unit: value.unit.to_string(), - data: value.data.into(), + name: value.name().to_string(), + description: value.description().to_string(), + unit: value.unit().to_string(), + data: value.data().into(), }; // Sort the datapoints so that we can compare them serde_metric.data.datapoints.sort(); @@ -567,18 +690,34 @@ pub(crate) mod test_utils { } } - impl From<&DataPoint> for SerdeMetricDataPoint + impl From<&GaugeDataPoint> for SerdeMetricDataPoint + where + T: Into + Copy, + { + fn from(value: &GaugeDataPoint) -> Self { + SerdeMetricDataPoint { + value: Some(value.value().into()), + sum: None, + count: None, + attributes: value + .attributes() + .map(|kv| (kv.key.to_string(), Self::convert(&kv.value))) + .collect(), + } + } + } + + impl From<&SumDataPoint> for SerdeMetricDataPoint where - T: Into + Clone, + T: Into + Copy, { - fn from(value: &DataPoint) -> Self { + fn from(value: &SumDataPoint) -> Self { SerdeMetricDataPoint { - value: Some(value.value.clone().into()), + value: Some(value.value().into()), sum: None, count: None, attributes: value - .attributes - .iter() + .attributes() .map(|kv| (kv.key.to_string(), Self::convert(&kv.value))) .collect(), } @@ -597,35 +736,44 @@ pub(crate) mod test_utils { Array::I64(v) => v.into(), Array::F64(v) => v.into(), Array::String(v) => v.iter().map(|v| v.to_string()).collect::>().into(), + _ => unreachable!("unexpected opentelemetry::Array variant"), }, + _ => unreachable!("unexpected opentelemetry::Value variant"), } } } impl From<&HistogramDataPoint> for SerdeMetricDataPoint where - T: Into + Clone, + T: Into + Copy, { fn from(value: &HistogramDataPoint) -> Self { SerdeMetricDataPoint { - sum: Some(value.sum.clone().into()), + sum: Some(value.sum().into()), value: None, - count: Some(value.count), + count: Some(value.count()), attributes: value - .attributes - .iter() + .attributes() .map(|kv| (kv.key.to_string(), Self::convert(&kv.value))) .collect(), } } } - impl From> for SerdeMetricData { - fn from(value: Box) -> Self { + impl From<&AggregatedMetrics> for SerdeMetricData { + fn from(value: &AggregatedMetrics) -> Self { let mut metric_data = SerdeMetricData::default(); - Self::extract_datapoints::(&mut metric_data, value.as_ref()); - Self::extract_datapoints::(&mut metric_data, value.as_ref()); - Self::extract_datapoints::(&mut metric_data, value.as_ref()); + match value { + AggregatedMetrics::F64(data) => { + Self::extract_datapoints_f64(&mut metric_data, data) + } + AggregatedMetrics::U64(data) => { + Self::extract_datapoints_u64(&mut metric_data, data) + } + AggregatedMetrics::I64(data) => { + Self::extract_datapoints_i64(&mut metric_data, data) + } + } metric_data } } @@ -1081,7 +1229,7 @@ macro_rules! metric { builder = builder.with_unit($unit); } - builder.init() + builder.build() }; if cache_callsite { @@ -1257,36 +1405,36 @@ macro_rules! assert_counter_not_exists { ($($name:ident).+, $value: ty, $($attr_key:literal = $attr_value:expr),+) => { let attributes = &[$(opentelemetry::KeyValue::new($attr_key, $attr_value)),+]; - let result = crate::metrics::collect_metrics().metric_exists::<$value>(stringify!($($name).+), crate::metrics::test_utils::MetricType::Counter, attributes); + let result = crate::metrics::collect_metrics().metric_exists(stringify!($($name).+), crate::metrics::test_utils::MetricType::Counter, attributes); assert_no_metric!(result, $name, None, None, None, attributes); }; ($($name:ident).+, $value: ty, $($($attr_key:ident).+ = $attr_value:expr),+) => { let attributes = &[$(opentelemetry::KeyValue::new(stringify!($($attr_key).+), $attr_value)),+]; - let result = crate::metrics::collect_metrics().metric_exists::<$value>(stringify!($($name).+), crate::metrics::test_utils::MetricType::Counter, attributes); + let result = crate::metrics::collect_metrics().metric_exists(stringify!($($name).+), crate::metrics::test_utils::MetricType::Counter, attributes); assert_no_metric!(result, $name, None, None, None, attributes); }; ($name:literal, $value: ty, $($attr_key:literal = $attr_value:expr),+) => { let attributes = &[$(opentelemetry::KeyValue::new($attr_key, $attr_value)),+]; - let result = crate::metrics::collect_metrics().metric_exists::<$value>($name, crate::metrics::test_utils::MetricType::Counter, attributes); + let result = crate::metrics::collect_metrics().metric_exists($name, crate::metrics::test_utils::MetricType::Counter, attributes); assert_no_metric!(result, $name, None, None, None, attributes); }; ($name:literal, $value: ty, $($($attr_key:ident).+ = $attr_value:expr),+) => { let attributes = &[$(opentelemetry::KeyValue::new(stringify!($($attr_key).+), $attr_value)),+]; - let result = crate::metrics::collect_metrics().metric_exists::<$value>($name, crate::metrics::test_utils::MetricType::Counter, attributes); + let result = crate::metrics::collect_metrics().metric_exists($name, crate::metrics::test_utils::MetricType::Counter, attributes); assert_no_metric!(result, $name, None, None, None, attributes); }; ($name:literal, $value: ty, $attributes: expr) => { - let result = crate::metrics::collect_metrics().metric_exists::<$value>($name, crate::metrics::test_utils::MetricType::Counter, $attributes); + let result = crate::metrics::collect_metrics().metric_exists($name, crate::metrics::test_utils::MetricType::Counter, $attributes); assert_no_metric!(result, $name, None, None, None, &$attributes); }; ($name:literal, $value: ty) => { - let result = crate::metrics::collect_metrics().metric_exists::<$value>($name, crate::metrics::test_utils::MetricType::Counter, &[]); + let result = crate::metrics::collect_metrics().metric_exists($name, crate::metrics::test_utils::MetricType::Counter, &[]); assert_no_metric!(result, $name, None, None, None, &[]); }; } @@ -1444,30 +1592,30 @@ macro_rules! assert_histogram_exists { ($($name:ident).+, $value: ty, $($attr_key:literal = $attr_value:expr),+) => { let attributes = &[$(opentelemetry::KeyValue::new($attr_key, $attr_value)),+]; - let result = crate::metrics::collect_metrics().metric_exists::<$value>(stringify!($($name).+), crate::metrics::test_utils::MetricType::Histogram, attributes); + let result = crate::metrics::collect_metrics().metric_exists(stringify!($($name).+), crate::metrics::test_utils::MetricType::Histogram, attributes); assert_metric!(result, $name, None, None, None, attributes); }; ($($name:ident).+, $value: ty, $($($attr_key:ident).+ = $attr_value:expr),+) => { let attributes = &[$(opentelemetry::KeyValue::new(stringify!($($attr_key).+), $attr_value)),+]; - let result = crate::metrics::collect_metrics().metric_exists::<$value>(stringify!($($name).+), crate::metrics::test_utils::MetricType::Histogram, attributes); + let result = crate::metrics::collect_metrics().metric_exists(stringify!($($name).+), crate::metrics::test_utils::MetricType::Histogram, attributes); assert_metric!(result, $name, None, None, None, attributes); }; ($name:literal, $value: ty, $($attr_key:literal = $attr_value:expr),+) => { let attributes = &[$(opentelemetry::KeyValue::new($attr_key, $attr_value)),+]; - let result = crate::metrics::collect_metrics().metric_exists::<$value>($name, crate::metrics::test_utils::MetricType::Histogram, attributes); + let result = crate::metrics::collect_metrics().metric_exists($name, crate::metrics::test_utils::MetricType::Histogram, attributes); assert_metric!(result, $name, None, None, None, attributes); }; ($name:literal, $value: ty, $($($attr_key:ident).+ = $attr_value:expr),+) => { let attributes = &[$(opentelemetry::KeyValue::new(stringify!($($attr_key).+), $attr_value)),+]; - let result = crate::metrics::collect_metrics().metric_exists::<$value>($name, crate::metrics::test_utils::MetricType::Histogram, attributes); + let result = crate::metrics::collect_metrics().metric_exists($name, crate::metrics::test_utils::MetricType::Histogram, attributes); assert_metric!(result, $name, None, None, None, attributes); }; ($name:literal, $value: ty) => { - let result = crate::metrics::collect_metrics().metric_exists::<$value>($name, crate::metrics::test_utils::MetricType::Histogram, &[]); + let result = crate::metrics::collect_metrics().metric_exists($name, crate::metrics::test_utils::MetricType::Histogram, &[]); assert_metric!(result, $name, None, None, None, &[]); }; } @@ -1481,30 +1629,30 @@ macro_rules! assert_histogram_not_exists { ($($name:ident).+, $value: ty, $($attr_key:literal = $attr_value:expr),+) => { let attributes = &[$(opentelemetry::KeyValue::new($attr_key, $attr_value)),+]; - let result = crate::metrics::collect_metrics().metric_exists::<$value>(stringify!($($name).+), crate::metrics::test_utils::MetricType::Histogram, attributes); + let result = crate::metrics::collect_metrics().metric_exists(stringify!($($name).+), crate::metrics::test_utils::MetricType::Histogram, attributes); assert_no_metric!(result, $name, None, None, None, attributes); }; ($($name:ident).+, $value: ty, $($($attr_key:ident).+ = $attr_value:expr),+) => { let attributes = &[$(opentelemetry::KeyValue::new(stringify!($($attr_key).+), $attr_value)),+]; - let result = crate::metrics::collect_metrics().metric_exists::<$value>(stringify!($($name).+), crate::metrics::test_utils::MetricType::Histogram, attributes); + let result = crate::metrics::collect_metrics().metric_exists(stringify!($($name).+), crate::metrics::test_utils::MetricType::Histogram, attributes); assert_no_metric!(result, $name, None, None, None, attributes); }; ($name:literal, $value: ty, $($attr_key:literal = $attr_value:expr),+) => { let attributes = &[$(opentelemetry::KeyValue::new($attr_key, $attr_value)),+]; - let result = crate::metrics::collect_metrics().metric_exists::<$value>($name, crate::metrics::test_utils::MetricType::Histogram, attributes); + let result = crate::metrics::collect_metrics().metric_exists($name, crate::metrics::test_utils::MetricType::Histogram, attributes); assert_no_metric!(result, $name, None, None, None, attributes); }; ($name:literal, $value: ty, $($($attr_key:ident).+ = $attr_value:expr),+) => { let attributes = &[$(opentelemetry::KeyValue::new(stringify!($($attr_key).+), $attr_value)),+]; - let result = crate::metrics::collect_metrics().metric_exists::<$value>($name, crate::metrics::test_utils::MetricType::Histogram, attributes); + let result = crate::metrics::collect_metrics().metric_exists($name, crate::metrics::test_utils::MetricType::Histogram, attributes); assert_no_metric!(result, $name, None, None, None, attributes); }; ($name:literal, $value: ty) => { - let result = crate::metrics::collect_metrics().metric_exists::<$value>($name, crate::metrics::test_utils::MetricType::Histogram, &[]); + let result = crate::metrics::collect_metrics().metric_exists($name, crate::metrics::test_utils::MetricType::Histogram, &[]); assert_no_metric!(result, $name, None, None, None, &[]); }; } @@ -1595,10 +1743,7 @@ pub(crate) trait FutureMetricsExt { // We want to eagerly create the meter provider, the reason is that this will be shared among subtasks that use `with_current_meter_provider`. let _ = meter_provider_internal(); let result = self.await; - let _ = tokio::task::spawn_blocking(|| { - meter_provider_internal().shutdown(); - }) - .await; + let _ = tokio::task::spawn_blocking(|| meter_provider_internal().shutdown()).await; result } .boxed_local(), @@ -1665,7 +1810,7 @@ mod test { fn assert_unit(name: &str, unit: &str) { let collected_metrics = crate::metrics::collect_metrics(); let metric = collected_metrics.find(name).unwrap(); - assert_eq!(metric.unit, unit); + assert_eq!(metric.unit(), unit); } #[test] @@ -1675,13 +1820,13 @@ mod test { .meter("test") .u64_observable_gauge("test") .with_callback(|m| m.observe(5, &[])) - .init(); + .build(); assert_gauge!("test", 5); } #[test] fn test_gauge_record() { - let gauge = meter_provider().meter("test").u64_gauge("test").init(); + let gauge = meter_provider().meter("test").u64_gauge("test").build(); gauge.record(5, &[]); assert_gauge!("test", 5); } @@ -1942,7 +2087,7 @@ mod test { .meter("test") .u64_observable_gauge("test") .with_callback(|m| m.observe(5, &[])) - .init(); + .build(); assert_histogram_sum!("test", 1, "attr" = "val"); } .with_metrics() diff --git a/apollo-router/src/plugins/cache/entity.rs b/apollo-router/src/plugins/cache/entity.rs index 925ebbd80b..a9e2c02033 100644 --- a/apollo-router/src/plugins/cache/entity.rs +++ b/apollo-router/src/plugins/cache/entity.rs @@ -52,8 +52,8 @@ use crate::json_ext::Object; use crate::json_ext::Path; use crate::json_ext::PathElement; use crate::layers::ServiceBuilderExt; -use crate::plugin::Plugin; use crate::plugin::PluginInit; +use crate::plugin::PluginPrivate; use crate::plugins::authorization::CacheKeyMetadata; use crate::plugins::response_cache::plugin::find_matching_key_field_set; use crate::query_planner::OperationKind; @@ -71,7 +71,7 @@ pub(crate) const CONTEXT_CACHE_KEY: &str = "apollo_entity_cache::key"; /// Context key to enable support of surrogate cache key pub(crate) const CONTEXT_CACHE_KEYS: &str = "apollo::entity_cache::cached_keys_status"; -register_plugin!("apollo", "preview_entity_cache", EntityCache); +register_private_plugin!("apollo", "preview_entity_cache", EntityCache); #[derive(Clone)] pub(crate) struct EntityCache { @@ -98,6 +98,16 @@ impl Storage { pub(crate) fn get(&self, subgraph: &str) -> Option<&RedisCacheStorage> { self.subgraphs.get(subgraph).or(self.all.as_ref()) } + + /// Activate all Redis storages so they can start emitting metrics. + pub(crate) fn activate(&self) { + if let Some(all) = &self.all { + all.activate(); + } + for storage in self.subgraphs.values() { + storage.activate(); + } + } } /// Configuration for entity caching @@ -190,7 +200,7 @@ pub(crate) struct CacheHitMiss { } #[async_trait::async_trait] -impl Plugin for EntityCache { +impl PluginPrivate for EntityCache { type Config = Config; async fn new(init: PluginInit) -> Result @@ -462,6 +472,10 @@ impl Plugin for EntityCache { map } + + fn activate(&self) { + self.storage.activate(); + } } #[cfg(test)] diff --git a/apollo-router/src/plugins/cache/tests.rs b/apollo-router/src/plugins/cache/tests.rs index 3744fd3929..6d0ede9444 100644 --- a/apollo-router/src/plugins/cache/tests.rs +++ b/apollo-router/src/plugins/cache/tests.rs @@ -229,7 +229,7 @@ async fn insert() { })) .unwrap() .schema(SCHEMA) - .extra_plugin(entity_cache) + .extra_private_plugin(entity_cache) .build_supergraph() .await .unwrap(); @@ -273,7 +273,7 @@ async fn insert() { .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true } })) .unwrap() .schema(SCHEMA) - .extra_plugin(entity_cache) + .extra_private_plugin(entity_cache) .build_supergraph() .await .unwrap(); @@ -369,7 +369,7 @@ async fn insert_with_requires() { .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true } })) .unwrap() .schema(SCHEMA_REQUIRES) - .extra_plugin(entity_cache) + .extra_private_plugin(entity_cache) .extra_plugin(subgraphs) .build_supergraph() .await @@ -414,7 +414,7 @@ async fn insert_with_requires() { .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true } })) .unwrap() .schema(SCHEMA_REQUIRES) - .extra_plugin(entity_cache) + .extra_private_plugin(entity_cache) .build_supergraph() .await .unwrap(); @@ -501,7 +501,7 @@ async fn insert_with_nested_field_set() { .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true }, "experimental_mock_subgraphs": subgraphs.clone() })) .unwrap() .schema(SCHEMA_NESTED_KEYS) - .extra_plugin(entity_cache) + .extra_private_plugin(entity_cache) .build_supergraph() .await .unwrap(); @@ -551,7 +551,7 @@ async fn insert_with_nested_field_set() { .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true }, "experimental_mock_subgraphs": subgraphs.clone() })) .unwrap() .schema(SCHEMA_NESTED_KEYS) - .extra_plugin(entity_cache) + .extra_private_plugin(entity_cache) .build_supergraph() .await .unwrap(); @@ -621,7 +621,7 @@ async fn no_cache_control() { .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true } })) .unwrap() .schema(SCHEMA) - .extra_plugin(entity_cache) + .extra_private_plugin(entity_cache) .extra_plugin(subgraphs) .build_supergraph() .await @@ -649,7 +649,7 @@ async fn no_cache_control() { .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true } })) .unwrap() .schema(SCHEMA) - .extra_plugin(entity_cache) + .extra_private_plugin(entity_cache) .build_supergraph() .await .unwrap(); @@ -738,7 +738,7 @@ async fn private() { .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true } })) .unwrap() .schema(SCHEMA) - .extra_plugin(entity_cache.clone()) + .extra_private_plugin(entity_cache.clone()) .extra_plugin(subgraphs) .build_supergraph() .await @@ -767,7 +767,7 @@ async fn private() { .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true } })) .unwrap() .schema(SCHEMA) - .extra_plugin(entity_cache) + .extra_private_plugin(entity_cache) .build_supergraph() .await .unwrap(); @@ -896,7 +896,7 @@ async fn no_data() { .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true } })) .unwrap() .schema(SCHEMA) - .extra_plugin(entity_cache) + .extra_private_plugin(entity_cache) .extra_plugin(subgraphs) .build_supergraph() .await @@ -961,7 +961,7 @@ async fn no_data() { .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true } })) .unwrap() .schema(SCHEMA) - .extra_plugin(entity_cache) + .extra_private_plugin(entity_cache) .subgraph_hook(|name, service| { if name == "orga" { let mut subgraph = MockSubgraphService::new(); @@ -1085,7 +1085,7 @@ async fn missing_entities() { .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true } })) .unwrap() .schema(SCHEMA) - .extra_plugin(entity_cache) + .extra_private_plugin(entity_cache) .extra_plugin(subgraphs) .build_supergraph() .await @@ -1147,7 +1147,7 @@ async fn missing_entities() { .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true } })) .unwrap() .schema(SCHEMA) - .extra_plugin(entity_cache) + .extra_private_plugin(entity_cache) .extra_plugin(subgraphs) .build_supergraph() .await @@ -1210,7 +1210,7 @@ async fn invalidate() { .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true } })) .unwrap() .schema(SCHEMA) - .extra_plugin(entity_cache.clone()) + .extra_private_plugin(entity_cache.clone()) .extra_plugin(subgraphs) .build_supergraph() .await @@ -1233,7 +1233,7 @@ async fn invalidate() { .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true } })) .unwrap() .schema(SCHEMA) - .extra_plugin(entity_cache.clone()) + .extra_private_plugin(entity_cache.clone()) .build_supergraph() .await .unwrap(); @@ -1264,7 +1264,7 @@ async fn invalidate() { .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true } })) .unwrap() .schema(SCHEMA) - .extra_plugin(entity_cache) + .extra_private_plugin(entity_cache) .build_supergraph() .await .unwrap(); @@ -1344,7 +1344,7 @@ async fn no_cache_from_request() { .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true } })) .unwrap() .schema(SCHEMA) - .extra_plugin(entity_cache) + .extra_private_plugin(entity_cache) .extra_plugin(subgraphs) .build_supergraph() .await @@ -1377,7 +1377,7 @@ async fn no_cache_from_request() { })) .unwrap() .schema(SCHEMA) - .extra_plugin(entity_cache) + .extra_private_plugin(entity_cache) .build_supergraph() .await .unwrap(); @@ -1403,7 +1403,7 @@ async fn no_cache_from_request() { .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true } })) .unwrap() .schema(SCHEMA) - .extra_plugin(entity_cache) + .extra_private_plugin(entity_cache) .build_supergraph() .await .unwrap(); @@ -1488,7 +1488,7 @@ async fn no_store_from_request() { })) .unwrap() .schema(SCHEMA) - .extra_plugin(entity_cache) + .extra_private_plugin(entity_cache) .extra_plugin(subgraphs) .build_supergraph() .await @@ -1522,7 +1522,7 @@ async fn no_store_from_request() { .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true } })) .unwrap() .schema(SCHEMA) - .extra_plugin(entity_cache) + .extra_private_plugin(entity_cache) .build_supergraph() .await .unwrap(); diff --git a/apollo-router/src/plugins/connectors/tracing.rs b/apollo-router/src/plugins/connectors/tracing.rs index 99f5039098..0213a809a7 100644 --- a/apollo-router/src/plugins/connectors/tracing.rs +++ b/apollo-router/src/plugins/connectors/tracing.rs @@ -27,7 +27,7 @@ pub(crate) fn connect_spec_version_instrument( ) }) }) - .init() + .build() }) } diff --git a/apollo-router/src/plugins/fleet_detector.rs b/apollo-router/src/plugins/fleet_detector.rs index ecacff6f26..1611a5d90b 100644 --- a/apollo-router/src/plugins/fleet_detector.rs +++ b/apollo-router/src/plugins/fleet_detector.rs @@ -114,7 +114,7 @@ impl GaugeStore { .with_callback(move |i| { i.observe(1, &attributes); }) - .init(), + .build(), ); } // apollo.router.instance.cpu_freq @@ -136,7 +136,7 @@ impl GaugeStore { cpus.iter().map(|cpu| cpu.frequency()).sum::() / cpus.len() as u64; gauge.observe(cpu_freq, &[]) }) - .init(), + .build(), ); } // apollo.router.instance.cpu_count @@ -161,7 +161,7 @@ impl GaugeStore { ], ) }) - .init(), + .build(), ); } // apollo.router.instance.total_memory @@ -183,7 +183,7 @@ impl GaugeStore { ) }) .with_unit("bytes") - .init(), + .build(), ); } { @@ -204,7 +204,7 @@ impl GaugeStore { } gauge.observe(1, attributes.as_slice()) }) - .init(), + .build(), ) } GaugeStore::Active(gauges) diff --git a/apollo-router/src/plugins/response_cache/plugin.rs b/apollo-router/src/plugins/response_cache/plugin.rs index 3760323e97..20db32da90 100644 --- a/apollo-router/src/plugins/response_cache/plugin.rs +++ b/apollo-router/src/plugins/response_cache/plugin.rs @@ -142,6 +142,20 @@ impl StorageInterface { let storage = self.subgraphs.get(subgraph).or(self.all.as_ref())?; storage.get() } + + /// Activate all storages so they can start emitting metrics. + pub(crate) fn activate(&self) { + if let Some(all) = &self.all + && let Some(storage) = all.get() + { + storage.activate(); + } + for storage in self.subgraphs.values() { + if let Some(storage) = storage.get() { + storage.activate(); + } + } + } } #[cfg(all( @@ -361,7 +375,9 @@ impl PluginPrivate for ResponseCache { }) } - fn activate(&self) {} + fn activate(&self) { + self.storage.activate(); + } fn supergraph_service(&self, service: supergraph::BoxService) -> supergraph::BoxService { let debug = self.debug; diff --git a/apollo-router/src/plugins/response_cache/storage/redis.rs b/apollo-router/src/plugins/response_cache/storage/redis.rs index 377a2e2873..169712ef7b 100644 --- a/apollo-router/src/plugins/response_cache/storage/redis.rs +++ b/apollo-router/src/plugins/response_cache/storage/redis.rs @@ -100,6 +100,11 @@ impl Storage { Ok(s) } + /// Activate the Redis storage so it can start emitting metrics. + pub(crate) fn activate(&self) { + self.storage.activate(); + } + fn make_key(&self, key: K) -> String { self.storage.make_key(RedisKey(key)) } diff --git a/apollo-router/src/plugins/telemetry/apollo_exporter.rs b/apollo-router/src/plugins/telemetry/apollo_exporter.rs index e6b539e279..f2f6a62481 100644 --- a/apollo-router/src/plugins/telemetry/apollo_exporter.rs +++ b/apollo-router/src/plugins/telemetry/apollo_exporter.rs @@ -16,7 +16,7 @@ use http::header::CONTENT_ENCODING; use http::header::CONTENT_TYPE; use http::header::RETRY_AFTER; use http::header::USER_AGENT; -use opentelemetry::ExportError; +use opentelemetry_sdk::ExportError; use parking_lot::Mutex; pub(crate) use prost::*; use reqwest::Client; diff --git a/apollo-router/src/plugins/telemetry/apollo_otlp_exporter.rs b/apollo-router/src/plugins/telemetry/apollo_otlp_exporter.rs index 6926d5522e..86f5e84a68 100644 --- a/apollo-router/src/plugins/telemetry/apollo_otlp_exporter.rs +++ b/apollo-router/src/plugins/telemetry/apollo_otlp_exporter.rs @@ -1,8 +1,5 @@ use derivative::Derivative; -use futures::TryFutureExt; -use futures::future; -use futures::future::BoxFuture; -use opentelemetry::InstrumentationLibrary; +use opentelemetry::InstrumentationScope; use opentelemetry::KeyValue; use opentelemetry::trace::Event; use opentelemetry::trace::SpanContext; @@ -11,11 +8,12 @@ use opentelemetry::trace::TraceFlags; use opentelemetry::trace::TraceState; use opentelemetry_otlp::SpanExporterBuilder; use opentelemetry_otlp::WithExportConfig; +use opentelemetry_otlp::WithTonicConfig; use opentelemetry_sdk::Resource; -use opentelemetry_sdk::export::trace::ExportResult; -use opentelemetry_sdk::export::trace::SpanData; -use opentelemetry_sdk::export::trace::SpanExporter; +use opentelemetry_sdk::error::OTelSdkResult; +use opentelemetry_sdk::trace::SpanData; use opentelemetry_sdk::trace::SpanEvents; +use opentelemetry_sdk::trace::SpanExporter; use opentelemetry_sdk::trace::SpanLinks; use sys_info::hostname; use tonic::metadata::MetadataMap; @@ -46,10 +44,7 @@ use crate::plugins::telemetry::tracing::apollo_telemetry::APOLLO_PRIVATE_OPERATI #[derive(Derivative)] #[derivative(Debug)] pub(crate) struct ApolloOtlpExporter { - batch_config: BatchProcessorConfig, - endpoint: Url, - apollo_key: String, - intrumentation_library: InstrumentationLibrary, + instrumentation_scope: InstrumentationScope, #[derivative(Debug = "ignore")] otlp_exporter: opentelemetry_otlp::SpanExporter, errors_configuration: ErrorsConfiguration, @@ -71,47 +66,44 @@ impl ApolloOtlpExporter { let mut metadata = MetadataMap::new(); metadata.insert("apollo.api.key", MetadataValue::try_from(apollo_key)?); let mut otlp_exporter = match protocol { - Protocol::Grpc => SpanExporterBuilder::from( - opentelemetry_otlp::new_exporter() - .tonic() - .with_tls_config(ClientTlsConfig::new().with_native_roots()) - .with_timeout(batch_config.max_export_timeout) - .with_endpoint(endpoint.to_string()) - .with_metadata(metadata) - .with_compression(opentelemetry_otlp::Compression::Gzip), - ) - .build_span_exporter()?, + Protocol::Grpc => SpanExporterBuilder::new() + .with_tonic() + .with_tls_config(ClientTlsConfig::new().with_native_roots()) + .with_timeout(batch_config.max_export_timeout) + .with_endpoint(endpoint.to_string()) + .with_metadata(metadata) + .with_compression(opentelemetry_otlp::Compression::Gzip) + .build()?, // So far only using HTTP path for testing - the Studio backend only accepts GRPC today. - Protocol::Http => SpanExporterBuilder::from( - opentelemetry_otlp::new_exporter() - .http() - .with_timeout(batch_config.max_export_timeout) - .with_endpoint(endpoint.to_string()), - ) - .build_span_exporter()?, + Protocol::Http => SpanExporterBuilder::new() + .with_http() + .with_timeout(batch_config.max_export_timeout) + .with_endpoint(endpoint.to_string()) + .build()?, }; - otlp_exporter.set_resource(&Resource::new([ - KeyValue::new("apollo.router.id", router_id()), - KeyValue::new("apollo.graph.ref", apollo_graph_ref.to_string()), - KeyValue::new("apollo.schema.id", schema_id.to_string()), - KeyValue::new( - "apollo.user.agent", - format!( - "{}@{}", - std::env!("CARGO_PKG_NAME"), - std::env!("CARGO_PKG_VERSION") - ), - ), - KeyValue::new("apollo.client.host", hostname()?), - KeyValue::new("apollo.client.uname", get_uname()?), - ])); + otlp_exporter.set_resource( + &Resource::builder_empty() + .with_attributes([ + KeyValue::new("apollo.router.id", router_id()), + KeyValue::new("apollo.graph.ref", apollo_graph_ref.to_string()), + KeyValue::new("apollo.schema.id", schema_id.to_string()), + KeyValue::new( + "apollo.user.agent", + format!( + "{}@{}", + std::env!("CARGO_PKG_NAME"), + std::env!("CARGO_PKG_VERSION") + ), + ), + KeyValue::new("apollo.client.host", hostname()?), + KeyValue::new("apollo.client.uname", get_uname()?), + ]) + .build(), + ); Ok(Self { - endpoint: endpoint.clone(), - batch_config: batch_config.clone(), - apollo_key: apollo_key.to_string(), - intrumentation_library: InstrumentationLibrary::builder(GLOBAL_TRACER_NAME) + instrumentation_scope: InstrumentationScope::builder(GLOBAL_TRACER_NAME) .with_version(format!( "{}@{}", std::env!("CARGO_PKG_NAME"), @@ -185,6 +177,7 @@ impl ApolloOtlpExporter { TraceState::default(), ), parent_span_id: span.parent_span_id, + parent_span_is_remote: false, span_kind: span.span_kind.clone(), name: span.name.clone(), start_time: span.start_time, @@ -197,7 +190,7 @@ impl ApolloOtlpExporter { events: Self::extract_span_events(&span), links: SpanLinks::default(), status: span.status, - instrumentation_lib: self.intrumentation_library.clone(), + instrumentation_scope: self.instrumentation_scope.clone(), dropped_attributes_count: span.droppped_attribute_count, } } @@ -238,6 +231,7 @@ impl ApolloOtlpExporter { TraceState::default(), ), parent_span_id: span.parent_span_id, + parent_span_is_remote: false, span_kind: span.span_kind.clone(), name: span.name.clone(), start_time: span.start_time, @@ -250,27 +244,27 @@ impl ApolloOtlpExporter { events: Self::extract_span_events(&span), links: SpanLinks::default(), status, - instrumentation_lib: self.intrumentation_library.clone(), + instrumentation_scope: self.instrumentation_scope.clone(), dropped_attributes_count: span.droppped_attribute_count, } } +} - pub(crate) fn export(&mut self, spans: Vec) -> BoxFuture<'static, ExportResult> { - let fut = self.otlp_exporter.export(spans); - Box::pin(fut.and_then(|_| { - // re-use the metric we already have in apollo_exporter but attach the protocol - u64_counter!( - "apollo.router.telemetry.studio.reports", - "The number of reports submitted to Studio by the Router", - 1, - report.type = ROUTER_REPORT_TYPE_TRACES, - report.protocol = ROUTER_TRACING_PROTOCOL_OTLP - ); - future::ready(Ok(())) - })) +impl SpanExporter for ApolloOtlpExporter { + async fn export(&self, batch: Vec) -> OTelSdkResult { + self.otlp_exporter.export(batch).await?; + // re-use the metric we already have in apollo_exporter but attach the protocol + u64_counter!( + "apollo.router.telemetry.studio.reports", + "The number of reports submitted to Studio by the Router", + 1, + report.type = ROUTER_REPORT_TYPE_TRACES, + report.protocol = ROUTER_TRACING_PROTOCOL_OTLP + ); + Ok(()) } - pub(crate) fn shutdown(&mut self) { - self.otlp_exporter.shutdown() + fn shutdown_with_timeout(&mut self, timeout: std::time::Duration) -> OTelSdkResult { + self.otlp_exporter.shutdown_with_timeout(timeout) } } diff --git a/apollo-router/src/plugins/telemetry/config.rs b/apollo-router/src/plugins/telemetry/config.rs index 51d48b8d51..685345b9d4 100644 --- a/apollo-router/src/plugins/telemetry/config.rs +++ b/apollo-router/src/plugins/telemetry/config.rs @@ -7,12 +7,9 @@ use http::HeaderName; use num_traits::ToPrimitive; use opentelemetry::Array; use opentelemetry::Value; -use opentelemetry::metrics::MetricsError; use opentelemetry_sdk::metrics::Aggregation; use opentelemetry_sdk::metrics::Instrument; use opentelemetry_sdk::metrics::Stream; -use opentelemetry_sdk::metrics::View; -use opentelemetry_sdk::metrics::new_view; use opentelemetry_sdk::trace::SpanLimits; use schemars::JsonSchema; use serde::Deserialize; @@ -161,36 +158,76 @@ pub(crate) struct MetricView { pub(crate) allowed_attribute_keys: Option>, } -impl TryInto> for MetricView { - type Error = MetricsError; +impl MetricView { + /// Creates a default view for a named instrument with histogram aggregation. + pub(crate) fn default_histogram(name: String, boundaries: Vec) -> Self { + Self { + name, + rename: None, + description: None, + unit: None, + aggregation: Some(MetricAggregation::Histogram { + buckets: boundaries, + }), + allowed_attribute_keys: None, + } + } - fn try_into(self) -> Result, Self::Error> { - let aggregation = self.aggregation.map(|aggregation| match aggregation { - MetricAggregation::Histogram { buckets } => Aggregation::ExplicitBucketHistogram { - boundaries: buckets, - record_min_max: true, - }, - MetricAggregation::Drop => Aggregation::Drop, - }); - let instrument = Instrument::new().name(self.name); - let mut mask = Stream::new(); + /// Merges user-provided overrides into this view configuration. + /// User-specified (`Some`) fields take precedence; unspecified (`None`) fields + /// retain the values from `self`. + pub(crate) fn merge(self, user: Self) -> Self { + Self { + name: self.name, + rename: user.rename.or(self.rename), + description: user.description.or(self.description), + unit: user.unit.or(self.unit), + aggregation: user.aggregation.or(self.aggregation), + allowed_attribute_keys: user.allowed_attribute_keys.or(self.allowed_attribute_keys), + } + } + + /// Builds a Stream from this view configuration. + /// Use this when you've already matched the instrument by name. + pub(crate) fn into_stream(self) -> Stream { + let mut stream = Stream::builder(); if let Some(new_name) = self.rename { - mask = mask.name(new_name); + stream = stream.with_name(new_name); } if let Some(desc) = self.description { - mask = mask.description(desc); + stream = stream.with_description(desc); } - if let Some(unit) = self.unit { - mask = mask.unit(unit); + if let Some(u) = self.unit { + stream = stream.with_unit(u); } - if let Some(aggregation) = aggregation { - mask = mask.aggregation(aggregation); + if let Some(agg) = self.aggregation { + let aggregation = match agg { + MetricAggregation::Histogram { buckets } => Aggregation::ExplicitBucketHistogram { + boundaries: buckets, + record_min_max: true, + }, + MetricAggregation::Drop => Aggregation::Drop, + }; + stream = stream.with_aggregation(aggregation); } - if let Some(allowed_attribute_keys) = self.allowed_attribute_keys { - mask = mask.allowed_attribute_keys(allowed_attribute_keys.into_iter().map(Key::new)); + if let Some(keys) = self.allowed_attribute_keys { + stream = stream.with_allowed_attribute_keys(keys.into_iter().map(Key::new)); } + stream.build().expect("Failed to build metric view") + } - new_view(instrument, mask) + /// Converts this MetricView into a view function for OTel SDK 0.31+ + pub(crate) fn into_view_fn( + self, + ) -> impl Fn(&Instrument) -> Option + Send + Sync + 'static { + let name = self.name.clone(); + let view = self; + move |instrument: &Instrument| { + if instrument.name() != name { + return None; + } + Some(view.clone().into_stream()) + } } } @@ -648,6 +685,7 @@ impl From for AttributeArray { opentelemetry::Array::String(v) => { AttributeArray::String(v.into_iter().map(|v| v.into()).collect()) } + _ => unreachable!("unexpected opentelemetry::Array variant"), } } } @@ -689,32 +727,35 @@ impl From for opentelemetry_sdk::trace::Sampler { } } -impl From<&TracingCommon> for opentelemetry_sdk::trace::Config { - fn from(config: &TracingCommon) -> Self { - let mut common = opentelemetry_sdk::trace::Config::default(); - - let mut sampler: opentelemetry_sdk::trace::Sampler = config.sampler.clone().into(); - if config.parent_based_sampler { +impl TracingCommon { + /// Configures a TracerProviderBuilder with sampler, span limits, and resource settings. + pub(crate) fn configure_tracer_provider_builder( + &self, + builder: opentelemetry_sdk::trace::TracerProviderBuilder, + ) -> opentelemetry_sdk::trace::TracerProviderBuilder { + let mut sampler: opentelemetry_sdk::trace::Sampler = self.sampler.clone().into(); + if self.parent_based_sampler { sampler = parent_based(sampler); } - if config.preview_datadog_agent_sampling.unwrap_or_default() { - common = common.with_sampler(DatadogAgentSampling::new( + + let builder = builder + .with_span_limits(SpanLimits { + max_events_per_span: self.max_events_per_span, + max_attributes_per_span: self.max_attributes_per_span, + max_links_per_span: self.max_links_per_span, + max_attributes_per_event: self.max_attributes_per_event, + max_attributes_per_link: self.max_attributes_per_link, + }) + .with_resource(self.to_resource()); + + if self.preview_datadog_agent_sampling.unwrap_or_default() { + builder.with_sampler(DatadogAgentSampling::new( sampler, - config.parent_based_sampler, - )); + self.parent_based_sampler, + )) } else { - common = common.with_sampler(sampler); + builder.with_sampler(sampler) } - - common = common.with_max_events_per_span(config.max_events_per_span); - common = common.with_max_attributes_per_span(config.max_attributes_per_span); - common = common.with_max_links_per_span(config.max_links_per_span); - common = common.with_max_attributes_per_event(config.max_attributes_per_event); - common = common.with_max_attributes_per_link(config.max_attributes_per_link); - - // Take the default first, then config, then env resources, then env variable. Last entry wins - common = common.with_resource(config.to_resource()); - common } } @@ -834,6 +875,13 @@ impl Conf { #[cfg(test)] mod tests { + + use opentelemetry::metrics::MeterProvider; + use opentelemetry_sdk::metrics::InMemoryMetricExporter; + use opentelemetry_sdk::metrics::MeterProviderBuilder; + use opentelemetry_sdk::metrics::data::MetricData; + use opentelemetry_sdk::metrics::periodic_reader_with_async_runtime::PeriodicReader; + use opentelemetry_sdk::runtime; use serde_json::json; use super::*; @@ -938,4 +986,329 @@ mod tests { assert_eq!(view.unit, Some("s".to_string())); assert!(view.aggregation.is_some()); } + + #[test] + fn test_default_histogram_creates_view_with_buckets() { + let boundaries = vec![0.1, 0.5, 1.0, 5.0]; + let view = MetricView::default_histogram("my.metric".to_string(), boundaries.clone()); + + assert_eq!(view.name, "my.metric"); + assert_eq!(view.rename, None); + assert_eq!(view.description, None); + assert_eq!(view.unit, None); + assert_eq!( + view.aggregation, + Some(MetricAggregation::Histogram { + buckets: boundaries + }) + ); + assert_eq!(view.allowed_attribute_keys, None); + } + + #[test] + fn test_merge_user_overrides_all_fields() { + let default = + MetricView::default_histogram("my.histogram".to_string(), vec![0.1, 0.5, 1.0]); + let user = MetricView { + name: "my.histogram".to_string(), + rename: Some("renamed.histogram".to_string()), + description: Some("User description".to_string()), + unit: Some("ms".to_string()), + aggregation: Some(MetricAggregation::Histogram { + buckets: vec![1.0, 5.0, 10.0], + }), + allowed_attribute_keys: Some(HashSet::from(["key1".to_string()])), + }; + + let merged = default.merge(user); + assert_eq!(merged.name, "my.histogram"); + assert_eq!(merged.rename, Some("renamed.histogram".to_string())); + assert_eq!(merged.description, Some("User description".to_string())); + assert_eq!(merged.unit, Some("ms".to_string())); + assert_eq!( + merged.aggregation, + Some(MetricAggregation::Histogram { + buckets: vec![1.0, 5.0, 10.0] + }) + ); + assert_eq!( + merged.allowed_attribute_keys, + Some(HashSet::from(["key1".to_string()])) + ); + } + + #[test] + fn test_merge_user_specifies_nothing_preserves_defaults() { + let default_buckets = vec![0.1, 0.5, 1.0]; + let default = + MetricView::default_histogram("my.histogram".to_string(), default_buckets.clone()); + let user = MetricView { + name: "my.histogram".to_string(), + rename: None, + description: None, + unit: None, + aggregation: None, + allowed_attribute_keys: None, + }; + + let merged = default.merge(user); + assert_eq!(merged.name, "my.histogram"); + assert_eq!(merged.rename, None); + assert_eq!(merged.description, None); + assert_eq!(merged.unit, None); + assert_eq!( + merged.aggregation, + Some(MetricAggregation::Histogram { + buckets: default_buckets + }), + "default histogram aggregation should be preserved when user specifies none" + ); + assert_eq!(merged.allowed_attribute_keys, None); + } + + #[test] + fn test_merge_partial_override_preserves_default_aggregation() { + let default_buckets = vec![0.001, 0.005, 0.015, 0.05, 0.1]; + let default = MetricView::default_histogram( + "http.server.request.duration".to_string(), + default_buckets.clone(), + ); + let user = MetricView { + name: "http.server.request.duration".to_string(), + rename: None, + description: Some("Custom description".to_string()), + unit: None, + aggregation: None, + allowed_attribute_keys: Some(HashSet::from([ + "http.method".to_string(), + "http.status_code".to_string(), + ])), + }; + + let merged = default.merge(user); + assert_eq!( + merged.aggregation, + Some(MetricAggregation::Histogram { + buckets: default_buckets + }), + "default histogram buckets should be inherited when user doesn't specify aggregation" + ); + assert_eq!(merged.description, Some("Custom description".to_string())); + assert_eq!( + merged.allowed_attribute_keys, + Some(HashSet::from([ + "http.method".to_string(), + "http.status_code".to_string(), + ])) + ); + } + + #[test] + fn test_merge_user_drop_overrides_default_histogram() { + let default = + MetricView::default_histogram("noisy.metric".to_string(), vec![0.1, 0.5, 1.0]); + let user = MetricView { + name: "noisy.metric".to_string(), + rename: None, + description: None, + unit: None, + aggregation: Some(MetricAggregation::Drop), + allowed_attribute_keys: None, + }; + + let merged = default.merge(user); + assert_eq!( + merged.aggregation, + Some(MetricAggregation::Drop), + "user Drop aggregation should override default histogram" + ); + } + + /// Helper to extract histogram bounds from exported metrics + fn get_histogram_bounds( + exporter: &InMemoryMetricExporter, + metric_name: &str, + ) -> Option> { + let metrics = exporter.get_finished_metrics().ok()?; + for resource_metrics in metrics { + for scope_metrics in resource_metrics.scope_metrics() { + for metric in scope_metrics.metrics() { + if metric.name() == metric_name + && let opentelemetry_sdk::metrics::data::AggregatedMetrics::F64( + MetricData::Histogram(histogram), + ) = metric.data() + && let Some(dp) = histogram.data_points().next() + { + return Some(dp.bounds().collect()); + } + } + } + } + None + } + + /// Helper to check if a metric exists in exported metrics + fn metric_exists(exporter: &InMemoryMetricExporter, metric_name: &str) -> bool { + let Ok(metrics) = exporter.get_finished_metrics() else { + return false; + }; + metrics + .iter() + .flat_map(|rm| rm.scope_metrics()) + .flat_map(|sm| sm.metrics()) + .any(|m| m.name() == metric_name) + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_user_custom_buckets_are_applied() { + let exporter = InMemoryMetricExporter::default(); + let custom_buckets = vec![0.005, 0.05, 0.5, 5.0]; + + // Create a view with custom histogram buckets + let view = MetricView { + name: "test.histogram".to_string(), + rename: None, + description: None, + unit: None, + aggregation: Some(MetricAggregation::Histogram { + buckets: custom_buckets.clone(), + }), + allowed_attribute_keys: None, + }; + + let meter_provider = MeterProviderBuilder::default() + .with_reader(PeriodicReader::builder(exporter.clone(), runtime::Tokio).build()) + .with_view(view.into_view_fn()) + .build(); + + // Record a histogram value + let meter = meter_provider.meter("test"); + let histogram = meter.f64_histogram("test.histogram").build(); + histogram.record(0.1, &[]); + + meter_provider.force_flush().unwrap(); + + let bounds = + get_histogram_bounds(&exporter, "test.histogram").expect("histogram should exist"); + assert_eq!( + bounds, custom_buckets, + "histogram should use user-specified custom buckets" + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_merged_view_inherits_default_buckets() { + let exporter = InMemoryMetricExporter::default(); + let default_buckets = vec![0.001, 0.01, 0.1, 1.0, 10.0]; + + // Create a default view with histogram buckets + let default_view = + MetricView::default_histogram("test.histogram".to_string(), default_buckets.clone()); + + // User view specifies only description, not aggregation + let user_view = MetricView { + name: "test.histogram".to_string(), + rename: None, + description: Some("Custom description".to_string()), + unit: None, + aggregation: None, // No aggregation specified - should inherit defaults + allowed_attribute_keys: None, + }; + + // Merge views - user view should inherit default buckets + let merged = default_view.merge(user_view); + + let meter_provider = MeterProviderBuilder::default() + .with_reader(PeriodicReader::builder(exporter.clone(), runtime::Tokio).build()) + .with_view(merged.into_view_fn()) + .build(); + + let meter = meter_provider.meter("test"); + let histogram = meter.f64_histogram("test.histogram").build(); + histogram.record(0.05, &[]); + + meter_provider.force_flush().unwrap(); + + let bounds = + get_histogram_bounds(&exporter, "test.histogram").expect("histogram should exist"); + assert_eq!( + bounds, default_buckets, + "merged view should inherit default buckets when user doesn't specify aggregation" + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_drop_aggregation_suppresses_metric() { + let exporter = InMemoryMetricExporter::default(); + + // Create a view with Drop aggregation + let view = MetricView { + name: "dropped.histogram".to_string(), + rename: None, + description: None, + unit: None, + aggregation: Some(MetricAggregation::Drop), + allowed_attribute_keys: None, + }; + + let meter_provider = MeterProviderBuilder::default() + .with_reader(PeriodicReader::builder(exporter.clone(), runtime::Tokio).build()) + .with_view(view.into_view_fn()) + .build(); + + let meter = meter_provider.meter("test"); + let histogram = meter.f64_histogram("dropped.histogram").build(); + histogram.record(1.0, &[]); + + meter_provider.force_flush().unwrap(); + + assert!( + !metric_exists(&exporter, "dropped.histogram"), + "metric with Drop aggregation should not be exported" + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn test_user_buckets_override_merged_defaults() { + let exporter = InMemoryMetricExporter::default(); + let default_buckets = vec![0.001, 0.01, 0.1, 1.0, 10.0]; + let user_buckets = vec![1.0, 5.0, 10.0, 50.0]; + + // Create a default view with histogram buckets + let default_view = + MetricView::default_histogram("test.histogram".to_string(), default_buckets); + + // User view specifies custom aggregation - should override defaults + let user_view = MetricView { + name: "test.histogram".to_string(), + rename: None, + description: None, + unit: None, + aggregation: Some(MetricAggregation::Histogram { + buckets: user_buckets.clone(), + }), + allowed_attribute_keys: None, + }; + + // Merge views - user aggregation should take precedence + let merged = default_view.merge(user_view); + + let meter_provider = MeterProviderBuilder::default() + .with_reader(PeriodicReader::builder(exporter.clone(), runtime::Tokio).build()) + .with_view(merged.into_view_fn()) + .build(); + + let meter = meter_provider.meter("test"); + let histogram = meter.f64_histogram("test.histogram").build(); + histogram.record(2.5, &[]); + + meter_provider.force_flush().unwrap(); + + let bounds = + get_histogram_bounds(&exporter, "test.histogram").expect("histogram should exist"); + assert_eq!( + bounds, user_buckets, + "user-specified buckets should override default buckets in merged view" + ); + } } diff --git a/apollo-router/src/plugins/telemetry/config_new/apollo/instruments.rs b/apollo-router/src/plugins/telemetry/config_new/apollo/instruments.rs index 4196811d41..67b7d15659 100644 --- a/apollo-router/src/plugins/telemetry/config_new/apollo/instruments.rs +++ b/apollo-router/src/plugins/telemetry/config_new/apollo/instruments.rs @@ -313,7 +313,7 @@ fn create_subgraph_and_connector_shared_static_instruments() -> HashMap Option { opentelemetry::Value::String(s) => s.as_str().parse::().ok(), opentelemetry::Value::Bool(_) => None, opentelemetry::Value::Array(_) => None, + _ => unreachable!("unexpected opentelemetry::Value variant"), } } @@ -1821,12 +1823,12 @@ where return; } - let attrs: Vec = inner + let attrs = inner .selectors .as_ref() - .map(|s| s.on_response(response).into_iter().collect()) + .map(|s| s.on_response(response)) .unwrap_or_default(); - inner.attributes.extend(attrs); + extend_attributes(&mut inner.attributes, attrs); if let Some(selected_value) = inner .selector @@ -1876,12 +1878,7 @@ where // Response event may be called multiple times so we don't extend inner.attributes let mut attrs = inner.attributes.clone(); if let Some(selectors) = inner.selectors.as_ref() { - attrs.extend( - selectors - .on_response_event(response, ctx) - .into_iter() - .collect::>(), - ); + extend_attributes(&mut attrs, selectors.on_response_event(response, ctx)); } if let Some(selected_value) = inner @@ -1933,12 +1930,7 @@ where let mut attrs = inner.attributes.clone(); if let Some(selectors) = inner.selectors.as_ref() { - attrs.extend( - selectors - .on_error(error, ctx) - .into_iter() - .collect::>(), - ); + extend_attributes(&mut attrs, selectors.on_error(error, ctx)); } let increment = match &inner.increment { @@ -2257,12 +2249,12 @@ where } return; } - let attrs: Vec = inner + let attrs = inner .selectors .as_ref() - .map(|s| s.on_response(response).into_iter().collect()) + .map(|s| s.on_response(response)) .unwrap_or_default(); - inner.attributes.extend(attrs); + extend_attributes(&mut inner.attributes, attrs); if let Some(selected_value) = inner .selector .as_ref() @@ -2313,12 +2305,7 @@ where // Response event may be called multiple times so we don't extend inner.attributes let mut attrs: Vec = inner.attributes.clone(); if let Some(selectors) = inner.selectors.as_ref() { - attrs.extend( - selectors - .on_response_event(response, ctx) - .into_iter() - .collect::>(), - ); + extend_attributes(&mut attrs, selectors.on_response_event(response, ctx)); } if let Some(selected_value) = inner @@ -2367,12 +2354,10 @@ where fn on_error(&self, error: &BoxError, ctx: &Context) { let mut inner = self.inner.lock(); - let mut attrs: Vec = inner - .selectors - .as_ref() - .map(|s| s.on_error(error, ctx).into_iter().collect()) - .unwrap_or_default(); - attrs.append(&mut inner.attributes); + let mut attrs = inner.attributes.clone(); + if let Some(selectors) = inner.selectors.as_ref() { + extend_attributes(&mut attrs, selectors.on_error(error, ctx)); + } let increment = match &inner.increment { Increment::Unit | Increment::EventUnit | Increment::FieldUnit => { diff --git a/apollo-router/src/plugins/telemetry/config_new/mod.rs b/apollo-router/src/plugins/telemetry/config_new/mod.rs index 5b616bbf74..37652baea5 100644 --- a/apollo-router/src/plugins/telemetry/config_new/mod.rs +++ b/apollo-router/src/plugins/telemetry/config_new/mod.rs @@ -173,7 +173,9 @@ pub(crate) fn trace_id() -> Option { pub(crate) fn get_baggage(key: &str) -> Option { let context = Span::current().context(); let baggage = context.baggage(); - baggage.get(key).cloned() + baggage + .get(key) + .map(|v| opentelemetry::Value::String(v.clone())) } pub(crate) trait ToOtelValue { @@ -248,6 +250,7 @@ impl From for AttributeValue { opentelemetry::Value::F64(v) => AttributeValue::F64(v), opentelemetry::Value::String(v) => AttributeValue::String(v.into()), opentelemetry::Value::Array(v) => AttributeValue::Array(v.into()), + _ => unreachable!("unexpected opentelemetry::Value variant"), } } } @@ -311,7 +314,7 @@ mod test { tracing::subscriber::with_default(subscriber, || { let span_context = SpanContext::new( TraceId::from(42), - SpanId::from_u64(42), + SpanId::from(42), TraceFlags::default(), false, TraceState::default(), @@ -322,7 +325,7 @@ mod test { let span = span!(tracing::Level::INFO, "test"); let _guard = span.enter(); let trace_id = trace_id(); - assert_eq!(trace_id, Some(TraceId::from_u128(42))); + assert_eq!(trace_id, Some(TraceId::from(42))); }); } @@ -332,8 +335,8 @@ mod test { let subscriber = tracing_subscriber::registry().with(otel::layer()); tracing::subscriber::with_default(subscriber, || { let span_context = SpanContext::new( - TraceId::from_u128(42), - SpanId::from_u64(42), + TraceId::from(42), + SpanId::from(42), TraceFlags::default(), false, TraceState::default(), @@ -344,7 +347,7 @@ mod test { let span = span!(tracing::Level::INFO, "test"); let _guard = span.enter(); let trace_id = trace_id(); - assert_eq!(trace_id, Some(TraceId::from_u128(42))); + assert_eq!(trace_id, Some(TraceId::from(42))); }); } diff --git a/apollo-router/src/plugins/telemetry/config_new/router/attributes.rs b/apollo-router/src/plugins/telemetry/config_new/router/attributes.rs index a43292b433..e17ba8ab0b 100644 --- a/apollo-router/src/plugins/telemetry/config_new/router/attributes.rs +++ b/apollo-router/src/plugins/telemetry/config_new/router/attributes.rs @@ -126,8 +126,8 @@ mod test { let subscriber = tracing_subscriber::registry().with(otel::layer()); subscriber::with_default(subscriber, || { let span_context = SpanContext::new( - TraceId::from_u128(42), - SpanId::from_u64(42), + TraceId::from(42), + SpanId::from(42), TraceFlags::default().with_sampled(true), false, TraceState::default(), diff --git a/apollo-router/src/plugins/telemetry/config_new/router/selectors.rs b/apollo-router/src/plugins/telemetry/config_new/router/selectors.rs index e2b9f8cee3..3ce467928e 100644 --- a/apollo-router/src/plugins/telemetry/config_new/router/selectors.rs +++ b/apollo-router/src/plugins/telemetry/config_new/router/selectors.rs @@ -747,8 +747,8 @@ mod test { default: Some("defaulted".into()), }; let span_context = SpanContext::new( - TraceId::from_u128(42), - SpanId::from_u64(42), + TraceId::from(42), + SpanId::from(42), // Make sure it's sampled if not, it won't create anything at the otel layer TraceFlags::default().with_sampled(true), false, @@ -801,8 +801,8 @@ mod test { ); let span_context = SpanContext::new( - TraceId::from_u128(42), - SpanId::from_u64(42), + TraceId::from(42), + SpanId::from(42), TraceFlags::default().with_sampled(true), false, TraceState::default(), diff --git a/apollo-router/src/plugins/telemetry/config_new/router_overhead/instruments.rs b/apollo-router/src/plugins/telemetry/config_new/router_overhead/instruments.rs index 98c1095fe6..334e69829c 100644 --- a/apollo-router/src/plugins/telemetry/config_new/router_overhead/instruments.rs +++ b/apollo-router/src/plugins/telemetry/config_new/router_overhead/instruments.rs @@ -36,7 +36,7 @@ pub(crate) fn create_static_instrument(enabled: bool) -> Option<(String, StaticI .with_description( "Router processing overhead (time not spent waiting for subgraphs or connectors to respond).", ) - .init(), + .build(), ), )) } diff --git a/apollo-router/src/plugins/telemetry/config_new/subgraph/selectors.rs b/apollo-router/src/plugins/telemetry/config_new/subgraph/selectors.rs index 61508af716..fec06ad282 100644 --- a/apollo-router/src/plugins/telemetry/config_new/subgraph/selectors.rs +++ b/apollo-router/src/plugins/telemetry/config_new/subgraph/selectors.rs @@ -1235,8 +1235,8 @@ mod test { default: Some("defaulted".into()), }; let span_context = SpanContext::new( - TraceId::from_u128(42), - SpanId::from_u64(42), + TraceId::from(42), + SpanId::from(42), // Make sure it's sampled if not, it won't create anything at the otel layer TraceFlags::default().with_sampled(true), false, diff --git a/apollo-router/src/plugins/telemetry/config_new/supergraph/selectors.rs b/apollo-router/src/plugins/telemetry/config_new/supergraph/selectors.rs index ad54a09c0d..ee3736387c 100644 --- a/apollo-router/src/plugins/telemetry/config_new/supergraph/selectors.rs +++ b/apollo-router/src/plugins/telemetry/config_new/supergraph/selectors.rs @@ -917,8 +917,8 @@ mod test { default: Some("defaulted".into()), }; let span_context = SpanContext::new( - TraceId::from_u128(42), - SpanId::from_u64(42), + TraceId::from(42), + SpanId::from(42), // Make sure it's sampled if not, it won't create anything at the otel layer TraceFlags::default().with_sampled(true), false, diff --git a/apollo-router/src/plugins/telemetry/dynamic_attribute.rs b/apollo-router/src/plugins/telemetry/dynamic_attribute.rs index d6b37e1946..6e1fd30939 100644 --- a/apollo-router/src/plugins/telemetry/dynamic_attribute.rs +++ b/apollo-router/src/plugins/telemetry/dynamic_attribute.rs @@ -22,6 +22,7 @@ use super::formatters::APOLLO_PRIVATE_PREFIX; use super::otel::OtelData; use super::otel::layer::str_to_span_kind; use super::otel::layer::str_to_status; +use super::utils::upsert_attribute; use crate::plugins::telemetry::reload::otel::IsSampled; #[derive(Debug, Default)] @@ -256,13 +257,14 @@ impl EventDynAttribute for ::tracing::Span { Some(event_attributes) => { // No need to use the upsert function here as they're going into a Map event_attributes.extend( - attributes.map(|KeyValue { key, value }| (key, value)), + attributes + .map(|KeyValue { key, value, .. }| (key, value)), ); } None => { otel_data.event_attributes = Some( attributes - .map(|KeyValue { key, value }| (key, value)) + .map(|KeyValue { key, value, .. }| (key, value)) .collect(), ); } @@ -302,15 +304,6 @@ impl EventDynAttribute for ::tracing::Span { } } -/// Replace existing attribute with same key, or add new one -fn upsert_attribute(attributes: &mut Vec, kv: KeyValue) { - if let Some(existing) = attributes.iter_mut().find(|a| a.key == kv.key) { - *existing = kv; - } else { - attributes.push(kv); - } -} - #[cfg(test)] mod tests { use std::borrow::Cow; diff --git a/apollo-router/src/plugins/telemetry/error_handler.rs b/apollo-router/src/plugins/telemetry/error_handler.rs deleted file mode 100644 index 435d0ee70c..0000000000 --- a/apollo-router/src/plugins/telemetry/error_handler.rs +++ /dev/null @@ -1,484 +0,0 @@ -use std::fmt::Debug; -use std::time::Duration; -use std::time::Instant; - -use async_trait::async_trait; -use dashmap::DashMap; -use futures::future::BoxFuture; -use once_cell::sync::OnceCell; -use opentelemetry::metrics::MetricsError; -use opentelemetry_sdk::export::trace::ExportResult; -use opentelemetry_sdk::export::trace::SpanData; -use opentelemetry_sdk::export::trace::SpanExporter; -use opentelemetry_sdk::metrics::Aggregation; -use opentelemetry_sdk::metrics::InstrumentKind; -use opentelemetry_sdk::metrics::data::ResourceMetrics; -use opentelemetry_sdk::metrics::data::Temporality; -use opentelemetry_sdk::metrics::exporter::PushMetricsExporter; -use opentelemetry_sdk::metrics::reader::AggregationSelector; -use opentelemetry_sdk::metrics::reader::TemporalitySelector; - -#[derive(Eq, PartialEq, Hash)] -enum ErrorType { - Trace, - Metric, - Other, -} -static OTEL_ERROR_LAST_LOGGED: OnceCell> = OnceCell::new(); - -pub(crate) fn handle_error>(err: T) { - // We have to rate limit these errors because when they happen they are very frequent. - // Use a dashmap to store the message type with the last time it was logged. - handle_error_with_map(err, OTEL_ERROR_LAST_LOGGED.get_or_init(DashMap::new)); -} - -// Allow for map injection to avoid using global map in tests -fn handle_error_with_map>( - err: T, - last_logged_map: &DashMap, -) { - let err = err.into(); - - // We don't want the dashmap to get big, so we key the error messages by type. - let error_type = match err { - opentelemetry::global::Error::Trace(_) => ErrorType::Trace, - opentelemetry::global::Error::Metric(_) => ErrorType::Metric, - _ => ErrorType::Other, - }; - #[cfg(not(test))] - let threshold = Duration::from_secs(10); - #[cfg(test)] - let threshold = Duration::from_millis(100); - - if let opentelemetry::global::Error::Metric(err) = &err { - // For now we have to suppress Metrics error: reader is shut down or not registered - // https://github.com/open-telemetry/opentelemetry-rust/issues/1244 - - if err.to_string() == "Metrics error: reader is shut down or not registered" { - return; - } - - // Keep track of the number of cardinality overflow errors otel emits. This can be removed after upgrading to 0.28.0 when the cardinality limit is removed. - // The version upgrade will also cause this log to be removed from our visibility even if we were set up custom a cardinality limit. - // https://github.com/open-telemetry/opentelemetry-rust/pull/2528 - if err - .to_string() - .contains("Maximum data points for metric stream exceeded. Entry added to overflow.") - { - u64_counter!( - "apollo.router.telemetry.metrics.cardinality_overflow", - "A count of how often a telemetry metric hit the hard cardinality limit", - 1 - ); - } - } - - // Copy here so that we don't retain a mutable reference into the dashmap and lock the shard - let now = Instant::now(); - let last_logged = *last_logged_map - .entry(error_type) - .and_modify(|last_logged| { - if last_logged.elapsed() > threshold { - *last_logged = now; - } - }) - .or_insert_with(|| now); - - if last_logged == now { - // These events are logged with explicitly no parent. This allows them to be detached from traces. - match err { - opentelemetry::global::Error::Trace(err) => { - ::tracing::error!("OpenTelemetry trace error occurred: {}", err) - } - opentelemetry::global::Error::Metric(err) => { - if let MetricsError::Other(msg) = &err { - if msg.contains("Warning") { - ::tracing::warn!(parent: None, "OpenTelemetry metric warning occurred: {}", msg); - return; - } - - // TODO: We should be able to remove this after upgrading to 0.26.0, which addresses the double-shutdown - // called out in https://github.com/open-telemetry/opentelemetry-rust/issues/1661 - if msg == "metrics provider already shut down" { - return; - } - } - ::tracing::error!(parent: None, "OpenTelemetry metric error occurred: {}", err); - } - opentelemetry::global::Error::Other(err) => { - ::tracing::error!(parent: None, "OpenTelemetry error occurred: {}", err) - } - other => { - ::tracing::error!(parent: None, "OpenTelemetry error occurred: {:?}", other) - } - } - } -} - -/// Wrapper that modifies trace export errors to include exporter name -pub(crate) struct NamedSpanExporter { - name: &'static str, - inner: E, -} - -impl NamedSpanExporter { - pub(crate) fn new(inner: E, name: &'static str) -> Self { - Self { name, inner } - } -} - -impl Debug for NamedSpanExporter { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("NamedSpanExporter") - .field("name", &self.name) - .finish() - } -} - -impl SpanExporter for NamedSpanExporter { - fn export(&mut self, batch: Vec) -> BoxFuture<'static, ExportResult> { - let name = self.name; - let fut = self.inner.export(batch); - Box::pin(async move { - fut.await.map_err(|err| { - let modified = format!("[{} traces] {}", name, err); - opentelemetry::trace::TraceError::from(modified) - }) - }) - } - - fn shutdown(&mut self) { - self.inner.shutdown() - } - - fn set_resource(&mut self, resource: &opentelemetry_sdk::Resource) { - self.inner.set_resource(resource) - } -} - -/// Wrapper that modifies metrics export errors to include exporter name -pub(crate) struct NamedMetricsExporter { - name: &'static str, - inner: E, -} - -impl NamedMetricsExporter { - pub(crate) fn new(inner: E, name: &'static str) -> Self { - Self { name, inner } - } -} - -impl Debug for NamedMetricsExporter { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("NamedMetricsExporter") - .field("name", &self.name) - .finish() - } -} - -impl AggregationSelector for NamedMetricsExporter { - fn aggregation(&self, kind: InstrumentKind) -> Aggregation { - self.inner.aggregation(kind) - } -} - -impl TemporalitySelector for NamedMetricsExporter { - fn temporality(&self, kind: InstrumentKind) -> Temporality { - self.inner.temporality(kind) - } -} - -fn prefix_metrics_error(name: &'static str, err: MetricsError) -> MetricsError { - match err { - MetricsError::Other(msg) => MetricsError::Other(format!("[{} metrics] {}", name, msg)), - MetricsError::Config(msg) => MetricsError::Config(format!("[{} metrics] {}", name, msg)), - MetricsError::ExportErr(inner) => { - MetricsError::Other(format!("[{} metrics] {}", name, inner)) - } - // Don't modify instrument configuration errors - not related to export - MetricsError::InvalidInstrumentConfiguration(msg) => { - MetricsError::InvalidInstrumentConfiguration(msg) - } - _ => MetricsError::Other(format!("[{} metrics] {}", name, err)), - } -} - -#[async_trait] -impl PushMetricsExporter for NamedMetricsExporter { - async fn export(&self, metrics: &mut ResourceMetrics) -> opentelemetry::metrics::Result<()> { - self.inner - .export(metrics) - .await - .map_err(|err| prefix_metrics_error(self.name, err)) - } - - async fn force_flush(&self) -> opentelemetry::metrics::Result<()> { - self.inner - .force_flush() - .await - .map_err(|err| prefix_metrics_error(self.name, err)) - } - - fn shutdown(&self) -> opentelemetry::metrics::Result<()> { - self.inner - .shutdown() - .map_err(|err| prefix_metrics_error(self.name, err)) - } -} - -#[cfg(test)] -mod tests { - use std::fmt::Debug; - use std::ops::DerefMut; - use std::sync::Arc; - use std::time::Duration; - - use dashmap::DashMap; - use futures::future::BoxFuture; - use opentelemetry::metrics::MetricsError; - use opentelemetry_sdk::export::trace::SpanData; - use opentelemetry_sdk::export::trace::SpanExporter; - use opentelemetry_sdk::metrics::data::ResourceMetrics; - use opentelemetry_sdk::metrics::exporter::PushMetricsExporter; - use parking_lot::Mutex; - use tracing_core::Event; - use tracing_core::Field; - use tracing_core::Subscriber; - use tracing_core::field::Visit; - use tracing_futures::WithSubscriber; - use tracing_subscriber::Layer; - use tracing_subscriber::layer::Context; - use tracing_subscriber::layer::SubscriberExt; - - use crate::metrics::FutureMetricsExt; - use crate::plugins::telemetry::error_handler::handle_error_with_map; - - #[tokio::test] - async fn test_handle_error_throttling() { - let error_map = DashMap::new(); - // Set up a fake subscriber so we can check log events. If this is useful then maybe it can be factored out into something reusable - #[derive(Default)] - struct TestVisitor { - log_entries: Vec, - } - - #[derive(Default, Clone)] - struct TestLayer { - visitor: Arc>, - } - impl TestLayer { - fn assert_log_entry_count(&self, message: &str, expected: usize) { - let log_entries = self.visitor.lock().log_entries.clone(); - let actual = log_entries.iter().filter(|e| e.contains(message)).count(); - assert_eq!(actual, expected); - } - } - impl Visit for TestVisitor { - fn record_debug(&mut self, field: &Field, value: &dyn Debug) { - self.log_entries - .push(format!("{}={:?}", field.name(), value)); - } - } - - impl Layer for TestLayer - where - S: Subscriber, - Self: 'static, - { - fn on_event(&self, event: &Event<'_>, _ctx: Context<'_, S>) { - event.record(self.visitor.lock().deref_mut()) - } - } - - let test_layer = TestLayer::default(); - - async { - // Log twice rapidly, they should get deduped - handle_error_with_map( - opentelemetry::global::Error::Other("other error".to_string()), - &error_map, - ); - handle_error_with_map( - opentelemetry::global::Error::Other("other error".to_string()), - &error_map, - ); - handle_error_with_map( - opentelemetry::global::Error::Trace("trace error".to_string().into()), - &error_map, - ); - } - .with_subscriber(tracing_subscriber::registry().with(test_layer.clone())) - .await; - - test_layer.assert_log_entry_count("other error", 1); - test_layer.assert_log_entry_count("trace error", 1); - - // Sleep a bit and then log again, it should get logged - tokio::time::sleep(Duration::from_millis(200)).await; - async { - handle_error_with_map( - opentelemetry::global::Error::Other("other error".to_string()), - &error_map, - ); - } - .with_subscriber(tracing_subscriber::registry().with(test_layer.clone())) - .await; - test_layer.assert_log_entry_count("other error", 2); - } - - #[tokio::test] - async fn test_cardinality_overflow_1() { - async { - let error_map = DashMap::new(); - let msg = "Warning: Maximum data points for metric stream exceeded. Entry added to overflow. Subsequent overflows to same metric until next collect will not be logged."; - handle_error_with_map( - opentelemetry::global::Error::Metric(opentelemetry::metrics::MetricsError::Other(msg.to_string())), - &error_map, - ); - - assert_counter!( - "apollo.router.telemetry.metrics.cardinality_overflow", - 1 - ); - } - .with_metrics() - .await; - } - - #[tokio::test] - async fn test_cardinality_overflow_2() { - async { - let error_map = DashMap::new(); - let msg = - "Warning: Maximum data points for metric stream exceeded. Entry added to overflow."; - handle_error_with_map( - opentelemetry::global::Error::Metric(opentelemetry::metrics::MetricsError::Other( - msg.to_string(), - )), - &error_map, - ); - - assert_counter!("apollo.router.telemetry.metrics.cardinality_overflow", 1); - } - .with_metrics() - .await; - } - - // Mock span exporter to test failures - #[derive(Debug)] - struct FailingSpanExporter; - - impl SpanExporter for FailingSpanExporter { - fn export( - &mut self, - _batch: Vec, - ) -> BoxFuture<'static, opentelemetry_sdk::export::trace::ExportResult> { - Box::pin(async { Err(opentelemetry::trace::TraceError::from("connection failed")) }) - } - - fn shutdown(&mut self) {} - - fn set_resource(&mut self, _resource: &opentelemetry_sdk::Resource) {} - } - - #[tokio::test] - async fn test_named_span_exporter_adds_prefix() { - let inner = FailingSpanExporter; - let mut named = super::NamedSpanExporter::new(inner, "test-exporter"); - - let result = named.export(vec![]).await; - - assert!(result.is_err()); - let err = result.unwrap_err(); - let err_msg = err.to_string(); - assert!(err_msg.contains("[test-exporter traces]")); - assert!(err_msg.contains("connection failed")); - } - - // Mock metrics exporter to test failures - #[derive(Debug)] - struct FailingMetricsExporter { - error_type: &'static str, - } - - #[async_trait::async_trait] - impl PushMetricsExporter for FailingMetricsExporter { - async fn export( - &self, - _metrics: &mut ResourceMetrics, - ) -> opentelemetry::metrics::Result<()> { - match self.error_type { - "other" => Err(MetricsError::Other("export failed".to_string())), - "config" => Err(MetricsError::Config("invalid config".to_string())), - _ => Ok(()), - } - } - - async fn force_flush(&self) -> opentelemetry::metrics::Result<()> { - Ok(()) - } - - fn shutdown(&self) -> opentelemetry::metrics::Result<()> { - Ok(()) - } - } - - impl opentelemetry_sdk::metrics::reader::AggregationSelector for FailingMetricsExporter { - fn aggregation( - &self, - _kind: opentelemetry_sdk::metrics::InstrumentKind, - ) -> opentelemetry_sdk::metrics::Aggregation { - opentelemetry_sdk::metrics::Aggregation::Default - } - } - - impl opentelemetry_sdk::metrics::reader::TemporalitySelector for FailingMetricsExporter { - fn temporality( - &self, - _kind: opentelemetry_sdk::metrics::InstrumentKind, - ) -> opentelemetry_sdk::metrics::data::Temporality { - opentelemetry_sdk::metrics::data::Temporality::Cumulative - } - } - - fn empty_resource_metrics() -> ResourceMetrics { - use opentelemetry_sdk::Resource; - ResourceMetrics { - resource: Resource::empty(), - scope_metrics: vec![], - } - } - - #[tokio::test] - async fn test_named_metrics_exporter_adds_prefix() { - let inner = FailingMetricsExporter { - error_type: "other", - }; - let named = super::NamedMetricsExporter::new(inner, "test-exporter"); - - let result = named.export(&mut empty_resource_metrics()).await; - - assert!(result.is_err()); - let err = result.unwrap_err(); - match err { - MetricsError::Other(msg) => { - assert!(msg.contains("[test-exporter metrics]")); - assert!(msg.contains("export failed")); - } - _ => panic!("Expected MetricsError::Other, got: {:?}", err), - } - } - - #[test] - fn test_prefix_metrics_error() { - let err = MetricsError::Config("bad config".to_string()); - let prefixed = super::prefix_metrics_error("test-exporter", err); - - match prefixed { - MetricsError::Config(msg) => { - assert_eq!(msg, "[test-exporter metrics] bad config"); - } - _ => panic!("Expected Config variant"), - } - } -} diff --git a/apollo-router/src/plugins/telemetry/fmt_layer.rs b/apollo-router/src/plugins/telemetry/fmt_layer.rs index c791568df7..f608ce48bb 100644 --- a/apollo-router/src/plugins/telemetry/fmt_layer.rs +++ b/apollo-router/src/plugins/telemetry/fmt_layer.rs @@ -280,6 +280,7 @@ mod tests { use apollo_federation::connectors::runtime::responses::MappedResponse; use http::HeaderValue; use http::header::CONTENT_LENGTH; + use opentelemetry_sdk::Resource; use parking_lot::Mutex; use parking_lot::MutexGuard; use tests::events::EventLevel; @@ -553,7 +554,7 @@ connector: display_resource: false, ..Default::default() }; - let format = Json::new(Default::default(), json_format); + let format = Json::new(Resource::builder_empty().build(), json_format); let fmt_layer = FmtLayer::new(format, buff.clone()).boxed(); ::tracing::subscriber::with_default( @@ -574,7 +575,7 @@ connector: ansi_escape_codes: false, ..Default::default() }; - let format = Text::new(Default::default(), text_format); + let format = Text::new(Resource::builder_empty().build(), text_format); let fmt_layer = FmtLayer::new(format, buff.clone()).boxed(); ::tracing::subscriber::with_default( @@ -592,7 +593,7 @@ connector: ansi_escape_codes: false, ..Default::default() }; - let format = Text::new(Default::default(), text_format); + let format = Text::new(Resource::builder_empty().build(), text_format); let fmt_layer = FmtLayer::new(format, buff.clone()).boxed(); ::tracing::subscriber::with_default( @@ -641,7 +642,7 @@ connector: display_resource: false, ..Default::default() }; - let format = Json::new(Default::default(), text_format); + let format = Json::new(Resource::builder_empty().build(), text_format); let fmt_layer = FmtLayer::new(format, buff.clone()).boxed(); ::tracing::subscriber::with_default( @@ -690,7 +691,7 @@ connector: display_resource: false, ..Default::default() }; - let format = Json::new(Default::default(), text_format); + let format = Json::new(Resource::builder_empty().build(), text_format); let fmt_layer = FmtLayer::new(format, buff.clone()).boxed(); let event_config: events::Events = serde_yaml::from_str(EVENT_CONFIGURATION).unwrap(); @@ -937,7 +938,7 @@ connector: display_resource: false, ..Default::default() }; - let format = Json::new(Default::default(), text_format); + let format = Json::new(Resource::builder_empty().build(), text_format); let fmt_layer = FmtLayer::new( RateLimitFormatter::new(format, &RateLimit::default()), buff.clone(), @@ -1043,7 +1044,7 @@ subgraph: ansi_escape_codes: false, ..Default::default() }; - let format = Text::new(Default::default(), text_format); + let format = Text::new(Resource::builder_empty().build(), text_format); let fmt_layer = FmtLayer::new(format, buff.clone()).boxed(); let event_config: events::Events = serde_yaml::from_str(EVENT_CONFIGURATION).unwrap(); diff --git a/apollo-router/src/plugins/telemetry/formatters/json.rs b/apollo-router/src/plugins/telemetry/formatters/json.rs index 961d549a9b..01fe65c0ad 100644 --- a/apollo-router/src/plugins/telemetry/formatters/json.rs +++ b/apollo-router/src/plugins/telemetry/formatters/json.rs @@ -180,6 +180,10 @@ where let array = array.iter().map(|a| a.as_str()).collect::>(); serializer.serialize_entry(kv.key.as_str(), &array)?; } + _ => { + // If otel adds more types, we should add support + unreachable!("unexpected opentelemetry::Value variant") + } } } } @@ -276,7 +280,7 @@ where event_attributes .take() .into_iter() - .map(|KeyValue { key, value }| (key, value)) + .map(|KeyValue { key, value, .. }| (key, value)) .collect() }) } diff --git a/apollo-router/src/plugins/telemetry/formatters/mod.rs b/apollo-router/src/plugins/telemetry/formatters/mod.rs index 7d4dd9671a..5c2192c5c4 100644 --- a/apollo-router/src/plugins/telemetry/formatters/mod.rs +++ b/apollo-router/src/plugins/telemetry/formatters/mod.rs @@ -250,7 +250,9 @@ pub(crate) fn to_list(resource: Resource) -> Vec<(String, serde_json::Value)> { .map(|s| serde_json::Value::String(s.to_string())) .collect(), ), + _ => unreachable!("unexpected opentelemetry::Array variant"), }, + _ => unreachable!("unexpected opentelemetry::Value variant"), }, ) }) diff --git a/apollo-router/src/plugins/telemetry/formatters/text.rs b/apollo-router/src/plugins/telemetry/formatters/text.rs index 2766b78866..70c471c76d 100644 --- a/apollo-router/src/plugins/telemetry/formatters/text.rs +++ b/apollo-router/src/plugins/telemetry/formatters/text.rs @@ -394,7 +394,7 @@ where event_attributes .take() .into_iter() - .map(|KeyValue { key, value }| (key, value)) + .map(|KeyValue { key, value, .. }| (key, value)) .collect() }) } diff --git a/apollo-router/src/plugins/telemetry/metrics/allocation/mod.rs b/apollo-router/src/plugins/telemetry/metrics/allocation/mod.rs index eb6f4e02a0..d52fd7f2d8 100644 --- a/apollo-router/src/plugins/telemetry/metrics/allocation/mod.rs +++ b/apollo-router/src/plugins/telemetry/metrics/allocation/mod.rs @@ -36,20 +36,35 @@ pub(crate) fn register_memory_allocation_views(builder: &mut MetricsBuilder) { }; // Register view for router request memory metric - let request_view = opentelemetry_sdk::metrics::new_view( - Instrument::new().name("apollo.router.request.memory"), - Stream::new().aggregation(aggregation.clone()), - ) - .unwrap(); - builder.with_view(MeterProviderType::Public, Box::new(request_view)); + let agg_clone = aggregation.clone(); + builder.with_view(MeterProviderType::Public, move |instrument: &Instrument| { + if instrument.name() == "apollo.router.request.memory" { + Some( + Stream::builder() + .with_aggregation(agg_clone.clone()) + .build() + .expect("Failed to create stream for apollo.router.request.memory metric"), + ) + } else { + None + } + }); // Register view for query planner memory metric - let query_planner_view = opentelemetry_sdk::metrics::new_view( - Instrument::new().name("apollo.router.query_planner.memory"), - Stream::new().aggregation(aggregation), - ) - .unwrap(); - builder.with_view(MeterProviderType::Public, Box::new(query_planner_view)); + builder.with_view(MeterProviderType::Public, move |instrument: &Instrument| { + if instrument.name() == "apollo.router.query_planner.memory" { + Some( + Stream::builder() + .with_aggregation(aggregation.clone()) + .build() + .expect( + "Failed to create stream for apollo.router.query_planner.memory metric", + ), + ) + } else { + None + } + }); } /// Tower layer that adds memory allocation tracking to router requests. diff --git a/apollo-router/src/plugins/telemetry/metrics/apollo/mod.rs b/apollo-router/src/plugins/telemetry/metrics/apollo/mod.rs index ce139d9552..72c8a7d092 100644 --- a/apollo-router/src/plugins/telemetry/metrics/apollo/mod.rs +++ b/apollo-router/src/plugins/telemetry/metrics/apollo/mod.rs @@ -4,12 +4,17 @@ use std::sync::atomic::Ordering; use std::time::Duration; use opentelemetry::KeyValue; -use opentelemetry_otlp::MetricsExporterBuilder; +use opentelemetry_otlp::MetricExporter; use opentelemetry_otlp::WithExportConfig; +use opentelemetry_otlp::WithTonicConfig; use opentelemetry_sdk::Resource; -use opentelemetry_sdk::metrics::PeriodicReader; +use opentelemetry_sdk::metrics::Aggregation; +use opentelemetry_sdk::metrics::Instrument; +use opentelemetry_sdk::metrics::InstrumentKind; +use opentelemetry_sdk::metrics::Stream; +use opentelemetry_sdk::metrics::Temporality; +use opentelemetry_sdk::metrics::periodic_reader_with_async_runtime::PeriodicReader; use opentelemetry_sdk::runtime; -use prometheus::exponential_buckets; use sys_info::hostname; use tonic::metadata::MetadataMap; use tonic::transport::ClientTlsConfig; @@ -25,9 +30,8 @@ use crate::plugins::telemetry::apollo_exporter::ApolloExporter; use crate::plugins::telemetry::apollo_exporter::get_uname; use crate::plugins::telemetry::config::ApolloMetricsReferenceMode; use crate::plugins::telemetry::config::Conf; -use crate::plugins::telemetry::error_handler::NamedMetricsExporter; -use crate::plugins::telemetry::metrics::CustomAggregationSelector; -use crate::plugins::telemetry::otlp::CustomTemporalitySelector; +use crate::plugins::telemetry::metrics::NamedMetricExporter; +use crate::plugins::telemetry::metrics::OverflowMetricExporter; use crate::plugins::telemetry::otlp::Protocol; use crate::plugins::telemetry::otlp::TelemetryDataKind; use crate::plugins::telemetry::otlp::process_endpoint; @@ -37,12 +41,23 @@ use crate::plugins::telemetry::reload::metrics::MetricsConfigurator; pub(crate) mod histogram; pub(crate) mod studio; +/// Default histogram buckets for Apollo metrics fn default_buckets() -> Vec { vec![ 0.001, 0.005, 0.015, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 1.0, 5.0, 10.0, ] } +/// Exponential buckets for Apollo realtime metrics. +/// +/// This aggregation uses the Apollo histogram format where a duration, x, in μs is +/// counted in the bucket of index max(0, min(ceil(ln(x)/ln(1.1)), 383)). +/// Returns buckets from ~1.4ms to ~5min. +fn realtime_buckets() -> Vec { + prometheus::exponential_buckets(0.001399084909, 1.1, 129) + .expect("failed to generate exponential buckets") +} + impl MetricsConfigurator for Config { fn config(conf: &Conf) -> &Self { &conf.apollo @@ -116,15 +131,15 @@ impl Config { let mut metadata = MetadataMap::new(); metadata.insert("apollo.api.key", key.parse()?); let exporter = match otlp_protocol { - Protocol::Grpc => MetricsExporterBuilder::Tonic( - opentelemetry_otlp::new_exporter() - .tonic() - .with_tls_config(ClientTlsConfig::new().with_native_roots()) - .with_endpoint(endpoint.as_str()) - .with_timeout(batch_config.max_export_timeout) - .with_metadata(metadata.clone()) - .with_compression(opentelemetry_otlp::Compression::Gzip), - ), + Protocol::Grpc => MetricExporter::builder() + .with_tonic() + .with_tls_config(ClientTlsConfig::new().with_native_roots()) + .with_endpoint(endpoint.as_str()) + .with_timeout(batch_config.max_export_timeout) + .with_metadata(metadata.clone()) + .with_compression(opentelemetry_otlp::Compression::Gzip) + .with_temporality(Temporality::Delta) + .build()?, // While Apollo doesn't use the HTTP protocol, we support it here for // use in tests to enable WireMock. Protocol::Http => { @@ -133,104 +148,124 @@ impl Config { &TelemetryDataKind::Metrics, &Protocol::Http, )?; - let mut otlp_exporter = opentelemetry_otlp::new_exporter() - .http() - .with_protocol(opentelemetry_otlp::Protocol::Grpc) - .with_timeout(batch_config.max_export_timeout); + let mut builder = MetricExporter::builder() + .with_http() + .with_timeout(batch_config.max_export_timeout) + .with_temporality(Temporality::Delta); if let Some(endpoint) = maybe_endpoint { - otlp_exporter = otlp_exporter.with_endpoint(endpoint); + builder = builder.with_endpoint(endpoint); } - MetricsExporterBuilder::Http(otlp_exporter) + builder.build()? } - } - .build_metrics_exporter( - Box::new(CustomTemporalitySelector( - opentelemetry_sdk::metrics::data::Temporality::Delta, - )), - Box::new( - CustomAggregationSelector::builder() - .boundaries(default_buckets()) - .build(), - ), - )?; - // MetricsExporterBuilder does not implement Clone, so we need to create a new builder for the realtime exporter + }; + // MetricExporter builder does not implement Clone, so we need to create a new builder for the realtime exporter let realtime_exporter = match otlp_protocol { - Protocol::Grpc => MetricsExporterBuilder::Tonic( - opentelemetry_otlp::new_exporter() - .tonic() - .with_tls_config(ClientTlsConfig::new().with_native_roots()) - .with_endpoint(endpoint.as_str()) - .with_timeout(batch_config.max_export_timeout) - .with_metadata(metadata.clone()) - .with_compression(opentelemetry_otlp::Compression::Gzip), - ), + Protocol::Grpc => MetricExporter::builder() + .with_tonic() + .with_tls_config(ClientTlsConfig::new().with_native_roots()) + .with_endpoint(endpoint.as_str()) + .with_timeout(batch_config.max_export_timeout) + .with_metadata(metadata.clone()) + .with_compression(opentelemetry_otlp::Compression::Gzip) + .with_temporality(Temporality::Delta) + .build()?, Protocol::Http => { let maybe_endpoint = process_endpoint( &Some(endpoint.to_string()), &TelemetryDataKind::Metrics, &Protocol::Http, )?; - let mut otlp_exporter = opentelemetry_otlp::new_exporter() - .http() - .with_protocol(opentelemetry_otlp::Protocol::Grpc) - .with_timeout(batch_config.max_export_timeout); + let mut builder = MetricExporter::builder() + .with_http() + .with_timeout(batch_config.max_export_timeout) + .with_temporality(Temporality::Delta); if let Some(endpoint) = maybe_endpoint { - otlp_exporter = otlp_exporter.with_endpoint(endpoint); + builder = builder.with_endpoint(endpoint); } - MetricsExporterBuilder::Http(otlp_exporter) + builder.build()? } - } - .build_metrics_exporter( - Box::new(CustomTemporalitySelector( - opentelemetry_sdk::metrics::data::Temporality::Delta, - )), - // This aggregation uses the Apollo histogram format where a duration, x, in μs is - // counted in the bucket of index max(0, min(ceil(ln(x)/ln(1.1)), 383)). - Box::new( - CustomAggregationSelector::builder() - .boundaries( - // Returns [~1.4ms ... ~5min] - exponential_buckets(0.001399084909, 1.1, 129).unwrap(), - ) - .build(), - ), - )?; - let named_exporter = NamedMetricsExporter::new(exporter, "apollo"); - let named_realtime_exporter = NamedMetricsExporter::new(realtime_exporter, "apollo"); + }; + // Wrap with overflow detection, then error prefixing + let named_exporter = + NamedMetricExporter::new(OverflowMetricExporter::new_push(exporter), "apollo"); + let named_realtime_exporter = NamedMetricExporter::new( + OverflowMetricExporter::new_push(realtime_exporter), + "apollo", + ); let default_reader = PeriodicReader::builder(named_exporter, runtime::Tokio) .with_interval(Duration::from_secs(60)) - .with_timeout(batch_config.max_export_timeout) .build(); let realtime_reader = PeriodicReader::builder(named_realtime_exporter, runtime::Tokio) .with_interval(batch_config.scheduled_delay) - .with_timeout(batch_config.max_export_timeout) .build(); - let resource = Resource::new([ - KeyValue::new("apollo.router.id", router_id()), - KeyValue::new("apollo.graph.ref", reference.to_string()), - KeyValue::new("apollo.schema.id", schema_id.to_string()), - KeyValue::new( - "apollo.user.agent", - format!( - "{}@{}", - std::env!("CARGO_PKG_NAME"), - std::env!("CARGO_PKG_VERSION") + let resource = Resource::builder_empty() + .with_attributes([ + KeyValue::new("apollo.router.id", router_id()), + KeyValue::new("apollo.graph.ref", reference.to_string()), + KeyValue::new("apollo.schema.id", schema_id.to_string()), + KeyValue::new( + "apollo.user.agent", + format!( + "{}@{}", + std::env!("CARGO_PKG_NAME"), + std::env!("CARGO_PKG_VERSION") + ), ), - ), - KeyValue::new("apollo.client.host", hostname()?), - KeyValue::new("apollo.client.uname", get_uname()?), - ]); + KeyValue::new("apollo.client.host", hostname()?), + KeyValue::new("apollo.client.uname", get_uname()?), + ]) + .build(); builder .with_reader(MeterProviderType::Apollo, default_reader) .with_resource(MeterProviderType::Apollo, resource.clone()); + // Configure histogram buckets for Apollo metrics + let apollo_buckets = default_buckets(); + builder.with_view(MeterProviderType::Apollo, move |instrument: &Instrument| { + if instrument.kind() == InstrumentKind::Histogram { + Some( + Stream::builder() + .with_aggregation(Aggregation::ExplicitBucketHistogram { + boundaries: apollo_buckets.clone(), + record_min_max: true, + }) + .build() + .expect("Failed to create stream for apollo metrics"), + ) + } else { + None + } + }); + builder .with_reader(MeterProviderType::ApolloRealtime, realtime_reader) .with_resource(MeterProviderType::ApolloRealtime, resource.clone()); + + // Configure exponential histogram buckets for Apollo realtime metrics + let realtime_histogram_buckets = realtime_buckets(); + builder.with_view( + MeterProviderType::ApolloRealtime, + move |instrument: &Instrument| { + if instrument.kind() == InstrumentKind::Histogram { + Some( + Stream::builder() + .with_aggregation(Aggregation::ExplicitBucketHistogram { + boundaries: realtime_histogram_buckets.clone(), + record_min_max: true, + }) + .build() + .expect("Failed to create stream for apollo realtime metrics"), + ) + } else { + None + } + }, + ); + Ok(()) } diff --git a/apollo-router/src/plugins/telemetry/metrics/mod.rs b/apollo-router/src/plugins/telemetry/metrics/mod.rs index 0c5d5fad16..dea0d7bead 100644 --- a/apollo-router/src/plugins/telemetry/metrics/mod.rs +++ b/apollo-router/src/plugins/telemetry/metrics/mod.rs @@ -1,44 +1,10 @@ -use opentelemetry_sdk::metrics::Aggregation; -use opentelemetry_sdk::metrics::InstrumentKind; -use opentelemetry_sdk::metrics::reader::AggregationSelector; pub(crate) mod allocation; pub(crate) mod apollo; pub(crate) mod local_type_stats; +mod named; pub(crate) mod otlp; +mod overflow; pub(crate) mod prometheus; -#[derive(Clone, Default, Debug)] -pub(crate) struct CustomAggregationSelector { - boundaries: Vec, - record_min_max: bool, -} - -#[buildstructor::buildstructor] -impl CustomAggregationSelector { - #[builder] - pub(crate) fn new( - boundaries: Vec, - record_min_max: Option, - ) -> CustomAggregationSelector { - Self { - boundaries, - record_min_max: record_min_max.unwrap_or(true), - } - } -} - -impl AggregationSelector for CustomAggregationSelector { - fn aggregation(&self, kind: InstrumentKind) -> Aggregation { - match kind { - InstrumentKind::Counter - | InstrumentKind::UpDownCounter - | InstrumentKind::ObservableCounter - | InstrumentKind::ObservableUpDownCounter => Aggregation::Sum, - InstrumentKind::Gauge | InstrumentKind::ObservableGauge => Aggregation::LastValue, - InstrumentKind::Histogram => Aggregation::ExplicitBucketHistogram { - boundaries: self.boundaries.clone(), - record_min_max: self.record_min_max, - }, - } - } -} +pub(crate) use named::NamedMetricExporter; +pub(crate) use overflow::OverflowMetricExporter; diff --git a/apollo-router/src/plugins/telemetry/metrics/named.rs b/apollo-router/src/plugins/telemetry/metrics/named.rs new file mode 100644 index 0000000000..7b3fd82efc --- /dev/null +++ b/apollo-router/src/plugins/telemetry/metrics/named.rs @@ -0,0 +1,136 @@ +//! Named metric exporter wrapper that prefixes error messages with exporter name. +//! +//! This wrapper helps identify which exporter produced an error when multiple +//! exporters are configured. + +use std::fmt::Debug; +use std::time::Duration; + +use opentelemetry_sdk::error::OTelSdkError; +use opentelemetry_sdk::error::OTelSdkResult; +use opentelemetry_sdk::metrics::Temporality; +use opentelemetry_sdk::metrics::data::ResourceMetrics; +use opentelemetry_sdk::metrics::exporter::PushMetricExporter; + +/// Wrapper that modifies metric export errors to include exporter name. +pub(crate) struct NamedMetricExporter { + name: &'static str, + inner: T, +} + +impl NamedMetricExporter { + pub(crate) fn new(inner: T, name: &'static str) -> Self { + Self { name, inner } + } +} + +impl Debug for NamedMetricExporter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("NamedMetricExporter") + .field("name", &self.name) + .field("inner", &self.inner) + .finish() + } +} + +fn prefix_otel_error(name: &'static str, err: OTelSdkError) -> OTelSdkError { + match err { + OTelSdkError::AlreadyShutdown => OTelSdkError::AlreadyShutdown, + OTelSdkError::Timeout(d) => OTelSdkError::Timeout(d), + OTelSdkError::InternalFailure(msg) => { + OTelSdkError::InternalFailure(format!("[{} metrics] {}", name, msg)) + } + } +} + +impl PushMetricExporter for NamedMetricExporter { + fn export( + &self, + metrics: &ResourceMetrics, + ) -> impl std::future::Future + Send { + let name = self.name; + let fut = self.inner.export(metrics); + async move { fut.await.map_err(|err| prefix_otel_error(name, err)) } + } + + fn force_flush(&self) -> OTelSdkResult { + self.inner + .force_flush() + .map_err(|err| prefix_otel_error(self.name, err)) + } + + fn shutdown_with_timeout(&self, timeout: Duration) -> OTelSdkResult { + self.inner + .shutdown_with_timeout(timeout) + .map_err(|err| prefix_otel_error(self.name, err)) + } + + fn temporality(&self) -> Temporality { + self.inner.temporality() + } +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use opentelemetry_sdk::error::OTelSdkError; + use opentelemetry_sdk::error::OTelSdkResult; + use opentelemetry_sdk::metrics::Temporality; + use opentelemetry_sdk::metrics::data::ResourceMetrics; + use opentelemetry_sdk::metrics::exporter::PushMetricExporter; + + use super::*; + + #[derive(Debug)] + struct FailingMetricExporter; + + impl PushMetricExporter for FailingMetricExporter { + async fn export(&self, _metrics: &ResourceMetrics) -> OTelSdkResult { + Err(OTelSdkError::InternalFailure("export failed".to_string())) + } + + fn force_flush(&self) -> OTelSdkResult { + Ok(()) + } + + fn shutdown_with_timeout(&self, _timeout: Duration) -> OTelSdkResult { + Ok(()) + } + + fn temporality(&self) -> Temporality { + Temporality::Cumulative + } + } + + #[tokio::test] + async fn test_named_metric_exporter_adds_prefix() { + let inner = FailingMetricExporter; + let named = NamedMetricExporter::new(inner, "test-exporter"); + + let result = named.export(&ResourceMetrics::default()).await; + + assert!(result.is_err()); + let err = result.unwrap_err(); + match err { + OTelSdkError::InternalFailure(msg) => { + assert!(msg.contains("[test-exporter metrics]")); + assert!(msg.contains("export failed")); + } + _ => panic!("Expected InternalFailure, got: {:?}", err), + } + } + + #[test] + fn test_prefix_otel_error() { + let err = OTelSdkError::InternalFailure("bad config".to_string()); + let prefixed = prefix_otel_error("test-exporter", err); + + match prefixed { + OTelSdkError::InternalFailure(msg) => { + assert_eq!(msg, "[test-exporter metrics] bad config"); + } + _ => panic!("Expected InternalFailure variant"), + } + } +} diff --git a/apollo-router/src/plugins/telemetry/metrics/otlp.rs b/apollo-router/src/plugins/telemetry/metrics/otlp.rs index 443c135b3c..f1216e374d 100644 --- a/apollo-router/src/plugins/telemetry/metrics/otlp.rs +++ b/apollo-router/src/plugins/telemetry/metrics/otlp.rs @@ -1,13 +1,16 @@ -use opentelemetry_otlp::MetricsExporterBuilder; -use opentelemetry_sdk::metrics::PeriodicReader; +use opentelemetry_otlp::MetricExporter; +use opentelemetry_otlp::WithExportConfig; +use opentelemetry_sdk::metrics::periodic_reader_with_async_runtime::PeriodicReader; use opentelemetry_sdk::runtime; use tower::BoxError; use crate::metrics::aggregation::MeterProviderType; use crate::plugins::telemetry::config::Conf; -use crate::plugins::telemetry::error_handler::NamedMetricsExporter; -use crate::plugins::telemetry::metrics::CustomAggregationSelector; +use crate::plugins::telemetry::metrics::NamedMetricExporter; +use crate::plugins::telemetry::metrics::OverflowMetricExporter; +use crate::plugins::telemetry::otlp::Protocol; use crate::plugins::telemetry::otlp::TelemetryDataKind; +use crate::plugins::telemetry::otlp::process_endpoint; use crate::plugins::telemetry::reload::metrics::MetricsBuilder; use crate::plugins::telemetry::reload::metrics::MetricsConfigurator; @@ -21,25 +24,83 @@ impl MetricsConfigurator for super::super::otlp::Config { } fn configure(&self, builder: &mut MetricsBuilder) -> Result<(), BoxError> { - let exporter_builder: MetricsExporterBuilder = self.exporter(TelemetryDataKind::Metrics)?; - let exporter = exporter_builder.build_metrics_exporter( - (&self.temporality).into(), - Box::new( - CustomAggregationSelector::builder() - .boundaries(builder.metrics_common().buckets.clone()) - .build(), - ), - )?; - - let named_exporter = NamedMetricsExporter::new(exporter, "otlp"); + // Apply env var overrides to the config + let config = self.clone().with_metrics_env_overrides()?; + + let exporter = config.build_metric_exporter()?; + + // Wrap with overflow detection, then error prefixing + let named_exporter = + NamedMetricExporter::new(OverflowMetricExporter::new_push(exporter), "otlp"); builder.with_reader( MeterProviderType::Public, PeriodicReader::builder(named_exporter, runtime::Tokio) - .with_interval(self.batch_processor.scheduled_delay) - .with_timeout(self.batch_processor.max_export_timeout) + .with_interval(config.batch_processor.scheduled_delay) .build(), ); Ok(()) } } + +impl super::super::otlp::Config { + fn build_metric_exporter(&self) -> Result { + match self.protocol { + Protocol::Grpc => self.build_grpc_metric_exporter(), + Protocol::Http => self.build_http_metric_exporter(), + } + } + + fn build_grpc_metric_exporter(&self) -> Result { + use http::Uri; + use opentelemetry_otlp::WithTonicConfig; + use tonic::metadata::MetadataMap; + + let endpoint_opt = + process_endpoint(&self.endpoint, &TelemetryDataKind::Metrics, &self.protocol)?; + let tls_config_opt = if let Some(endpoint) = &endpoint_opt { + if !endpoint.is_empty() { + let tls_url = Uri::try_from(endpoint)?; + Some(self.grpc.clone().to_tls_config(&tls_url)?) + } else { + None + } + } else { + None + }; + + let mut exporter_builder = MetricExporter::builder() + .with_tonic() + .with_temporality(self.temporality.into()) + .with_timeout(self.batch_processor.max_export_timeout) + .with_metadata(MetadataMap::from_headers(self.grpc.metadata.clone())); + + if let Some(endpoint) = endpoint_opt { + exporter_builder = exporter_builder.with_endpoint(endpoint); + } + if let Some(tls_config) = tls_config_opt { + exporter_builder = exporter_builder.with_tls_config(tls_config); + } + + Ok(exporter_builder.build()?) + } + + fn build_http_metric_exporter(&self) -> Result { + use opentelemetry_otlp::WithHttpConfig; + + let endpoint_opt = + process_endpoint(&self.endpoint, &TelemetryDataKind::Metrics, &self.protocol)?; + + let mut exporter_builder = MetricExporter::builder() + .with_http() + .with_temporality(self.temporality.into()) + .with_timeout(self.batch_processor.max_export_timeout) + .with_headers(self.http.headers.clone()); + + if let Some(endpoint) = endpoint_opt { + exporter_builder = exporter_builder.with_endpoint(endpoint); + } + + Ok(exporter_builder.build()?) + } +} diff --git a/apollo-router/src/plugins/telemetry/metrics/overflow.rs b/apollo-router/src/plugins/telemetry/metrics/overflow.rs new file mode 100644 index 0000000000..759f3862c7 --- /dev/null +++ b/apollo-router/src/plugins/telemetry/metrics/overflow.rs @@ -0,0 +1,330 @@ +//! Cardinality overflow detection for metric exporters. +//! +//! When OpenTelemetry SDK exceeds cardinality limits for a metric, it aggregates +//! overflow measurements into a special data point marked with `otel.metric.overflow=true`. +//! This module provides wrappers that detect those overflow data points and increment +//! a counter to make the overflow visible to monitoring systems. + +use std::fmt::Debug; +use std::sync::Weak; +use std::time::Duration; + +use opentelemetry::Value; +use opentelemetry_sdk::error::OTelSdkResult; +use opentelemetry_sdk::metrics::InstrumentKind; +use opentelemetry_sdk::metrics::Pipeline; +use opentelemetry_sdk::metrics::Temporality; +use opentelemetry_sdk::metrics::data::AggregatedMetrics; +use opentelemetry_sdk::metrics::data::Metric; +use opentelemetry_sdk::metrics::data::MetricData; +use opentelemetry_sdk::metrics::data::ResourceMetrics; +use opentelemetry_sdk::metrics::exporter::PushMetricExporter; +use opentelemetry_sdk::metrics::reader::MetricReader; + +const OTEL_METRIC_OVERFLOW_KEY: &str = "otel.metric.overflow"; +const CARDINALITY_OVERFLOW_METRIC: &str = "apollo.router.telemetry.metrics.cardinality_overflow"; + +/// Wrapper for metric exporters and readers that detects cardinality overflow. +/// +/// Implements `PushMetricExporter` when `T: PushMetricExporter` and +/// `MetricReader` when `T: MetricReader`. +pub(crate) struct OverflowMetricExporter { + inner: T, +} + +impl Clone for OverflowMetricExporter { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + } + } +} + +impl OverflowMetricExporter { + /// Create a new overflow-detecting wrapper for push-based exporters. + pub(crate) fn new_push(inner: T) -> Self { + Self { inner } + } + + /// Create a new overflow-detecting wrapper for pull-based readers. + pub(crate) fn new_pull(inner: T) -> Self { + Self { inner } + } +} + +impl Debug for OverflowMetricExporter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OverflowMetricExporter") + .field("inner", &self.inner) + .finish() + } +} + +/// Implementation for push-based exporters (OTLP, Apollo, etc.) +impl PushMetricExporter for OverflowMetricExporter { + fn export( + &self, + metrics: &ResourceMetrics, + ) -> impl std::future::Future + Send { + report_cardinality_overflow(metrics); + self.inner.export(metrics) + } + + fn force_flush(&self) -> OTelSdkResult { + self.inner.force_flush() + } + + fn shutdown_with_timeout(&self, timeout: Duration) -> OTelSdkResult { + self.inner.shutdown_with_timeout(timeout) + } + + fn temporality(&self) -> Temporality { + self.inner.temporality() + } +} + +/// Implementation for pull-based readers (Prometheus) +impl MetricReader for OverflowMetricExporter { + fn register_pipeline(&self, pipeline: Weak) { + self.inner.register_pipeline(pipeline) + } + + fn collect(&self, rm: &mut ResourceMetrics) -> OTelSdkResult { + let result = self.inner.collect(rm); + if result.is_ok() { + report_cardinality_overflow(rm); + } + result + } + + fn force_flush(&self) -> OTelSdkResult { + self.inner.force_flush() + } + + fn shutdown(&self) -> OTelSdkResult { + self.inner.shutdown() + } + + fn shutdown_with_timeout(&self, timeout: Duration) -> OTelSdkResult { + self.inner.shutdown_with_timeout(timeout) + } + + fn temporality(&self, kind: InstrumentKind) -> Temporality { + self.inner.temporality(kind) + } +} + +/// Check for cardinality overflow in metrics and report via counter. +fn report_cardinality_overflow(metrics: &ResourceMetrics) { + for scope_metrics in metrics.scope_metrics() { + for metric in scope_metrics.metrics() { + // Skip our own overflow counter to avoid recursion + if metric.name() == CARDINALITY_OVERFLOW_METRIC { + continue; + } + if has_overflow_data_point(metric) { + u64_counter_with_unit!( + "apollo.router.telemetry.metrics.cardinality_overflow", + "Counts metrics that have exceeded their cardinality limit", + "count", + 1, + [opentelemetry::KeyValue::new( + "metric.name", + metric.name().to_string(), + )] + ); + } + } + } +} + +/// Check if a metric has any data points with the overflow attribute. +fn has_overflow_data_point(metric: &Metric) -> bool { + match metric.data() { + AggregatedMetrics::F64(data) => has_overflow_in_metric_data(data), + AggregatedMetrics::U64(data) => has_overflow_in_metric_data(data), + AggregatedMetrics::I64(data) => has_overflow_in_metric_data(data), + } +} + +/// Check if any data point in a MetricData has the overflow attribute. +fn has_overflow_in_metric_data(data: &MetricData) -> bool { + match data { + MetricData::Gauge(gauge) => gauge + .data_points() + .any(|dp| has_overflow_attribute(dp.attributes())), + MetricData::Sum(sum) => sum + .data_points() + .any(|dp| has_overflow_attribute(dp.attributes())), + MetricData::Histogram(hist) => hist + .data_points() + .any(|dp| has_overflow_attribute(dp.attributes())), + MetricData::ExponentialHistogram(exp_hist) => exp_hist + .data_points() + .any(|dp| has_overflow_attribute(dp.attributes())), + } +} + +/// Check if attributes contain the overflow marker. +fn has_overflow_attribute<'a>(attrs: impl Iterator) -> bool { + attrs + .into_iter() + .any(|kv| kv.key.as_str() == OTEL_METRIC_OVERFLOW_KEY && kv.value == Value::Bool(true)) +} + +#[cfg(test)] +mod tests { + use opentelemetry::KeyValue; + use opentelemetry::Value; + use opentelemetry::metrics::MeterProvider; + use opentelemetry_sdk::Resource; + use opentelemetry_sdk::metrics::InMemoryMetricExporter; + use opentelemetry_sdk::metrics::SdkMeterProvider; + use opentelemetry_sdk::metrics::Stream; + use opentelemetry_sdk::metrics::data::ResourceMetrics; + use opentelemetry_sdk::metrics::exporter::PushMetricExporter; + use opentelemetry_sdk::metrics::reader::MetricReader; + + use super::*; + use crate::metrics::FutureMetricsExt; + use crate::metrics::test_utils::ClonableManualReader; + + #[test] + fn detects_overflow_attribute() { + let attrs = [ + KeyValue::new("http.method", "GET"), + KeyValue::new(OTEL_METRIC_OVERFLOW_KEY, true), + ]; + assert!(has_overflow_attribute(attrs.iter())); + } + + #[test] + fn no_overflow_when_attribute_missing() { + let attrs = [ + KeyValue::new("http.method", "GET"), + KeyValue::new("http.status_code", 200), + ]; + assert!(!has_overflow_attribute(attrs.iter())); + } + + #[test] + fn no_overflow_when_attribute_is_false() { + let attrs = [KeyValue::new(OTEL_METRIC_OVERFLOW_KEY, false)]; + assert!(!has_overflow_attribute(attrs.iter())); + } + + #[test] + fn no_overflow_when_attribute_is_wrong_type() { + let attrs = [KeyValue::new( + OTEL_METRIC_OVERFLOW_KEY, + Value::String("true".into()), + )]; + assert!(!has_overflow_attribute(attrs.iter())); + } + + #[test] + fn no_overflow_on_empty_attributes() { + let attrs: Vec = vec![]; + assert!(!has_overflow_attribute(attrs.iter())); + } + + #[tokio::test] + async fn increments_counter_on_cardinality_overflow() { + async { + // Create a meter provider with a very low cardinality limit for our test metric + let reader = ClonableManualReader::default(); + let provider = SdkMeterProvider::builder() + .with_reader(reader.clone()) + .with_resource(Resource::builder_empty().build()) + .with_view(|instrument: &opentelemetry_sdk::metrics::Instrument| { + if instrument.name() == "test.overflow.metric" { + Some( + Stream::builder() + .with_cardinality_limit(2) // Very low limit to trigger overflow + .build() + .expect("valid stream"), + ) + } else { + None + } + }) + .build(); + + // Record metrics that will exceed the cardinality limit + let meter = provider.meter("test"); + let counter = meter.u64_counter("test.overflow.metric").build(); + + // Record with 3 different attribute sets to exceed limit of 2 + counter.add(1, &[opentelemetry::KeyValue::new("key", "value1")]); + counter.add(1, &[opentelemetry::KeyValue::new("key", "value2")]); + counter.add(1, &[opentelemetry::KeyValue::new("key", "value3")]); // This should overflow + + // Collect metrics from the test provider + let mut resource_metrics = ResourceMetrics::default(); + reader.collect(&mut resource_metrics).unwrap(); + + // Export through OverflowMetricExporter which should detect overflow and increment counter + let inner_exporter = InMemoryMetricExporter::default(); + let exporter = OverflowMetricExporter::new_push(inner_exporter); + exporter.export(&resource_metrics).await.unwrap(); + + // Verify the overflow counter was incremented + assert_counter!( + "apollo.router.telemetry.metrics.cardinality_overflow", + 1, + "metric.name" = "test.overflow.metric" + ); + } + .with_metrics() + .await + } + + #[tokio::test] + async fn pull_reader_increments_counter_on_overflow() { + async { + // Create a cloneable reader wrapped with overflow detection (simulates Prometheus path) + let inner_reader = ClonableManualReader::default(); + let reader = OverflowMetricExporter::new_pull(inner_reader); + + // Clone the reader before passing to builder so we can call collect() later + let reader_for_collect = reader.clone(); + + let provider = SdkMeterProvider::builder() + .with_reader(reader) + .with_resource(Resource::builder_empty().build()) + .with_view(|instrument: &opentelemetry_sdk::metrics::Instrument| { + if instrument.name() == "test.pull.overflow.metric" { + Some( + Stream::builder() + .with_cardinality_limit(2) + .build() + .expect("valid stream"), + ) + } else { + None + } + }) + .build(); + + // Record metrics that exceed cardinality limit + let meter = provider.meter("test"); + let counter = meter.u64_counter("test.pull.overflow.metric").build(); + counter.add(1, &[opentelemetry::KeyValue::new("key", "value1")]); + counter.add(1, &[opentelemetry::KeyValue::new("key", "value2")]); + counter.add(1, &[opentelemetry::KeyValue::new("key", "value3")]); // Overflow + + // Collect via the wrapped reader (simulates Prometheus scrape triggering overflow detection) + let mut resource_metrics = ResourceMetrics::default(); + reader_for_collect.collect(&mut resource_metrics).unwrap(); + + // Verify the overflow counter was incremented + assert_counter!( + "apollo.router.telemetry.metrics.cardinality_overflow", + 1, + "metric.name" = "test.pull.overflow.metric" + ); + } + .with_metrics() + .await + } +} diff --git a/apollo-router/src/plugins/telemetry/metrics/prometheus.rs b/apollo-router/src/plugins/telemetry/metrics/prometheus.rs index 125ff7c1e9..287adee13b 100644 --- a/apollo-router/src/plugins/telemetry/metrics/prometheus.rs +++ b/apollo-router/src/plugins/telemetry/metrics/prometheus.rs @@ -15,7 +15,7 @@ use tower_service::Service; use crate::ListenAddr; use crate::metrics::aggregation::MeterProviderType; use crate::plugins::telemetry::config::Conf; -use crate::plugins::telemetry::metrics::CustomAggregationSelector; +use crate::plugins::telemetry::metrics::OverflowMetricExporter; use crate::plugins::telemetry::reload::metrics::MetricsBuilder; use crate::plugins::telemetry::reload::metrics::MetricsConfigurator; use crate::services::router; @@ -78,18 +78,15 @@ impl MetricsConfigurator for Config { let registry = Registry::new(); let exporter = opentelemetry_prometheus::exporter() - .with_aggregation_selector( - CustomAggregationSelector::builder() - .boundaries(builder.metrics_common().buckets.clone()) - .record_min_max(true) - .build(), - ) .with_resource_selector(self.resource_selector) .with_registry(registry.clone()) .build()?; - builder.with_reader(MeterProviderType::Public, exporter); + // Wrap with overflow detection to increment cardinality_overflow counter on pull + let reader = OverflowMetricExporter::new_pull(exporter); + builder.with_reader(MeterProviderType::Public, reader); builder.with_prometheus_registry(registry); + Ok(()) } } diff --git a/apollo-router/src/plugins/telemetry/mod.rs b/apollo-router/src/plugins/telemetry/mod.rs index c0696fb7aa..2a158de22f 100644 --- a/apollo-router/src/plugins/telemetry/mod.rs +++ b/apollo-router/src/plugins/telemetry/mod.rs @@ -15,7 +15,6 @@ use config_new::connector::instruments::ConnectorInstruments; use config_new::instruments::InstrumentsConfig; use config_new::instruments::StaticInstrument; use config_new::router_overhead; -use error_handler::handle_error; use futures::StreamExt; use futures::future::BoxFuture; use futures::future::ready; @@ -153,7 +152,6 @@ pub(crate) mod consts; pub(crate) mod dynamic_attribute; mod endpoint; mod error_counter; -mod error_handler; mod fmt_layer; pub(crate) mod formatters; mod logging; @@ -238,7 +236,7 @@ impl LruSizeInstrument { gauge.observe(value.load(std::sync::atomic::Ordering::Relaxed), &[]); } }) - .init(); + .build(); Self { value, @@ -317,9 +315,6 @@ impl PluginPrivate for Telemetry { } } - opentelemetry::global::set_error_handler(handle_error) - .expect("otel error handler lock poisoned, fatal"); - let mut config = init.config; config.instrumentation.spans.update_defaults(); config.instrumentation.instruments.update_defaults(); @@ -2177,13 +2172,16 @@ mod tests { .full_config(full_config) .build(), ) + .with_metrics() .await .unwrap(); } #[tokio::test(flavor = "multi_thread")] async fn config_serialization() { - create_plugin_with_config(include_str!("testdata/config.router.yaml")).await; + create_plugin_with_config(include_str!("testdata/config.router.yaml")) + .with_metrics() + .await; } #[tokio::test(flavor = "multi_thread")] @@ -2192,6 +2190,7 @@ mod tests { let plugin = create_plugin_with_config(include_str!( "testdata/full_config_all_features_enabled.router.yaml" )) + .with_metrics() .await; let features = enabled_features(plugin.as_ref()); assert!( @@ -2207,6 +2206,7 @@ mod tests { let plugin = create_plugin_with_config(include_str!( "testdata/full_config_all_features_enabled_response_cache.router.yaml" )) + .with_metrics() .await; let features = enabled_features(plugin.as_ref()); assert!( @@ -2222,6 +2222,7 @@ mod tests { let plugin = create_plugin_with_config(include_str!( "testdata/full_config_all_features_explicitly_disabled.router.yaml" )) + .with_metrics() .await; let features = enabled_features(plugin.as_ref()); assert!( @@ -2241,6 +2242,7 @@ mod tests { let plugin = create_plugin_with_config(include_str!( "testdata/full_config_all_features_defaults.router.yaml" )) + .with_metrics() .await; let features = enabled_features(plugin.as_ref()); assert!( @@ -2260,6 +2262,7 @@ mod tests { let plugin = create_plugin_with_config(include_str!( "testdata/full_config_apq_enabled_partial_defaults.router.yaml" )) + .with_metrics() .await; let features = enabled_features(plugin.as_ref()); assert!( @@ -2271,6 +2274,7 @@ mod tests { let plugin = create_plugin_with_config(include_str!( "testdata/full_config_apq_disabled_partial_defaults.router.yaml" )) + .with_metrics() .await; let features = enabled_features(plugin.as_ref()); assert!( @@ -2297,7 +2301,7 @@ mod tests { assert_counter!( "http.request", - 1, + 1.0, "another_test" = "my_default_value", "my_value" = 2, "myname" = "label_value", @@ -2348,7 +2352,7 @@ mod tests { assert_counter!( "http.request", - 1, + 1.0, "another_test" = "my_default_value", "error" = "nope", "myname" = "label_value", @@ -2517,7 +2521,6 @@ mod tests { "http.response.status_code" = 400, "acme.my_attribute" = "application/json", "error.type" = "Bad Request", - "http.response.status_code" = 400, "network.protocol.version" = "HTTP/1.1" ); } @@ -2836,6 +2839,7 @@ mod tests { let plugin = create_plugin_with_config(include_str!( "testdata/config.field_instrumentation_sampler.router.yaml" )) + .with_metrics() .await; let ftv1_counter = Arc::new(AtomicUsize::new(0)); @@ -3217,7 +3221,7 @@ mod tests { let mut injected = Injected(HashMap::new()); let _ctx = opentelemetry::Context::new() .with_remote_span_context(SpanContext::new( - TraceId::from_u128(0x04f9e396465c4840bc2bf493b8b1a7fc), + TraceId::from(0x04f9e396465c4840bc2bf493b8b1a7fc), SpanId::INVALID, TraceFlags::default(), false, diff --git a/apollo-router/src/plugins/telemetry/otel/layer.rs b/apollo-router/src/plugins/telemetry/otel/layer.rs index 9e8f623656..07265a3c75 100644 --- a/apollo-router/src/plugins/telemetry/otel/layer.rs +++ b/apollo-router/src/plugins/telemetry/otel/layer.rs @@ -250,7 +250,7 @@ impl field::Visit for SpanEventVisitor<'_, '_> { if self.exception_config.record { self.event_builder .attributes - .push(Key::new(FIELD_EXCEPTION_MESSAGE).string(error_msg.clone())); + .push(KeyValue::new(FIELD_EXCEPTION_MESSAGE, error_msg.clone())); // NOTE: This is actually not the stacktrace of the exception. This is // the "source chain". It represents the hierarchy of errors from the @@ -258,9 +258,10 @@ impl field::Visit for SpanEventVisitor<'_, '_> { // of the callsites in the code that led to the error happening. // `std::error::Error::backtrace` is a nightly-only API and cannot be // used here until the feature is stabilized. - self.event_builder - .attributes - .push(Key::new(FIELD_EXCEPTION_STACKTRACE).array(chain.clone())); + self.event_builder.attributes.push(KeyValue::new( + FIELD_EXCEPTION_STACKTRACE, + Value::Array(chain.clone().into()), + )); } if self.exception_config.propagate @@ -283,10 +284,11 @@ impl field::Visit for SpanEventVisitor<'_, '_> { self.event_builder .attributes - .push(Key::new(field.name()).string(error_msg)); - self.event_builder - .attributes - .push(Key::new(format!("{}.chain", field.name())).array(chain)); + .push(KeyValue::new(field.name(), error_msg)); + self.event_builder.attributes.push(KeyValue::new( + format!("{}.chain", field.name()), + Value::Array(chain.into()), + )); } } @@ -367,7 +369,7 @@ impl field::Visit for SpanAttributeVisitor<'_> { OTEL_STATUS_MESSAGE => { self.span_builder.status = otel::Status::error(format!("{value:?}")) } - _ => self.record(Key::new(field.name()).string(format!("{value:?}"))), + _ => self.record(KeyValue::new(field.name(), format!("{value:?}"))), } } @@ -391,7 +393,7 @@ impl field::Visit for SpanAttributeVisitor<'_> { let error_msg = value.to_string(); if self.exception_config.record { - self.record(Key::new(FIELD_EXCEPTION_MESSAGE).string(error_msg.clone())); + self.record(KeyValue::new(FIELD_EXCEPTION_MESSAGE, error_msg.clone())); // NOTE: This is actually not the stacktrace of the exception. This is // the "source chain". It represents the hierarchy of errors from the @@ -399,11 +401,17 @@ impl field::Visit for SpanAttributeVisitor<'_> { // of the callsites in the code that led to the error happening. // `std::error::Error::backtrace` is a nightly-only API and cannot be // used here until the feature is stabilized. - self.record(Key::new(FIELD_EXCEPTION_STACKTRACE).array(chain.clone())); + self.record(KeyValue::new( + FIELD_EXCEPTION_STACKTRACE, + Value::Array(chain.clone().into()), + )); } - self.record(Key::new(field.name()).string(error_msg)); - self.record(Key::new(format!("{}.chain", field.name())).array(chain)); + self.record(KeyValue::new(field.name(), error_msg)); + self.record(KeyValue::new( + format!("{}.chain", field.name()), + Value::Array(chain.into()), + )); } } @@ -939,9 +947,7 @@ where // Performing read operations before getting a write lock to avoid a deadlock // See https://github.com/tokio-rs/tracing/issues/763 let meta = event.metadata(); - let target = Key::new("target"); - - let target = target.string(meta.target()); + let target = KeyValue::new("target", meta.target()); let mut extensions = span.extensions_mut(); let mut otel_data = extensions.get_mut::(); @@ -950,7 +956,7 @@ where let mut otel_event = otel::Event::new( String::new(), SystemTime::now(), - vec![Key::new("level").string(meta.level().as_str()), target], + vec![KeyValue::new("level", meta.level().as_str()), target], 0, ); let mut span_event_visit = SpanEventVisitor { diff --git a/apollo-router/src/plugins/telemetry/otel/mod.rs b/apollo-router/src/plugins/telemetry/otel/mod.rs index a13327472b..04229fc63f 100644 --- a/apollo-router/src/plugins/telemetry/otel/mod.rs +++ b/apollo-router/src/plugins/telemetry/otel/mod.rs @@ -1,6 +1,5 @@ /// Implementation of the trace::Layer as a source of OpenTelemetry data. pub(crate) mod layer; -pub(crate) mod named_runtime_channel; /// Span extension which enables OpenTelemetry context management. pub(crate) mod span_ext; /// Protocols for OpenTelemetry Tracers that are compatible with Tracing diff --git a/apollo-router/src/plugins/telemetry/otel/tracer.rs b/apollo-router/src/plugins/telemetry/otel/tracer.rs index 163b9791aa..697bba0167 100644 --- a/apollo-router/src/plugins/telemetry/otel/tracer.rs +++ b/apollo-router/src/plugins/telemetry/otel/tracer.rs @@ -157,15 +157,14 @@ mod tests { use opentelemetry::trace::SpanBuilder; use opentelemetry::trace::SpanId; use opentelemetry::trace::TracerProvider as _; - use opentelemetry_sdk::trace::Config; use opentelemetry_sdk::trace::Sampler; - use opentelemetry_sdk::trace::TracerProvider; + use opentelemetry_sdk::trace::SdkTracerProvider; use super::*; #[test] fn assigns_default_trace_id_if_missing() { - let provider = TracerProvider::default(); + let provider = SdkTracerProvider::default(); let tracer = provider.tracer("test"); let mut builder = SpanBuilder::from_name("empty".to_string()); builder.span_id = Some(SpanId::from(1u64)); @@ -210,9 +209,7 @@ mod tests { #[test] fn sampled_context() { for (name, sampler, parent_cx, previous_sampling_result, is_sampled) in sampler_data() { - let provider = TracerProvider::builder() - .with_config(Config::default().with_sampler(sampler)) - .build(); + let provider = SdkTracerProvider::builder().with_sampler(sampler).build(); let tracer = provider.tracer("test"); let mut builder = SpanBuilder::from_name("parent".to_string()); builder.sampling_result = previous_sampling_result; diff --git a/apollo-router/src/plugins/telemetry/otlp.rs b/apollo-router/src/plugins/telemetry/otlp.rs index ce80e06333..03378e2fc5 100644 --- a/apollo-router/src/plugins/telemetry/otlp.rs +++ b/apollo-router/src/plugins/telemetry/otlp.rs @@ -2,16 +2,11 @@ use std::collections::HashMap; use http::Uri; -use opentelemetry_otlp::HttpExporterBuilder; -use opentelemetry_otlp::TonicExporterBuilder; -use opentelemetry_otlp::WithExportConfig; -use opentelemetry_sdk::metrics::InstrumentKind; -use opentelemetry_sdk::metrics::reader::TemporalitySelector; +use opentelemetry_sdk::metrics::Temporality as SdkTemporality; use schemars::JsonSchema; use serde::Deserialize; use serde::Serialize; use serde_json::Value; -use tonic::metadata::MetadataMap; use tonic::transport::Certificate; use tonic::transport::ClientTlsConfig; use tonic::transport::Identity; @@ -53,6 +48,74 @@ pub(crate) struct Config { pub(crate) temporality: Temporality, } +impl Config { + /// Apply OTEL_EXPORTER_OTLP_* environment variable overrides for traces. + /// Env vars take precedence over config values. + pub(crate) fn with_tracing_env_overrides(self) -> Result { + let endpoint = std::env::var("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") + .ok() + .or_else(|| std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").ok()) + .or(self.endpoint); + + let protocol = Self::parse_protocol_env( + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL", + "OTEL_EXPORTER_OTLP_PROTOCOL", + self.protocol, + )?; + + Ok(Config { + endpoint, + protocol, + ..self + }) + } + + /// Apply OTEL_EXPORTER_OTLP_* environment variable overrides for metrics. + /// Env vars take precedence over config values. + pub(crate) fn with_metrics_env_overrides(self) -> Result { + let endpoint = std::env::var("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT") + .ok() + .or_else(|| std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").ok()) + .or(self.endpoint); + + let protocol = Self::parse_protocol_env( + "OTEL_EXPORTER_OTLP_METRICS_PROTOCOL", + "OTEL_EXPORTER_OTLP_PROTOCOL", + self.protocol, + )?; + + Ok(Config { + endpoint, + protocol, + ..self + }) + } + + fn parse_protocol_env( + specific_var: &str, + general_var: &str, + default: Protocol, + ) -> Result { + let (var_name, value) = if let Ok(v) = std::env::var(specific_var) { + (specific_var, v) + } else if let Ok(v) = std::env::var(general_var) { + (general_var, v) + } else { + return Ok(default); + }; + + match value.to_lowercase().as_str() { + "grpc" => Ok(Protocol::Grpc), + "http/protobuf" | "http" => Ok(Protocol::Http), + _ => Err(format!( + "invalid value '{}' for {}, expected 'grpc' or 'http/protobuf'", + value, var_name + ) + .into()), + } + } +} + #[derive(Copy, Clone, Debug)] pub(crate) enum TelemetryDataKind { Traces, @@ -153,56 +216,6 @@ pub(super) fn process_endpoint( .transpose() } -impl Config { - pub(crate) fn exporter + From>( - &self, - kind: TelemetryDataKind, - ) -> Result { - match self.protocol { - Protocol::Grpc => { - let endpoint_opt = process_endpoint(&self.endpoint, &kind, &self.protocol)?; - // Figure out if we need to set tls config for our exporter - let tls_config_opt = if let Some(endpoint) = &endpoint_opt { - if !endpoint.is_empty() { - let tls_url = Uri::try_from(endpoint)?; - Some(self.grpc.clone().to_tls_config(&tls_url)?) - } else { - None - } - } else { - None - }; - - let mut exporter = opentelemetry_otlp::new_exporter() - .tonic() - .with_protocol(opentelemetry_otlp::Protocol::Grpc) - .with_timeout(self.batch_processor.max_export_timeout) - .with_metadata(MetadataMap::from_headers(self.grpc.metadata.clone())); - if let Some(endpoint) = endpoint_opt { - exporter = exporter.with_endpoint(endpoint); - } - if let Some(tls_config) = tls_config_opt { - exporter = exporter.with_tls_config(tls_config); - } - Ok(exporter.into()) - } - Protocol::Http => { - let endpoint_opt = process_endpoint(&self.endpoint, &kind, &self.protocol)?; - let headers = self.http.headers.clone(); - let mut exporter: HttpExporterBuilder = opentelemetry_otlp::new_exporter() - .http() - .with_protocol(opentelemetry_otlp::Protocol::Grpc) - .with_timeout(self.batch_processor.max_export_timeout) - .with_headers(headers); - if let Some(endpoint) = endpoint_opt { - exporter = exporter.with_endpoint(endpoint); - } - Ok(exporter.into()) - } - } - } -} - #[derive(Debug, Clone, Deserialize, Serialize, Default, JsonSchema, PartialEq)] #[serde(deny_unknown_fields, default)] pub(crate) struct HttpExporter { @@ -280,7 +293,7 @@ pub(crate) enum Protocol { Http, } -#[derive(Debug, Default, Clone, Deserialize, Serialize, JsonSchema, PartialEq)] +#[derive(Debug, Default, Clone, Deserialize, Serialize, JsonSchema, PartialEq, Copy)] #[serde(deny_unknown_fields, rename_all = "snake_case")] pub(crate) enum Temporality { /// Export cumulative metrics. @@ -290,155 +303,19 @@ pub(crate) enum Temporality { Delta, } -pub(crate) struct CustomTemporalitySelector( - pub(crate) opentelemetry_sdk::metrics::data::Temporality, -); - -impl TemporalitySelector for CustomTemporalitySelector { - fn temporality(&self, kind: InstrumentKind) -> opentelemetry_sdk::metrics::data::Temporality { - // Up/down counters should always use cumulative temporality to ensure they are sent as aggregates - // rather than deltas, which prevents drift issues. - // See https://github.com/open-telemetry/opentelemetry-specification/blob/a1c13d59bb7d0fb086df2b3e1eaec9df9efef6cc/specification/metrics/sdk_exporters/otlp.md#additional-configuration for mor information - match kind { - InstrumentKind::UpDownCounter | InstrumentKind::ObservableUpDownCounter => { - opentelemetry_sdk::metrics::data::Temporality::Cumulative - } - _ => self.0, +impl From for SdkTemporality { + fn from(value: Temporality) -> Self { + match value { + Temporality::Cumulative => SdkTemporality::Cumulative, + Temporality::Delta => SdkTemporality::Delta, } } } -impl From<&Temporality> for Box { - fn from(value: &Temporality) -> Self { - Box::new(match value { - Temporality::Cumulative => { - CustomTemporalitySelector(opentelemetry_sdk::metrics::data::Temporality::Cumulative) - } - Temporality::Delta => { - CustomTemporalitySelector(opentelemetry_sdk::metrics::data::Temporality::Delta) - } - }) - } -} - #[cfg(test)] mod tests { - use opentelemetry_sdk::metrics::data::Temporality as SdkTemporality; - use super::*; - #[test] - fn test_updown_counter_temporality_override() { - // Test that up/down counters always get cumulative temporality regardless of configuration - let delta_selector = CustomTemporalitySelector(SdkTemporality::Delta); - let cumulative_selector = CustomTemporalitySelector(SdkTemporality::Cumulative); - - // UpDownCounter should always be cumulative - assert_eq!( - delta_selector.temporality(InstrumentKind::UpDownCounter), - SdkTemporality::Cumulative, - "UpDownCounter should always use cumulative temporality even with delta config" - ); - assert_eq!( - cumulative_selector.temporality(InstrumentKind::UpDownCounter), - SdkTemporality::Cumulative, - "UpDownCounter should use cumulative temporality with cumulative config" - ); - - // ObservableUpDownCounter should always be cumulative - assert_eq!( - delta_selector.temporality(InstrumentKind::ObservableUpDownCounter), - SdkTemporality::Cumulative, - "ObservableUpDownCounter should always use cumulative temporality even with delta config" - ); - assert_eq!( - cumulative_selector.temporality(InstrumentKind::ObservableUpDownCounter), - SdkTemporality::Cumulative, - "ObservableUpDownCounter should use cumulative temporality with cumulative config" - ); - } - - #[test] - fn test_counter_temporality_respects_config() { - // Test that regular counters respect the configured temporality - let delta_selector = CustomTemporalitySelector(SdkTemporality::Delta); - let cumulative_selector = CustomTemporalitySelector(SdkTemporality::Cumulative); - - // Counter should respect configuration - assert_eq!( - delta_selector.temporality(InstrumentKind::Counter), - SdkTemporality::Delta, - "Counter should use delta temporality with delta config" - ); - assert_eq!( - cumulative_selector.temporality(InstrumentKind::Counter), - SdkTemporality::Cumulative, - "Counter should use cumulative temporality with cumulative config" - ); - - // ObservableCounter should respect configuration - assert_eq!( - delta_selector.temporality(InstrumentKind::ObservableCounter), - SdkTemporality::Delta, - "ObservableCounter should use delta temporality with delta config" - ); - assert_eq!( - cumulative_selector.temporality(InstrumentKind::ObservableCounter), - SdkTemporality::Cumulative, - "ObservableCounter should use cumulative temporality with cumulative config" - ); - } - - #[test] - fn test_gauge_temporality_respects_config() { - // Test that gauges respect the configured temporality (gauges are not forced to cumulative) - let delta_selector = CustomTemporalitySelector(SdkTemporality::Delta); - let cumulative_selector = CustomTemporalitySelector(SdkTemporality::Cumulative); - - // Gauge should respect configuration - assert_eq!( - delta_selector.temporality(InstrumentKind::Gauge), - SdkTemporality::Delta, - "Gauge should use delta temporality with delta config" - ); - assert_eq!( - cumulative_selector.temporality(InstrumentKind::Gauge), - SdkTemporality::Cumulative, - "Gauge should use cumulative temporality with cumulative config" - ); - - // ObservableGauge should respect configuration - assert_eq!( - delta_selector.temporality(InstrumentKind::ObservableGauge), - SdkTemporality::Delta, - "ObservableGauge should use delta temporality with delta config" - ); - assert_eq!( - cumulative_selector.temporality(InstrumentKind::ObservableGauge), - SdkTemporality::Cumulative, - "ObservableGauge should use cumulative temporality with cumulative config" - ); - } - - #[test] - fn test_histogram_temporality_respects_config() { - // Test that histograms respect the configured temporality - let delta_selector = CustomTemporalitySelector(SdkTemporality::Delta); - let cumulative_selector = CustomTemporalitySelector(SdkTemporality::Cumulative); - - // Histogram should respect configuration - assert_eq!( - delta_selector.temporality(InstrumentKind::Histogram), - SdkTemporality::Delta, - "Histogram should use delta temporality with delta config" - ); - assert_eq!( - cumulative_selector.temporality(InstrumentKind::Histogram), - SdkTemporality::Cumulative, - "Histogram should use cumulative temporality with cumulative config" - ); - } - #[test] fn endpoint_grpc_defaulting_no_scheme() { let url = Url::parse("api.apm.com:433").unwrap(); diff --git a/apollo-router/src/plugins/telemetry/reload/activation.rs b/apollo-router/src/plugins/telemetry/reload/activation.rs index 82e993ba60..2d9f6d4b64 100644 --- a/apollo-router/src/plugins/telemetry/reload/activation.rs +++ b/apollo-router/src/plugins/telemetry/reload/activation.rs @@ -25,10 +25,12 @@ use std::collections::HashMap; use std::sync::LazyLock; +use opentelemetry::InstrumentationScope; use opentelemetry::propagation::TextMapCompositePropagator; use opentelemetry::trace::TracerProvider; use parking_lot::Mutex; use prometheus::Registry; +use tokio::task::block_in_place; use tokio::task::spawn_blocking; use tracing_subscriber::Layer; @@ -46,7 +48,7 @@ use crate::plugins::telemetry::reload::otel::reload_fmt; /// then atomically applies them during the activation phase via [`Activation::commit()`]. pub(crate) struct Activation { /// The new tracer provider. None means leave the existing one - new_trace_provider: Option, + new_trace_provider: Option, /// The new tracer propagator. None means leave the existing one new_trace_propagator: Option, @@ -135,7 +137,7 @@ impl Activation { pub(crate) fn with_tracer_provider( &mut self, - tracer_provider: opentelemetry_sdk::trace::TracerProvider, + tracer_provider: opentelemetry_sdk::trace::SdkTracerProvider, ) { self.new_trace_provider = Some(tracer_provider); #[cfg(test)] @@ -191,17 +193,16 @@ impl Activation { && let Some(tracer_provider) = self.new_trace_provider.take() { // Build a new tracer from the provider and hot-swap it into the tracing subscriber - let tracer = tracer_provider - .tracer_builder(GLOBAL_TRACER_NAME) + let scope = InstrumentationScope::builder(GLOBAL_TRACER_NAME) .with_version(env!("CARGO_PKG_VERSION")) .build(); + let tracer = tracer_provider.tracer_with_scope(scope); hot_tracer.reload(tracer); - // Install the new provider globally and safely drop the old one in a blocking task - let last_provider = opentelemetry::global::set_tracer_provider(tracer_provider); - spawn_blocking(move || { - drop(last_provider); - }); + // Install the new provider globally. The old provider is returned and must be + // dropped in a blocking task to avoid deadlocking the async runtime during shutdown. + // block_in_place is used to ensure that no tasks after this point use the old tracer provider. + block_in_place(move || opentelemetry::global::set_tracer_provider(tracer_provider)); } } diff --git a/apollo-router/src/plugins/telemetry/reload/builder.rs b/apollo-router/src/plugins/telemetry/reload/builder.rs index 7059ad402f..9755d1393a 100644 --- a/apollo-router/src/plugins/telemetry/reload/builder.rs +++ b/apollo-router/src/plugins/telemetry/reload/builder.rs @@ -103,11 +103,12 @@ impl<'a> Builder<'a> { crate::plugins::telemetry::metrics::allocation::register_memory_allocation_views( &mut builder, ); - builder.configure_views(MeterProviderType::Public)?; + builder.configure_views(MeterProviderType::Public); let (prometheus_registry, meter_providers, _) = builder.build(); self.activation .with_prometheus_registry(prometheus_registry); + self.activation.add_meter_providers(meter_providers); } // Always create Prometheus endpoint if we have a registry (either new or existing). @@ -171,7 +172,7 @@ impl<'a> Builder<'a> { aggregation: Some(crate::plugins::telemetry::config::MetricAggregation::Drop), allowed_attribute_keys: None, } - .try_into()?, + .into_view_fn(), ); } @@ -244,8 +245,6 @@ impl<'a> Builder<'a> { #[cfg(test)] mod tests { - use std::collections::HashSet; - use super::*; use crate::plugins::telemetry::apollo; use crate::plugins::telemetry::config::Exporters; @@ -425,10 +424,14 @@ mod tests { instr.logging_layer_set, "First run should set logging layer" ); - // One meter provider is added for memory tracking - assert_eq!( - instr.meter_providers_added, - HashSet::from([MeterProviderType::Public]) + // A noop Public provider is always set when metrics config changes, + // even without exporters. This ensures any previous provider is replaced + // during hot reload when exporters are disabled. + assert!( + instr + .meter_providers_added + .contains(&MeterProviderType::Public), + "Public meter provider (noop) should be set on first run" ); } diff --git a/apollo-router/src/plugins/telemetry/reload/metrics.rs b/apollo-router/src/plugins/telemetry/reload/metrics.rs index 00d95f7877..989782f6eb 100644 --- a/apollo-router/src/plugins/telemetry/reload/metrics.rs +++ b/apollo-router/src/plugins/telemetry/reload/metrics.rs @@ -20,11 +20,16 @@ //! - `Public` - For standard metrics exposed via Prometheus or sent to OTLP collectors //! - `Apollo`/`ApolloRealtime` - For metrics sent to Apollo Studio with specific filtering +use std::collections::HashSet; + use ahash::HashMap; use opentelemetry_sdk::Resource; +use opentelemetry_sdk::metrics::Aggregation; +use opentelemetry_sdk::metrics::Instrument; +use opentelemetry_sdk::metrics::InstrumentKind; use opentelemetry_sdk::metrics::MeterProviderBuilder; use opentelemetry_sdk::metrics::SdkMeterProvider; -use opentelemetry_sdk::metrics::View; +use opentelemetry_sdk::metrics::Stream; use prometheus::Registry; use tower::BoxError; @@ -33,6 +38,7 @@ use crate::metrics::aggregation::MeterProviderType; use crate::metrics::filter::FilterMeterProvider; use crate::plugins::telemetry::apollo_exporter::Sender; use crate::plugins::telemetry::config::Conf; +use crate::plugins::telemetry::config::MetricView; use crate::plugins::telemetry::config::MetricsCommon; /// Trait for metric exporters to contribute to meter provider construction @@ -49,6 +55,10 @@ pub(crate) trait MetricsConfigurator { pub(crate) struct MetricsBuilder<'a> { pub(super) meter_provider_builders: HashMap, + /// Tracks which provider types have readers configured. + /// Only providers with readers are returned from build() to avoid OTel SDK errors + /// when observable instruments are registered with providers that have no readers. + providers_with_readers: HashSet, apollo_metrics_sender: Sender, prometheus_registry: Option, metrics_common: &'a MetricsCommon, @@ -68,6 +78,11 @@ impl<'a> MetricsBuilder<'a> { self.meter_provider_builders .into_iter() .map(|(k, v)| { + // Providers without readers get a noop to avoid OTel SDK errors + // when observable instruments are registered. + if !self.providers_with_readers.contains(&k) { + return (k, FilterMeterProvider::noop()); + } ( k, match k { @@ -98,6 +113,7 @@ impl<'a> MetricsBuilder<'a> { Self { meter_provider_builders: HashMap::default(), + providers_with_readers: HashSet::new(), resource, apollo_metrics_sender: Sender::default(), prometheus_registry: None, @@ -125,14 +141,19 @@ impl<'a> MetricsBuilder<'a> { ) -> &mut Self { let meter_provider = self.meter_provider(meter_provider_type); *meter_provider = std::mem::take(meter_provider).with_reader(reader); + // Track that this provider has a reader - only providers with readers will be returned + self.providers_with_readers.insert(meter_provider_type); self } - pub(crate) fn with_view( + pub(crate) fn with_view( &mut self, meter_provider_type: MeterProviderType, - view: Box, - ) -> &mut Self { + view: T, + ) -> &mut Self + where + T: Fn(&Instrument) -> Option + Send + Sync + 'static, + { let meter_provider = self.meter_provider(meter_provider_type); *meter_provider = std::mem::take(meter_provider).with_view(view); self @@ -172,13 +193,39 @@ impl<'a> MetricsBuilder<'a> { }) } - pub(crate) fn configure_views( - &mut self, - meter_provider_type: MeterProviderType, - ) -> Result<(), BoxError> { - for metric_view in self.metrics_common().views.clone() { - self.with_view(meter_provider_type, metric_view.try_into()?); - } - Ok(()) + pub(crate) fn configure_views(&mut self, meter_provider_type: MeterProviderType) { + let boundaries = self.metrics_common().buckets.clone(); + + // Pre-merge user views with default histogram aggregation + let merged_views: HashMap = self + .metrics_common() + .views + .clone() + .into_iter() + .map(|v| { + let name = v.name.clone(); + let default_view = MetricView::default_histogram(name.clone(), boundaries.clone()); + (name, default_view.merge(v)) + }) + .collect(); + + // Single view that handles both user-configured and default histogram views + self.with_view(meter_provider_type, move |instrument: &Instrument| { + merged_views + .get(instrument.name()) + .cloned() + .map(|view| view.into_stream()) + .or_else(|| { + (instrument.kind() == InstrumentKind::Histogram).then(|| { + Stream::builder() + .with_aggregation(Aggregation::ExplicitBucketHistogram { + boundaries: boundaries.clone(), + record_min_max: true, + }) + .build() + .expect("Failed to create stream for default histogram bucket view") + }) + }) + }); } } diff --git a/apollo-router/src/plugins/telemetry/reload/mod.rs b/apollo-router/src/plugins/telemetry/reload/mod.rs index c46edcb4e6..0065a1a272 100644 --- a/apollo-router/src/plugins/telemetry/reload/mod.rs +++ b/apollo-router/src/plugins/telemetry/reload/mod.rs @@ -48,6 +48,7 @@ pub(crate) mod activation; pub(crate) mod builder; pub(crate) mod metrics; pub(crate) mod otel; +pub(crate) mod rate_limit; pub(crate) mod tracing; /// Prepares telemetry components for activation (Phase 1 of reload lifecycle). diff --git a/apollo-router/src/plugins/telemetry/reload/otel.rs b/apollo-router/src/plugins/telemetry/reload/otel.rs index 62f44f7c26..f18f9902e9 100644 --- a/apollo-router/src/plugins/telemetry/reload/otel.rs +++ b/apollo-router/src/plugins/telemetry/reload/otel.rs @@ -18,6 +18,7 @@ //! - Dynamic attribute layer for request-scoped attributes //! - OpenTelemetry layer for distributed tracing //! - Format layer for structured logging (JSON or text based on TTY) +//! - Rate limiting layer for OpenTelemetry internal log messages //! - Environment filter for log level control //! //! ## Reloading @@ -26,10 +27,12 @@ //! entire subscriber stack, which would require restarting the application. use std::io::IsTerminal; +use std::time::Duration; use anyhow::anyhow; use once_cell::sync::OnceCell; use opentelemetry::Context; +use opentelemetry::InstrumentationScope; use opentelemetry::trace::SpanContext; use opentelemetry::trace::SpanId; use opentelemetry::trace::TraceContextExt; @@ -54,6 +57,7 @@ use crate::plugins::telemetry::formatters::text::Text; use crate::plugins::telemetry::otel; use crate::plugins::telemetry::otel::OpenTelemetryLayer; use crate::plugins::telemetry::otel::PreSampledTracer; +use crate::plugins::telemetry::reload::rate_limit::RateLimitLayer; use crate::plugins::telemetry::tracing::reload::ReloadTracer; use crate::tracer::TraceId; @@ -79,9 +83,8 @@ static FMT_LAYER_HANDLE: OnceCell< pub(crate) fn init_telemetry(log_level: &str) -> anyhow::Result<()> { let hot_tracer = ReloadTracer::new( - opentelemetry_sdk::trace::TracerProvider::default() - .tracer_builder("noop") - .build(), + opentelemetry_sdk::trace::SdkTracerProvider::default() + .tracer_with_scope(InstrumentationScope::builder("noop").build()), ); let opentelemetry_layer = otel::layer().with_tracer(hot_tracer.clone()); @@ -98,7 +101,8 @@ pub(crate) fn init_telemetry(log_level: &str) -> anyhow::Result<()> { OPENTELEMETRY_TRACER_HANDLE .get_or_try_init(move || { // manually filter salsa logs because some of them run at the INFO level https://github.com/salsa-rs/salsa/issues/425 - let log_level = format!("{log_level},salsa=error"); + // filter opentelemetry internal logs to warn level (OTel 0.31 emits INFO logs for provider setup) + let log_level = format!("{log_level},salsa=error,opentelemetry=warn"); tracing::debug!("Running the router with log level set to {log_level}"); // Env filter is separate because of https://github.com/tokio-rs/tracing/issues/1629 // the tracing registry is only created once @@ -107,6 +111,11 @@ pub(crate) fn init_telemetry(log_level: &str) -> anyhow::Result<()> { .with(opentelemetry_layer) .with(fmt_layer) .with(WarnLegacyMetricsLayer) + // Rate limit OpenTelemetry internal log messages to avoid log spam when things go wrong + .with(RateLimitLayer::new( + "opentelemetry", + Duration::from_secs(10), + )) .with(EnvFilter::try_new(log_level)?) .try_init()?; diff --git a/apollo-router/src/plugins/telemetry/reload/rate_limit.rs b/apollo-router/src/plugins/telemetry/reload/rate_limit.rs new file mode 100644 index 0000000000..f328e30dca --- /dev/null +++ b/apollo-router/src/plugins/telemetry/reload/rate_limit.rs @@ -0,0 +1,343 @@ +//! Rate limiting layer for log messages. +//! +//! Some libraries can emit frequent error/warning messages when things go wrong (e.g., export +//! failures, shutdown issues). This module provides a tracing layer that rate-limits these +//! messages to avoid log spam while still surfacing important errors. +//! +//! The rate limiting works by tracking each unique callsite (log location) and only allowing +//! one message per callsite per time window (10 seconds by default). + +use std::time::Duration; +use std::time::Instant; + +use dashmap::DashMap; +use tracing::Subscriber; +use tracing::callsite::Identifier; +use tracing_subscriber::Layer; +use tracing_subscriber::layer::Context; + +/// State for tracking rate limiting of a single callsite. +#[derive(Default)] +struct RateLimitState { + /// Last time a message from this callsite was logged. None means never logged. + last_logged: Option, + /// Count of suppressed messages since last log. + suppressed_count: u64, +} + +/// A tracing layer that rate-limits log messages from a specific target prefix. +/// +/// This layer intercepts events from targets starting with a configurable prefix and applies +/// rate limiting based on callsite. Messages from each unique callsite are allowed through +/// at most once per time window, with suppressed message counts reported periodically. +pub(crate) struct RateLimitLayer { + /// Target prefix to match (e.g., "opentelemetry"). + target_prefix: &'static str, + /// Rate limit states keyed by callsite identifier. + states: DashMap, + /// Time window for rate limiting. + threshold: Duration, +} + +impl RateLimitLayer { + /// Create a new rate limiting layer for the specified target prefix and time window. + pub(crate) fn new(target_prefix: &'static str, threshold: Duration) -> Self { + Self { + target_prefix, + states: DashMap::new(), + threshold, + } + } + + /// Returns the total number of suppressed messages across all callsites. + #[cfg(test)] + fn suppressed_count(&self) -> u64 { + self.states.iter().map(|e| e.suppressed_count).sum() + } + + /// Check if a message should be allowed through. + /// + /// Returns `true` if the message should be logged, `false` if rate limited. + fn is_allowed(&self, callsite: Identifier) -> bool { + let now = Instant::now(); + + let mut entry = self.states.entry(callsite.clone()).or_default(); + let state = entry.value_mut(); + + let allowed = match state.last_logged { + None => true, // First message from this callsite + Some(last) => now.duration_since(last) >= self.threshold, + }; + + if allowed { + state.last_logged = Some(now); + state.suppressed_count = 0; + } else { + state.suppressed_count += 1; + } + + allowed + } +} + +impl Layer for RateLimitLayer +where + S: Subscriber + for<'lookup> tracing_subscriber::registry::LookupSpan<'lookup>, +{ + fn event_enabled(&self, event: &tracing::Event<'_>, _ctx: Context<'_, S>) -> bool { + let metadata = event.metadata(); + + // Only apply rate limiting to matching targets + if !metadata.target().starts_with(self.target_prefix) { + return true; + } + + // Only rate limit WARN and ERROR level messages + // DEBUG/INFO/TRACE are already filtered by log level + if *metadata.level() > tracing::Level::WARN { + return true; + } + + self.is_allowed(metadata.callsite()) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + use std::sync::atomic::AtomicUsize; + use std::sync::atomic::Ordering; + use std::time::Duration; + + use tracing::Subscriber; + use tracing_subscriber::Layer; + use tracing_subscriber::layer::Context; + use tracing_subscriber::layer::SubscriberExt; + + use super::*; + + /// Test layer that counts events. + struct CountingLayer { + count: Arc, + } + + impl Layer for CountingLayer { + fn on_event(&self, _event: &tracing::Event<'_>, _ctx: Context<'_, S>) { + self.count.fetch_add(1, Ordering::Relaxed); + } + } + + #[test] + fn test_rate_limiting_suppresses_rapid_messages() { + let count = Arc::new(AtomicUsize::new(0)); + let rate_limiter = RateLimitLayer::new("opentelemetry", Duration::from_millis(100)); + + let subscriber = tracing_subscriber::registry() + .with(rate_limiter) + .with(CountingLayer { + count: count.clone(), + }); + + tracing::subscriber::with_default(subscriber, || { + // Log multiple messages rapidly + for _ in 0..10 { + tracing::warn!(target: "opentelemetry::trace::exporter", "export failed"); + } + }); + + // Only one message should have gotten through + assert_eq!(count.load(Ordering::Relaxed), 1); + } + + #[test] + fn test_different_callsites_not_suppressed() { + let count = Arc::new(AtomicUsize::new(0)); + let rate_limiter = RateLimitLayer::new("opentelemetry", Duration::from_millis(100)); + + let subscriber = tracing_subscriber::registry() + .with(rate_limiter) + .with(CountingLayer { + count: count.clone(), + }); + + tracing::subscriber::with_default(subscriber, || { + // Each line is a different callsite, so each should get through + tracing::warn!(target: "opentelemetry::trace::exporter", "trace error"); + tracing::warn!(target: "opentelemetry::metrics::exporter", "metric error"); + tracing::warn!(target: "opentelemetry::other", "other error"); + }); + + // Each callsite should log once + assert_eq!(count.load(Ordering::Relaxed), 3); + } + + #[test] + fn test_non_otel_messages_not_affected() { + let count = Arc::new(AtomicUsize::new(0)); + let rate_limiter = RateLimitLayer::new("opentelemetry", Duration::from_millis(100)); + + let subscriber = tracing_subscriber::registry() + .with(rate_limiter) + .with(CountingLayer { + count: count.clone(), + }); + + tracing::subscriber::with_default(subscriber, || { + // Log non-otel messages + for _ in 0..10 { + tracing::warn!(target: "apollo_router", "normal message"); + } + }); + + // All messages should get through + assert_eq!(count.load(Ordering::Relaxed), 10); + } + + #[test] + fn test_messages_allowed_after_threshold() { + let count = Arc::new(AtomicUsize::new(0)); + let rate_limiter = RateLimitLayer::new("opentelemetry", Duration::from_millis(50)); + + let subscriber = tracing_subscriber::registry() + .with(rate_limiter) + .with(CountingLayer { + count: count.clone(), + }); + + // Helper to emit from same callsite + fn emit_otel_warning() { + tracing::warn!(target: "opentelemetry::trace", "message"); + } + + tracing::subscriber::with_default(subscriber, || { + emit_otel_warning(); // First - allowed + emit_otel_warning(); // Second - suppressed (same callsite, within threshold) + + // Wait for threshold + std::thread::sleep(Duration::from_millis(60)); + + emit_otel_warning(); // Third - allowed (threshold elapsed) + }); + + // First and third messages should get through + assert_eq!(count.load(Ordering::Relaxed), 2); + } + + #[test] + fn test_debug_level_not_rate_limited() { + let count = Arc::new(AtomicUsize::new(0)); + let rate_limiter = RateLimitLayer::new("opentelemetry", Duration::from_millis(100)); + + let subscriber = tracing_subscriber::registry() + .with(rate_limiter) + .with(CountingLayer { + count: count.clone(), + }); + + tracing::subscriber::with_default(subscriber, || { + // Debug level messages should not be rate limited + for _ in 0..5 { + tracing::debug!(target: "opentelemetry::trace", "debug message"); + } + }); + + // All debug messages should get through (rate limiting only applies to WARN+) + assert_eq!(count.load(Ordering::Relaxed), 5); + } + + #[test] + fn test_warn_level_is_rate_limited() { + let count = Arc::new(AtomicUsize::new(0)); + let rate_limiter = RateLimitLayer::new("opentelemetry", Duration::from_millis(100)); + + let subscriber = tracing_subscriber::registry() + .with(rate_limiter) + .with(CountingLayer { + count: count.clone(), + }); + + tracing::subscriber::with_default(subscriber, || { + // WARN level should be rate limited + for _ in 0..10 { + tracing::warn!(target: "opentelemetry::trace::exporter", "export warning"); + } + }); + + // Only one message should have gotten through + assert_eq!(count.load(Ordering::Relaxed), 1); + } + + #[test] + fn test_error_level_is_rate_limited() { + let count = Arc::new(AtomicUsize::new(0)); + let rate_limiter = RateLimitLayer::new("opentelemetry", Duration::from_millis(100)); + + let subscriber = tracing_subscriber::registry() + .with(rate_limiter) + .with(CountingLayer { + count: count.clone(), + }); + + tracing::subscriber::with_default(subscriber, || { + // ERROR level should also be rate limited + for _ in 0..10 { + tracing::error!(target: "opentelemetry::trace::exporter", "export error"); + } + }); + + // Only one message should have gotten through + assert_eq!(count.load(Ordering::Relaxed), 1); + } + + #[test] + fn test_info_level_not_rate_limited() { + let count = Arc::new(AtomicUsize::new(0)); + let rate_limiter = RateLimitLayer::new("opentelemetry", Duration::from_millis(100)); + + let subscriber = tracing_subscriber::registry() + .with(rate_limiter) + .with(CountingLayer { + count: count.clone(), + }); + + tracing::subscriber::with_default(subscriber, || { + // INFO level messages should not be rate limited (only WARN and ERROR are) + for _ in 0..5 { + tracing::info!(target: "opentelemetry::trace", "info message"); + } + }); + + // All INFO messages should get through + assert_eq!(count.load(Ordering::Relaxed), 5); + } + + #[test] + fn test_suppression_count_tracked() { + // Test is_allowed directly to verify counting + let rate_limiter = RateLimitLayer::new("opentelemetry", Duration::from_millis(100)); + + // Create a fake callsite identifier for testing + static TEST_CALLSITE: tracing_core::callsite::DefaultCallsite = + tracing_core::callsite::DefaultCallsite::new(&TEST_META); + static TEST_META: tracing_core::Metadata<'static> = tracing_core::metadata! { + name: "test", + target: "opentelemetry::test", + level: tracing_core::Level::WARN, + fields: &[], + callsite: &TEST_CALLSITE, + kind: tracing_core::metadata::Kind::EVENT, + }; + + let callsite = TEST_META.callsite(); + + // First call should be allowed + assert!(rate_limiter.is_allowed(callsite.clone())); + assert_eq!(rate_limiter.suppressed_count(), 0); + + // Subsequent calls within threshold should be suppressed + for i in 1..=9 { + assert!(!rate_limiter.is_allowed(callsite.clone())); + assert_eq!(rate_limiter.suppressed_count(), i); + } + } +} diff --git a/apollo-router/src/plugins/telemetry/reload/tracing.rs b/apollo-router/src/plugins/telemetry/reload/tracing.rs index 3b6f2744dd..31106e6aa4 100644 --- a/apollo-router/src/plugins/telemetry/reload/tracing.rs +++ b/apollo-router/src/plugins/telemetry/reload/tracing.rs @@ -22,8 +22,8 @@ use opentelemetry::propagation::TextMapCompositePropagator; use opentelemetry::propagation::TextMapPropagator; +use opentelemetry_sdk::trace::SdkTracerProvider; use opentelemetry_sdk::trace::SpanProcessor; -use opentelemetry_sdk::trace::TracerProvider; use tower::BoxError; use crate::plugins::telemetry::CustomTraceIdPropagator; @@ -37,16 +37,18 @@ use crate::plugins::telemetry::config_new::spans::Spans; pub(crate) struct TracingBuilder<'a> { common: &'a TracingCommon, spans: &'a Spans, - builder: opentelemetry_sdk::trace::Builder, + builder: opentelemetry_sdk::trace::TracerProviderBuilder, } impl<'a> TracingBuilder<'a> { pub(crate) fn new(config: &'a Conf) -> Self { + let common = &config.exporters.tracing.common; Self { - common: &config.exporters.tracing.common, + common, spans: &config.instrumentation.spans, - builder: opentelemetry_sdk::trace::TracerProvider::builder() - .with_config((&config.exporters.tracing.common).into()), + builder: common.configure_tracer_provider_builder( + opentelemetry_sdk::trace::SdkTracerProvider::builder(), + ), } } @@ -70,7 +72,7 @@ impl<'a> TracingBuilder<'a> { self.builder = builder.with_span_processor(span_processor); } - pub(crate) fn build(self) -> TracerProvider { + pub(crate) fn build(self) -> SdkTracerProvider { self.builder.build() } } diff --git a/apollo-router/src/plugins/telemetry/resource.rs b/apollo-router/src/plugins/telemetry/resource.rs index 338e22d8f2..1c8ebf14d8 100644 --- a/apollo-router/src/plugins/telemetry/resource.rs +++ b/apollo-router/src/plugins/telemetry/resource.rs @@ -1,6 +1,5 @@ use std::collections::BTreeMap; use std::env; -use std::time::Duration; use opentelemetry::KeyValue; use opentelemetry_sdk::Resource; @@ -11,11 +10,29 @@ use crate::plugins::telemetry::config::AttributeValue; const UNKNOWN_SERVICE: &str = "unknown_service"; const OTEL_SERVICE_NAME: &str = "OTEL_SERVICE_NAME"; +/// Resource detector that adds a default service name, +/// which is normally expected to be overwritten by other detectors. +struct DefaultServiceNameDetector; +impl ResourceDetector for DefaultServiceNameDetector { + fn detect(&self) -> Resource { + let executable_name = executable_name(); + let default_service_name = executable_name + .map(|name| format!("{UNKNOWN_SERVICE}:{name}")) + .unwrap_or_else(|| UNKNOWN_SERVICE.to_string()); + Resource::builder_empty() + .with_attribute(KeyValue::new( + opentelemetry_semantic_conventions::resource::SERVICE_NAME, + default_service_name, + )) + .build() + } +} + /// This resource detector fills out things like the default service version and executable name. /// Users can always override them via config. struct StaticResourceDetector; impl ResourceDetector for StaticResourceDetector { - fn detect(&self, _timeout: Duration) -> Resource { + fn detect(&self) -> Resource { let mut config_resources = vec![]; config_resources.push(KeyValue::new( opentelemetry_semantic_conventions::resource::SERVICE_VERSION, @@ -29,20 +46,24 @@ impl ResourceDetector for StaticResourceDetector { executable_name, )); } - Resource::new(config_resources) + Resource::builder_empty() + .with_attributes(config_resources) + .build() } } struct EnvServiceNameDetector; // Used instead of SdkProvidedResourceDetector impl ResourceDetector for EnvServiceNameDetector { - fn detect(&self, _timeout: Duration) -> Resource { + fn detect(&self) -> Resource { match env::var(OTEL_SERVICE_NAME) { - Ok(service_name) if !service_name.is_empty() => Resource::new(vec![KeyValue::new( - opentelemetry_semantic_conventions::resource::SERVICE_NAME, - service_name, - )]), - Ok(_) | Err(_) => Resource::new(vec![]), // return empty resource + Ok(service_name) if !service_name.is_empty() => Resource::builder_empty() + .with_attribute(KeyValue::new( + opentelemetry_semantic_conventions::resource::SERVICE_NAME, + service_name, + )) + .build(), + Ok(_) | Err(_) => Resource::builder_empty().build(), // return empty resource } } } @@ -60,33 +81,15 @@ pub trait ConfigResource { service_namespace: self.service_namespace().clone(), resources: self.resource().clone(), }; - // Last one wins - let resource = Resource::from_detectors( - Duration::from_secs(0), - vec![ - Box::new(StaticResourceDetector), - Box::new(config_resource_detector), - Box::new(EnvResourceDetector::new()), - Box::new(EnvServiceNameDetector), - ], - ); - - // Default service name - if resource - .get(opentelemetry_semantic_conventions::resource::SERVICE_NAME.into()) - .is_none() - { - let executable_name = executable_name(); - resource.merge(&Resource::new(vec![KeyValue::new( - opentelemetry_semantic_conventions::resource::SERVICE_NAME, - executable_name - .map(|executable_name| format!("{UNKNOWN_SERVICE}:{executable_name}")) - .unwrap_or_else(|| UNKNOWN_SERVICE.to_string()), - )])) - } else { - resource - } + let detectors: Vec> = vec![ + Box::new(DefaultServiceNameDetector), + Box::new(StaticResourceDetector), + Box::new(config_resource_detector), + Box::new(EnvResourceDetector::new()), + Box::new(EnvServiceNameDetector), + ]; + Resource::builder_empty().with_detectors(&detectors).build() } } @@ -104,7 +107,7 @@ struct ConfigResourceDetector { } impl ResourceDetector for ConfigResourceDetector { - fn detect(&self, _timeout: Duration) -> Resource { + fn detect(&self) -> Resource { let mut config_resources = vec![]; // For config resources last entry wins @@ -136,7 +139,9 @@ impl ResourceDetector for ConfigResourceDetector { service_name.to_string(), )); } - Resource::new(config_resources) + Resource::builder_empty() + .with_attributes(config_resources) + .build() } } diff --git a/apollo-router/src/plugins/telemetry/tracing/apollo.rs b/apollo-router/src/plugins/telemetry/tracing/apollo.rs index 1e7b67ea62..99cd47f294 100644 --- a/apollo-router/src/plugins/telemetry/tracing/apollo.rs +++ b/apollo-router/src/plugins/telemetry/tracing/apollo.rs @@ -1,5 +1,5 @@ //! Tracing configuration for apollo telemetry. -use opentelemetry_sdk::trace::BatchSpanProcessor; +use opentelemetry_sdk::trace::span_processor_with_async_runtime::BatchSpanProcessor; use serde::Serialize; use tower::BoxError; @@ -7,11 +7,11 @@ use crate::plugins::telemetry::apollo::Config; use crate::plugins::telemetry::apollo::router_id; use crate::plugins::telemetry::apollo_exporter::proto::reports::Trace; use crate::plugins::telemetry::config::Conf; -use crate::plugins::telemetry::error_handler::NamedSpanExporter; -use crate::plugins::telemetry::otel::named_runtime_channel::NamedTokioRuntime; use crate::plugins::telemetry::reload::tracing::TracingBuilder; use crate::plugins::telemetry::reload::tracing::TracingConfigurator; use crate::plugins::telemetry::span_factory::SpanMode; +use crate::plugins::telemetry::tracing::NamedSpanExporter; +use crate::plugins::telemetry::tracing::NamedTokioRuntime; use crate::plugins::telemetry::tracing::apollo_telemetry; impl TracingConfigurator for Config { diff --git a/apollo-router/src/plugins/telemetry/tracing/apollo_telemetry.rs b/apollo-router/src/plugins/telemetry/tracing/apollo_telemetry.rs index 8b153f16d8..ea4481a502 100644 --- a/apollo-router/src/plugins/telemetry/tracing/apollo_telemetry.rs +++ b/apollo-router/src/plugins/telemetry/tracing/apollo_telemetry.rs @@ -3,16 +3,12 @@ use std::collections::HashMap; use std::collections::HashSet; use std::io::Cursor; use std::num::NonZeroUsize; -use std::sync::Arc; use std::time::SystemTime; use std::time::SystemTimeError; -use async_trait::async_trait; use base64::Engine as _; use base64::prelude::BASE64_STANDARD; use derivative::Derivative; -use futures::FutureExt; -use futures::future::BoxFuture; use http::HeaderMap; use http::HeaderValue; use http::header::CACHE_CONTROL; @@ -24,12 +20,13 @@ use opentelemetry::Value; use opentelemetry::trace::SpanId; use opentelemetry::trace::SpanKind; use opentelemetry::trace::Status; -use opentelemetry::trace::TraceError; use opentelemetry::trace::TraceId; use opentelemetry_sdk::Resource; -use opentelemetry_sdk::export::trace::ExportResult; -use opentelemetry_sdk::export::trace::SpanData; -use opentelemetry_sdk::export::trace::SpanExporter; +use opentelemetry_sdk::error::OTelSdkError; +use opentelemetry_sdk::error::OTelSdkResult; +use opentelemetry_sdk::trace::SpanData; +use opentelemetry_sdk::trace::SpanExporter; +use parking_lot::Mutex; use prost::Message; use rand::Rng; use serde::de::DeserializeOwned; @@ -287,7 +284,7 @@ impl LightSpanData { None => value .attributes .into_iter() - .map(|KeyValue { key, value }| (key, value)) + .map(|KeyValue { key, value, .. }| (key, value)) .collect(), Some(attr_names) => value .attributes @@ -349,47 +346,18 @@ impl LightSpanData { #[derive(Derivative)] #[derivative(Debug)] pub(crate) struct Exporter { - spans_by_parent_id: LruCache>, + span_cache: Mutex, /// An externally updateable gauge for "apollo.router.exporter.span.lru.size". span_lru_size_instrument: LruSizeInstrument, #[derivative(Debug = "ignore")] - report_exporter: Option>, + report_exporter: Option, #[derivative(Debug = "ignore")] otlp_exporter: Option, otlp_tracing_ratio: f64, - field_execution_weight: f64, - errors_configuration: ErrorsConfiguration, - use_legacy_request_span: bool, - include_span_names: HashSet<&'static str>, include_attr_names: Option>, include_attr_event_names: Option>, } -#[derive(Debug)] -enum TreeData { - Request(Result, Error>), - SubscriptionEvent(Result, Error>), - Router { - http: Box, - client_name: Option, - client_version: Option, - duration_ns: u64, - }, - Supergraph { - operation_signature: String, - operation_name: String, - variables_json: HashMap, - limits: Option, - }, - QueryPlanNode(QueryPlanNode), - DeferPrimary(DeferNodePrimary), - DeferDeferred(DeferredNode), - ConditionIf(Option), - ConditionElse(Option), - Execution(String), - Trace(Option, Error>>), -} - #[buildstructor::buildstructor] impl Exporter { #[builder] @@ -425,11 +393,23 @@ impl Exporter { let span_lru_size_instrument = LruSizeInstrument::new("apollo.router.exporter.span.lru.size"); - Ok(Self { + let span_cache = SpanCache { spans_by_parent_id: LruCache::new(buffer_size), + field_execution_weight: match field_execution_sampler { + SamplerOption::Always(Sampler::AlwaysOn) => 1.0, + SamplerOption::Always(Sampler::AlwaysOff) => 0.0, + SamplerOption::TraceIdRatioBased(ratio) => 1.0 / ratio, + }, + use_legacy_request_span: use_legacy_request_span.unwrap_or_default(), + include_span_names: REPORTS_INCLUDE_SPANS.into(), + errors_configuration: errors_configuration.clone(), + }; + + Ok(Self { + span_cache: Mutex::new(span_cache), span_lru_size_instrument, report_exporter: if otlp_tracing_ratio < 1f64 { - Some(Arc::new(ApolloExporter::new( + Some(ApolloExporter::new( endpoint, &batch_processor_config.into(), apollo_key, @@ -437,7 +417,7 @@ impl Exporter { schema_id, router_id, metrics_reference_mode, - )?)) + )?) } else { None }, @@ -455,14 +435,6 @@ impl Exporter { None }, otlp_tracing_ratio, - field_execution_weight: match field_execution_sampler { - SamplerOption::Always(Sampler::AlwaysOn) => 1.0, - SamplerOption::Always(Sampler::AlwaysOff) => 0.0, - SamplerOption::TraceIdRatioBased(ratio) => 1.0 / ratio, - }, - errors_configuration: errors_configuration.clone(), - use_legacy_request_span: use_legacy_request_span.unwrap_or_default(), - include_span_names: REPORTS_INCLUDE_SPANS.into(), include_attr_names: if otlp_tracing_ratio > 0f64 { Some(HashSet::from_iter( [&REPORTS_INCLUDE_ATTRS[..], &OTLP_EXT_INCLUDE_ATTRS[..]].concat(), @@ -477,6 +449,179 @@ impl Exporter { }, }) } +} + +impl SpanExporter for Exporter { + /// Export spans to apollo telemetry + async fn export(&self, batch: Vec) -> OTelSdkResult { + // Exporting to apollo means that we must have complete trace as the entire trace must be built. + // We do what we can, and if there are any traces that are not complete then we keep them for the next export event. + // We may get spans that simply don't complete. These need to be cleaned up after a period. It's the price of using ftv1. + let mut traces: Vec<(String, proto::reports::Trace)> = Vec::new(); + let mut otlp_trace_spans: Vec> = Vec::new(); + + // Decide whether to send via OTLP or reports proto based on the sampling config. Roll dice if using a percentage rollout. + let send_otlp = self.otlp_exporter.is_some() + && rand::rng().random_range(0.0..1.0) < self.otlp_tracing_ratio; + let send_reports = self.report_exporter.is_some() && !send_otlp; + + { + let mut span_cache = self.span_cache.lock(); + for span in batch { + if span.name == SUBSCRIPTION_EVENT_SPAN_NAME + || span + .attributes + .iter() + .any(|kv| kv.key == APOLLO_PRIVATE_REQUEST) + { + let root_span: LightSpanData = LightSpanData::from_span_data( + span, + &self.include_attr_names, + &self.include_attr_event_names, + ); + if send_otlp { + let grouped_trace_spans = span_cache.group_by_trace(root_span); + if let Some(trace) = self + .otlp_exporter + .as_ref() + .expect("otlp exporter required") + .prepare_for_export(grouped_trace_spans) + { + otlp_trace_spans.push(trace); + } + } else if send_reports { + match span_cache.extract_traces(root_span) { + Ok(extracted_traces) => { + for mut trace in extracted_traces { + let mut operation_signature = Default::default(); + std::mem::swap(&mut trace.signature, &mut operation_signature); + if !operation_signature.is_empty() { + traces.push((operation_signature, trace)); + } + } + } + Err(Error::MultipleErrors(errors)) => { + tracing::error!( + "failed to construct trace: {}, skipping", + Error::MultipleErrors(errors) + ); + } + Err(error) => { + tracing::error!("failed to construct trace: {}, skipping", error); + } + } + } + } else if span.parent_span_id != SpanId::INVALID { + // Not a root span, we may need it later so stash it. + span_cache.insert(LightSpanData::from_span_data( + span, + &self.include_attr_names, + &self.include_attr_event_names, + )); + } + } + + // Note this won't be correct anymore if there is any way outside of `.export()` + // to affect the size of the cache. + self.span_lru_size_instrument + .update(span_cache.len() as u64); + } + + if send_otlp && !otlp_trace_spans.is_empty() { + self.otlp_exporter + .as_ref() + .expect("expected an otel exporter") + .export(otlp_trace_spans.into_iter().flatten().collect()) + .await + } else if send_reports && !traces.is_empty() { + let mut report = telemetry::apollo::Report::default(); + report += SingleReport::Traces(TracesReport { traces }); + self.report_exporter + .as_ref() + .expect("expected an apollo exporter") + .submit_report(report) + .await + .map_err(|e| OTelSdkError::InternalFailure(e.to_string())) + } else { + Ok(()) + } + } + + fn shutdown_with_timeout(&mut self, timeout: std::time::Duration) -> OTelSdkResult { + // Currently only handled in the OTLP case. + if let Some(exporter) = &mut self.otlp_exporter { + exporter.shutdown_with_timeout(timeout) + } else { + Ok(()) + } + } + + fn force_flush(&mut self) -> OTelSdkResult { + Ok(()) + } + + fn set_resource(&mut self, _resource: &Resource) { + // This is intentionally a NOOP. The reason for this is that we do not allow users to set the resource attributes + // for telemetry that is sent to Apollo. To do so would expose potential private information that the user did not intend for us. + } +} + +#[derive(Debug)] +enum TreeData { + Request(Result, Error>), + SubscriptionEvent(Result, Error>), + Router { + http: Box, + client_name: Option, + client_version: Option, + duration_ns: u64, + }, + Supergraph { + operation_signature: String, + operation_name: String, + variables_json: HashMap, + limits: Option, + }, + QueryPlanNode(QueryPlanNode), + DeferPrimary(DeferNodePrimary), + DeferDeferred(DeferredNode), + ConditionIf(Option), + ConditionElse(Option), + Execution(String), + Trace(Option, Error>>), +} + +/// Accumulate span data so we can build full trace reports for Apollo Studio telemetry once a +/// trace is complete. +/// +/// Normally we'd send spans off to APMs whenever we can, and the APM builds the full trace +/// progressively, but the Apollo backend doesn't do this. +#[derive(Debug)] +struct SpanCache { + spans_by_parent_id: LruCache>, + field_execution_weight: f64, + use_legacy_request_span: bool, + include_span_names: HashSet<&'static str>, + // We have a reference to error configuration here to do last-minute redaction of subgraph + // errors (yeah...) + errors_configuration: ErrorsConfiguration, +} + +impl SpanCache { + fn insert(&mut self, span: LightSpanData) { + // This is sad, but with LRU there is no `get_insert_mut` so a double lookup is required + // It is safe to expect the entry to exist as we just inserted it, however capacity of the LRU must not be 0. + let len = self + .spans_by_parent_id + .get_or_insert(span.parent_span_id, || { + LruCache::new(NonZeroUsize::new(50).unwrap()) + }) + .len(); + self.spans_by_parent_id + .get_mut(&span.parent_span_id) + .expect("capacity of cache was zero") + .push(len, span); + } fn extract_root_traces( &mut self, @@ -915,6 +1060,11 @@ impl Exporter { _ => child_nodes, }) } + + /// Returns the size of the span LRU cache. + fn len(&self) -> usize { + self.spans_by_parent_id.len() + } } fn extract_limits(span: &LightSpanData) -> Limits { @@ -1175,133 +1325,6 @@ fn extract_http_data(span: &LightSpanData) -> (Http, Option) { ) } -#[async_trait] -impl SpanExporter for Exporter { - /// Export spans to apollo telemetry - fn export(&mut self, batch: Vec) -> BoxFuture<'static, ExportResult> { - // Exporting to apollo means that we must have complete trace as the entire trace must be built. - // We do what we can, and if there are any traces that are not complete then we keep them for the next export event. - // We may get spans that simply don't complete. These need to be cleaned up after a period. It's the price of using ftv1. - let mut traces: Vec<(String, proto::reports::Trace)> = Vec::new(); - let mut otlp_trace_spans: Vec> = Vec::new(); - - // Decide whether to send via OTLP or reports proto based on the sampling config. Roll dice if using a percentage rollout. - let send_otlp = self.otlp_exporter.is_some() - && rand::rng().random_range(0.0..1.0) < self.otlp_tracing_ratio; - let send_reports = self.report_exporter.is_some() && !send_otlp; - - for span in batch { - if span.name == SUBSCRIPTION_EVENT_SPAN_NAME - || span - .attributes - .iter() - .any(|kv| kv.key == APOLLO_PRIVATE_REQUEST) - { - let root_span: LightSpanData = LightSpanData::from_span_data( - span, - &self.include_attr_names, - &self.include_attr_event_names, - ); - if send_otlp { - let grouped_trace_spans = self.group_by_trace(root_span); - if let Some(trace) = self - .otlp_exporter - .as_ref() - .expect("otlp exporter required") - .prepare_for_export(grouped_trace_spans) - { - otlp_trace_spans.push(trace); - } - } else if send_reports { - match self.extract_traces(root_span) { - Ok(extracted_traces) => { - for mut trace in extracted_traces { - let mut operation_signature = Default::default(); - std::mem::swap(&mut trace.signature, &mut operation_signature); - if !operation_signature.is_empty() { - traces.push((operation_signature, trace)); - } - } - } - Err(Error::MultipleErrors(errors)) => { - tracing::error!( - "failed to construct trace: {}, skipping", - Error::MultipleErrors(errors) - ); - } - Err(error) => { - tracing::error!("failed to construct trace: {}, skipping", error); - } - } - } - } else if span.parent_span_id != SpanId::INVALID { - // Not a root span, we may need it later so stash it. - - // This is sad, but with LRU there is no `get_insert_mut` so a double lookup is required - // It is safe to expect the entry to exist as we just inserted it, however capacity of the LRU must not be 0. - let len = self - .spans_by_parent_id - .get_or_insert(span.parent_span_id, || { - LruCache::new(NonZeroUsize::new(50).unwrap()) - }) - .len(); - self.spans_by_parent_id - .get_mut(&span.parent_span_id) - .expect("capacity of cache was zero") - .push( - len, - LightSpanData::from_span_data( - span, - &self.include_attr_names, - &self.include_attr_event_names, - ), - ); - } - } - - // Note this won't be correct anymore if there is any way outside of `.export()` - // to affect the size of the cache. - self.span_lru_size_instrument - .update(self.spans_by_parent_id.len() as u64); - - if send_otlp && !otlp_trace_spans.is_empty() { - self.otlp_exporter - .as_mut() - .expect("expected an otel exporter") - .export(otlp_trace_spans.into_iter().flatten().collect()) - } else if send_reports && !traces.is_empty() { - let mut report = telemetry::apollo::Report::default(); - report += SingleReport::Traces(TracesReport { traces }); - let exporter = self - .report_exporter - .as_ref() - .expect("expected an apollo exporter") - .clone(); - async move { - exporter - .submit_report(report) - .await - .map_err(|e| TraceError::ExportFailed(Box::new(e))) - } - .boxed() - } else { - async { ExportResult::Ok(()) }.boxed() - } - } - - fn shutdown(&mut self) { - // Currently only handled in the OTLP case. - if let Some(exporter) = &mut self.otlp_exporter { - exporter.shutdown() - }; - } - - fn set_resource(&mut self, _resource: &Resource) { - // This is intentionally a NOOP. The reason for this is that we do not allow users to set the resource attributes - // for telemetry that is sent to Apollo. To do so would expose potential private information that the user did not intend for us. - } -} - trait ChildNodes { fn remove_first_query_plan_node(&mut self) -> Option; fn remove_query_plan_nodes(&mut self) -> Vec; @@ -1813,8 +1836,8 @@ mod test { #[test] fn test_extract_limits() { let mut span = LightSpanData { - trace_id: TraceId::from_u128(0), - span_id: SpanId::from_u64(1), + trace_id: TraceId::from(0), + span_id: SpanId::from(1), parent_span_id: SpanId::INVALID, span_kind: SpanKind::Client, name: Default::default(), @@ -1857,8 +1880,8 @@ mod test { #[test] fn test_extract_cache_control() { let mut span = LightSpanData { - trace_id: TraceId::from_u128(0), - span_id: SpanId::from_u64(1), + trace_id: TraceId::from(0), + span_id: SpanId::from(1), parent_span_id: SpanId::INVALID, span_kind: SpanKind::Client, name: Default::default(), diff --git a/apollo-router/src/plugins/telemetry/tracing/datadog/agent_sampling.rs b/apollo-router/src/plugins/telemetry/tracing/datadog/agent_sampling.rs index 9b523c4b39..24cef9ddb4 100644 --- a/apollo-router/src/plugins/telemetry/tracing/datadog/agent_sampling.rs +++ b/apollo-router/src/plugins/telemetry/tracing/datadog/agent_sampling.rs @@ -154,7 +154,7 @@ mod tests { let result = datadog_sampler.should_sample( None, - TraceId::from_u128(1), + TraceId::from(1), "test_span", &SpanKind::Internal, &[], @@ -191,7 +191,7 @@ mod tests { let result = datadog_sampler.should_sample( None, - TraceId::from_u128(1), + TraceId::from(1), "test_span", &SpanKind::Internal, &[], @@ -228,7 +228,7 @@ mod tests { let result = datadog_sampler.should_sample( None, - TraceId::from_u128(1), + TraceId::from(1), "test_span", &SpanKind::Internal, &[], @@ -266,7 +266,7 @@ mod tests { let result = datadog_sampler.should_sample( Some(&Context::new()), - TraceId::from_u128(1), + TraceId::from(1), "test_span", &SpanKind::Internal, &[], @@ -304,13 +304,13 @@ mod tests { let result = datadog_sampler.should_sample( Some(&Context::new().with_remote_span_context(SpanContext::new( - TraceId::from_u128(1), - SpanId::from_u64(1), + TraceId::from(1), + SpanId::from(1), TraceFlags::SAMPLED, true, TraceState::default().with_priority_sampling(SamplingPriority::UserReject), ))), - TraceId::from_u128(1), + TraceId::from(1), "test_span", &SpanKind::Internal, &[], @@ -348,13 +348,13 @@ mod tests { let result = datadog_sampler.should_sample( Some(&Context::new().with_remote_span_context(SpanContext::new( - TraceId::from_u128(1), - SpanId::from_u64(1), + TraceId::from(1), + SpanId::from(1), TraceFlags::default(), true, TraceState::default().with_priority_sampling(SamplingPriority::UserReject), ))), - TraceId::from_u128(1), + TraceId::from(1), "test_span", &SpanKind::Internal, &[], diff --git a/apollo-router/src/plugins/telemetry/tracing/datadog/mod.rs b/apollo-router/src/plugins/telemetry/tracing/datadog/mod.rs index 8633e5bf8f..4baa475c52 100644 --- a/apollo-router/src/plugins/telemetry/tracing/datadog/mod.rs +++ b/apollo-router/src/plugins/telemetry/tracing/datadog/mod.rs @@ -10,7 +10,6 @@ use std::time::Duration; pub(crate) use agent_sampling::DatadogAgentSampling; use ahash::HashMap; use ahash::HashMapExt; -use futures::future::BoxFuture; use http::Uri; use opentelemetry::Key; use opentelemetry::KeyValue; @@ -18,9 +17,10 @@ use opentelemetry::Value; use opentelemetry::trace::SpanContext; use opentelemetry::trace::SpanKind; use opentelemetry_sdk::Resource; -use opentelemetry_sdk::export::trace::ExportResult; -use opentelemetry_sdk::export::trace::SpanData; -use opentelemetry_sdk::export::trace::SpanExporter; +use opentelemetry_sdk::error::OTelSdkResult; +use opentelemetry_sdk::trace::SpanData; +use opentelemetry_sdk::trace::SpanExporter; +use opentelemetry_sdk::trace::span_processor_with_async_runtime::BatchSpanProcessor; use opentelemetry_semantic_conventions::resource::SERVICE_NAME; use opentelemetry_semantic_conventions::resource::SERVICE_VERSION; use schemars::JsonSchema; @@ -40,11 +40,12 @@ use crate::plugins::telemetry::consts::SUBGRAPH_REQUEST_SPAN_NAME; use crate::plugins::telemetry::consts::SUBGRAPH_SPAN_NAME; use crate::plugins::telemetry::consts::SUPERGRAPH_SPAN_NAME; use crate::plugins::telemetry::endpoint::UriEndpoint; -use crate::plugins::telemetry::error_handler::NamedSpanExporter; -use crate::plugins::telemetry::otel::named_runtime_channel::NamedTokioRuntime; use crate::plugins::telemetry::reload::tracing::TracingBuilder; use crate::plugins::telemetry::reload::tracing::TracingConfigurator; +use crate::plugins::telemetry::resource::ConfigResource; use crate::plugins::telemetry::tracing::BatchProcessorConfig; +use crate::plugins::telemetry::tracing::NamedSpanExporter; +use crate::plugins::telemetry::tracing::NamedTokioRuntime; use crate::plugins::telemetry::tracing::SpanProcessorExt; use crate::plugins::telemetry::tracing::datadog_exporter; use crate::plugins::telemetry::tracing::datadog_exporter::DatadogTraceState; @@ -66,6 +67,10 @@ fn default_resource_mappings() -> HashMap { const ENV_KEY: Key = Key::from_static_str("env"); const DEFAULT_ENDPOINT: &str = "http://127.0.0.1:8126"; +const DD_TRACE_AGENT_URL: &str = "DD_TRACE_AGENT_URL"; +const DD_AGENT_HOST: &str = "DD_AGENT_HOST"; +const DD_TRACE_AGENT_PORT: &str = "DD_TRACE_AGENT_PORT"; + #[derive(Debug, Clone, Deserialize, JsonSchema, serde_derive_default::Default, PartialEq)] #[serde(deny_unknown_fields)] #[schemars(rename = "DatadogConfig")] @@ -117,6 +122,37 @@ fn default_true() -> bool { true } +impl Config { + /// Apply environment variable overrides for the endpoint. + /// Supports `DD_TRACE_AGENT_URL`, or `DD_AGENT_HOST` + `DD_TRACE_AGENT_PORT`. + fn endpoint_with_env_override(&self) -> Result { + // DD_TRACE_AGENT_URL takes precedence + if let Ok(url) = std::env::var(DD_TRACE_AGENT_URL) { + return url.parse::().map_err(|e| { + format!("invalid URI in {}: '{}': {}", DD_TRACE_AGENT_URL, url, e).into() + }); + } + + // Fall back to DD_AGENT_HOST + DD_TRACE_AGENT_PORT + if let Ok(host) = std::env::var(DD_AGENT_HOST) { + let port = std::env::var(DD_TRACE_AGENT_PORT).unwrap_or_else(|_| "8126".to_string()); + let url = format!("http://{}:{}", host, port); + return url.parse::().map_err(|e| { + format!( + "invalid URI from {} and {}: '{}': {}", + DD_AGENT_HOST, DD_TRACE_AGENT_PORT, url, e + ) + .into() + }); + } + + // Fall back to config + Ok(self + .endpoint + .to_full_uri(&Uri::from_static(DEFAULT_ENDPOINT))) + } +} + impl TracingConfigurator for Config { fn config(conf: &Conf) -> &Self { &conf.exporters.tracing.datadog @@ -128,7 +164,7 @@ impl TracingConfigurator for Config { fn configure(&self, builder: &mut TracingBuilder) -> Result<(), BoxError> { tracing::info!("Configuring Datadog tracing: {}", self.batch_processor); - let common: opentelemetry_sdk::trace::Config = builder.tracing_common().into(); + let resource = builder.tracing_common().to_resource(); // Precompute representation otel Keys for the mappings so that we don't do heap allocation for each span let resource_mappings = self.enable_span_mapping.then(|| { @@ -141,9 +177,7 @@ impl TracingConfigurator for Config { }); let fixed_span_names = self.fixed_span_names; - let endpoint = &self - .endpoint - .to_full_uri(&Uri::from_static(DEFAULT_ENDPOINT)); + let endpoint = self.endpoint_with_env_override()?; let exporter = datadog_exporter::new_pipeline() .with_agent_endpoint(endpoint.to_string().trim_end_matches('/')) @@ -163,6 +197,7 @@ impl TracingConfigurator for Config { && let Some(KeyValue { key: _, value: Value::String(v), + .. }) = span.attributes.iter().find(|kv| kv.key == *mapping) { return v.as_str(); @@ -188,20 +223,19 @@ impl TracingConfigurator for Config { &span.name }) .with( - &common.resource.get(SERVICE_NAME.into()), + &resource.get(&SERVICE_NAME.into()), |builder, service_name| { // Datadog exporter incorrectly ignores the service name in the resource // Set it explicitly here builder.with_service_name(service_name.as_str()) }, ) - .with(&common.resource.get(ENV_KEY), |builder, env| { + .with(&resource.get(&ENV_KEY), |builder, env| { builder.with_env(env.as_str()) }) .with_version( - common - .resource - .get(SERVICE_VERSION.into()) + resource + .get(&SERVICE_VERSION.into()) .expect("cargo version is set as a resource default;qed") .to_string(), ) @@ -212,7 +246,6 @@ impl TracingConfigurator for Config { .pool_idle_timeout(Duration::from_millis(1)) .build()?, ) - .with_trace_config(common) .build_exporter()?; // Use the default span metrics and override with the ones from the config @@ -225,13 +258,11 @@ impl TracingConfigurator for Config { }; let named_exporter = NamedSpanExporter::new(wrapper, "datadog"); - let batch_processor = opentelemetry_sdk::trace::BatchSpanProcessor::builder( - named_exporter, - NamedTokioRuntime::new("datadog-tracing"), - ) - .with_batch_config(self.batch_processor.clone().into()) - .build() - .filtered(); + let batch_processor = + BatchSpanProcessor::builder(named_exporter, NamedTokioRuntime::new("datadog-tracing")) + .with_batch_config(self.batch_processor.clone().with_env_overrides()?.into()) + .build() + .filtered(); if builder .tracing_common() @@ -258,7 +289,10 @@ impl Debug for ExporterWrapper { } impl SpanExporter for ExporterWrapper { - fn export(&mut self, mut batch: Vec) -> BoxFuture<'static, ExportResult> { + fn export( + &self, + mut batch: Vec, + ) -> impl std::future::Future + Send { // Here we do some special processing of the spans before passing them to the delegate // In particular we default the span.kind to the span kind, and also override the trace measure status if we need to. for span in &mut batch { @@ -303,10 +337,10 @@ impl SpanExporter for ExporterWrapper { } self.delegate.export(batch) } - fn shutdown(&mut self) { + fn shutdown(&mut self) -> OTelSdkResult { self.delegate.shutdown() } - fn force_flush(&mut self) -> BoxFuture<'static, ExportResult> { + fn force_flush(&mut self) -> OTelSdkResult { self.delegate.force_flush() } fn set_resource(&mut self, resource: &Resource) { diff --git a/apollo-router/src/plugins/telemetry/tracing/datadog/span_processor.rs b/apollo-router/src/plugins/telemetry/tracing/datadog/span_processor.rs index e362ca967c..fe4cc4d006 100644 --- a/apollo-router/src/plugins/telemetry/tracing/datadog/span_processor.rs +++ b/apollo-router/src/plugins/telemetry/tracing/datadog/span_processor.rs @@ -1,9 +1,11 @@ +use std::time::Duration; + use opentelemetry::Context; use opentelemetry::trace::SpanContext; -use opentelemetry::trace::TraceResult; use opentelemetry_sdk::Resource; -use opentelemetry_sdk::export::trace::SpanData; +use opentelemetry_sdk::error::OTelSdkResult; use opentelemetry_sdk::trace::Span; +use opentelemetry_sdk::trace::SpanData; use opentelemetry_sdk::trace::SpanProcessor; /// When using the Datadog agent we need spans to always be exported. However, the batch span processor will only export spans that are sampled. @@ -39,12 +41,12 @@ impl SpanProcessor for DatadogSpanProcessor { self.delegate.on_end(span) } - fn force_flush(&self) -> TraceResult<()> { + fn force_flush(&self) -> OTelSdkResult { self.delegate.force_flush() } - fn shutdown(&self) -> TraceResult<()> { - self.delegate.shutdown() + fn shutdown_with_timeout(&self, timeout: Duration) -> OTelSdkResult { + self.delegate.shutdown_with_timeout(timeout) } fn set_resource(&mut self, resource: &Resource) { @@ -55,6 +57,7 @@ impl SpanProcessor for DatadogSpanProcessor { #[cfg(test)] mod tests { use std::sync::Arc; + use std::time::Duration; use std::time::SystemTime; use opentelemetry::Context; @@ -62,6 +65,7 @@ mod tests { use opentelemetry::trace::SpanKind; use opentelemetry::trace::TraceFlags; use opentelemetry::trace::TraceId; + use opentelemetry_sdk::error::OTelSdkResult; use opentelemetry_sdk::trace::SpanEvents; use opentelemetry_sdk::trace::SpanLinks; use opentelemetry_sdk::trace::SpanProcessor; @@ -89,11 +93,11 @@ mod tests { self.spans.lock().push(span); } - fn force_flush(&self) -> TraceResult<()> { + fn force_flush(&self) -> OTelSdkResult { Ok(()) } - fn shutdown(&self) -> TraceResult<()> { + fn shutdown_with_timeout(&self, _timeout: Duration) -> OTelSdkResult { Ok(()) } } @@ -103,15 +107,16 @@ mod tests { let mock_processor = MockSpanProcessor::new(); let processor = DatadogSpanProcessor::new(mock_processor.clone()); let span_context = SpanContext::new( - TraceId::from_u128(1), - SpanId::from_u64(1), + TraceId::from(1), + SpanId::from(1), TraceFlags::default(), false, Default::default(), ); let span_data = SpanData { span_context, - parent_span_id: SpanId::from_u64(1), + parent_span_id: SpanId::from(1), + parent_span_is_remote: false, span_kind: SpanKind::Client, name: Default::default(), start_time: SystemTime::now(), @@ -120,7 +125,7 @@ mod tests { events: SpanEvents::default(), links: SpanLinks::default(), status: Default::default(), - instrumentation_lib: Default::default(), + instrumentation_scope: Default::default(), dropped_attributes_count: 0, }; diff --git a/apollo-router/src/plugins/telemetry/tracing/datadog_exporter/exporter/intern.rs b/apollo-router/src/plugins/telemetry/tracing/datadog_exporter/exporter/intern.rs index d63fb9a42e..3c47fbb84f 100644 --- a/apollo-router/src/plugins/telemetry/tracing/datadog_exporter/exporter/intern.rs +++ b/apollo-router/src/plugins/telemetry/tracing/datadog_exporter/exporter/intern.rs @@ -34,7 +34,9 @@ impl Hash for InternValue<'_> { } } opentelemetry::Array::String(x) => x.hash(state), + _ => unreachable!("unexpected opentelemetry::Array variant"), }, + _ => unreachable!("unexpected opentelemetry::Value variant"), }, } } @@ -108,7 +110,9 @@ impl InternValue<'_> { opentelemetry::Array::String(x) => { Self::write_generic_array(payload, reusable_buffer, x) } + _ => unreachable!("unexpected opentelemetry::Array variant"), }, + _ => unreachable!("unexpected opentelemetry::Value variant"), }, } } diff --git a/apollo-router/src/plugins/telemetry/tracing/datadog_exporter/exporter/mod.rs b/apollo-router/src/plugins/telemetry/tracing/datadog_exporter/exporter/mod.rs index 5bace8a3f9..80e4e09665 100644 --- a/apollo-router/src/plugins/telemetry/tracing/datadog_exporter/exporter/mod.rs +++ b/apollo-router/src/plugins/telemetry/tracing/datadog_exporter/exporter/mod.rs @@ -1,31 +1,23 @@ mod intern; mod model; -use std::borrow::Cow; use std::fmt::Debug; use std::fmt::Formatter; use std::sync::Arc; -use std::time::Duration; -use futures::future::BoxFuture; +use bytes::Bytes; pub use model::ApiVersion; pub use model::Error; pub use model::FieldMappingFn; -use opentelemetry::KeyValue; -use opentelemetry::global; -use opentelemetry::trace::TraceError; -use opentelemetry::trace::TracerProvider; use opentelemetry_http::HttpClient; use opentelemetry_http::ResponseExt; use opentelemetry_sdk::Resource; -use opentelemetry_sdk::export::trace::ExportResult; -use opentelemetry_sdk::export::trace::SpanData; -use opentelemetry_sdk::export::trace::SpanExporter; +use opentelemetry_sdk::error::OTelSdkError; +use opentelemetry_sdk::error::OTelSdkResult; use opentelemetry_sdk::resource::ResourceDetector; use opentelemetry_sdk::resource::SdkProvidedResourceDetector; -use opentelemetry_sdk::runtime::RuntimeChannel; -use opentelemetry_sdk::trace::Config; -use opentelemetry_sdk::trace::Tracer; +use opentelemetry_sdk::trace::SpanData; +use opentelemetry_sdk::trace::SpanExporter; use opentelemetry_semantic_conventions as semcov; use url::Url; @@ -97,10 +89,7 @@ impl DatadogExporter { } } - fn build_request( - &self, - mut batch: Vec, - ) -> Result>, TraceError> { + fn build_request(&self, mut batch: Vec) -> Result>, Error> { let traces: Vec<&[SpanData]> = group_into_traces(&mut batch); let trace_count = traces.len(); let data = self.api_version.encode( @@ -152,7 +141,6 @@ pub fn new_pipeline() -> DatadogPipelineBuilder { /// Builder for `ExporterConfig` struct. pub struct DatadogPipelineBuilder { agent_endpoint: String, - trace_config: Option, api_version: ApiVersion, client: Option>, mapping: Mapping, @@ -163,7 +151,6 @@ impl Default for DatadogPipelineBuilder { fn default() -> Self { DatadogPipelineBuilder { agent_endpoint: DEFAULT_AGENT_ENDPOINT.to_string(), - trace_config: None, mapping: Mapping::empty(), api_version: ApiVersion::Version05, unified_tags: UnifiedTags::new(), @@ -176,7 +163,6 @@ impl Debug for DatadogPipelineBuilder { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.debug_struct("DatadogExporter") .field("agent_endpoint", &self.agent_endpoint) - .field("trace_config", &self.trace_config) .field("client", &self.client) .field("resource_mapping", &mapping_debug(&self.mapping.resource)) .field("name_mapping", &mapping_debug(&self.mapping.name)) @@ -192,43 +178,24 @@ impl DatadogPipelineBuilder { /// Building a new exporter. /// /// This is useful if you are manually constructing a pipeline. - pub fn build_exporter(mut self) -> Result { - let (_, service_name) = self.build_config_and_service_name(); + pub fn build_exporter(self) -> Result { + let service_name = self.get_service_name(); self.build_exporter_with_service_name(service_name) } - fn build_config_and_service_name(&mut self) -> (Config, String) { - let service_name = self.unified_tags.service(); - if let Some(service_name) = service_name { - let config = if let Some(mut cfg) = self.trace_config.take() { - cfg.resource = Cow::Owned(Resource::new( - cfg.resource - .iter() - .filter(|(k, _v)| k.as_str() != semcov::resource::SERVICE_NAME) - .map(|(k, v)| KeyValue::new(k.clone(), v.clone())), - )); - cfg - } else { - Config::default().with_resource(Resource::empty()) - }; - (config, service_name) - } else { - let service_name = SdkProvidedResourceDetector - .detect(Duration::from_secs(0)) - .get(semcov::resource::SERVICE_NAME.into()) - .unwrap() - .to_string(); - ( - // use a empty resource to prevent TracerProvider to assign a service name. - Config::default().with_resource(Resource::empty()), - service_name, - ) - } + fn get_service_name(&self) -> String { + self.unified_tags.service().unwrap_or_else(|| { + SdkProvidedResourceDetector + .detect() + .get(&semcov::resource::SERVICE_NAME.into()) + .map(|v| v.to_string()) + .unwrap_or_else(|| "unknown_service".to_string()) + }) } // parse the endpoint and append the path based on versions. // keep the query and host the same. - fn build_endpoint(agent_endpoint: &str, version: &str) -> Result { + fn build_endpoint(agent_endpoint: &str, version: &str) -> Result { // build agent endpoint based on version let mut endpoint = agent_endpoint .parse::() @@ -242,13 +209,13 @@ impl DatadogPipelineBuilder { let path_str = paths.join("/"); endpoint.set_path(path_str.as_str()); - Ok(endpoint.as_str().parse().map_err::(Into::into)?) + endpoint.as_str().parse().map_err::(Into::into) } fn build_exporter_with_service_name( self, service_name: String, - ) -> Result { + ) -> Result { if let Some(client) = self.client { let model_config = ModelConfig { service_name }; @@ -262,45 +229,10 @@ impl DatadogPipelineBuilder { ); Ok(exporter) } else { - Err(Error::NoHttpClient.into()) + Err(Error::NoHttpClient) } } - /// Install the Datadog trace exporter pipeline using a simple span processor. - pub fn install_simple(mut self) -> Result { - let (config, service_name) = self.build_config_and_service_name(); - let exporter = self.build_exporter_with_service_name(service_name)?; - let mut provider_builder = - opentelemetry_sdk::trace::TracerProvider::builder().with_simple_exporter(exporter); - provider_builder = provider_builder.with_config(config); - let provider = provider_builder.build(); - let tracer = provider - .tracer_builder("opentelemetry-datadog") - .with_version(env!("CARGO_PKG_VERSION")) - .with_schema_url(semcov::SCHEMA_URL) - .build(); - let _ = global::set_tracer_provider(provider); - Ok(tracer) - } - - /// Install the Datadog trace exporter pipeline using a batch span processor with the specified - /// runtime. - pub fn install_batch(mut self, runtime: R) -> Result { - let (config, service_name) = self.build_config_and_service_name(); - let exporter = self.build_exporter_with_service_name(service_name)?; - let mut provider_builder = opentelemetry_sdk::trace::TracerProvider::builder() - .with_batch_exporter(exporter, runtime); - provider_builder = provider_builder.with_config(config); - let provider = provider_builder.build(); - let tracer = provider - .tracer_builder("opentelemetry-datadog") - .with_version(env!("CARGO_PKG_VERSION")) - .with_schema_url(semcov::SCHEMA_URL) - .build(); - let _ = global::set_tracer_provider(provider); - Ok(tracer) - } - /// Assign the service name under which to group traces pub fn with_service_name>(mut self, service_name: T) -> Self { self.unified_tags.set_service(Some(service_name.into())); @@ -333,12 +265,6 @@ impl DatadogPipelineBuilder { self } - /// Assign the SDK trace configuration - pub fn with_trace_config(mut self, config: Config) -> Self { - self.trace_config = Some(config); - self - } - /// Set version of Datadog trace ingestion API pub fn with_api_version(mut self, api_version: ApiVersion) -> Self { self.api_version = api_version; @@ -402,21 +328,31 @@ fn group_into_traces(spans: &mut [SpanData]) -> Vec<&[SpanData]> { async fn send_request( client: Arc, request: http::Request>, -) -> ExportResult { - let _ = client.send(request).await?.error_for_status()?; +) -> OTelSdkResult { + let (parts, body) = request.into_parts(); + let request = http::Request::from_parts(parts, Bytes::from(body)); + client + .send_bytes(request) + .await + .map_err(|e| OTelSdkError::InternalFailure(format!("datadog http error: {}", e)))? + .error_for_status() + .map_err(|e| OTelSdkError::InternalFailure(format!("datadog http error: {}", e)))?; Ok(()) } impl SpanExporter for DatadogExporter { /// Export spans to datadog-agent - fn export(&mut self, batch: Vec) -> BoxFuture<'static, ExportResult> { - let request = match self.build_request(batch) { - Ok(req) => req, - Err(err) => return Box::pin(std::future::ready(Err(err))), - }; - + fn export( + &self, + batch: Vec, + ) -> impl std::future::Future + Send { + let request = self.build_request(batch); let client = self.client.clone(); - Box::pin(send_request(client, request)) + async move { + let request = request + .map_err(|e| OTelSdkError::InternalFailure(format!("datadog error: {}", e)))?; + send_request(client, request).await + } } fn set_resource(&mut self, resource: &Resource) { @@ -444,6 +380,11 @@ fn mapping_debug(f: &Option) -> String { #[cfg(test)] mod tests { + use bytes::Bytes; + use http::Request; + use http::Response; + use opentelemetry_http::HttpError; + use super::*; use crate::plugins::telemetry::tracing::datadog_exporter::ApiVersion::Version05; use crate::plugins::telemetry::tracing::datadog_exporter::exporter::model::tests::get_span; @@ -498,10 +439,7 @@ mod tests { #[async_trait::async_trait] impl HttpClient for DummyClient { - async fn send( - &self, - _request: http::Request>, - ) -> Result, opentelemetry_http::HttpError> { + async fn send_bytes(&self, _request: Request) -> Result, HttpError> { Ok(http::Response::new("dummy response".into())) } } @@ -513,22 +451,4 @@ mod tests { .build_exporter() .unwrap(); } - - #[test] - fn test_install_simple() { - new_pipeline() - .with_service_name("test_service") - .with_http_client(DummyClient) - .install_simple() - .unwrap(); - } - - #[test] - fn test_install_batch() { - new_pipeline() - .with_service_name("test_service") - .with_http_client(DummyClient) - .install_batch(opentelemetry_sdk::runtime::AsyncStd {}) - .unwrap(); - } } diff --git a/apollo-router/src/plugins/telemetry/tracing/datadog_exporter/exporter/model/mod.rs b/apollo-router/src/plugins/telemetry/tracing/datadog_exporter/exporter/model/mod.rs index dfd3649dd1..e2d102afa5 100644 --- a/apollo-router/src/plugins/telemetry/tracing/datadog_exporter/exporter/model/mod.rs +++ b/apollo-router/src/plugins/telemetry/tracing/datadog_exporter/exporter/model/mod.rs @@ -2,9 +2,7 @@ use std::fmt::Debug; use http::uri; use opentelemetry_sdk::Resource; -use opentelemetry_sdk::export::ExportError; -use opentelemetry_sdk::export::trace::SpanData; -use opentelemetry_sdk::export::trace::{self}; +use opentelemetry_sdk::trace::SpanData; use url::ParseError; use self::unified_tags::UnifiedTags; @@ -68,7 +66,7 @@ fn default_service_name_mapping<'a>(_span: &'a SpanData, config: &'a ModelConfig } fn default_name_mapping<'a>(span: &'a SpanData, _config: &'a ModelConfig) -> &'a str { - span.instrumentation_lib.name.as_ref() + span.instrumentation_scope.name() } fn default_resource_mapping<'a>(span: &'a SpanData, _config: &'a ModelConfig) -> &'a str { @@ -98,12 +96,6 @@ pub enum Error { Other(String), } -impl ExportError for Error { - fn exporter_name(&self) -> &'static str { - "datadog" - } -} - impl From for Error { fn from(_: rmp::encode::ValueWriteError) -> Self { Self::MessagePackError @@ -150,7 +142,7 @@ impl ApiVersion { pub(crate) fn encode( self, model_config: &ModelConfig, - traces: Vec<&[trace::SpanData]>, + traces: Vec<&[SpanData]>, mapping: &Mapping, unified_tags: &UnifiedTags, resource: Option<&Resource>, @@ -201,6 +193,7 @@ pub(crate) mod tests { use std::time::SystemTime; use base64::Engine; + use opentelemetry::InstrumentationScope; use opentelemetry::KeyValue; use opentelemetry::trace::SpanContext; use opentelemetry::trace::SpanId; @@ -209,21 +202,19 @@ pub(crate) mod tests { use opentelemetry::trace::TraceFlags; use opentelemetry::trace::TraceId; use opentelemetry::trace::TraceState; - use opentelemetry_sdk::InstrumentationLibrary; use opentelemetry_sdk::trace::SpanEvents; use opentelemetry_sdk::trace::SpanLinks; - use opentelemetry_sdk::{self}; use super::*; - fn get_traces() -> Vec> { + fn get_traces() -> Vec> { vec![vec![get_span(7, 1, 99)]] } - pub(crate) fn get_span(trace_id: u128, parent_span_id: u64, span_id: u64) -> trace::SpanData { + pub(crate) fn get_span(trace_id: u128, parent_span_id: u64, span_id: u64) -> SpanData { let span_context = SpanContext::new( - TraceId::from_u128(trace_id), - SpanId::from_u64(span_id), + TraceId::from_bytes(trace_id.to_be_bytes()), + SpanId::from_bytes(span_id.to_be_bytes()), TraceFlags::default(), false, TraceState::default(), @@ -236,11 +227,12 @@ pub(crate) mod tests { KeyValue::new("span.type", "web"), KeyValue::new("host.name", "test"), ]; - let instrumentation_lib = InstrumentationLibrary::builder("component").build(); + let instrumentation_scope = InstrumentationScope::builder("component").build(); - trace::SpanData { + SpanData { span_context, - parent_span_id: SpanId::from_u64(parent_span_id), + parent_span_id: SpanId::from_bytes(parent_span_id.to_be_bytes()), + parent_span_is_remote: false, span_kind: SpanKind::Client, name: "resource".into(), start_time, @@ -249,7 +241,7 @@ pub(crate) mod tests { events: SpanEvents::default(), links: SpanLinks::default(), status: Status::Ok, - instrumentation_lib, + instrumentation_scope, dropped_attributes_count: 0, } } diff --git a/apollo-router/src/plugins/telemetry/tracing/datadog_exporter/exporter/model/v03.rs b/apollo-router/src/plugins/telemetry/tracing/datadog_exporter/exporter/model/v03.rs index e29a4b9c00..1d6612ba8f 100644 --- a/apollo-router/src/plugins/telemetry/tracing/datadog_exporter/exporter/model/v03.rs +++ b/apollo-router/src/plugins/telemetry/tracing/datadog_exporter/exporter/model/v03.rs @@ -3,7 +3,7 @@ use std::time::SystemTime; use opentelemetry::KeyValue; use opentelemetry::trace::Status; use opentelemetry_sdk::Resource; -use opentelemetry_sdk::export::trace::SpanData; +use opentelemetry_sdk::trace::SpanData; use crate::plugins::telemetry::tracing::datadog_exporter::Error; use crate::plugins::telemetry::tracing::datadog_exporter::ModelConfig; @@ -111,7 +111,7 @@ where rmp::encode::write_str(&mut encoded, value.as_str().as_ref())?; } } - for KeyValue { key, value } in span.attributes.iter() { + for KeyValue { key, value, .. } in span.attributes.iter() { rmp::encode::write_str(&mut encoded, key.as_str())?; rmp::encode::write_str(&mut encoded, value.as_str().as_ref())?; } diff --git a/apollo-router/src/plugins/telemetry/tracing/datadog_exporter/exporter/model/v05.rs b/apollo-router/src/plugins/telemetry/tracing/datadog_exporter/exporter/model/v05.rs index 5bd8f24e0e..664de43aac 100644 --- a/apollo-router/src/plugins/telemetry/tracing/datadog_exporter/exporter/model/v05.rs +++ b/apollo-router/src/plugins/telemetry/tracing/datadog_exporter/exporter/model/v05.rs @@ -3,7 +3,7 @@ use std::time::SystemTime; use opentelemetry::KeyValue; use opentelemetry::trace::Status; use opentelemetry_sdk::Resource; -use opentelemetry_sdk::export::trace::SpanData; +use opentelemetry_sdk::trace::SpanData; use super::unified_tags::UnifiedTagField; use super::unified_tags::UnifiedTags; @@ -198,7 +198,7 @@ where .unwrap_or(0); let mut span_type = interner.intern(""); - for KeyValue { key, value } in &span.attributes { + for KeyValue { key, value, .. } in &span.attributes { if key.as_str() == "span.type" { span_type = interner.intern_value(value); break; @@ -253,7 +253,7 @@ where write_unified_tags(&mut encoded, interner, unified_tags)?; - for KeyValue { key, value } in span.attributes.iter() { + for KeyValue { key, value, .. } in span.attributes.iter() { rmp::encode::write_u32(&mut encoded, interner.intern(key.as_str()))?; rmp::encode::write_u32(&mut encoded, interner.intern_value(value))?; } diff --git a/apollo-router/src/plugins/telemetry/tracing/datadog_exporter/mod.rs b/apollo-router/src/plugins/telemetry/tracing/datadog_exporter/mod.rs index f2d5c21aef..b487f5b930 100644 --- a/apollo-router/src/plugins/telemetry/tracing/datadog_exporter/mod.rs +++ b/apollo-router/src/plugins/telemetry/tracing/datadog_exporter/mod.rs @@ -491,12 +491,12 @@ pub(crate) mod propagator { (vec![], SpanContext::empty_context()), (vec![(DATADOG_SAMPLING_PRIORITY_HEADER, "0")], SpanContext::empty_context()), (vec![(DATADOG_TRACE_ID_HEADER, "garbage")], SpanContext::empty_context()), - (vec![(DATADOG_TRACE_ID_HEADER, "1234"), (DATADOG_PARENT_ID_HEADER, "garbage")], SpanContext::new(TraceId::from_u128(1234), SpanId::INVALID, TRACE_FLAG_DEFERRED, true, TraceState::default())), - (vec![(DATADOG_TRACE_ID_HEADER, "1234"), (DATADOG_PARENT_ID_HEADER, "12")], SpanContext::new(TraceId::from_u128(1234), SpanId::from_u64(12), TRACE_FLAG_DEFERRED, true, TraceState::default())), - (vec![(DATADOG_TRACE_ID_HEADER, "1234"), (DATADOG_PARENT_ID_HEADER, "12"), (DATADOG_SAMPLING_PRIORITY_HEADER, "-1")], SpanContext::new(TraceId::from_u128(1234), SpanId::from_u64(12), TraceFlags::default(), true, DatadogTraceStateBuilder::default().with_priority_sampling(SamplingPriority::UserReject).build())), - (vec![(DATADOG_TRACE_ID_HEADER, "1234"), (DATADOG_PARENT_ID_HEADER, "12"), (DATADOG_SAMPLING_PRIORITY_HEADER, "0")], SpanContext::new(TraceId::from_u128(1234), SpanId::from_u64(12), TraceFlags::default(), true, DatadogTraceStateBuilder::default().with_priority_sampling(SamplingPriority::AutoReject).build())), - (vec![(DATADOG_TRACE_ID_HEADER, "1234"), (DATADOG_PARENT_ID_HEADER, "12"), (DATADOG_SAMPLING_PRIORITY_HEADER, "1")], SpanContext::new(TraceId::from_u128(1234), SpanId::from_u64(12), TraceFlags::SAMPLED, true, DatadogTraceStateBuilder::default().with_priority_sampling(SamplingPriority::AutoKeep).build())), - (vec![(DATADOG_TRACE_ID_HEADER, "1234"), (DATADOG_PARENT_ID_HEADER, "12"), (DATADOG_SAMPLING_PRIORITY_HEADER, "2")], SpanContext::new(TraceId::from_u128(1234), SpanId::from_u64(12), TraceFlags::SAMPLED, true, DatadogTraceStateBuilder::default().with_priority_sampling(SamplingPriority::UserKeep).build())), + (vec![(DATADOG_TRACE_ID_HEADER, "1234"), (DATADOG_PARENT_ID_HEADER, "garbage")], SpanContext::new(TraceId::from_bytes(1234u128.to_be_bytes()), SpanId::INVALID, TRACE_FLAG_DEFERRED, true, TraceState::default())), + (vec![(DATADOG_TRACE_ID_HEADER, "1234"), (DATADOG_PARENT_ID_HEADER, "12")], SpanContext::new(TraceId::from_bytes(1234u128.to_be_bytes()), SpanId::from_bytes(12u64.to_be_bytes()), TRACE_FLAG_DEFERRED, true, TraceState::default())), + (vec![(DATADOG_TRACE_ID_HEADER, "1234"), (DATADOG_PARENT_ID_HEADER, "12"), (DATADOG_SAMPLING_PRIORITY_HEADER, "-1")], SpanContext::new(TraceId::from_bytes(1234u128.to_be_bytes()), SpanId::from_bytes(12u64.to_be_bytes()), TraceFlags::default(), true, DatadogTraceStateBuilder::default().with_priority_sampling(SamplingPriority::UserReject).build())), + (vec![(DATADOG_TRACE_ID_HEADER, "1234"), (DATADOG_PARENT_ID_HEADER, "12"), (DATADOG_SAMPLING_PRIORITY_HEADER, "0")], SpanContext::new(TraceId::from_bytes(1234u128.to_be_bytes()), SpanId::from_bytes(12u64.to_be_bytes()), TraceFlags::default(), true, DatadogTraceStateBuilder::default().with_priority_sampling(SamplingPriority::AutoReject).build())), + (vec![(DATADOG_TRACE_ID_HEADER, "1234"), (DATADOG_PARENT_ID_HEADER, "12"), (DATADOG_SAMPLING_PRIORITY_HEADER, "1")], SpanContext::new(TraceId::from_bytes(1234u128.to_be_bytes()), SpanId::from_bytes(12u64.to_be_bytes()), TraceFlags::SAMPLED, true, DatadogTraceStateBuilder::default().with_priority_sampling(SamplingPriority::AutoKeep).build())), + (vec![(DATADOG_TRACE_ID_HEADER, "1234"), (DATADOG_PARENT_ID_HEADER, "12"), (DATADOG_SAMPLING_PRIORITY_HEADER, "2")], SpanContext::new(TraceId::from_bytes(1234u128.to_be_bytes()), SpanId::from_bytes(12u64.to_be_bytes()), TraceFlags::SAMPLED, true, DatadogTraceStateBuilder::default().with_priority_sampling(SamplingPriority::UserKeep).build())), ] } @@ -507,11 +507,11 @@ pub(crate) mod propagator { (vec![], SpanContext::new(TraceId::INVALID, SpanId::INVALID, TRACE_FLAG_DEFERRED, true, TraceState::default())), (vec![], SpanContext::new(TraceId::from_hex("1234").unwrap(), SpanId::INVALID, TRACE_FLAG_DEFERRED, true, TraceState::default())), (vec![], SpanContext::new(TraceId::from_hex("1234").unwrap(), SpanId::INVALID, TraceFlags::SAMPLED, true, TraceState::default())), - (vec![(DATADOG_TRACE_ID_HEADER, "1234"), (DATADOG_PARENT_ID_HEADER, "12")], SpanContext::new(TraceId::from_u128(1234), SpanId::from_u64(12), TRACE_FLAG_DEFERRED, true, TraceState::default())), - (vec![(DATADOG_TRACE_ID_HEADER, "1234"), (DATADOG_PARENT_ID_HEADER, "12"), (DATADOG_SAMPLING_PRIORITY_HEADER, "-1")], SpanContext::new(TraceId::from_u128(1234), SpanId::from_u64(12), TraceFlags::default(), true, DatadogTraceStateBuilder::default().with_priority_sampling(SamplingPriority::UserReject).build())), - (vec![(DATADOG_TRACE_ID_HEADER, "1234"), (DATADOG_PARENT_ID_HEADER, "12"), (DATADOG_SAMPLING_PRIORITY_HEADER, "0")], SpanContext::new(TraceId::from_u128(1234), SpanId::from_u64(12), TraceFlags::default(), true, DatadogTraceStateBuilder::default().with_priority_sampling(SamplingPriority::AutoReject).build())), - (vec![(DATADOG_TRACE_ID_HEADER, "1234"), (DATADOG_PARENT_ID_HEADER, "12"), (DATADOG_SAMPLING_PRIORITY_HEADER, "1")], SpanContext::new(TraceId::from_u128(1234), SpanId::from_u64(12), TraceFlags::SAMPLED, true, DatadogTraceStateBuilder::default().with_priority_sampling(SamplingPriority::AutoKeep).build())), - (vec![(DATADOG_TRACE_ID_HEADER, "1234"), (DATADOG_PARENT_ID_HEADER, "12"), (DATADOG_SAMPLING_PRIORITY_HEADER, "2")], SpanContext::new(TraceId::from_u128(1234), SpanId::from_u64(12), TraceFlags::SAMPLED, true, DatadogTraceStateBuilder::default().with_priority_sampling(SamplingPriority::UserKeep).build())), + (vec![(DATADOG_TRACE_ID_HEADER, "1234"), (DATADOG_PARENT_ID_HEADER, "12")], SpanContext::new(TraceId::from_bytes(1234u128.to_be_bytes()), SpanId::from_bytes(12u64.to_be_bytes()), TRACE_FLAG_DEFERRED, true, TraceState::default())), + (vec![(DATADOG_TRACE_ID_HEADER, "1234"), (DATADOG_PARENT_ID_HEADER, "12"), (DATADOG_SAMPLING_PRIORITY_HEADER, "-1")], SpanContext::new(TraceId::from_bytes(1234u128.to_be_bytes()), SpanId::from_bytes(12u64.to_be_bytes()), TraceFlags::default(), true, DatadogTraceStateBuilder::default().with_priority_sampling(SamplingPriority::UserReject).build())), + (vec![(DATADOG_TRACE_ID_HEADER, "1234"), (DATADOG_PARENT_ID_HEADER, "12"), (DATADOG_SAMPLING_PRIORITY_HEADER, "0")], SpanContext::new(TraceId::from_bytes(1234u128.to_be_bytes()), SpanId::from_bytes(12u64.to_be_bytes()), TraceFlags::default(), true, DatadogTraceStateBuilder::default().with_priority_sampling(SamplingPriority::AutoReject).build())), + (vec![(DATADOG_TRACE_ID_HEADER, "1234"), (DATADOG_PARENT_ID_HEADER, "12"), (DATADOG_SAMPLING_PRIORITY_HEADER, "1")], SpanContext::new(TraceId::from_bytes(1234u128.to_be_bytes()), SpanId::from_bytes(12u64.to_be_bytes()), TraceFlags::SAMPLED, true, DatadogTraceStateBuilder::default().with_priority_sampling(SamplingPriority::AutoKeep).build())), + (vec![(DATADOG_TRACE_ID_HEADER, "1234"), (DATADOG_PARENT_ID_HEADER, "12"), (DATADOG_SAMPLING_PRIORITY_HEADER, "2")], SpanContext::new(TraceId::from_bytes(1234u128.to_be_bytes()), SpanId::from_bytes(12u64.to_be_bytes()), TraceFlags::SAMPLED, true, DatadogTraceStateBuilder::default().with_priority_sampling(SamplingPriority::UserKeep).build())), ] } diff --git a/apollo-router/src/plugins/telemetry/tracing/mod.rs b/apollo-router/src/plugins/telemetry/tracing/mod.rs index c689bf8d47..01d2ecad3c 100644 --- a/apollo-router/src/plugins/telemetry/tracing/mod.rs +++ b/apollo-router/src/plugins/telemetry/tracing/mod.rs @@ -3,15 +3,16 @@ use std::fmt::Formatter; use std::time::Duration; use opentelemetry::Context; -use opentelemetry::trace::TraceResult; use opentelemetry_sdk::Resource; -use opentelemetry_sdk::export::trace::SpanData; +use opentelemetry_sdk::error::OTelSdkResult; use opentelemetry_sdk::trace::BatchConfig; use opentelemetry_sdk::trace::BatchConfigBuilder; use opentelemetry_sdk::trace::Span; +use opentelemetry_sdk::trace::SpanData; use opentelemetry_sdk::trace::SpanProcessor; use schemars::JsonSchema; use serde::Deserialize; +use tower::BoxError; use super::formatters::APOLLO_CONNECTOR_PREFIX; use super::formatters::APOLLO_PRIVATE_PREFIX; @@ -22,10 +23,14 @@ pub(crate) mod apollo_telemetry; pub(crate) mod datadog; #[allow(unreachable_pub, dead_code)] pub(crate) mod datadog_exporter; +mod named; pub(crate) mod otlp; pub(crate) mod reload; pub(crate) mod zipkin; +pub(crate) use named::NamedSpanExporter; +pub(crate) use named::NamedTokioRuntime; + #[derive(Debug)] struct ApolloFilterSpanProcessor { delegate: T, @@ -59,12 +64,12 @@ impl SpanProcessor for ApolloFilterSpanProcessor { } } - fn force_flush(&self) -> TraceResult<()> { + fn force_flush(&self) -> OTelSdkResult { self.delegate.force_flush() } - fn shutdown(&self) -> TraceResult<()> { - self.delegate.shutdown() + fn shutdown_with_timeout(&self, timeout: Duration) -> OTelSdkResult { + self.delegate.shutdown_with_timeout(timeout) } fn set_resource(&mut self, resource: &Resource) { @@ -150,14 +155,70 @@ fn max_concurrent_exports_default() -> usize { 1 } +impl BatchProcessorConfig { + /// Apply OTEL_BSP_* environment variable overrides to this config. + /// This should be used for third-party exporters (OTLP, Datadog, Zipkin) + /// but NOT for Apollo exporters. + pub(crate) fn with_env_overrides(self) -> Result { + Ok(BatchProcessorConfig { + scheduled_delay: Self::parse_duration_env( + "OTEL_BSP_SCHEDULE_DELAY", + self.scheduled_delay, + )?, + max_queue_size: Self::parse_usize_env("OTEL_BSP_MAX_QUEUE_SIZE", self.max_queue_size)?, + max_export_batch_size: Self::parse_usize_env( + "OTEL_BSP_MAX_EXPORT_BATCH_SIZE", + self.max_export_batch_size, + )?, + max_export_timeout: Self::parse_duration_env( + "OTEL_BSP_EXPORT_TIMEOUT", + self.max_export_timeout, + )?, + max_concurrent_exports: Self::parse_usize_env( + "OTEL_BSP_MAX_CONCURRENT_EXPORTS", + self.max_concurrent_exports, + )?, + }) + } + + fn parse_duration_env(var: &str, default: Duration) -> Result { + match std::env::var(var) { + Ok(value) => { + let millis = value.parse::().map_err(|e| { + format!( + "invalid value '{}' for {}, expected milliseconds: {}", + value, var, e + ) + })?; + Ok(Duration::from_millis(millis)) + } + Err(_) => Ok(default), + } + } + + fn parse_usize_env(var: &str, default: usize) -> Result { + match std::env::var(var) { + Ok(value) => value.parse::().map_err(|e| { + format!( + "invalid value '{}' for {}, expected integer: {}", + value, var, e + ) + .into() + }), + Err(_) => Ok(default), + } + } +} + impl From for BatchConfig { fn from(config: BatchProcessorConfig) -> Self { BatchConfigBuilder::default() .with_scheduled_delay(config.scheduled_delay) .with_max_queue_size(config.max_queue_size) .with_max_export_batch_size(config.max_export_batch_size) - .with_max_export_timeout(config.max_export_timeout) + // Concurrent exports and export timeout require experimental_trace_batch_span_processor_with_async_runtime feature .with_max_concurrent_exports(config.max_concurrent_exports) + .with_max_export_timeout(config.max_export_timeout) .build() } } diff --git a/apollo-router/src/plugins/telemetry/otel/named_runtime_channel.rs b/apollo-router/src/plugins/telemetry/tracing/named.rs similarity index 54% rename from apollo-router/src/plugins/telemetry/otel/named_runtime_channel.rs rename to apollo-router/src/plugins/telemetry/tracing/named.rs index c49b76e332..3770150204 100644 --- a/apollo-router/src/plugins/telemetry/otel/named_runtime_channel.rs +++ b/apollo-router/src/plugins/telemetry/tracing/named.rs @@ -1,43 +1,94 @@ +//! Named wrappers for OpenTelemetry components. +//! +//! This module provides wrappers that add exporter name context to errors and metrics: +//! - `NamedSpanExporter`: Prefixes export error messages with exporter name +//! - `NamedTokioRuntime`: Emits metrics when batch processor channel operations fail + use std::fmt::Debug; +use std::future::Future; use std::time::Duration; -use futures::future::BoxFuture; +use opentelemetry_sdk::error::OTelSdkError; +use opentelemetry_sdk::error::OTelSdkResult; use opentelemetry_sdk::runtime::Runtime; use opentelemetry_sdk::runtime::RuntimeChannel; use opentelemetry_sdk::runtime::Tokio; use opentelemetry_sdk::runtime::TrySend; use opentelemetry_sdk::runtime::TrySendError; +use opentelemetry_sdk::trace::SpanData; +use opentelemetry_sdk::trace::SpanExporter; + +/// Wrapper that modifies trace export errors to include exporter name. +pub(crate) struct NamedSpanExporter { + name: &'static str, + inner: E, +} + +impl NamedSpanExporter { + pub(crate) fn new(inner: E, name: &'static str) -> Self { + Self { name, inner } + } +} + +impl Debug for NamedSpanExporter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("NamedSpanExporter") + .field("name", &self.name) + .finish() + } +} + +impl SpanExporter for NamedSpanExporter { + fn export( + &self, + batch: Vec, + ) -> impl std::future::Future + Send { + let name = self.name; + let fut = self.inner.export(batch); + async move { + fut.await + .map_err(|err| OTelSdkError::InternalFailure(format!("[{} traces] {}", name, err))) + } + } -/// Wraps an otel tokio runtime to provide a name in the error messages and metrics + fn shutdown(&mut self) -> OTelSdkResult { + self.inner.shutdown() + } + + fn force_flush(&mut self) -> OTelSdkResult { + self.inner.force_flush() + } + + fn set_resource(&mut self, resource: &opentelemetry_sdk::Resource) { + self.inner.set_resource(resource) + } +} + +/// Wraps the Tokio runtime to emit metrics when batch processor channel operations fail. +/// +/// This enables the `apollo.router.telemetry.batch_processor.errors` metric to be +/// emitted with the exporter name when spans are dropped due to a full or closed channel. #[derive(Debug, Clone)] pub(crate) struct NamedTokioRuntime { name: &'static str, - parent: Tokio, } impl NamedTokioRuntime { pub(crate) fn new(name: &'static str) -> Self { - Self { - name, - parent: Tokio, - } + Self { name } } } impl Runtime for NamedTokioRuntime { - type Interval = ::Interval; - type Delay = ::Delay; - - fn interval(&self, duration: Duration) -> Self::Interval { - self.parent.interval(duration) + fn spawn(&self, future: F) + where + F: Future + Send + 'static, + { + Tokio.spawn(future) } - fn spawn(&self, future: BoxFuture<'static, ()>) { - self.parent.spawn(future) - } - - fn delay(&self, duration: Duration) -> Self::Delay { - self.parent.delay(duration) + fn delay(&self, duration: Duration) -> impl Future + Send + 'static { + Tokio.delay(duration) } } @@ -57,6 +108,7 @@ impl RuntimeChannel for NamedTokioRuntime { } } +/// A channel sender that emits metrics when send operations fail. #[derive(Debug)] pub(crate) struct NamedSender { name: &'static str, @@ -66,8 +118,8 @@ pub(crate) struct NamedSender { } impl NamedSender { - fn new(name: &'static str, sender: tokio::sync::mpsc::Sender) -> NamedSender { - NamedSender { + fn new(name: &'static str, sender: tokio::sync::mpsc::Sender) -> Self { + Self { name, channel_full_message: format!( "cannot send message to batch processor '{name}' as the channel is full" @@ -84,7 +136,6 @@ impl TrySend for NamedSender { type Message = T; fn try_send(&self, item: Self::Message) -> Result<(), TrySendError> { - // Convert the error into something that has a name self.sender.try_send(item).map_err(|err| { let error = match &err { tokio::sync::mpsc::error::TrySendError::Full(_) => "channel full", @@ -109,16 +160,57 @@ impl TrySend for NamedSender { }) } } + #[cfg(test)] mod tests { + use opentelemetry_sdk::error::OTelSdkError; + use opentelemetry_sdk::error::OTelSdkResult; + use opentelemetry_sdk::trace::SpanData; + use opentelemetry_sdk::trace::SpanExporter; + use super::*; use crate::metrics::FutureMetricsExt; + #[derive(Debug)] + struct FailingSpanExporter; + + impl SpanExporter for FailingSpanExporter { + async fn export(&self, _batch: Vec) -> OTelSdkResult { + Err(OTelSdkError::InternalFailure( + "connection failed".to_string(), + )) + } + + fn shutdown(&mut self) -> OTelSdkResult { + Ok(()) + } + + fn force_flush(&mut self) -> OTelSdkResult { + Ok(()) + } + + fn set_resource(&mut self, _resource: &opentelemetry_sdk::Resource) {} + } + + #[tokio::test] + async fn test_named_span_exporter_adds_prefix() { + let inner = FailingSpanExporter; + let named = NamedSpanExporter::new(inner, "test-exporter"); + + let result = named.export(vec![]).await; + + assert!(result.is_err()); + let err = result.unwrap_err(); + let err_msg = err.to_string(); + assert!(err_msg.contains("[test-exporter traces]")); + assert!(err_msg.contains("connection failed")); + } + #[tokio::test] - async fn test_channel_full_error_metrics() { + async fn test_named_runtime_channel_full_emits_metric() { async { let runtime = NamedTokioRuntime::new("test_processor"); - let (sender, mut _receiver) = runtime.batch_message_channel(1); + let (sender, _receiver) = runtime.batch_message_channel::<&str>(1); // Fill the channel sender.try_send("first").expect("should send first message"); @@ -139,10 +231,10 @@ mod tests { } #[tokio::test] - async fn test_channel_closed_error_metrics() { + async fn test_named_runtime_channel_closed_emits_metric() { async { let runtime = NamedTokioRuntime::new("test_processor"); - let (sender, receiver) = runtime.batch_message_channel(1); + let (sender, receiver) = runtime.batch_message_channel::<&str>(1); // Drop receiver to close channel drop(receiver); @@ -162,10 +254,10 @@ mod tests { } #[tokio::test] - async fn test_successful_message_send() { + async fn test_named_runtime_successful_send_no_metric() { async { let runtime = NamedTokioRuntime::new("test_processor"); - let (sender, _receiver) = runtime.batch_message_channel(1); + let (sender, _receiver) = runtime.batch_message_channel::<&str>(1); let result = sender.try_send("message"); assert!(result.is_ok()); diff --git a/apollo-router/src/plugins/telemetry/tracing/otlp.rs b/apollo-router/src/plugins/telemetry/tracing/otlp.rs index ea90c0f69d..380f27394f 100644 --- a/apollo-router/src/plugins/telemetry/tracing/otlp.rs +++ b/apollo-router/src/plugins/telemetry/tracing/otlp.rs @@ -1,16 +1,23 @@ //! Configuration for Otlp tracing. use std::result::Result; -use opentelemetry_otlp::SpanExporterBuilder; -use opentelemetry_sdk::trace::BatchSpanProcessor; +use http::Uri; +use opentelemetry_otlp::WithExportConfig; +use opentelemetry_otlp::WithHttpConfig; +use opentelemetry_otlp::WithTonicConfig; +use opentelemetry_sdk::trace::span_processor_with_async_runtime::BatchSpanProcessor; +use tonic::metadata::MetadataMap; use tower::BoxError; use crate::plugins::telemetry::config::Conf; -use crate::plugins::telemetry::error_handler::NamedSpanExporter; -use crate::plugins::telemetry::otel::named_runtime_channel::NamedTokioRuntime; +use crate::plugins::telemetry::otlp::Config; +use crate::plugins::telemetry::otlp::Protocol; use crate::plugins::telemetry::otlp::TelemetryDataKind; +use crate::plugins::telemetry::otlp::process_endpoint; use crate::plugins::telemetry::reload::tracing::TracingBuilder; use crate::plugins::telemetry::reload::tracing::TracingConfigurator; +use crate::plugins::telemetry::tracing::NamedSpanExporter; +use crate::plugins::telemetry::tracing::NamedTokioRuntime; use crate::plugins::telemetry::tracing::SpanProcessorExt; impl TracingConfigurator for super::super::otlp::Config { @@ -23,11 +30,14 @@ impl TracingConfigurator for super::super::otlp::Config { } fn configure(&self, builder: &mut TracingBuilder) -> Result<(), BoxError> { - let exporter: SpanExporterBuilder = self.exporter(TelemetryDataKind::Traces)?; - let named_exporter = NamedSpanExporter::new(exporter.build_span_exporter()?, "otlp"); + // Apply env var overrides to the config + let config = self.clone().with_tracing_env_overrides()?; + + let exporter = config.build_span_exporter()?; + let named_exporter = NamedSpanExporter::new(exporter, "otlp"); let batch_span_processor = BatchSpanProcessor::builder(named_exporter, NamedTokioRuntime::new("otlp-tracing")) - .with_batch_config(self.batch_processor.clone().into()) + .with_batch_config(config.batch_processor.clone().with_env_overrides()?.into()) .build() .filtered(); @@ -44,3 +54,55 @@ impl TracingConfigurator for super::super::otlp::Config { Ok(()) } } + +impl Config { + pub(crate) fn build_span_exporter(&self) -> Result { + match self.protocol { + Protocol::Grpc => self.build_grpc_span_exporter(), + Protocol::Http => self.build_http_span_exporter(), + } + } + + fn build_grpc_span_exporter(&self) -> Result { + let endpoint_opt = + process_endpoint(&self.endpoint, &TelemetryDataKind::Traces, &self.protocol)?; + let tls_config_opt = if let Some(endpoint) = &endpoint_opt { + if !endpoint.is_empty() { + let tls_url = Uri::try_from(endpoint)?; + Some(self.grpc.clone().to_tls_config(&tls_url)?) + } else { + None + } + } else { + None + }; + + let mut exporter_builder = opentelemetry_otlp::SpanExporter::builder() + .with_tonic() + .with_timeout(self.batch_processor.max_export_timeout) + .with_metadata(MetadataMap::from_headers(self.grpc.metadata.clone())); + + if let Some(endpoint) = endpoint_opt { + exporter_builder = exporter_builder.with_endpoint(endpoint); + } + if let Some(tls_config) = tls_config_opt { + exporter_builder = exporter_builder.with_tls_config(tls_config); + } + Ok(exporter_builder.build()?) + } + + fn build_http_span_exporter(&self) -> Result { + let endpoint_opt = + process_endpoint(&self.endpoint, &TelemetryDataKind::Traces, &self.protocol)?; + + let mut exporter_builder = opentelemetry_otlp::SpanExporter::builder() + .with_http() + .with_timeout(self.batch_processor.max_export_timeout) + .with_headers(self.http.headers.clone()); + + if let Some(endpoint) = endpoint_opt { + exporter_builder = exporter_builder.with_endpoint(endpoint); + } + Ok(exporter_builder.build()?) + } +} diff --git a/apollo-router/src/plugins/telemetry/tracing/zipkin.rs b/apollo-router/src/plugins/telemetry/tracing/zipkin.rs index a25da681fc..e91d0f1cd1 100644 --- a/apollo-router/src/plugins/telemetry/tracing/zipkin.rs +++ b/apollo-router/src/plugins/telemetry/tracing/zipkin.rs @@ -1,23 +1,38 @@ -//! Configuration for zipkin tracing. +//! Configuration for Zipkin tracing. +//! +//! # Deprecation Notice +//! +//! The native Zipkin exporter is deprecated and will be removed in the next major release +//! of the Router. Zipkin supports OTLP ingestion, and OpenTelemetry is deprecating native +//! Zipkin exporters in favor of OTLP. Users should migrate to the OTLP exporter instead. +//! +//! # Known Limitations +//! +//! The upstream `opentelemetry-zipkin` crate (v0.31) does not currently support setting +//! the service name on the Zipkin `localEndpoint`. This means traces exported to Zipkin +//! will not have a service name associated with them. +//! +//! See: use std::sync::LazyLock; use http::Uri; -use opentelemetry_sdk::trace::BatchSpanProcessor; -use opentelemetry_semantic_conventions::resource::SERVICE_NAME; +use opentelemetry_sdk::trace::span_processor_with_async_runtime::BatchSpanProcessor; +use opentelemetry_zipkin::ZipkinExporter; use schemars::JsonSchema; use serde::Deserialize; use tower::BoxError; use crate::plugins::telemetry::config::Conf; -use crate::plugins::telemetry::config::GenericWith; use crate::plugins::telemetry::endpoint::UriEndpoint; -use crate::plugins::telemetry::error_handler::NamedSpanExporter; -use crate::plugins::telemetry::otel::named_runtime_channel::NamedTokioRuntime; use crate::plugins::telemetry::reload::tracing::TracingBuilder; use crate::plugins::telemetry::reload::tracing::TracingConfigurator; use crate::plugins::telemetry::tracing::BatchProcessorConfig; +use crate::plugins::telemetry::tracing::NamedSpanExporter; +use crate::plugins::telemetry::tracing::NamedTokioRuntime; use crate::plugins::telemetry::tracing::SpanProcessorExt; +const OTEL_EXPORTER_ZIPKIN_ENDPOINT: &str = "OTEL_EXPORTER_ZIPKIN_ENDPOINT"; + static DEFAULT_ENDPOINT: LazyLock = LazyLock::new(|| Uri::from_static("http://127.0.0.1:9411/api/v2/spans")); @@ -37,6 +52,24 @@ pub(crate) struct Config { pub(crate) batch_processor: BatchProcessorConfig, } +impl Config { + /// Apply environment variable overrides. + /// Supports `OTEL_EXPORTER_ZIPKIN_ENDPOINT`. + fn endpoint_with_env_override(&self) -> Result { + if let Ok(endpoint) = std::env::var(OTEL_EXPORTER_ZIPKIN_ENDPOINT) { + endpoint.parse::().map_err(|e| { + format!( + "invalid URI in {}: '{}': {}", + OTEL_EXPORTER_ZIPKIN_ENDPOINT, endpoint, e + ) + .into() + }) + } else { + Ok(self.endpoint.to_full_uri(&DEFAULT_ENDPOINT)) + } + } +} + impl TracingConfigurator for Config { fn config(conf: &Conf) -> &Self { &conf.exporters.tracing.zipkin @@ -48,25 +81,18 @@ impl TracingConfigurator for Config { fn configure(&self, builder: &mut TracingBuilder) -> Result<(), BoxError> { tracing::info!("configuring Zipkin tracing: {}", self.batch_processor); - let common: opentelemetry_sdk::trace::Config = builder.tracing_common().into(); - let endpoint = &self.endpoint.to_full_uri(&DEFAULT_ENDPOINT); - let exporter = opentelemetry_zipkin::new_pipeline() + let endpoint = self.endpoint_with_env_override()?; + // TODO: The upstream opentelemetry-zipkin crate (v0.31) removed support for setting + // service_name on the localEndpoint. Track upstream fix or implement workaround. + // See: https://github.com/open-telemetry/opentelemetry-rust/issues/381 + let exporter = ZipkinExporter::builder() .with_collector_endpoint(endpoint.to_string()) - .with( - &common.resource.get(SERVICE_NAME.into()), - |builder, service_name| { - // Zipkin exporter incorrectly ignores the service name in the resource - // Set it explicitly here - builder.with_service_name(service_name.as_str()) - }, - ) - .with_trace_config(common) - .init_exporter()?; + .build()?; let named_exporter = NamedSpanExporter::new(exporter, "zipkin"); builder.with_span_processor( BatchSpanProcessor::builder(named_exporter, NamedTokioRuntime::new("zipkin-tracing")) - .with_batch_config(self.batch_processor.clone().into()) + .with_batch_config(self.batch_processor.clone().with_env_overrides()?.into()) .build() .filtered(), ); diff --git a/apollo-router/src/plugins/telemetry/utils.rs b/apollo-router/src/plugins/telemetry/utils.rs index 50d12ac81b..d3eed3eeab 100644 --- a/apollo-router/src/plugins/telemetry/utils.rs +++ b/apollo-router/src/plugins/telemetry/utils.rs @@ -1,6 +1,8 @@ use std::time::Duration; use std::time::Instant; +use opentelemetry::KeyValue; + /// Timer implementing Drop to automatically compute the duration between the moment it has been created until it's dropped ///```ignore /// Timer::new(|duration| { @@ -39,3 +41,23 @@ where self.f.take().expect("f must exist")(self.start.elapsed()) } } + +/// Replace existing attribute with same key, or add new one. +/// This is needed because OTel 0.31+ uses Vec instead of HashMap for attributes. +pub(crate) fn upsert_attribute(attributes: &mut Vec, kv: KeyValue) { + if let Some(existing) = attributes.iter_mut().find(|a| a.key == kv.key) { + *existing = kv; + } else { + attributes.push(kv); + } +} + +/// Extends attributes with new values, updating existing keys instead of duplicating. +pub(crate) fn extend_attributes( + attrs: &mut Vec, + new_attrs: impl IntoIterator, +) { + for kv in new_attrs { + upsert_attribute(attrs, kv); + } +} diff --git a/apollo-router/src/query_planner/query_planner_service.rs b/apollo-router/src/query_planner/query_planner_service.rs index 7f2c058b31..d76c2c1af9 100644 --- a/apollo-router/src/query_planner/query_planner_service.rs +++ b/apollo-router/src/query_planner/query_planner_service.rs @@ -109,7 +109,7 @@ fn federation_version_instrument(federation_version: Option) -> ObservableG )], ); }) - .init() + .build() } impl QueryPlannerService { diff --git a/apollo-router/src/tracer.rs b/apollo-router/src/tracer.rs index 13da01fd0d..e1809a43a4 100644 --- a/apollo-router/src/tracer.rs +++ b/apollo-router/src/tracer.rs @@ -115,17 +115,15 @@ mod test { #[tokio::test] async fn it_returns_valid_trace_id() { + use opentelemetry::InstrumentationScope; + let _guard = TRACING_LOCK.lock(); // Create a tracing layer with the configured tracer - let provider = opentelemetry_sdk::trace::TracerProvider::builder() - .with_simple_exporter( - opentelemetry_stdout::SpanExporter::builder() - .with_writer(std::io::stdout()) - .build(), - ) + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() + .with_simple_exporter(opentelemetry_stdout::SpanExporter::default()) .build(); - let tracer = provider.tracer_builder("noop").build(); + let tracer = provider.tracer_with_scope(InstrumentationScope::builder("noop").build()); let telemetry = otel::layer().force_sampling().with_tracer(tracer); // Use the tracing subscriber `Registry`, or any other subscriber @@ -145,10 +143,11 @@ mod test { let my_id = TraceId::maybe_new(); assert!(my_id.is_none()); // Create a tracing layer with the configured tracer - let provider = opentelemetry_sdk::trace::TracerProvider::builder() + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() .with_simple_exporter(opentelemetry_stdout::SpanExporter::default()) .build(); - let tracer = provider.tracer_builder("noop").build(); + let tracer = provider + .tracer_with_scope(opentelemetry::InstrumentationScope::builder("noop").build()); let telemetry = otel::layer().force_sampling().with_tracer(tracer); // Use the tracing subscriber `Registry`, or any other subscriber // that impls `LookupSpan` @@ -168,10 +167,11 @@ mod test { fn it_correctly_compares_valid_and_valid_trace_id() { let _guard = TRACING_LOCK.lock(); // Create a tracing layer with the configured tracer - let provider = opentelemetry_sdk::trace::TracerProvider::builder() + let provider = opentelemetry_sdk::trace::SdkTracerProvider::builder() .with_simple_exporter(opentelemetry_stdout::SpanExporter::default()) .build(); - let tracer = provider.tracer_builder("noop").build(); + let tracer = provider + .tracer_with_scope(opentelemetry::InstrumentationScope::builder("noop").build()); let telemetry = otel::layer().force_sampling().with_tracer(tracer); // Use the tracing subscriber `Registry`, or any other subscriber // that impls `LookupSpan` diff --git a/apollo-router/tests/common.rs b/apollo-router/tests/common.rs index 56789f99c1..be92b72d2d 100644 --- a/apollo-router/tests/common.rs +++ b/apollo-router/tests/common.rs @@ -29,17 +29,14 @@ use opentelemetry::trace::SpanContext; use opentelemetry::trace::TraceContextExt; use opentelemetry::trace::TraceId; use opentelemetry::trace::TracerProvider as OtherTracerProvider; -use opentelemetry_otlp::HttpExporterBuilder; -use opentelemetry_otlp::Protocol; -use opentelemetry_otlp::SpanExporterBuilder; use opentelemetry_otlp::WithExportConfig; use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; use opentelemetry_sdk::Resource; +use opentelemetry_sdk::runtime; use opentelemetry_sdk::testing::trace::NoopSpanExporter; use opentelemetry_sdk::trace::BatchConfigBuilder; -use opentelemetry_sdk::trace::BatchSpanProcessor; -use opentelemetry_sdk::trace::Config; -use opentelemetry_sdk::trace::TracerProvider; +use opentelemetry_sdk::trace::SdkTracerProvider; +use opentelemetry_sdk::trace::span_processor_with_async_runtime::BatchSpanProcessor; use opentelemetry_semantic_conventions::resource::SERVICE_NAME; use parking_lot::Mutex; use prost::Message; @@ -209,8 +206,8 @@ pub struct IntegrationTest { telemetry: Telemetry, extra_propagator: Telemetry, - pub _tracer_provider_client: TracerProvider, - pub _tracer_provider_subgraph: TracerProvider, + pub _tracer_provider_client: SdkTracerProvider, + pub _tracer_provider_subgraph: SdkTracerProvider, subscriber_client: Dispatch, _subgraph_overrides: HashMap, @@ -374,27 +371,24 @@ pub enum Telemetry { } impl Telemetry { - fn tracer_provider(&self, service_name: &str) -> TracerProvider { - let config = Config::default().with_resource(Resource::new(vec![KeyValue::new( - SERVICE_NAME, - service_name.to_string(), - )])); + fn tracer_provider(&self, service_name: &str) -> SdkTracerProvider { + let resource = Resource::builder_empty() + .with_attributes([KeyValue::new(SERVICE_NAME, service_name.to_string())]) + .build(); match self { Telemetry::Otlp { endpoint: Some(endpoint), - } => TracerProvider::builder() - .with_config(config) + } => SdkTracerProvider::builder() + .with_resource(resource) .with_span_processor( BatchSpanProcessor::builder( - SpanExporterBuilder::Http( - HttpExporterBuilder::default() - .with_endpoint(endpoint) - .with_protocol(Protocol::HttpBinary), - ) - .build_span_exporter() - .expect("otlp pipeline failed"), - opentelemetry_sdk::runtime::Tokio, + opentelemetry_otlp::SpanExporter::builder() + .with_http() + .with_endpoint(endpoint) + .build() + .expect("otlp pipeline failed"), + runtime::Tokio, ) .with_batch_config( BatchConfigBuilder::default() @@ -404,15 +398,15 @@ impl Telemetry { .build(), ) .build(), - Telemetry::Datadog => TracerProvider::builder() - .with_config(config) + Telemetry::Datadog => SdkTracerProvider::builder() + .with_resource(resource) .with_span_processor( BatchSpanProcessor::builder( opentelemetry_datadog::new_pipeline() .with_service_name(service_name) .build_exporter() .expect("datadog pipeline failed"), - opentelemetry_sdk::runtime::Tokio, + runtime::Tokio, ) .with_batch_config( BatchConfigBuilder::default() @@ -422,15 +416,15 @@ impl Telemetry { .build(), ) .build(), - Telemetry::Zipkin => TracerProvider::builder() - .with_config(config) + Telemetry::Zipkin => SdkTracerProvider::builder() + .with_resource(resource) .with_span_processor( BatchSpanProcessor::builder( - opentelemetry_zipkin::new_pipeline() - .with_service_name(service_name) - .init_exporter() + opentelemetry_zipkin::ZipkinExporter::builder() + .with_collector_endpoint("http://127.0.0.1:9411/api/v2/spans") + .build() .expect("zipkin pipeline failed"), - opentelemetry_sdk::runtime::Tokio, + runtime::Tokio, ) .with_batch_config( BatchConfigBuilder::default() @@ -440,8 +434,8 @@ impl Telemetry { .build(), ) .build(), - Telemetry::None | Telemetry::Otlp { endpoint: None } => TracerProvider::builder() - .with_config(config) + Telemetry::None | Telemetry::Otlp { endpoint: None } => SdkTracerProvider::builder() + .with_resource(resource) .with_simple_exporter(NoopSpanExporter::default()) .build(), } @@ -686,12 +680,12 @@ impl IntegrationTest { Mock::given(method(Method::POST)) .and(path("/v1/metrics")) .and(move |req: &wiremock::Request| { - // Decode the OTLP request + // Decode the OTLP request and forward to the channel if let Ok(msg) = ExportMetricsServiceRequest::decode(req.body.as_ref()) { - // We don't care about the result of send here let _ = apollo_otlp_metrics_tx.try_send(msg); } - false + // Always match so we return 200 OK + true }) .respond_with(ResponseTemplate::new(200)) .mount(&apollo_otlp_server) @@ -728,7 +722,7 @@ impl IntegrationTest { } } - fn dispatch(tracer_provider: &TracerProvider) -> Dispatch { + fn dispatch(tracer_provider: &SdkTracerProvider) -> Dispatch { let tracer = tracer_provider.tracer("tracer"); let tracing_layer = tracing_opentelemetry::layer() .with_tracer(tracer) @@ -1120,19 +1114,27 @@ impl IntegrationTest { let mut metrics = Vec::new(); while Instant::now() < deadline { - if let Some(msg) = self.apollo_otlp_metrics_rx.recv().await { - // Only break once we see a batch with metrics in it - if msg - .resource_metrics - .iter() - .any(|rm| !rm.scope_metrics.is_empty()) - { - metrics.push(msg); + let remaining = deadline.saturating_duration_since(Instant::now()); + match tokio::time::timeout(remaining, self.apollo_otlp_metrics_rx.recv()).await { + Ok(Some(msg)) => { + // Only break once we see a batch with metrics in it + if msg + .resource_metrics + .iter() + .any(|rm| !rm.scope_metrics.is_empty()) + { + metrics.push(msg); + break; + } + } + Ok(None) => { + // channel closed + break; + } + Err(_) => { + // timeout elapsed break; } - } else { - // channel closed - break; } } @@ -1562,16 +1564,12 @@ impl IntegrationTest { pub(crate) fn force_flush(&self) { let tracer_provider_client = self._tracer_provider_client.clone(); let tracer_provider_subgraph = self._tracer_provider_subgraph.clone(); - for r in tracer_provider_subgraph.force_flush() { - if let Err(e) = r { - eprintln!("failed to flush subgraph tracer: {e}"); - } + if let Err(e) = tracer_provider_subgraph.force_flush() { + eprintln!("failed to flush subgraph tracer: {e}"); } - for r in tracer_provider_client.force_flush() { - if let Err(e) = r { - eprintln!("failed to flush client tracer: {e}"); - } + if let Err(e) = tracer_provider_client.force_flush() { + eprintln!("failed to flush client tracer: {e}"); } } diff --git a/apollo-router/tests/integration/lifecycle.rs b/apollo-router/tests/integration/lifecycle.rs index 81dc9743d7..a726344265 100644 --- a/apollo-router/tests/integration/lifecycle.rs +++ b/apollo-router/tests/integration/lifecycle.rs @@ -143,7 +143,7 @@ async fn test_force_config_reload_via_chaos() -> Result<(), BoxError> { let mut router = IntegrationTest::builder() .config( "experimental_chaos: - force_config_reload: 1s", + force_config_reload: 2s", ) .build() .await; @@ -159,7 +159,7 @@ async fn test_force_schema_reload_via_chaos() -> Result<(), BoxError> { let mut router = IntegrationTest::builder() .config( "experimental_chaos: - force_schema_reload: 1s", + force_schema_reload: 2s", ) .build() .await; diff --git a/apollo-router/tests/integration/telemetry/apollo_otel_metrics.rs b/apollo-router/tests/integration/telemetry/apollo_otel_metrics.rs index 5717d386f7..1e1213c9c5 100644 --- a/apollo-router/tests/integration/telemetry/apollo_otel_metrics.rs +++ b/apollo-router/tests/integration/telemetry/apollo_otel_metrics.rs @@ -42,8 +42,10 @@ async fn test_validation_error_emits_metric() { telemetry: apollo: experimental_otlp_metrics_protocol: http - batch_processor: - scheduled_delay: 10ms + metrics: + otlp: + batch_processor: + scheduled_delay: 100ms errors: preview_extended_error_metrics: enabled "#, @@ -63,7 +65,7 @@ async fn test_validation_error_emits_metric() { assert!(response.contains(expected_error_code)); let metrics = router - .wait_for_emitted_otel_metrics(Duration::from_millis(20)) + .wait_for_emitted_otel_metrics(Duration::from_secs(2)) .await; assert!(!metrics.is_empty()); assert_metrics_contain( @@ -96,8 +98,10 @@ async fn test_subgraph_http_error_emits_metric() { telemetry: apollo: experimental_otlp_metrics_protocol: http - batch_processor: - scheduled_delay: 10ms + metrics: + otlp: + batch_processor: + scheduled_delay: 100ms errors: preview_extended_error_metrics: enabled include_subgraph_errors: @@ -124,7 +128,7 @@ async fn test_subgraph_http_error_emits_metric() { assert!(response.contains(expected_error_code)); let metrics = router - .wait_for_emitted_otel_metrics(Duration::from_millis(20)) + .wait_for_emitted_otel_metrics(Duration::from_secs(2)) .await; assert!(!metrics.is_empty()); @@ -163,8 +167,10 @@ async fn test_subgraph_layer_error_emits_metric() { telemetry: apollo: experimental_otlp_metrics_protocol: http - batch_processor: - scheduled_delay: 10ms + metrics: + otlp: + batch_processor: + scheduled_delay: 100ms errors: preview_extended_error_metrics: enabled "#, @@ -201,7 +207,7 @@ async fn test_subgraph_layer_error_emits_metric() { .await; let metrics = router - .wait_for_emitted_otel_metrics(Duration::from_millis(200)) + .wait_for_emitted_otel_metrics(Duration::from_secs(2)) .await; assert!(!metrics.is_empty()); @@ -240,8 +246,10 @@ async fn test_subgraph_layer_entities_error_emits_metric() { telemetry: apollo: experimental_otlp_metrics_protocol: http - batch_processor: - scheduled_delay: 10ms + metrics: + otlp: + batch_processor: + scheduled_delay: 100ms errors: preview_extended_error_metrics: enabled "#, @@ -278,7 +286,7 @@ async fn test_subgraph_layer_entities_error_emits_metric() { .await; let metrics = router - .wait_for_emitted_otel_metrics(Duration::from_millis(20)) + .wait_for_emitted_otel_metrics(Duration::from_secs(2)) .await; assert!(!metrics.is_empty()); @@ -318,8 +326,10 @@ async fn test_include_subgraph_error_disabled_does_not_redact_error_metrics() { telemetry: apollo: experimental_otlp_metrics_protocol: http - batch_processor: - scheduled_delay: 10ms + metrics: + otlp: + batch_processor: + scheduled_delay: 100ms errors: preview_extended_error_metrics: enabled include_subgraph_errors: @@ -358,7 +368,7 @@ async fn test_include_subgraph_error_disabled_does_not_redact_error_metrics() { .await; let metrics = router - .wait_for_emitted_otel_metrics(Duration::from_millis(20)) + .wait_for_emitted_otel_metrics(Duration::from_secs(2)) .await; assert!(!metrics.is_empty()); @@ -398,8 +408,10 @@ async fn test_supergraph_layer_error_emits_metric() { telemetry: apollo: experimental_otlp_metrics_protocol: http - batch_processor: - scheduled_delay: 10ms + metrics: + otlp: + batch_processor: + scheduled_delay: 100ms errors: preview_extended_error_metrics: enabled supergraph: @@ -423,7 +435,7 @@ async fn test_supergraph_layer_error_emits_metric() { .await; let metrics = router - .wait_for_emitted_otel_metrics(Duration::from_millis(20)) + .wait_for_emitted_otel_metrics(Duration::from_secs(2)) .await; assert!(!metrics.is_empty()); @@ -461,8 +473,10 @@ async fn test_execution_layer_error_emits_metric() { telemetry: apollo: experimental_otlp_metrics_protocol: http - batch_processor: - scheduled_delay: 10ms + metrics: + otlp: + batch_processor: + scheduled_delay: 100ms errors: preview_extended_error_metrics: enabled forbid_mutations: true @@ -487,7 +501,7 @@ async fn test_execution_layer_error_emits_metric() { ).await; let metrics = router - .wait_for_emitted_otel_metrics(Duration::from_millis(20)) + .wait_for_emitted_otel_metrics(Duration::from_secs(2)) .await; assert!(!metrics.is_empty()); @@ -526,8 +540,10 @@ async fn test_router_layer_error_emits_metric() { telemetry: apollo: experimental_otlp_metrics_protocol: http - batch_processor: - scheduled_delay: 10ms + metrics: + otlp: + batch_processor: + scheduled_delay: 100ms errors: preview_extended_error_metrics: enabled csrf: @@ -553,7 +569,7 @@ async fn test_router_layer_error_emits_metric() { .await; let metrics = router - .wait_for_emitted_otel_metrics(Duration::from_millis(20)) + .wait_for_emitted_otel_metrics(Duration::from_secs(2)) .await; assert!(!metrics.is_empty()); @@ -590,7 +606,6 @@ async fn test_apollo_studio_metrics_not_affected_by_rename() { let expected_error_code = "CSRF_ERROR"; let expected_client_name = "CLIENT_NAME"; let expected_client_version = "v0.14"; - let mut router = IntegrationTest::builder() .telemetry(Telemetry::Otlp { endpoint: None }) .config( @@ -598,8 +613,10 @@ async fn test_apollo_studio_metrics_not_affected_by_rename() { telemetry: apollo: experimental_otlp_metrics_protocol: http - batch_processor: - scheduled_delay: 10ms + metrics: + otlp: + batch_processor: + scheduled_delay: 100ms errors: preview_extended_error_metrics: enabled exporters: @@ -616,10 +633,8 @@ async fn test_apollo_studio_metrics_not_affected_by_rename() { ) .build() .await; - router.start().await; router.assert_started().await; - router .execute_query( Query::builder() @@ -630,13 +645,11 @@ async fn test_apollo_studio_metrics_not_affected_by_rename() { .build(), ) .await; - let metrics = router - .wait_for_emitted_otel_metrics(Duration::from_millis(20)) + .wait_for_emitted_otel_metrics(Duration::from_secs(2)) .await; assert!(!metrics.is_empty()); - assert_metrics_contain( &metrics, Metric::builder() @@ -669,8 +682,10 @@ async fn test_subgraph_request_emits_histogram() { telemetry: apollo: experimental_otlp_metrics_protocol: http - batch_processor: - scheduled_delay: 10ms + metrics: + otlp: + batch_processor: + scheduled_delay: 100ms subgraph_metrics: true include_subgraph_errors: all: true @@ -692,7 +707,7 @@ async fn test_subgraph_request_emits_histogram() { .await; let metrics = router - .wait_for_emitted_otel_metrics(Duration::from_millis(20)) + .wait_for_emitted_otel_metrics(Duration::from_secs(2)) .await; assert!(!metrics.is_empty()); assert_metrics_contain( @@ -729,8 +744,10 @@ async fn test_failed_subgraph_request_emits_histogram() { telemetry: apollo: experimental_otlp_metrics_protocol: http - batch_processor: - scheduled_delay: 10ms + metrics: + otlp: + batch_processor: + scheduled_delay: 100ms subgraph_metrics: true include_subgraph_errors: all: true @@ -753,7 +770,7 @@ async fn test_failed_subgraph_request_emits_histogram() { .await; let metrics = router - .wait_for_emitted_otel_metrics(Duration::from_millis(20)) + .wait_for_emitted_otel_metrics(Duration::from_secs(2)) .await; assert!(!metrics.is_empty()); assert_metrics_contain( @@ -791,8 +808,10 @@ async fn test_connector_request_emits_histogram() { telemetry: apollo: experimental_otlp_metrics_protocol: http - batch_processor: - scheduled_delay: 10ms + metrics: + otlp: + batch_processor: + scheduled_delay: 100ms subgraph_metrics: true include_subgraph_errors: all: true @@ -828,7 +847,7 @@ async fn test_connector_request_emits_histogram() { .await; let metrics = router - .wait_for_emitted_otel_metrics(Duration::from_millis(20)) + .wait_for_emitted_otel_metrics(Duration::from_secs(2)) .await; assert!(!metrics.is_empty()); assert_metrics_contain( @@ -867,8 +886,10 @@ async fn test_failed_connector_request_emits_histogram() { telemetry: apollo: experimental_otlp_metrics_protocol: http - batch_processor: - scheduled_delay: 10ms + metrics: + otlp: + batch_processor: + scheduled_delay: 100ms subgraph_metrics: true traffic_shaping: connector: @@ -906,7 +927,7 @@ async fn test_failed_connector_request_emits_histogram() { .await; let metrics = router - .wait_for_emitted_otel_metrics(Duration::from_millis(20)) + .wait_for_emitted_otel_metrics(Duration::from_secs(2)) .await; assert!(!metrics.is_empty()); assert_metrics_contain( diff --git a/apollo-router/tests/integration/telemetry/datadog.rs b/apollo-router/tests/integration/telemetry/datadog.rs index 2c4a089899..710ac2216b 100644 --- a/apollo-router/tests/integration/telemetry/datadog.rs +++ b/apollo-router/tests/integration/telemetry/datadog.rs @@ -706,7 +706,7 @@ async fn test_header_propagator_override() -> Result<(), BoxError> { .build() .await; - let trace_id = opentelemetry::trace::TraceId::from_u128(uuid::Uuid::new_v4().as_u128()); + let trace_id = opentelemetry::trace::TraceId::from(uuid::Uuid::new_v4().as_u128()); router.start().await; router.assert_started().await; diff --git a/apollo-router/tests/integration/telemetry/otlp/mod.rs b/apollo-router/tests/integration/telemetry/otlp/mod.rs index ed9e718f28..f93d1a5079 100644 --- a/apollo-router/tests/integration/telemetry/otlp/mod.rs +++ b/apollo-router/tests/integration/telemetry/otlp/mod.rs @@ -95,7 +95,7 @@ impl Verifier for OtlpTraceSpec<'_> { } } }).filter(|t| { - let datadog_trace_id = TraceId::from_u128(trace_id.to_datadog() as u128); + let datadog_trace_id = TraceId::from(trace_id.to_datadog() as u128); let trace_found1 = !t.select_path(&format!("$..[?(@.traceId == '{trace_id}')]")).unwrap_or_default().is_empty(); let trace_found2 = !t.select_path(&format!("$..[?(@.traceId == '{datadog_trace_id}')]")).unwrap_or_default().is_empty(); trace_found1 | trace_found2 diff --git a/apollo-router/tests/integration/telemetry/otlp/tracing.rs b/apollo-router/tests/integration/telemetry/otlp/tracing.rs index 0c862ac830..bc41c26c59 100644 --- a/apollo-router/tests/integration/telemetry/otlp/tracing.rs +++ b/apollo-router/tests/integration/telemetry/otlp/tracing.rs @@ -35,7 +35,6 @@ async fn test_trace_error() -> Result<(), BoxError> { router.start().await; router.assert_started().await; - router.assert_log_contained("OpenTelemetry trace error occurred: cannot send message to batch processor 'otlp-tracing' as the channel is full"); router.assert_metrics_contains(r#"apollo_router_telemetry_batch_processor_errors_total{error="channel full",name="otlp-tracing",otel_scope_name="apollo/router"}"#, None).await; router.graceful_shutdown().await; diff --git a/apollo-router/tests/integration/telemetry/zipkin.rs b/apollo-router/tests/integration/telemetry/zipkin.rs index ad5445938f..a76d093109 100644 --- a/apollo-router/tests/integration/telemetry/zipkin.rs +++ b/apollo-router/tests/integration/telemetry/zipkin.rs @@ -68,22 +68,29 @@ impl Verifier for ZipkinTraceSpec { } fn verify_services(&self, trace: &Value) -> Result<(), axum::BoxError> { - let actual_services: HashSet = trace - .select_path("$..serviceName")? + // Note: opentelemetry-zipkin 0.31 has a bug where it doesn't set localEndpoint.serviceName + // from the Resource. Instead of checking service names, we verify that spans exist + // by checking for expected span names. + // See: https://github.com/open-telemetry/opentelemetry-rust-contrib/issues/XXX + let span_names: HashSet = trace + .select_path("$..name")? .into_iter() - .filter_map(|service| service.as_string()) + .filter_map(|name| name.as_string()) .collect(); - tracing::debug!("found services {:?}", actual_services); + tracing::debug!("found span names {:?}", span_names); - let expected_services = self - .trace_spec - .services + // Verify we have spans from client, router, and subgraph by checking for characteristic span names + let has_client_span = span_names.iter().any(|n| n == "client_request"); + let has_router_span = span_names .iter() - .map(|s| s.to_string()) - .collect::>(); - if actual_services != expected_services { + .any(|n| n == "router" || n == "supergraph"); + let has_subgraph_span = span_names + .iter() + .any(|n| n == "subgraph server" || n.starts_with("subgraph")); + + if !has_client_span || !has_router_span || !has_subgraph_span { return Err(BoxError::from(format!( - "incomplete traces, got {actual_services:?} expected {expected_services:?}" + "incomplete traces, expected spans from client/router/subgraph, got span names: {span_names:?}" ))); } Ok(()) @@ -118,25 +125,15 @@ impl Verifier for ZipkinTraceSpec { } async fn get_trace(&self, trace_id: TraceId) -> Result { - let params = url::form_urlencoded::Serializer::new(String::new()) - .append_pair( - "service", - self.trace_spec - .services - .first() - .expect("expected root service"), - ) - .finish(); - let id = trace_id.to_string(); - let url = format!("http://localhost:9411/api/v2/trace/{id}?{params}"); + let url = format!("http://localhost:9411/api/v2/trace/{id}"); println!("url: {}", url); let value: serde_json::Value = reqwest::get(url) .await - .map_err(|e| anyhow!("failed to contact datadog; {}", e))? + .map_err(|e| anyhow!("failed to contact zipkin; {}", e))? .json() .await - .map_err(|e| anyhow!("failed to contact datadog; {}", e))?; + .map_err(|e| anyhow!("failed to contact zipkin; {}", e))?; Ok(value) } diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__batch_send_header-2.snap b/apollo-router/tests/snapshots/apollo_otel_traces__batch_send_header-2.snap index e50a3122a0..b5c7cb4c0a 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__batch_send_header-2.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__batch_send_header-2.snap @@ -24,6 +24,7 @@ resourceSpans: value: stringValue: "[redacted]" droppedAttributesCount: 0 + entityRefs: [] scopeSpans: - scope: name: apollo-router @@ -35,7 +36,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query one kind: 2 startTimeUnixNano: "[start_time]" @@ -77,7 +78,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -95,7 +96,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -113,7 +114,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -131,7 +132,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -149,7 +150,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -167,7 +168,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -185,7 +186,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -203,7 +204,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -239,7 +240,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -257,7 +258,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -275,7 +276,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -293,7 +294,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -311,7 +312,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -332,7 +333,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -350,7 +351,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: sequence kind: 1 startTimeUnixNano: "[start_time]" @@ -368,7 +369,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -392,7 +393,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -413,7 +414,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -431,7 +432,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -455,7 +456,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -476,7 +477,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -497,7 +498,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -521,7 +522,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -542,7 +543,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -560,7 +561,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -584,7 +585,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -605,7 +606,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -626,7 +627,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -650,7 +651,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -671,7 +672,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -689,7 +690,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -713,7 +714,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -734,7 +735,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -770,7 +771,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -788,7 +789,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -806,7 +807,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -824,7 +825,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -842,7 +843,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -863,7 +864,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -881,7 +882,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: sequence kind: 1 startTimeUnixNano: "[start_time]" @@ -899,7 +900,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -923,7 +924,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -944,7 +945,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -962,7 +963,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -986,7 +987,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1007,7 +1008,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -1028,7 +1029,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -1052,7 +1053,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -1073,7 +1074,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -1091,7 +1092,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1115,7 +1116,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1136,7 +1137,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -1157,7 +1158,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -1181,7 +1182,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -1202,7 +1203,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -1220,7 +1221,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1244,7 +1245,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__batch_send_header.snap b/apollo-router/tests/snapshots/apollo_otel_traces__batch_send_header.snap index e50a3122a0..b5c7cb4c0a 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__batch_send_header.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__batch_send_header.snap @@ -24,6 +24,7 @@ resourceSpans: value: stringValue: "[redacted]" droppedAttributesCount: 0 + entityRefs: [] scopeSpans: - scope: name: apollo-router @@ -35,7 +36,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query one kind: 2 startTimeUnixNano: "[start_time]" @@ -77,7 +78,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -95,7 +96,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -113,7 +114,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -131,7 +132,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -149,7 +150,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -167,7 +168,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -185,7 +186,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -203,7 +204,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -239,7 +240,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -257,7 +258,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -275,7 +276,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -293,7 +294,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -311,7 +312,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -332,7 +333,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -350,7 +351,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: sequence kind: 1 startTimeUnixNano: "[start_time]" @@ -368,7 +369,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -392,7 +393,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -413,7 +414,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -431,7 +432,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -455,7 +456,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -476,7 +477,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -497,7 +498,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -521,7 +522,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -542,7 +543,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -560,7 +561,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -584,7 +585,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -605,7 +606,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -626,7 +627,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -650,7 +651,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -671,7 +672,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -689,7 +690,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -713,7 +714,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -734,7 +735,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -770,7 +771,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -788,7 +789,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -806,7 +807,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -824,7 +825,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -842,7 +843,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -863,7 +864,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -881,7 +882,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: sequence kind: 1 startTimeUnixNano: "[start_time]" @@ -899,7 +900,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -923,7 +924,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -944,7 +945,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -962,7 +963,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -986,7 +987,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1007,7 +1008,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -1028,7 +1029,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -1052,7 +1053,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -1073,7 +1074,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -1091,7 +1092,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1115,7 +1116,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1136,7 +1137,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -1157,7 +1158,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -1181,7 +1182,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -1202,7 +1203,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -1220,7 +1221,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1244,7 +1245,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__batch_trace_id-2.snap b/apollo-router/tests/snapshots/apollo_otel_traces__batch_trace_id-2.snap index c46b22fab1..5a2168bc97 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__batch_trace_id-2.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__batch_trace_id-2.snap @@ -24,6 +24,7 @@ resourceSpans: value: stringValue: "[redacted]" droppedAttributesCount: 0 + entityRefs: [] scopeSpans: - scope: name: apollo-router @@ -35,7 +36,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query one kind: 2 startTimeUnixNano: "[start_time]" @@ -77,7 +78,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -95,7 +96,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -113,7 +114,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -131,7 +132,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -149,7 +150,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -167,7 +168,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -185,7 +186,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -203,7 +204,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -239,7 +240,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -257,7 +258,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -275,7 +276,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -293,7 +294,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -311,7 +312,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -332,7 +333,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -350,7 +351,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: sequence kind: 1 startTimeUnixNano: "[start_time]" @@ -368,7 +369,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -392,7 +393,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -413,7 +414,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -431,7 +432,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -455,7 +456,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -476,7 +477,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -497,7 +498,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -521,7 +522,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -542,7 +543,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -560,7 +561,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -584,7 +585,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -605,7 +606,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -626,7 +627,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -650,7 +651,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -671,7 +672,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -689,7 +690,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -713,7 +714,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -734,7 +735,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -770,7 +771,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -788,7 +789,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -806,7 +807,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -824,7 +825,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -842,7 +843,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -863,7 +864,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -881,7 +882,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: sequence kind: 1 startTimeUnixNano: "[start_time]" @@ -899,7 +900,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -923,7 +924,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -944,7 +945,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -962,7 +963,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -986,7 +987,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1007,7 +1008,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -1028,7 +1029,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -1052,7 +1053,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -1073,7 +1074,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -1091,7 +1092,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1115,7 +1116,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1136,7 +1137,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -1157,7 +1158,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -1181,7 +1182,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -1202,7 +1203,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -1220,7 +1221,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1244,7 +1245,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__batch_trace_id.snap b/apollo-router/tests/snapshots/apollo_otel_traces__batch_trace_id.snap index c46b22fab1..5a2168bc97 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__batch_trace_id.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__batch_trace_id.snap @@ -24,6 +24,7 @@ resourceSpans: value: stringValue: "[redacted]" droppedAttributesCount: 0 + entityRefs: [] scopeSpans: - scope: name: apollo-router @@ -35,7 +36,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query one kind: 2 startTimeUnixNano: "[start_time]" @@ -77,7 +78,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -95,7 +96,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -113,7 +114,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -131,7 +132,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -149,7 +150,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -167,7 +168,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -185,7 +186,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -203,7 +204,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -239,7 +240,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -257,7 +258,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -275,7 +276,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -293,7 +294,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -311,7 +312,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -332,7 +333,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -350,7 +351,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: sequence kind: 1 startTimeUnixNano: "[start_time]" @@ -368,7 +369,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -392,7 +393,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -413,7 +414,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -431,7 +432,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -455,7 +456,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -476,7 +477,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -497,7 +498,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -521,7 +522,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -542,7 +543,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -560,7 +561,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -584,7 +585,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -605,7 +606,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -626,7 +627,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -650,7 +651,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -671,7 +672,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -689,7 +690,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -713,7 +714,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -734,7 +735,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -770,7 +771,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -788,7 +789,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -806,7 +807,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -824,7 +825,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -842,7 +843,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -863,7 +864,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -881,7 +882,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: sequence kind: 1 startTimeUnixNano: "[start_time]" @@ -899,7 +900,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -923,7 +924,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -944,7 +945,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -962,7 +963,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -986,7 +987,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1007,7 +1008,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -1028,7 +1029,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -1052,7 +1053,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -1073,7 +1074,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -1091,7 +1092,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1115,7 +1116,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1136,7 +1137,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -1157,7 +1158,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -1181,7 +1182,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -1202,7 +1203,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -1220,7 +1221,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1244,7 +1245,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__client_name-2.snap b/apollo-router/tests/snapshots/apollo_otel_traces__client_name-2.snap index 9afd8a36ac..6635618626 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__client_name-2.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__client_name-2.snap @@ -24,6 +24,7 @@ resourceSpans: value: stringValue: "[redacted]" droppedAttributesCount: 0 + entityRefs: [] scopeSpans: - scope: name: apollo-router @@ -35,7 +36,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query kind: 2 startTimeUnixNano: "[start_time]" @@ -62,7 +63,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: router kind: 1 startTimeUnixNano: "[start_time]" @@ -104,7 +105,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -122,7 +123,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -140,7 +141,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -158,7 +159,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -176,7 +177,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -212,7 +213,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -230,7 +231,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -248,7 +249,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -266,7 +267,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -284,7 +285,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -305,7 +306,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -323,7 +324,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: sequence kind: 1 startTimeUnixNano: "[start_time]" @@ -341,7 +342,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -365,7 +366,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -392,7 +393,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -410,7 +411,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -434,7 +435,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -455,7 +456,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -476,7 +477,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -500,7 +501,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -527,7 +528,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -545,7 +546,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -569,7 +570,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -590,7 +591,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -611,7 +612,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -635,7 +636,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -662,7 +663,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -680,7 +681,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -704,7 +705,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__client_name.snap b/apollo-router/tests/snapshots/apollo_otel_traces__client_name.snap index 9afd8a36ac..6635618626 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__client_name.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__client_name.snap @@ -24,6 +24,7 @@ resourceSpans: value: stringValue: "[redacted]" droppedAttributesCount: 0 + entityRefs: [] scopeSpans: - scope: name: apollo-router @@ -35,7 +36,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query kind: 2 startTimeUnixNano: "[start_time]" @@ -62,7 +63,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: router kind: 1 startTimeUnixNano: "[start_time]" @@ -104,7 +105,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -122,7 +123,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -140,7 +141,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -158,7 +159,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -176,7 +177,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -212,7 +213,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -230,7 +231,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -248,7 +249,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -266,7 +267,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -284,7 +285,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -305,7 +306,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -323,7 +324,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: sequence kind: 1 startTimeUnixNano: "[start_time]" @@ -341,7 +342,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -365,7 +366,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -392,7 +393,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -410,7 +411,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -434,7 +435,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -455,7 +456,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -476,7 +477,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -500,7 +501,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -527,7 +528,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -545,7 +546,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -569,7 +570,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -590,7 +591,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -611,7 +612,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -635,7 +636,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -662,7 +663,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -680,7 +681,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -704,7 +705,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__client_version-2.snap b/apollo-router/tests/snapshots/apollo_otel_traces__client_version-2.snap index 30df031d34..1102f56b57 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__client_version-2.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__client_version-2.snap @@ -24,6 +24,7 @@ resourceSpans: value: stringValue: "[redacted]" droppedAttributesCount: 0 + entityRefs: [] scopeSpans: - scope: name: apollo-router @@ -35,7 +36,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query kind: 2 startTimeUnixNano: "[start_time]" @@ -62,7 +63,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: router kind: 1 startTimeUnixNano: "[start_time]" @@ -104,7 +105,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -122,7 +123,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -140,7 +141,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -158,7 +159,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -176,7 +177,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -212,7 +213,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -230,7 +231,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -248,7 +249,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -266,7 +267,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -284,7 +285,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -305,7 +306,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -323,7 +324,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: sequence kind: 1 startTimeUnixNano: "[start_time]" @@ -341,7 +342,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -365,7 +366,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -392,7 +393,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -410,7 +411,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -434,7 +435,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -455,7 +456,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -476,7 +477,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -500,7 +501,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -527,7 +528,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -545,7 +546,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -569,7 +570,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -590,7 +591,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -611,7 +612,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -635,7 +636,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -662,7 +663,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -680,7 +681,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -704,7 +705,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__client_version.snap b/apollo-router/tests/snapshots/apollo_otel_traces__client_version.snap index 30df031d34..1102f56b57 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__client_version.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__client_version.snap @@ -24,6 +24,7 @@ resourceSpans: value: stringValue: "[redacted]" droppedAttributesCount: 0 + entityRefs: [] scopeSpans: - scope: name: apollo-router @@ -35,7 +36,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query kind: 2 startTimeUnixNano: "[start_time]" @@ -62,7 +63,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: router kind: 1 startTimeUnixNano: "[start_time]" @@ -104,7 +105,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -122,7 +123,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -140,7 +141,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -158,7 +159,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -176,7 +177,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -212,7 +213,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -230,7 +231,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -248,7 +249,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -266,7 +267,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -284,7 +285,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -305,7 +306,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -323,7 +324,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: sequence kind: 1 startTimeUnixNano: "[start_time]" @@ -341,7 +342,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -365,7 +366,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -392,7 +393,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -410,7 +411,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -434,7 +435,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -455,7 +456,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -476,7 +477,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -500,7 +501,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -527,7 +528,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -545,7 +546,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -569,7 +570,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -590,7 +591,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -611,7 +612,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -635,7 +636,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -662,7 +663,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -680,7 +681,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -704,7 +705,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__condition_else-2.snap b/apollo-router/tests/snapshots/apollo_otel_traces__condition_else-2.snap index dfc88f0fd9..89328ddc22 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__condition_else-2.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__condition_else-2.snap @@ -24,6 +24,7 @@ resourceSpans: value: stringValue: "[redacted]" droppedAttributesCount: 0 + entityRefs: [] scopeSpans: - scope: name: apollo-router @@ -35,7 +36,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query kind: 2 startTimeUnixNano: "[start_time]" @@ -62,7 +63,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: router kind: 1 startTimeUnixNano: "[start_time]" @@ -104,7 +105,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -122,7 +123,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -140,7 +141,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -158,7 +159,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -176,7 +177,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -212,7 +213,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -230,7 +231,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -248,7 +249,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -266,7 +267,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -284,7 +285,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -305,7 +306,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -323,7 +324,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: condition kind: 1 startTimeUnixNano: "[start_time]" @@ -344,7 +345,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: condition_else kind: 1 startTimeUnixNano: "[start_time]" @@ -362,7 +363,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: sequence kind: 1 startTimeUnixNano: "[start_time]" @@ -380,7 +381,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -404,7 +405,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -431,7 +432,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -449,7 +450,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -473,7 +474,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -494,7 +495,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -515,7 +516,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -539,7 +540,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -566,7 +567,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -584,7 +585,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -608,7 +609,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -629,7 +630,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -650,7 +651,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -674,7 +675,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -701,7 +702,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -719,7 +720,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -743,7 +744,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__condition_else.snap b/apollo-router/tests/snapshots/apollo_otel_traces__condition_else.snap index dfc88f0fd9..89328ddc22 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__condition_else.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__condition_else.snap @@ -24,6 +24,7 @@ resourceSpans: value: stringValue: "[redacted]" droppedAttributesCount: 0 + entityRefs: [] scopeSpans: - scope: name: apollo-router @@ -35,7 +36,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query kind: 2 startTimeUnixNano: "[start_time]" @@ -62,7 +63,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: router kind: 1 startTimeUnixNano: "[start_time]" @@ -104,7 +105,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -122,7 +123,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -140,7 +141,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -158,7 +159,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -176,7 +177,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -212,7 +213,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -230,7 +231,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -248,7 +249,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -266,7 +267,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -284,7 +285,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -305,7 +306,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -323,7 +324,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: condition kind: 1 startTimeUnixNano: "[start_time]" @@ -344,7 +345,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: condition_else kind: 1 startTimeUnixNano: "[start_time]" @@ -362,7 +363,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: sequence kind: 1 startTimeUnixNano: "[start_time]" @@ -380,7 +381,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -404,7 +405,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -431,7 +432,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -449,7 +450,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -473,7 +474,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -494,7 +495,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -515,7 +516,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -539,7 +540,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -566,7 +567,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -584,7 +585,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -608,7 +609,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -629,7 +630,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -650,7 +651,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -674,7 +675,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -701,7 +702,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -719,7 +720,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -743,7 +744,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__condition_if-2.snap b/apollo-router/tests/snapshots/apollo_otel_traces__condition_if-2.snap index 4a1499c9b0..fe7c57b34e 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__condition_if-2.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__condition_if-2.snap @@ -24,6 +24,7 @@ resourceSpans: value: stringValue: "[redacted]" droppedAttributesCount: 0 + entityRefs: [] scopeSpans: - scope: name: apollo-router @@ -35,7 +36,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query kind: 2 startTimeUnixNano: "[start_time]" @@ -62,7 +63,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: router kind: 1 startTimeUnixNano: "[start_time]" @@ -104,7 +105,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -122,7 +123,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -140,7 +141,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -158,7 +159,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -176,7 +177,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -212,7 +213,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -230,7 +231,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -248,7 +249,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -266,7 +267,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -284,7 +285,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -305,7 +306,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -323,7 +324,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: condition kind: 1 startTimeUnixNano: "[start_time]" @@ -344,7 +345,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: condition_if kind: 1 startTimeUnixNano: "[start_time]" @@ -362,7 +363,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: defer kind: 1 startTimeUnixNano: "[start_time]" @@ -380,7 +381,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: defer_primary kind: 1 startTimeUnixNano: "[start_time]" @@ -398,7 +399,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -422,7 +423,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -449,7 +450,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -467,7 +468,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -491,7 +492,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -512,7 +513,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: defer_deferred kind: 1 startTimeUnixNano: "[start_time]" @@ -539,7 +540,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: sequence kind: 1 startTimeUnixNano: "[start_time]" @@ -557,7 +558,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -578,7 +579,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -602,7 +603,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -629,7 +630,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -647,7 +648,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -671,7 +672,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -692,7 +693,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -713,7 +714,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -737,7 +738,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -764,7 +765,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -782,7 +783,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -806,7 +807,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__condition_if.snap b/apollo-router/tests/snapshots/apollo_otel_traces__condition_if.snap index 4a1499c9b0..fe7c57b34e 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__condition_if.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__condition_if.snap @@ -24,6 +24,7 @@ resourceSpans: value: stringValue: "[redacted]" droppedAttributesCount: 0 + entityRefs: [] scopeSpans: - scope: name: apollo-router @@ -35,7 +36,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query kind: 2 startTimeUnixNano: "[start_time]" @@ -62,7 +63,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: router kind: 1 startTimeUnixNano: "[start_time]" @@ -104,7 +105,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -122,7 +123,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -140,7 +141,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -158,7 +159,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -176,7 +177,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -212,7 +213,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -230,7 +231,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -248,7 +249,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -266,7 +267,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -284,7 +285,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -305,7 +306,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -323,7 +324,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: condition kind: 1 startTimeUnixNano: "[start_time]" @@ -344,7 +345,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: condition_if kind: 1 startTimeUnixNano: "[start_time]" @@ -362,7 +363,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: defer kind: 1 startTimeUnixNano: "[start_time]" @@ -380,7 +381,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: defer_primary kind: 1 startTimeUnixNano: "[start_time]" @@ -398,7 +399,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -422,7 +423,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -449,7 +450,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -467,7 +468,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -491,7 +492,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -512,7 +513,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: defer_deferred kind: 1 startTimeUnixNano: "[start_time]" @@ -539,7 +540,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: sequence kind: 1 startTimeUnixNano: "[start_time]" @@ -557,7 +558,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -578,7 +579,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -602,7 +603,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -629,7 +630,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -647,7 +648,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -671,7 +672,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -692,7 +693,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -713,7 +714,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -737,7 +738,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -764,7 +765,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -782,7 +783,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -806,7 +807,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__connector-2.snap b/apollo-router/tests/snapshots/apollo_otel_traces__connector-2.snap index 35c5da51ad..027a1f0b1c 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__connector-2.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__connector-2.snap @@ -24,6 +24,7 @@ resourceSpans: value: stringValue: "[redacted]" droppedAttributesCount: 0 + entityRefs: [] scopeSpans: - scope: name: apollo-router @@ -35,7 +36,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query kind: 2 startTimeUnixNano: "[start_time]" @@ -62,7 +63,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: router kind: 1 startTimeUnixNano: "[start_time]" @@ -104,7 +105,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -122,7 +123,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -140,7 +141,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -158,7 +159,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -176,7 +177,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -212,7 +213,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -230,7 +231,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -248,7 +249,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -266,7 +267,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -284,7 +285,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -305,7 +306,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -323,7 +324,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -347,7 +348,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect kind: 1 startTimeUnixNano: "[start_time]" @@ -383,7 +384,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -401,7 +402,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__connector.snap b/apollo-router/tests/snapshots/apollo_otel_traces__connector.snap index 35c5da51ad..027a1f0b1c 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__connector.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__connector.snap @@ -24,6 +24,7 @@ resourceSpans: value: stringValue: "[redacted]" droppedAttributesCount: 0 + entityRefs: [] scopeSpans: - scope: name: apollo-router @@ -35,7 +36,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query kind: 2 startTimeUnixNano: "[start_time]" @@ -62,7 +63,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: router kind: 1 startTimeUnixNano: "[start_time]" @@ -104,7 +105,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -122,7 +123,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -140,7 +141,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -158,7 +159,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -176,7 +177,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -212,7 +213,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -230,7 +231,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -248,7 +249,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -266,7 +267,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -284,7 +285,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -305,7 +306,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -323,7 +324,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -347,7 +348,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect kind: 1 startTimeUnixNano: "[start_time]" @@ -383,7 +384,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -401,7 +402,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__connector_error-2.snap b/apollo-router/tests/snapshots/apollo_otel_traces__connector_error-2.snap index 032c230a1a..fdbdb72da3 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__connector_error-2.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__connector_error-2.snap @@ -24,6 +24,7 @@ resourceSpans: value: stringValue: "[redacted]" droppedAttributesCount: 0 + entityRefs: [] scopeSpans: - scope: name: apollo-router @@ -35,7 +36,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query kind: 2 startTimeUnixNano: "[start_time]" @@ -62,7 +63,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: router kind: 1 startTimeUnixNano: "[start_time]" @@ -104,7 +105,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -122,7 +123,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -140,7 +141,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -158,7 +159,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -176,7 +177,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -212,7 +213,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -230,7 +231,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -248,7 +249,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -266,7 +267,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -284,7 +285,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -305,7 +306,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -323,7 +324,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: sequence kind: 1 startTimeUnixNano: "[start_time]" @@ -341,7 +342,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -365,7 +366,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect kind: 1 startTimeUnixNano: "[start_time]" @@ -401,7 +402,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -419,7 +420,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -440,7 +441,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -461,7 +462,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -485,7 +486,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect kind: 1 startTimeUnixNano: "[start_time]" @@ -521,7 +522,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -552,7 +553,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -573,7 +574,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -604,7 +605,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -625,7 +626,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -656,7 +657,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -677,7 +678,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -708,7 +709,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -729,7 +730,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -760,7 +761,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -781,7 +782,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -812,7 +813,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -833,7 +834,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -864,7 +865,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -885,7 +886,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -916,7 +917,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -937,7 +938,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -968,7 +969,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -989,7 +990,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1020,7 +1021,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1041,7 +1042,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1072,7 +1073,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1093,7 +1094,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1124,7 +1125,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1145,7 +1146,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1176,7 +1177,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1197,7 +1198,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1228,7 +1229,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1249,7 +1250,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1280,7 +1281,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1301,7 +1302,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1332,7 +1333,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1353,7 +1354,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1384,7 +1385,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1405,7 +1406,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1436,7 +1437,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1457,7 +1458,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1488,7 +1489,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1509,7 +1510,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1540,7 +1541,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1561,7 +1562,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1592,7 +1593,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1613,7 +1614,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1644,7 +1645,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1665,7 +1666,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1696,7 +1697,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1717,7 +1718,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1748,7 +1749,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1769,7 +1770,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1800,7 +1801,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1821,7 +1822,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1852,7 +1853,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1873,7 +1874,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1904,7 +1905,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1925,7 +1926,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1956,7 +1957,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1977,7 +1978,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2008,7 +2009,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2029,7 +2030,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2060,7 +2061,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2081,7 +2082,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2112,7 +2113,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2133,7 +2134,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2164,7 +2165,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2185,7 +2186,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2216,7 +2217,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2237,7 +2238,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2268,7 +2269,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2289,7 +2290,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2320,7 +2321,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2341,7 +2342,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2372,7 +2373,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2393,7 +2394,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2424,7 +2425,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2445,7 +2446,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2476,7 +2477,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2497,7 +2498,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2528,7 +2529,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2549,7 +2550,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2580,7 +2581,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2601,7 +2602,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2632,7 +2633,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2653,7 +2654,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2684,7 +2685,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2705,7 +2706,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2736,7 +2737,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2757,7 +2758,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2788,7 +2789,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2809,7 +2810,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2840,7 +2841,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2861,7 +2862,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2892,7 +2893,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2913,7 +2914,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2944,7 +2945,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2965,7 +2966,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2996,7 +2997,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -3017,7 +3018,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -3048,7 +3049,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -3069,7 +3070,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -3100,7 +3101,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__connector_error.snap b/apollo-router/tests/snapshots/apollo_otel_traces__connector_error.snap index 032c230a1a..fdbdb72da3 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__connector_error.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__connector_error.snap @@ -24,6 +24,7 @@ resourceSpans: value: stringValue: "[redacted]" droppedAttributesCount: 0 + entityRefs: [] scopeSpans: - scope: name: apollo-router @@ -35,7 +36,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query kind: 2 startTimeUnixNano: "[start_time]" @@ -62,7 +63,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: router kind: 1 startTimeUnixNano: "[start_time]" @@ -104,7 +105,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -122,7 +123,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -140,7 +141,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -158,7 +159,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -176,7 +177,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -212,7 +213,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -230,7 +231,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -248,7 +249,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -266,7 +267,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -284,7 +285,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -305,7 +306,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -323,7 +324,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: sequence kind: 1 startTimeUnixNano: "[start_time]" @@ -341,7 +342,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -365,7 +366,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect kind: 1 startTimeUnixNano: "[start_time]" @@ -401,7 +402,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -419,7 +420,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -440,7 +441,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -461,7 +462,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -485,7 +486,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect kind: 1 startTimeUnixNano: "[start_time]" @@ -521,7 +522,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -552,7 +553,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -573,7 +574,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -604,7 +605,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -625,7 +626,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -656,7 +657,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -677,7 +678,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -708,7 +709,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -729,7 +730,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -760,7 +761,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -781,7 +782,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -812,7 +813,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -833,7 +834,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -864,7 +865,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -885,7 +886,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -916,7 +917,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -937,7 +938,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -968,7 +969,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -989,7 +990,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1020,7 +1021,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1041,7 +1042,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1072,7 +1073,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1093,7 +1094,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1124,7 +1125,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1145,7 +1146,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1176,7 +1177,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1197,7 +1198,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1228,7 +1229,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1249,7 +1250,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1280,7 +1281,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1301,7 +1302,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1332,7 +1333,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1353,7 +1354,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1384,7 +1385,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1405,7 +1406,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1436,7 +1437,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1457,7 +1458,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1488,7 +1489,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1509,7 +1510,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1540,7 +1541,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1561,7 +1562,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1592,7 +1593,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1613,7 +1614,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1644,7 +1645,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1665,7 +1666,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1696,7 +1697,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1717,7 +1718,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1748,7 +1749,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1769,7 +1770,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1800,7 +1801,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1821,7 +1822,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1852,7 +1853,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1873,7 +1874,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1904,7 +1905,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1925,7 +1926,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -1956,7 +1957,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -1977,7 +1978,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2008,7 +2009,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2029,7 +2030,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2060,7 +2061,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2081,7 +2082,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2112,7 +2113,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2133,7 +2134,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2164,7 +2165,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2185,7 +2186,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2216,7 +2217,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2237,7 +2238,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2268,7 +2269,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2289,7 +2290,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2320,7 +2321,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2341,7 +2342,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2372,7 +2373,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2393,7 +2394,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2424,7 +2425,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2445,7 +2446,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2476,7 +2477,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2497,7 +2498,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2528,7 +2529,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2549,7 +2550,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2580,7 +2581,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2601,7 +2602,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2632,7 +2633,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2653,7 +2654,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2684,7 +2685,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2705,7 +2706,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2736,7 +2737,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2757,7 +2758,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2788,7 +2789,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2809,7 +2810,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2840,7 +2841,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2861,7 +2862,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2892,7 +2893,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2913,7 +2914,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2944,7 +2945,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -2965,7 +2966,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -2996,7 +2997,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -3017,7 +3018,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -3048,7 +3049,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -3069,7 +3070,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: connect_request kind: 1 startTimeUnixNano: "[start_time]" @@ -3100,7 +3101,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__non_defer-2.snap b/apollo-router/tests/snapshots/apollo_otel_traces__non_defer-2.snap index 8f789400c3..aee07a18a8 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__non_defer-2.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__non_defer-2.snap @@ -24,6 +24,7 @@ resourceSpans: value: stringValue: "[redacted]" droppedAttributesCount: 0 + entityRefs: [] scopeSpans: - scope: name: apollo-router @@ -35,7 +36,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query kind: 2 startTimeUnixNano: "[start_time]" @@ -62,7 +63,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: router kind: 1 startTimeUnixNano: "[start_time]" @@ -104,7 +105,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -122,7 +123,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -140,7 +141,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -158,7 +159,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -176,7 +177,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -212,7 +213,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -230,7 +231,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -248,7 +249,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -266,7 +267,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -284,7 +285,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -305,7 +306,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -323,7 +324,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: sequence kind: 1 startTimeUnixNano: "[start_time]" @@ -341,7 +342,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -365,7 +366,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -392,7 +393,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -410,7 +411,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -434,7 +435,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -455,7 +456,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -476,7 +477,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -500,7 +501,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -527,7 +528,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -545,7 +546,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -569,7 +570,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -590,7 +591,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -611,7 +612,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -635,7 +636,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -662,7 +663,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -680,7 +681,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -704,7 +705,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__non_defer.snap b/apollo-router/tests/snapshots/apollo_otel_traces__non_defer.snap index 8f789400c3..aee07a18a8 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__non_defer.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__non_defer.snap @@ -24,6 +24,7 @@ resourceSpans: value: stringValue: "[redacted]" droppedAttributesCount: 0 + entityRefs: [] scopeSpans: - scope: name: apollo-router @@ -35,7 +36,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query kind: 2 startTimeUnixNano: "[start_time]" @@ -62,7 +63,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: router kind: 1 startTimeUnixNano: "[start_time]" @@ -104,7 +105,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -122,7 +123,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -140,7 +141,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -158,7 +159,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -176,7 +177,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -212,7 +213,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -230,7 +231,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -248,7 +249,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -266,7 +267,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -284,7 +285,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -305,7 +306,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -323,7 +324,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: sequence kind: 1 startTimeUnixNano: "[start_time]" @@ -341,7 +342,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -365,7 +366,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -392,7 +393,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -410,7 +411,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -434,7 +435,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -455,7 +456,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -476,7 +477,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -500,7 +501,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -527,7 +528,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -545,7 +546,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -569,7 +570,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -590,7 +591,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -611,7 +612,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -635,7 +636,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -662,7 +663,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -680,7 +681,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -704,7 +705,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__send_header-2.snap b/apollo-router/tests/snapshots/apollo_otel_traces__send_header-2.snap index 6bd45e39ee..b5fa69b6c7 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__send_header-2.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__send_header-2.snap @@ -24,6 +24,7 @@ resourceSpans: value: stringValue: "[redacted]" droppedAttributesCount: 0 + entityRefs: [] scopeSpans: - scope: name: apollo-router @@ -35,7 +36,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query kind: 2 startTimeUnixNano: "[start_time]" @@ -62,7 +63,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: router kind: 1 startTimeUnixNano: "[start_time]" @@ -104,7 +105,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -122,7 +123,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -140,7 +141,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -158,7 +159,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -176,7 +177,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -212,7 +213,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -230,7 +231,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -248,7 +249,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -266,7 +267,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -284,7 +285,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -305,7 +306,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -323,7 +324,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: sequence kind: 1 startTimeUnixNano: "[start_time]" @@ -341,7 +342,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -365,7 +366,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -392,7 +393,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -410,7 +411,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -434,7 +435,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -455,7 +456,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -476,7 +477,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -500,7 +501,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -527,7 +528,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -545,7 +546,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -569,7 +570,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -590,7 +591,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -611,7 +612,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -635,7 +636,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -662,7 +663,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -680,7 +681,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -704,7 +705,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__send_header.snap b/apollo-router/tests/snapshots/apollo_otel_traces__send_header.snap index 6bd45e39ee..b5fa69b6c7 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__send_header.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__send_header.snap @@ -24,6 +24,7 @@ resourceSpans: value: stringValue: "[redacted]" droppedAttributesCount: 0 + entityRefs: [] scopeSpans: - scope: name: apollo-router @@ -35,7 +36,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query kind: 2 startTimeUnixNano: "[start_time]" @@ -62,7 +63,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: router kind: 1 startTimeUnixNano: "[start_time]" @@ -104,7 +105,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -122,7 +123,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -140,7 +141,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -158,7 +159,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -176,7 +177,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -212,7 +213,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -230,7 +231,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -248,7 +249,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -266,7 +267,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -284,7 +285,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -305,7 +306,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -323,7 +324,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: sequence kind: 1 startTimeUnixNano: "[start_time]" @@ -341,7 +342,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -365,7 +366,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -392,7 +393,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -410,7 +411,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -434,7 +435,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -455,7 +456,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -476,7 +477,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -500,7 +501,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -527,7 +528,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -545,7 +546,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -569,7 +570,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -590,7 +591,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -611,7 +612,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -635,7 +636,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -662,7 +663,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -680,7 +681,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -704,7 +705,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__send_variable_value-2.snap b/apollo-router/tests/snapshots/apollo_otel_traces__send_variable_value-2.snap index e150c8a2cc..586c94083d 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__send_variable_value-2.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__send_variable_value-2.snap @@ -24,6 +24,7 @@ resourceSpans: value: stringValue: "[redacted]" droppedAttributesCount: 0 + entityRefs: [] scopeSpans: - scope: name: apollo-router @@ -35,7 +36,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query kind: 2 startTimeUnixNano: "[start_time]" @@ -62,7 +63,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: router kind: 1 startTimeUnixNano: "[start_time]" @@ -104,7 +105,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -122,7 +123,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -140,7 +141,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -158,7 +159,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -176,7 +177,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -212,7 +213,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -230,7 +231,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -248,7 +249,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -266,7 +267,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -284,7 +285,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -305,7 +306,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -323,7 +324,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: sequence kind: 1 startTimeUnixNano: "[start_time]" @@ -341,7 +342,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -365,7 +366,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -392,7 +393,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -410,7 +411,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -434,7 +435,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -455,7 +456,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -476,7 +477,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -500,7 +501,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -527,7 +528,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -545,7 +546,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -569,7 +570,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -590,7 +591,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -611,7 +612,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -635,7 +636,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -662,7 +663,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -680,7 +681,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -704,7 +705,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__send_variable_value.snap b/apollo-router/tests/snapshots/apollo_otel_traces__send_variable_value.snap index e150c8a2cc..586c94083d 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__send_variable_value.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__send_variable_value.snap @@ -24,6 +24,7 @@ resourceSpans: value: stringValue: "[redacted]" droppedAttributesCount: 0 + entityRefs: [] scopeSpans: - scope: name: apollo-router @@ -35,7 +36,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query kind: 2 startTimeUnixNano: "[start_time]" @@ -62,7 +63,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: router kind: 1 startTimeUnixNano: "[start_time]" @@ -104,7 +105,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -122,7 +123,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -140,7 +141,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -158,7 +159,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -176,7 +177,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -212,7 +213,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -230,7 +231,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -248,7 +249,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -266,7 +267,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -284,7 +285,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -305,7 +306,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -323,7 +324,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: sequence kind: 1 startTimeUnixNano: "[start_time]" @@ -341,7 +342,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -365,7 +366,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -392,7 +393,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -410,7 +411,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -434,7 +435,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -455,7 +456,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -476,7 +477,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -500,7 +501,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -527,7 +528,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -545,7 +546,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -569,7 +570,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -590,7 +591,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -611,7 +612,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -635,7 +636,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -662,7 +663,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -680,7 +681,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -704,7 +705,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__trace_id-2.snap b/apollo-router/tests/snapshots/apollo_otel_traces__trace_id-2.snap index 8f789400c3..aee07a18a8 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__trace_id-2.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__trace_id-2.snap @@ -24,6 +24,7 @@ resourceSpans: value: stringValue: "[redacted]" droppedAttributesCount: 0 + entityRefs: [] scopeSpans: - scope: name: apollo-router @@ -35,7 +36,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query kind: 2 startTimeUnixNano: "[start_time]" @@ -62,7 +63,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: router kind: 1 startTimeUnixNano: "[start_time]" @@ -104,7 +105,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -122,7 +123,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -140,7 +141,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -158,7 +159,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -176,7 +177,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -212,7 +213,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -230,7 +231,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -248,7 +249,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -266,7 +267,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -284,7 +285,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -305,7 +306,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -323,7 +324,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: sequence kind: 1 startTimeUnixNano: "[start_time]" @@ -341,7 +342,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -365,7 +366,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -392,7 +393,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -410,7 +411,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -434,7 +435,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -455,7 +456,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -476,7 +477,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -500,7 +501,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -527,7 +528,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -545,7 +546,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -569,7 +570,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -590,7 +591,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -611,7 +612,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -635,7 +636,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -662,7 +663,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -680,7 +681,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -704,7 +705,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__trace_id.snap b/apollo-router/tests/snapshots/apollo_otel_traces__trace_id.snap index 8f789400c3..aee07a18a8 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__trace_id.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__trace_id.snap @@ -24,6 +24,7 @@ resourceSpans: value: stringValue: "[redacted]" droppedAttributesCount: 0 + entityRefs: [] scopeSpans: - scope: name: apollo-router @@ -35,7 +36,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query kind: 2 startTimeUnixNano: "[start_time]" @@ -62,7 +63,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: router kind: 1 startTimeUnixNano: "[start_time]" @@ -104,7 +105,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -122,7 +123,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: parse_query kind: 1 startTimeUnixNano: "[start_time]" @@ -140,7 +141,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -158,7 +159,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -176,7 +177,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: supergraph kind: 1 startTimeUnixNano: "[start_time]" @@ -212,7 +213,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -230,7 +231,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: query_planning kind: 1 startTimeUnixNano: "[start_time]" @@ -248,7 +249,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job kind: 1 startTimeUnixNano: "[start_time]" @@ -266,7 +267,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: compute_job.execution kind: 1 startTimeUnixNano: "[start_time]" @@ -284,7 +285,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: execution kind: 1 startTimeUnixNano: "[start_time]" @@ -305,7 +306,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -323,7 +324,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: sequence kind: 1 startTimeUnixNano: "[start_time]" @@ -341,7 +342,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -365,7 +366,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -392,7 +393,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -410,7 +411,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -434,7 +435,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -455,7 +456,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -476,7 +477,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -500,7 +501,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -527,7 +528,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -545,7 +546,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -569,7 +570,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" @@ -590,7 +591,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: flatten kind: 1 startTimeUnixNano: "[start_time]" @@ -611,7 +612,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: fetch kind: 1 startTimeUnixNano: "[start_time]" @@ -635,7 +636,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph kind: 1 startTimeUnixNano: "[start_time]" @@ -662,7 +663,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: rhai_plugin kind: 1 startTimeUnixNano: "[start_time]" @@ -680,7 +681,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: subgraph_request kind: 3 startTimeUnixNano: "[start_time]" @@ -704,7 +705,7 @@ resourceSpans: spanId: "[span_id]" traceState: "" parentSpanId: "[span_id]" - flags: 1 + flags: 257 name: http_request kind: 3 startTimeUnixNano: "[start_time]" diff --git a/apollo-router/tests/telemetry_resource_tests.rs b/apollo-router/tests/telemetry_resource_tests.rs index 2c5c82ea27..cfddbf3ebb 100644 --- a/apollo-router/tests/telemetry_resource_tests.rs +++ b/apollo-router/tests/telemetry_resource_tests.rs @@ -9,7 +9,6 @@ use apollo_router::_private::telemetry::ConfigResource; use libtest_mimic::Arguments; use libtest_mimic::Failed; use libtest_mimic::Trial; -use opentelemetry::Key; fn main() { let mut args = Arguments::from_args(); @@ -53,7 +52,7 @@ fn test_empty() -> Result<(), Failed> { }; let resource = test_config.to_resource(); let service_name = resource - .get(opentelemetry_semantic_conventions::resource::SERVICE_NAME.into()) + .get(&opentelemetry_semantic_conventions::resource::SERVICE_NAME.into()) .unwrap(); assert!( service_name @@ -63,17 +62,17 @@ fn test_empty() -> Result<(), Failed> { ); assert!( resource - .get(opentelemetry_semantic_conventions::resource::SERVICE_NAMESPACE.into()) + .get(&opentelemetry_semantic_conventions::resource::SERVICE_NAMESPACE.into()) .is_none() ); assert_eq!( - resource.get(opentelemetry_semantic_conventions::resource::SERVICE_VERSION.into()), + resource.get(&opentelemetry_semantic_conventions::resource::SERVICE_VERSION.into()), Some(std::env!("CARGO_PKG_VERSION").into()) ); assert!( resource - .get(opentelemetry_semantic_conventions::resource::PROCESS_EXECUTABLE_NAME.into()) + .get(&opentelemetry_semantic_conventions::resource::PROCESS_EXECUTABLE_NAME.into()) .expect("expected excutable name") .as_str() .contains("telemetry_resources") @@ -102,15 +101,15 @@ fn test_config_resources() -> Result<(), Failed> { }; let resource = test_config.to_resource(); assert_eq!( - resource.get(opentelemetry_semantic_conventions::resource::SERVICE_NAME.into()), + resource.get(&opentelemetry_semantic_conventions::resource::SERVICE_NAME.into()), Some("override-service-name".into()) ); assert_eq!( - resource.get(opentelemetry_semantic_conventions::resource::SERVICE_NAMESPACE.into()), + resource.get(&opentelemetry_semantic_conventions::resource::SERVICE_NAMESPACE.into()), Some("override-namespace".into()) ); assert_eq!( - resource.get(Key::from_static_str("extra-key")), + resource.get(&"extra-key".into()), Some("extra-value".into()) ); Ok(()) @@ -124,11 +123,11 @@ fn test_service_name_service_namespace() -> Result<(), Failed> { }; let resource = test_config.to_resource(); assert_eq!( - resource.get(opentelemetry_semantic_conventions::resource::SERVICE_NAME.into()), + resource.get(&opentelemetry_semantic_conventions::resource::SERVICE_NAME.into()), Some("override-service-name".into()) ); assert_eq!( - resource.get(opentelemetry_semantic_conventions::resource::SERVICE_NAMESPACE.into()), + resource.get(&opentelemetry_semantic_conventions::resource::SERVICE_NAMESPACE.into()), Some("override-namespace".into()) ); Ok(()) @@ -150,7 +149,7 @@ fn test_service_name_override() -> Result<(), Failed> { resources: Default::default(), } .to_resource() - .get(opentelemetry_semantic_conventions::resource::SERVICE_NAME.into()) + .get(&opentelemetry_semantic_conventions::resource::SERVICE_NAME.into()) .unwrap() .as_str() .starts_with("unknown_service:telemetry_resources-") @@ -166,7 +165,7 @@ fn test_service_name_override() -> Result<(), Failed> { )]), } .to_resource() - .get(opentelemetry_semantic_conventions::resource::SERVICE_NAME.into()), + .get(&opentelemetry_semantic_conventions::resource::SERVICE_NAME.into()), Some("yaml-resource".into()) ); @@ -180,7 +179,7 @@ fn test_service_name_override() -> Result<(), Failed> { )]), } .to_resource() - .get(opentelemetry_semantic_conventions::resource::SERVICE_NAME.into()), + .get(&opentelemetry_semantic_conventions::resource::SERVICE_NAME.into()), Some("yaml-service-name".into()) ); @@ -198,7 +197,7 @@ fn test_service_name_override() -> Result<(), Failed> { )]), } .to_resource() - .get(opentelemetry_semantic_conventions::resource::SERVICE_NAME.into()), + .get(&opentelemetry_semantic_conventions::resource::SERVICE_NAME.into()), Some("env-resource".into()) ); @@ -216,7 +215,7 @@ fn test_service_name_override() -> Result<(), Failed> { )]), } .to_resource() - .get(opentelemetry_semantic_conventions::resource::SERVICE_NAME.into()), + .get(&opentelemetry_semantic_conventions::resource::SERVICE_NAME.into()), Some("env-service-name".into()) ); diff --git a/deny.toml b/deny.toml index 66561762ab..bfc53f96e6 100644 --- a/deny.toml +++ b/deny.toml @@ -24,11 +24,6 @@ git-fetch-with-cli = true ignore = [ "RUSTSEC-2023-0071", - # protobuf is used only through prometheus crates, enforced by - # a `[bans]` entry below. in the prometheus crates, only the protobuf - # encoder is used, while only the decoder is affected by this advisory. - "RUSTSEC-2024-0437", - # The following crates are unmaintained "RUSTSEC-2024-0320", # TODO replace the `yaml-rust` crate with a maintained equivalent "RUSTSEC-2024-0436", # TODO replace the `paste` crate with a maintained equivalent @@ -96,12 +91,7 @@ highlight = "all" # List of crates to deny deny = [ - { crate = "openssl-sys" }, - # Prevent adding new dependencies on protobuf that may use code with - # a security advisory in it (see `[advisories]`). - # If you *must* add a new crate to the "wrappers" here, carefully audit - # that it is *not* affected by any of the advisories above. - { crate = "protobuf:<3.7.2", wrappers = ["prometheus", "opentelemetry-prometheus"] }, + { name = "openssl-sys" }, ] # This section is considered when running `cargo deny check sources`. diff --git a/docs/source/routing/observability/router-telemetry-otel/apm-guides/zipkin/zipkin-traces.mdx b/docs/source/routing/observability/router-telemetry-otel/apm-guides/zipkin/zipkin-traces.mdx index a80812ef11..3f9f69c0c1 100644 --- a/docs/source/routing/observability/router-telemetry-otel/apm-guides/zipkin/zipkin-traces.mdx +++ b/docs/source/routing/observability/router-telemetry-otel/apm-guides/zipkin/zipkin-traces.mdx @@ -1,7 +1,7 @@ --- -title: Zipkin exporter -subtitle: Configure the Zipkin exporter for tracing -description: Enable and configure the Zipkin exporter for tracing in the Apollo GraphOS Router or Apollo Router Core. +title: Zipkin tracing +subtitle: Configure tracing for Zipkin +description: Enable and configure tracing for Zipkin in Apollo GraphOS Router or Apollo Router Core. context: - telemetry redirectFrom: @@ -10,13 +10,38 @@ redirectFrom: import BatchProcessorPreamble from '../../../../../../shared/batch-processor-preamble.mdx'; import BatchProcessorRef from '../../../../../../shared/batch-processor-ref.mdx'; -Enable and configure the [Zipkin](https://zipkin.io/) exporter for tracing in the GraphOS Router or Apollo Router Core. +Enable and configure tracing for [Zipkin](https://zipkin.io/) in GraphOS Router or Apollo Router Core. For general tracing configuration, refer to [Router Tracing Configuration](/router/configuration/telemetry/exporters/tracing/overview). -## Zipkin configuration +## OTLP configuration (recommended) -The router can be configured to export tracing data for Zipkin to either the default collector address or a URL: +Zipkin supports OTLP ingestion via [zipkin-otel](https://github.com/openzipkin-contrib/zipkin-otel), and OpenTelemetry is [deprecating the native Zipkin exporters](https://opentelemetry.io/blog/2025/deprecating-zipkin-exporters/) in favor of OTLP. Using the OTLP exporter is recommended: + +```yaml title="router.yaml" +telemetry: + exporters: + tracing: + otlp: + enabled: true + endpoint: "http://${env.ZIPKIN_HOST}:9411" + protocol: http +``` + +This sends traces to Zipkin using the OTLP protocol. + +See [OTLP configuration](/router/configuration/telemetry/exporters/tracing/otlp#configuration) for more details on settings. + +## Zipkin native configuration + + + The native Zipkin exporter is deprecated and will be removed in the next major version of Router. Zipkin supports OTLP ingestion, and OpenTelemetry is [deprecating native Zipkin exporters](https://opentelemetry.io/blog/2025/deprecating-zipkin-exporters/) in favor of OTLP. Use [OTLP configuration](#otlp-configuration-recommended) instead. + + + Zipkin native exporter does not currently support setting the service name on the Zipkin native exports. + + +You can configure the router to export tracing data to Zipkin using the native Zipkin exporter: ```yaml title="router.yaml" telemetry: @@ -24,36 +49,32 @@ telemetry: tracing: zipkin: enabled: true - - # Optional endpoint, either 'default' or a URL (Defaults to http://127.0.0.1:9411/api/v2/span) - endpoint: "http://${env.ZIPKIN_HOST}:9411/api/v2/spans}" + # Optional endpoint (defaults to http://127.0.0.1:9411/api/v2/spans) + endpoint: "http://${env.ZIPKIN_HOST}:9411/api/v2/spans" ``` ### `enabled` -Flag to enable the Zipkin exporter. - Set to true to enable the Zipkin exporter. Defaults to false. ### `endpoint` -The Zipkin endpoint address. Defaults to http://127.0.0.1:9411/api/v2/span +The Zipkin collector endpoint address. Defaults to `http://127.0.0.1:9411/api/v2/spans`. ### `batch_processor` -An example configuration using Zipkin with `batch_processor`: - -```yaml +```yaml title="router.yaml" telemetry: exporters: tracing: - zipkin: - batch_processor: + zipkin: + enabled: true + batch_processor: max_export_batch_size: 512 max_concurrent_exports: 1 - max_export_timeout: 30s + max_export_timeout: 30s max_queue_size: 2048 scheduled_delay: 5s ``` @@ -62,10 +83,10 @@ telemetry: -## Zipkin configuration reference +## Zipkin native configuration reference -| Attribute | Default | Description | -|-------------------|-------------------------------------|--------------------------------| -| `enabled` | `false` | Enable the Zipkin exporter. | -| `endpoint` | `http://127.0.0.1:9411/api/v2/span` | The endpoint to send spans to. | -| `batch_processor` | | The batch processor settings. | +| Attribute | Default | Description | +|-------------------|--------------------------------------|--------------------------------| +| `enabled` | `false` | Enable the Zipkin exporter. | +| `endpoint` | `http://127.0.0.1:9411/api/v2/spans` | The endpoint to send spans to. | +| `batch_processor` | | The batch processor settings. | diff --git a/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/standard-instruments.mdx b/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/standard-instruments.mdx index 0d9c7eac83..ff2f9086f6 100644 --- a/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/standard-instruments.mdx +++ b/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/standard-instruments.mdx @@ -260,7 +260,8 @@ Similar to the initial call to Uplink, the router does not record metrics for ca - `name`: One of `apollo-tracing`, `datadog-tracing`, `jaeger-collector`, `otlp-tracing`, `zipkin-tracing`. - `error`: One of `channel closed`, `channel full`. -- `apollo.router.telemetry.metrics.cardinality_overflow` - A count of how often a telemetry metric hit otel's hard cardinality limit. +- `apollo.router.telemetry.metrics.cardinality_overflow` - A count of how often a telemetry metric hit OpenTelemetry's cardinality limit. When a metric exceeds its cardinality limit, new attribute combinations are aggregated into an overflow bucket. + - `metric.name`: The name of the metric that exceeded its cardinality limit. ## Internals