diff --git a/.circleci/config.yml b/.circleci/config.yml index 42e906a9d2..1c1162518a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -56,6 +56,9 @@ parameters: type: string # update this as new versions of jaeger become available default: "1.33.0" + protoc_version: + type: string + default: "21.8" # These are common environment variables that we want to set on on all jobs. # While these could conceivably be set on the CircleCI project settings' @@ -85,6 +88,12 @@ commands: curl -L https://github.com/jaegertracing/jaeger/releases/download/v<< pipeline.parameters.jaeger_version >>/jaeger-<< pipeline.parameters.jaeger_version >>-linux-amd64.tar.gz --output jaeger.tar.gz tar -xf jaeger.tar.gz mv jaeger-<< pipeline.parameters.jaeger_version >>-linux-amd64 jaeger + - run: + name: Install protoc + command: | + curl -L https://github.com/protocolbuffers/protobuf/releases/download/v<< pipeline.parameters.protoc_version >>/protoc-<< pipeline.parameters.protoc_version >>-linux-x86_64.zip --output protoc.zip + unzip protoc.zip -d $HOME/.local + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" linux_arm_install_baseline: steps: - linux_install_baseline @@ -94,6 +103,12 @@ commands: curl -L https://github.com/jaegertracing/jaeger/releases/download/v<< pipeline.parameters.jaeger_version >>/jaeger-<< pipeline.parameters.jaeger_version >>-linux-arm64.tar.gz --output jaeger.tar.gz tar -xf jaeger.tar.gz mv jaeger-<< pipeline.parameters.jaeger_version >>-linux-arm64 jaeger + - run: + name: Install protoc + command: | + curl -L https://github.com/protocolbuffers/protobuf/releases/download/v<< pipeline.parameters.protoc_version >>/protoc-<< pipeline.parameters.protoc_version >>-linux-aarch_64.zip --output protoc.zip + unzip protoc.zip -d $HOME/.local + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" macos_install_baseline: steps: - run: @@ -103,6 +118,12 @@ commands: tar -xf jaeger.tar.gz mv jaeger-<< pipeline.parameters.jaeger_version >>-darwin-amd64 jaeger - install_minimal_rust + - run: + name: Install protoc + command: | + curl -L https://github.com/protocolbuffers/protobuf/releases/download/v<< pipeline.parameters.protoc_version >>/protoc-<< pipeline.parameters.protoc_version >>-osx-universal_binary.zip --output protoc.zip + unzip protoc.zip -d $HOME/.local + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" windows_install_baseline: steps: - run: @@ -112,6 +133,13 @@ commands: curl -L https://github.com/jaegertracing/jaeger/releases/download/v<< pipeline.parameters.jaeger_version >>/jaeger-<< pipeline.parameters.jaeger_version >>-windows-amd64.tar.gz --output jaeger.tar.gz tar -xf jaeger.tar.gz mv jaeger-<< pipeline.parameters.jaeger_version >>-windows-amd64 jaeger + - run: + name: Install protoc + shell: bash.exe + command: | + curl -L https://github.com/protocolbuffers/protobuf/releases/download/v<< pipeline.parameters.protoc_version >>/protoc-<< pipeline.parameters.protoc_version >>-win64.zip --output protoc.zip + unzip protoc.zip -d $HOME/.local + echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV" # This job makes sure everything is ready to run integration tests macos_prepare_env: @@ -145,6 +173,12 @@ commands: [net] git-fetch-with-cli = true "@ + - run: + name: Add protoc to env + command: | + $oldpath = (Get-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH).path + $newpath = โ€œ$oldpath;$HOME/.local/binโ€ + Set-ItemProperty -Path 'Registry::HKEY_LOCAL_MACHINE\System\CurrentControlSet\Control\Session Manager\Environment' -Name PATH -Value $newPath install_minimal_rust: steps: @@ -586,14 +620,17 @@ workflows: only: - dev - prepare_release: + name: "Prepare major release" release_type: "major" requires: - prepare_major_release_approval - prepare_release: + name: "Prepare minor release" release_type: "minor" requires: - prepare_minor_release_approval - prepare_release: + name: "Prepare patch release" release_type: "patch" requires: - prepare_patch_release_approval diff --git a/.gitignore b/.gitignore index 080c3ce480..ad362c28b0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ # Generated by Cargo # will have compiled files and executables -/target/ +**/target/ # These are backup files generated by rustfmt **/*.rs.bk diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ae542a7e5..fa4cb307ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,224 @@ All notable changes to Router will be documented in this file. This project adheres to [Semantic Versioning v2.0.0](https://semver.org/spec/v2.0.0.html). +# [1.6.0] - 2022-12-13 + +## โ— BREAKING โ— + +### Protoc now required to build ([Issue #1970](https://github.com/apollographql/router/issues/1970)) + +Protoc is now required to build Apollo Router. Upgrading to Open Telemetry 0.18 has enabled us to upgrade tonic which in turn no longer bundles protoc. +Users must install it themselves https://grpc.io/docs/protoc-installation/. + +By [@bryncooke](https://github.com/bryncooke) in https://github.com/apollographql/router/pull/1970 + +### Jaeger scheduled_delay moved to batch_processor->scheduled_delay ([Issue #2232](https://github.com/apollographql/router/issues/2232)) + +Jager config previously allowed configuration of scheduled_delay for batch span processor. To bring it in line with all other exporters this is now set using a batch_processor section. + +Before: +```yaml +telemetry: + tracing: + jaeger: + scheduled_delay: 100ms +``` + +After: +```yaml +telemetry: + tracing: + jaeger: + batch_processor: + scheduled_delay: 100ms +``` + +By [@bryncooke](https://github.com/bryncooke) in https://github.com/apollographql/router/pull/1970 + +## ๐Ÿš€ Features + +### Add support for experimental tooling ([Issue #2136](https://github.com/apollographql/router/issues/2136)) + +Display a message at startup listing used `experimental_` configurations with related GitHub discussions. +It also adds a new cli command `router config experimental` to display all available experimental configurations. + +By [@bnjjj](https://github.com/bnjjj) in https://github.com/apollographql/router/pull/2242 + +### Re-deploy router pods if the SuperGraph configmap changes ([PR #2223](https://github.com/apollographql/router/pull/2223)) +When setting the supergraph with the `supergraphFile` variable a `sha256` checksum is calculated and set as an annotation for the router pods. This will spin up new pods when the supergraph is mounted via config map and the schema has changed. + +Note: It is preferable to not have `--hot-reload` enabled with this feature since re-configuring the router during a pod restart is duplicating the work and may cause confusion in log messaging. + +By [@toneill818](https://github.com/toneill818) in https://github.com/apollographql/router/pull/2223 + +### Tracing batch span processor is now configurable ([Issue #2232](https://github.com/apollographql/router/issues/2232)) + +Exporting traces often requires performance tuning based on the throughput of the router, sampling settings and ingestion capability of tracing ingress. + +All exporters now support configuring the batch span processor in the router yaml. +```yaml +telemetry: + apollo: + batch_processor: + scheduled_delay: 100ms + max_concurrent_exports: 1000 + max_export_batch_size: 10000 + max_export_timeout: 100s + max_queue_size: 10000 + tracing: + jaeger|zipkin|otlp|datadog: + batch_processor: + scheduled_delay: 100ms + max_concurrent_exports: 1000 + max_export_batch_size: 10000 + max_export_timeout: 100s + max_queue_size: 10000 +``` + +See the Open Telemetry docs for more information. + +By [@bryncooke](https://github.com/bryncooke) in https://github.com/apollographql/router/pull/1970 + +### Add hot-reload support for Rhai scripts ([Issue #1071](https://github.com/apollographql/router/issues/1071)) + +The router will "watch" your "rhai.scripts" directory for changes and prompt an interpreter re-load if changes are detected. Changes are defined as: + + * creating a new file with a ".rhai" suffix + * modifying or removing an existing file with a ".rhai" suffix + +The watch is recursive, so files in sub-directories of the "rhai.scripts" directory are also watched. + +The Router attempts to identify errors in scripts before applying the changes. If errors are detected, these will be logged and the changes will not be applied to the runtime. Not all classes of error can be reliably detected, so check the log output of your router to make sure that changes have been applied. + +By [@garypen](https://github.com/garypen) in https://github.com/apollographql/router/pull/2198 + +### Add support for working with multi-value header keys to Rhai ([Issue #2211](https://github.com/apollographql/router/issues/2211), [Issue #2255](https://github.com/apollographql/router/issues/2255)) + +Adds support for setting a header map key with an array. This causes the HeaderMap key/values to be appended() to the map, rather than inserted(). + +Adds support for a new `values()` fn which retrieves multiple values for a HeaderMap key as an array. + +Example use from Rhai as: + +``` + response.headers["set-cookie"] = [ + "foo=bar; Domain=localhost; Path=/; Expires=Wed, 04 Jan 2023 17:25:27 GMT; HttpOnly; Secure; SameSite=None", + "foo2=bar2; Domain=localhost; Path=/; Expires=Wed, 04 Jan 2023 17:25:27 GMT; HttpOnly; Secure; SameSite=None", + ]; + response.headers.values("set-cookie"); // Returns the array of values +``` + +By [@garypen](https://github.com/garypen) in https://github.com/apollographql/router/pull/2219, https://github.com/apollographql/router/pull/2258 + +## ๐Ÿ› Fixes + +### Filter nullified deferred responses ([Issue #2213](https://github.com/apollographql/router/issues/2168)) + +[`@defer` spec updates](https://github.com/graphql/graphql-spec/compare/01d7b98f04810c9a9db4c0e53d3c4d54dbf10b82...f58632f496577642221c69809c32dd46b5398bd7#diff-0f02d73330245629f776bb875e5ca2b30978a716732abca136afdd028d5cd33cR448-R470) mandates that a deferred response should not be sent if its path points to an element of the response that was nullified in a previous payload. + +By [@Geal](https://github.com/geal) in https://github.com/apollographql/router/pull/2184 + +### Return root `__typename` when parts of a query with deferred fragment ([Issue #1677](https://github.com/apollographql/router/issues/1677)) + +With this query: + +```graphql +{ + __typename + fast + ...deferedFragment @defer +} + +fragment deferedFragment on Query { + slow +} +``` + +You will receive the first response chunk: + +```json +{"data":{"__typename": "Query", "fast":0},"hasNext":true} +``` + +By [@bnjjj](https://github.com/bnjjj) in https://github.com/apollographql/router/pull/2188 + + +### Wait for opentelemetry tracer provider to shutdown ([PR #2191](https://github.com/apollographql/router/pull/2191)) + +When we drop Telemetry we spawn a thread to perform the global opentelemetry trace provider shutdown. The documentation of this function indicates that "This will invoke the shutdown method on all span processors. span processors should export remaining spans before return". We should give that process some time to complete (5 seconds currently) before returning from the `drop`. This will provide more opportunity for spans to be exported. + +By [@garypen](https://github.com/garypen) in https://github.com/apollographql/router/pull/2191 +### Dispatch errors from the primary response to deferred responses ([Issue #1818](https://github.com/apollographql/router/issues/1818), [Issue #2185](https://github.com/apollographql/router/issues/2185)) + +When errors are generated during the primary execution, some may also be assigned to deferred responses. + +By [@Geal](https://github.com/geal) in https://github.com/apollographql/router/pull/2192 + +### Reconstruct deferred queries with knowledge about fragments ([Issue #2105](https://github.com/apollographql/router/issues/2105)) + +When we are using `@defer`, response formatting must apply on a subset of the query (primary or deferred), that is reconstructed from information provided by the query planner: a path into the response and a subselection. Previously, that path did not include information on fragment application, which resulted in query reconstruction issues if `@defer` was used under a fragment application on an interface. + +By [@Geal](https://github.com/geal) in https://github.com/apollographql/router/pull/2109 + +## ๐Ÿ›  Maintenance + +### Improve plugin registration predictability ([PR #2181](https://github.com/apollographql/router/pull/2181)) + +This replaces [ctor](https://crates.io/crates/ctor) with [linkme](https://crates.io/crates/linkme). `ctor` enables rust code to execute before `main`. This can be a source of undefined behaviour and we don't need our code to execute before `main`. `linkme` provides a registration mechanism that is perfect for this use case, so switching to use it makes the router more predictable, simpler to reason about and with a sound basis for future plugin enhancements. + +By [@garypen](https://github.com/garypen) in https://github.com/apollographql/router/pull/2181 + +### it_rate_limit_subgraph_requests fixed ([Issue #2213](https://github.com/apollographql/router/issues/2213)) + +This test was failing frequently due to it being a timing test being run in a single threaded tokio runtime. + +By [@bryncooke](https://github.com/bryncooke) in https://github.com/apollographql/router/pull/2218 + +### Update reports.proto protobuf definition ([PR #2247](https://github.com/apollographql/router/pull/2247)) + +Update the reports.proto file, and change the prompt to update the file with the correct new location. + +By [@o0Ignition0o](https://github.com/o0Ignition0o) in https://github.com/apollographql/router/pull/2247 +### Upgrade OpenTelemetry to 0.18 ([Issue #1970](https://github.com/apollographql/router/issues/1970)) + +Update to OpenTelemetry 0.18. + +By [@bryncooke](https://github.com/bryncooke) and [@bnjjj](https://github.com/bnjjj) in https://github.com/apollographql/router/pull/1970 and https://github.com/apollographql/router/pull/2236 + +### Remove spaceport ([Issue #2233](https://github.com/apollographql/router/issues/2233)) + +Removal significantly simplifies telemetry code and likely to increase performance and reliability. + +By [@bryncooke](https://github.com/bryncooke) in https://github.com/apollographql/router/pull/1970 + +### Update to Rust 1.65 ([Issue #2220](https://github.com/apollographql/router/issues/2220)) + +Rust MSRV incremented to 1.65. + +By [@bryncooke](https://github.com/bryncooke) in https://github.com/apollographql/router/pull/2221 and https://github.com/apollographql/router/pull/2240 + +### Improve automated release ([Pull #2220](https://github.com/apollographql/router/pull/2256)) + +Improved the automated release to: +* Update the scaffold files +* Improve the names of prepare release steps in circle. + +By [@bryncooke](https://github.com/bryncooke) in https://github.com/apollographql/router/pull/2256 + +### Use Elastic-2.0 license spdx ([PR #2055](https://github.com/apollographql/router/issues/2055)) + +Now that the Elastic-2.0 spdx is a valid identifier in the rust ecosystem, we can update the router references. + +By [@o0Ignition0o](https://github.com/o0Ignition0o) in https://github.com/apollographql/router/pull/2054 + +## ๐Ÿ“š Documentation +### Create yaml config design guidance ([Issue #2158](https://github.com/apollographql/router/issues/2158)) + +Added some yaml design guidance to help us create consistent yaml config for new and existing features. + +By [@bryncooke](https://github.com/bryncooke) in https://github.com/apollographql/router/pull/2159 + + # [1.5.0] - 2022-12-06 ## โ— BREAKING โ— diff --git a/Cargo.lock b/Cargo.lock index fd006763c4..ae122caa4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -127,7 +127,7 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b17d38f06e92256e9b0b271b878e20309822a587b2acfa234a60d36d92b6b43" dependencies = [ - "apollo-parser 0.3.2", + "apollo-parser 0.3.1", "thiserror", ] @@ -143,9 +143,9 @@ dependencies = [ [[package]] name = "apollo-parser" -version = "0.3.2" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d640c8fb7f9ab98a78a8086bb413d8ecf3ee44849976e1636e27265f09e9e544" +checksum = "7a8f6cc3fa1313e045538ed2ce72ba916d52b501cd81e636a0bd5cdc703a0c73" dependencies = [ "rowan", ] @@ -162,12 +162,14 @@ dependencies = [ [[package]] name = "apollo-router" -version = "1.5.0" +version = "1.6.0" dependencies = [ "access-json", "ansi_term", "anyhow", + "apollo-encoder 0.4.0", "apollo-parser 0.4.0", + "arc-swap", "askama", "async-compression", "async-trait", @@ -179,9 +181,7 @@ dependencies = [ "bytes", "clap 3.2.23", "console-subscriber", - "ctor", - "dashmap 5.4.0", - "deadpool", + "dashmap", "derivative", "derive_more", "dhat", @@ -206,6 +206,7 @@ dependencies = [ "jsonschema", "lazy_static", "libc", + "linkme", "lru", "maplit", "mediatype", @@ -227,8 +228,8 @@ dependencies = [ "paste", "pin-project-lite", "prometheus", - "prost 0.9.0", - "prost-types 0.9.0", + "prost", + "prost-types", "proteus", "rand", "redis", @@ -254,8 +255,8 @@ dependencies = [ "thiserror", "tokio", "tokio-stream", - "tokio-util 0.7.4", - "tonic 0.6.2", + "tokio-util", + "tonic", "tonic-build", "tower", "tower-http", @@ -277,7 +278,7 @@ dependencies = [ [[package]] name = "apollo-router-benchmarks" -version = "1.5.0" +version = "1.6.0" dependencies = [ "apollo-router", "async-trait", @@ -293,7 +294,7 @@ dependencies = [ [[package]] name = "apollo-router-scaffold" -version = "1.5.0" +version = "1.6.0" dependencies = [ "anyhow", "cargo-scaffold", @@ -327,6 +328,12 @@ dependencies = [ "derive_arbitrary", ] +[[package]] +name = "arc-swap" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "983cd8b9d4b02a6dc6ffa557262eb5858a27a0038ffffe21a0f133eaa819a164" + [[package]] name = "ascii" version = "0.9.3" @@ -833,7 +840,7 @@ dependencies = [ "memchr", "pin-project-lite", "tokio", - "tokio-util 0.7.4", + "tokio-util", ] [[package]] @@ -911,9 +918,9 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e57ff02e8ad8e06ab9731d5dc72dc23bef9200778eae1a89d555d8c42e5d4a86" dependencies = [ - "prost 0.11.2", - "prost-types 0.11.2", - "tonic 0.8.2", + "prost", + "prost-types", + "tonic", "tracing-core", ] @@ -929,13 +936,13 @@ dependencies = [ "futures", "hdrhistogram", "humantime", - "prost-types 0.11.2", + "prost-types", "serde", "serde_json", "thread_local", "tokio", "tokio-stream", - "tonic 0.8.2", + "tonic", "tracing", "tracing-core", "tracing-subscriber", @@ -1213,16 +1220,6 @@ dependencies = [ "serde", ] -[[package]] -name = "dashmap" -version = "4.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" -dependencies = [ - "cfg-if", - "num_cpus", -] - [[package]] name = "dashmap" version = "5.4.0" @@ -1237,28 +1234,6 @@ dependencies = [ "serde", ] -[[package]] -name = "deadpool" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "421fe0f90f2ab22016f32a9881be5134fdd71c65298917084b0c7477cbc3856e" -dependencies = [ - "async-trait", - "deadpool-runtime", - "num_cpus", - "retain_mut", - "tokio", -] - -[[package]] -name = "deadpool-runtime" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaa37046cc0f6c3cc6090fbdbf73ef0b8ef4cfcc37f6befc0020f63e8cf121e1" -dependencies = [ - "tokio", -] - [[package]] name = "deno_core" version = "0.142.0" @@ -1930,7 +1905,7 @@ dependencies = [ "indexmap", "slab", "tokio", - "tokio-util 0.7.4", + "tokio-util", "tracing", ] @@ -2177,17 +2152,17 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.23.1" +version = "0.23.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59df7c4e19c950e6e0e868dcc0a300b09a9b88e9ec55bd879ca819087a77355d" +checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" dependencies = [ "http", "hyper", "log", - "rustls 0.20.7", - "rustls-native-certs 0.6.2", + "rustls", + "rustls-native-certs", "tokio", - "tokio-rustls 0.23.4", + "tokio-rustls", ] [[package]] @@ -2276,9 +2251,9 @@ dependencies = [ [[package]] name = "insta" -version = "1.22.0" +version = "1.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "197f4e300af8b23664d4077bf5c40e0afa9ba66a567bb5a51d3def3c7b287d1c" +checksum = "e48b08a091dfe5b09a6a9688c468fdd5b4396e92ce09e2eb932f0884b02788a4" dependencies = [ "console 0.15.2", "lazy_static", @@ -2601,6 +2576,26 @@ dependencies = [ "serde", ] +[[package]] +name = "linkme" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c25ef733283514e2c267f019701d6c81537e5d1807d6b1d7ef17f17585db964b" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a82efd9655156d970e99312fd3a613b4be4f169b4c35f4463b7499a40d3ed24" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "lock_api" version = "0.4.9" @@ -2887,6 +2882,16 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi 0.3.9", +] + [[package]] name = "num" version = "0.4.0" @@ -3084,70 +3089,60 @@ dependencies = [ [[package]] name = "opentelemetry" -version = "0.17.0" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8" +checksum = "69d6c3d7288a106c0a363e4b0e8d308058d56902adefb16f4936f417ffef086e" dependencies = [ - "async-trait", - "crossbeam-channel", - "dashmap 4.0.2", - "fnv", - "futures-channel", - "futures-executor", - "futures-util", - "js-sys", - "lazy_static", - "percent-encoding", - "pin-project", - "rand", - "serde", - "thiserror", - "tokio", - "tokio-stream", + "opentelemetry_api", + "opentelemetry_sdk", ] [[package]] name = "opentelemetry-datadog" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "457462dc4cd365992c574c79181ff11ee6f66c5cbfb15a352217b4e0b35eac34" +checksum = "171770efa142d2a19455b7e985037f560b2e75461f822dd1688bfd83c14856f6" dependencies = [ "async-trait", + "futures-core", "http", "indexmap", "itertools", - "lazy_static", + "once_cell", "opentelemetry", "opentelemetry-http", "opentelemetry-semantic-conventions", "reqwest", "rmp", "thiserror", + "url", ] [[package]] name = "opentelemetry-http" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "449048140ee61e28f57abe6e9975eedc1f3a29855c7407bd6c12b18578863379" +checksum = "1edc79add46364183ece1a4542592ca593e6421c60807232f5b8f7a31703825d" dependencies = [ "async-trait", "bytes", "http", - "opentelemetry", + "opentelemetry_api", "reqwest", ] [[package]] name = "opentelemetry-jaeger" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8c0b12cd9e3f9b35b52f6e0dac66866c519b26f424f4bbf96e3fe8bfbdc5229" +checksum = "1e785d273968748578931e4dc3b4f5ec86b26e09d9e0d66b55adda7fce742f7a" dependencies = [ "async-trait", + "futures", + "futures-executor", "headers", "http", - "lazy_static", + "once_cell", "opentelemetry", "opentelemetry-http", "opentelemetry-semantic-conventions", @@ -3159,9 +3154,9 @@ dependencies = [ [[package]] name = "opentelemetry-otlp" -version = "0.10.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1a6ca9de4c8b00aa7f1a153bd76cb263287155cec642680d79d98706f3d28a" +checksum = "d1c928609d087790fc936a1067bdc310ae702bdf3b090c3f281b713622c8bbde" dependencies = [ "async-trait", "futures", @@ -3169,44 +3164,58 @@ dependencies = [ "http", "opentelemetry", "opentelemetry-http", - "prost 0.9.0", - "prost-build", + "opentelemetry-proto", + "prost", "reqwest", "thiserror", "tokio", - "tonic 0.6.2", - "tonic-build", + "tonic", ] [[package]] name = "opentelemetry-prometheus" -version = "0.10.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9328977e479cebe12ce0d3fcecdaea4721d234895a9440c5b5dfd113f0594ac6" +checksum = "06c3d833835a53cf91331d2cfb27e9121f5a95261f31f08a1f79ab31688b8da8" dependencies = [ "opentelemetry", "prometheus", "protobuf", ] +[[package]] +name = "opentelemetry-proto" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d61a2f56df5574508dd86aaca016c917489e589ece4141df1b5e349af8d66c28" +dependencies = [ + "futures", + "futures-util", + "opentelemetry", + "prost", + "tonic", + "tonic-build", +] + [[package]] name = "opentelemetry-semantic-conventions" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985cc35d832d412224b2cffe2f9194b1b89b6aa5d0bef76d080dce09d90e62bd" +checksum = "9b02e0230abb0ab6636d18e2ba8fa02903ea63772281340ccac18e0af3ec9eeb" dependencies = [ "opentelemetry", ] [[package]] name = "opentelemetry-zipkin" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb86c6e02de97a3a7ffa5d267e1ff3f0c930ccf8a31e286277a209af6ed6cfc1" +checksum = "9bd6a5d672fe50f682d801f6737a54a633834cf8c91be419c0c9cae8ac85bf4d" dependencies = [ "async-trait", + "futures-core", "http", - "lazy_static", + "once_cell", "opentelemetry", "opentelemetry-http", "opentelemetry-semantic-conventions", @@ -3217,6 +3226,44 @@ dependencies = [ "typed-builder", ] +[[package]] +name = "opentelemetry_api" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c24f96e21e7acc813c7a8394ee94978929db2bcc46cf6b5014fc612bf7760c22" +dependencies = [ + "fnv", + "futures-channel", + "futures-util", + "indexmap", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ca41c4933371b61c2a2f214bf16931499af4ec90543604ec828f7a625c09113" +dependencies = [ + "async-trait", + "crossbeam-channel", + "dashmap", + "fnv", + "futures-channel", + "futures-executor", + "futures-util", + "once_cell", + "opentelemetry_api", + "percent-encoding", + "rand", + "thiserror", + "tokio", + "tokio-stream", +] + [[package]] name = "ordered-float" version = "1.1.1" @@ -3241,6 +3288,12 @@ version = "6.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3baf96e39c5359d2eb0dd6ccb42c62b91d9678aa68160d261b9e0ccbf9e9dea9" +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "owo-colors" version = "3.5.0" @@ -3319,9 +3372,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1de2e551fb905ac83f73f7aedf2f0cb4a0da7e35efa24a202a936269f1f18e1" +checksum = "cf1c2c742266c2f1041c914ba65355a83ae8747b05f208319784083583494b4b" [[package]] name = "pem-rfc7468" @@ -3518,6 +3571,16 @@ dependencies = [ "termtree", ] +[[package]] +name = "prettyplease" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c142c0e46b57171fe0c528bee8c5b7569e80f0c17e377cd0e30ea57dbc11bb51" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro-crate" version = "1.2.1" @@ -3599,16 +3662,6 @@ dependencies = [ "tower", ] -[[package]] -name = "prost" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001" -dependencies = [ - "bytes", - "prost-derive 0.9.0", -] - [[package]] name = "prost" version = "0.11.2" @@ -3616,42 +3669,31 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0841812012b2d4a6145fae9a6af1534873c32aa67fff26bd09f8fa42c83f95a" dependencies = [ "bytes", - "prost-derive 0.11.2", + "prost-derive", ] [[package]] name = "prost-build" -version = "0.9.0" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5" +checksum = "e330bf1316db56b12c2bcfa399e8edddd4821965ea25ddb2c134b610b1c1c604" dependencies = [ "bytes", - "heck 0.3.3", + "heck 0.4.0", "itertools", "lazy_static", "log", "multimap", "petgraph", - "prost 0.9.0", - "prost-types 0.9.0", + "prettyplease", + "prost", + "prost-types", "regex", + "syn", "tempfile", "which", ] -[[package]] -name = "prost-derive" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "prost-derive" version = "0.11.2" @@ -3665,16 +3707,6 @@ dependencies = [ "syn", ] -[[package]] -name = "prost-types" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a" -dependencies = [ - "bytes", - "prost 0.9.0", -] - [[package]] name = "prost-types" version = "0.11.2" @@ -3682,7 +3714,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "747761bc3dc48f9a34553bf65605cf6cb6288ba219f3450b4275dbd81539551a" dependencies = [ "bytes", - "prost 0.11.2", + "prost", ] [[package]] @@ -3786,7 +3818,7 @@ dependencies = [ "ryu", "sha1 0.6.1", "tokio", - "tokio-util 0.7.4", + "tokio-util", "url", ] @@ -3885,16 +3917,16 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.20.7", - "rustls-native-certs 0.6.2", + "rustls", + "rustls-native-certs", "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", "tokio", "tokio-native-tls", - "tokio-rustls 0.23.4", - "tokio-util 0.7.4", + "tokio-rustls", + "tokio-util", "tower-service", "url", "wasm-bindgen", @@ -3904,12 +3936,6 @@ dependencies = [ "winreg", ] -[[package]] -name = "retain_mut" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4389f1d5789befaf6029ebd9f7dac4af7f7e3d61b69d4f30e2ac02b57e7712b0" - [[package]] name = "rfc6979" version = "0.3.1" @@ -4041,9 +4067,9 @@ dependencies = [ [[package]] name = "router-bridge" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d2434deb0c39a41b7b68f968c78517c59b1032894c2e013244ba18d9fb49b4" +checksum = "855a1971da25bf89dcdf00ce91ce2accbb9abfe247f040fc4064098685960f17" dependencies = [ "anyhow", "async-channel", @@ -4169,19 +4195,6 @@ dependencies = [ "semver 1.0.14", ] -[[package]] -name = "rustls" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7" -dependencies = [ - "base64 0.13.1", - "log", - "ring", - "sct 0.6.1", - "webpki 0.21.4", -] - [[package]] name = "rustls" version = "0.20.7" @@ -4190,20 +4203,8 @@ checksum = "539a2bfe908f471bfa933876bd1eb6a19cf2176d375f82ef7f99530a40e48c2c" dependencies = [ "log", "ring", - "sct 0.7.0", - "webpki 0.22.0", -] - -[[package]] -name = "rustls-native-certs" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092" -dependencies = [ - "openssl-probe", - "rustls 0.19.1", - "schannel", - "security-framework", + "sct", + "webpki", ] [[package]] @@ -4318,16 +4319,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" -[[package]] -name = "sct" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "sct" version = "0.7.0" @@ -4398,9 +4389,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "256b9932320c590e707b94576e3cc1f7c9024d0ee6612dfbcf1cb106cbe8e055" +checksum = "e326c9ec8042f1b5da33252c8a37e9ffbd2c9bef0155215b6e6c80c790e05f91" dependencies = [ "serde_derive", ] @@ -4417,9 +4408,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4eae9b04cbffdfd550eb462ed33bc6a1b68c935127d008b27444d08380f94e4" +checksum = "42a3df25b0713732468deadad63ab9da1f1fd75a48a15024b50363f128db627e" dependencies = [ "proc-macro2", "quote", @@ -4788,9 +4779,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.104" +version = "1.0.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ae548ec36cf198c0ef7710d3c230987c2d6d7bd98ad6edc0274462724c585ce" +checksum = "60b9b43d45702de4c839cb9b51d9f529c5dd26a4aff255b42b1ebc03e88ee908" dependencies = [ "proc-macro2", "quote", @@ -4982,9 +4973,9 @@ dependencies = [ [[package]] name = "thrift" -version = "0.15.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b82ca8f46f95b3ce96081fe3dd89160fdea970c254bb72925255d1b62aae692e" +checksum = "09678c4cdbb4eed72e18b7c2af1329c69825ed16fcbac62d083fc3e2b0590ff0" dependencies = [ "byteorder", "integer-encoding", @@ -5055,9 +5046,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.22.0" +version = "1.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d76ce4a75fb488c605c54bf610f221cea8b0dafb53333c1a67e8ee199dcd2ae3" +checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46" dependencies = [ "autocfg", "bytes", @@ -5071,7 +5062,7 @@ dependencies = [ "socket2", "tokio-macros", "tracing", - "winapi 0.3.9", + "windows-sys 0.42.0", ] [[package]] @@ -5105,26 +5096,15 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-rustls" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6" -dependencies = [ - "rustls 0.19.1", - "tokio", - "webpki 0.21.4", -] - [[package]] name = "tokio-rustls" version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ - "rustls 0.20.7", + "rustls", "tokio", - "webpki 0.22.0", + "webpki", ] [[package]] @@ -5136,7 +5116,7 @@ dependencies = [ "futures-core", "pin-project-lite", "tokio", - "tokio-util 0.7.4", + "tokio-util", ] [[package]] @@ -5152,20 +5132,6 @@ dependencies = [ "tokio-stream", ] -[[package]] -name = "tokio-util" -version = "0.6.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "log", - "pin-project-lite", - "tokio", -] - [[package]] name = "tokio-util" version = "0.7.4" @@ -5189,39 +5155,6 @@ dependencies = [ "serde", ] -[[package]] -name = "tonic" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff08f4649d10a70ffa3522ca559031285d8e421d727ac85c60825761818f5d0a" -dependencies = [ - "async-stream", - "async-trait", - "base64 0.13.1", - "bytes", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-timeout", - "percent-encoding", - "pin-project", - "prost 0.9.0", - "prost-derive 0.9.0", - "rustls-native-certs 0.5.0", - "tokio", - "tokio-rustls 0.22.0", - "tokio-stream", - "tokio-util 0.6.10", - "tower", - "tower-layer", - "tower-service", - "tracing", - "tracing-futures", -] - [[package]] name = "tonic" version = "0.8.2" @@ -5242,11 +5175,14 @@ dependencies = [ "hyper-timeout", "percent-encoding", "pin-project", - "prost 0.11.2", - "prost-derive 0.11.2", + "prost", + "prost-derive", + "rustls-native-certs", + "rustls-pemfile", "tokio", + "tokio-rustls", "tokio-stream", - "tokio-util 0.7.4", + "tokio-util", "tower", "tower-layer", "tower-service", @@ -5256,10 +5192,11 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.6.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9403f1bafde247186684b230dc6f38b5cd514584e8bec1dd32514be4745fa757" +checksum = "5bf5e9b9c0f7e0a7c027dcfaba7b2c60816c7049171f679d99ee2ff65d0de8c4" dependencies = [ + "prettyplease", "proc-macro2", "prost-build", "quote", @@ -5281,7 +5218,7 @@ dependencies = [ "rand", "slab", "tokio", - "tokio-util 0.7.4", + "tokio-util", "tower-layer", "tower-service", "tracing", @@ -5303,7 +5240,7 @@ dependencies = [ "http-range-header", "pin-project-lite", "tokio", - "tokio-util 0.7.4", + "tokio-util", "tower", "tower-layer", "tower-service", @@ -5338,9 +5275,9 @@ dependencies = [ [[package]] name = "tracing" -version = "0.1.35" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a400e31aa60b9d44a52a8ee0343b5b18566b03a8321e0d321f695cf56e940160" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" dependencies = [ "cfg-if", "log", @@ -5362,9 +5299,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.28" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b7358be39f2f274f322d2aaed611acc57f382e8eb1e5b48cb9ae30933495ce7" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" dependencies = [ "once_cell", "valuable", @@ -5395,9 +5332,9 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.17.4" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbbe89715c1dbbb790059e2565353978564924ee85017b5fff365c872ff6721f" +checksum = "21ebb87a95ea13271332df069020513ab70bdb5637ca42d6e492dc3bbbad48de" dependencies = [ "once_cell", "opentelemetry", @@ -5419,12 +5356,12 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60db860322da191b40952ad9affe65ea23e7dd6a5c442c2c42865810c6ab8e6b" +checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" dependencies = [ - "ansi_term", "matchers", + "nu-ansi-term", "once_cell", "regex", "serde", @@ -5796,16 +5733,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki" -version = "0.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "webpki" version = "0.22.0" @@ -5822,7 +5749,7 @@ version = "0.22.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "368bfe657969fb01238bb756d351dcade285e0f6fcbd36dcb23359a5169975be" dependencies = [ - "webpki 0.22.0", + "webpki", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index f72820b5ee..dc7d27dd4e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,3 +38,11 @@ incremental = false [profile.release-dhat] inherits = "release" debug = 1 + +# TODO: to delete +# [patch.crates-io] +# opentelemetry = { git = "https://github.com/open-telemetry/opentelemetry-rust.git", rev = "e5ef3552efab2bdbf2f838023c37461cd799ab2c"} +# opentelemetry-http = { git = "https://github.com/open-telemetry/opentelemetry-rust.git", rev = "e5ef3552efab2bdbf2f838023c37461cd799ab2c"} +# opentelemetry-jaeger = { git = "https://github.com/open-telemetry/opentelemetry-rust.git", rev = "e5ef3552efab2bdbf2f838023c37461cd799ab2c"} +# opentelemetry-zipkin = { git = "https://github.com/open-telemetry/opentelemetry-rust.git", rev = "e5ef3552efab2bdbf2f838023c37461cd799ab2c"} +# opentelemetry-datadog = { git = "https://github.com/open-telemetry/opentelemetry-rust.git", rev = "e5ef3552efab2bdbf2f838023c37461cd799ab2c"} \ No newline at end of file diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 66b203c412..ef12538e3d 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -18,7 +18,7 @@ The **Apollo Router** is a configurable, high-performance **graph router** for a ## Development -You will need a recent version of rust (`1.63` works well as of writing). +You will need a recent version of rust (`1.65` works well as of writing). Installing rust [using rustup](https://www.rust-lang.org/tools/install) is the recommended way to do it as it will install rustup, rustfmt and other goodies that are not always included by default in other rust distribution channels: @@ -27,6 +27,8 @@ goodies that are not always included by default in other rust distribution chann curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh ``` +In addition, you will need to [install protoc](https://grpc.io/docs/protoc-installation/). + Set up your git hooks: ```shell @@ -78,6 +80,11 @@ The CI checks require `cargo-deny` and `cargo-about` which can both be installed They also need you to have the federation-demo project up and running, as explained in the Getting started section above. +### Yaml configuration design + +If you are adding a new feature or modifying an existing feature then consult the [yaml design guidance](dev-docs/yaml-design-guidance.md) page. + + ### Investigating memory usage There are two features: `dhat-heap` and `dhat-ad-hoc` which may be enabled for investigating memory issues diff --git a/about.toml b/about.toml index c77910db9f..7facbe8b8f 100644 --- a/about.toml +++ b/about.toml @@ -5,8 +5,8 @@ accepted = [ "BSD-3-Clause", "CC0-1.0", "ISC", - "LicenseRef-ELv2", - "LicenseRef-webpki", + "Elastic-2.0", + "LicenseRef-ring", "MIT", "MPL-2.0", "Unicode-DFS-2016" @@ -25,35 +25,6 @@ workarounds = [ [ring] accepted = ["OpenSSL"] -# apollographql licenses -[xtask.clarify] -license = "LicenseRef-ELv2" -[[xtask.clarify.files]] -path = 'LICENSE' -license = 'LicenseRef-ELv2' -checksum = '6330b076d84694d0e8905c12d7a506e4ed8e5f4a7b0ddf41a3137483ff80be50' - -[apollo-router.clarify] -license = "LicenseRef-ELv2" -[[apollo-router.clarify.files]] -path = 'LICENSE' -license = 'LicenseRef-ELv2' -checksum = '6330b076d84694d0e8905c12d7a506e4ed8e5f4a7b0ddf41a3137483ff80be50' - -[router-bridge.clarify] -license = "LicenseRef-ELv2" -[[router-bridge.clarify.files]] -path = 'LICENSE' -license = 'LicenseRef-ELv2' -checksum = 'f527cb71b36ad7d828d0d1198ee0ab60db4170521a405661c0893f31b9962a6c' - -[apollo-spaceport.clarify] -license = "LicenseRef-ELv2" -[[apollo-spaceport.clarify.files]] -path = 'LICENSE' -license = 'LicenseRef-ELv2' -checksum = '6330b076d84694d0e8905c12d7a506e4ed8e5f4a7b0ddf41a3137483ff80be50' - [webpki.clarify] license = "ISC" [[webpki.clarify.files]] diff --git a/apollo-router-benchmarks/Cargo.toml b/apollo-router-benchmarks/Cargo.toml index 49e47e1ed2..d5f0c841e0 100644 --- a/apollo-router-benchmarks/Cargo.toml +++ b/apollo-router-benchmarks/Cargo.toml @@ -1,9 +1,9 @@ [package] name = "apollo-router-benchmarks" -version = "1.5.0" +version = "1.6.0" authors = ["Apollo Graph, Inc. "] edition = "2021" -license = "LicenseRef-ELv2" +license = "Elastic-2.0" publish = false # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/apollo-router-scaffold/Cargo.toml b/apollo-router-scaffold/Cargo.toml index e8d9b9061e..960bed71e4 100644 --- a/apollo-router-scaffold/Cargo.toml +++ b/apollo-router-scaffold/Cargo.toml @@ -1,9 +1,9 @@ [package] name = "apollo-router-scaffold" -version = "1.5.0" +version = "1.6.0" authors = ["Apollo Graph, Inc. "] edition = "2021" -license = "LicenseRef-ELv2" +license = "Elastic-2.0" publish = false [dependencies] diff --git a/apollo-router-scaffold/src/lib.rs b/apollo-router-scaffold/src/lib.rs index c26e8b2876..3823dc7762 100644 --- a/apollo-router-scaffold/src/lib.rs +++ b/apollo-router-scaffold/src/lib.rs @@ -169,7 +169,7 @@ mod test { // best effort to prepare the output directory let _ = std::fs::remove_dir_all(&output_dir); - copy_dir::copy_dir(&temp_dir, &output_dir) + copy_dir::copy_dir(temp_dir, &output_dir) .expect("couldn't copy test_scaffold_output directory"); anyhow::anyhow!( "scaffold test failed: {e}\nYou can find the scaffolded project at '{}'", diff --git a/apollo-router-scaffold/templates/base/Cargo.toml b/apollo-router-scaffold/templates/base/Cargo.toml index dcfc7a04e2..dc76d8ca05 100644 --- a/apollo-router-scaffold/templates/base/Cargo.toml +++ b/apollo-router-scaffold/templates/base/Cargo.toml @@ -22,7 +22,7 @@ apollo-router = { path ="{{integration_test}}apollo-router" } apollo-router = { git="https://github.com/apollographql/router.git", branch="{{branch}}" } {{else}} # Note if you update these dependencies then also update xtask/Cargo.toml -apollo-router = "1.4.0" +apollo-router = "1.6.0" {{/if}} {{/if}} async-trait = "0.1.52" @@ -32,7 +32,7 @@ serde = "1.0.136" serde_json = "1.0.79" tokio = { version = "1.17.0", features = ["full"] } tower = { version = "0.4.12", features = ["full"] } -tracing = "0.1.35" +tracing = "0.1.37" # this makes build scripts and proc macros faster to compile [profile.dev.build-override] diff --git a/apollo-router-scaffold/templates/base/xtask/Cargo.toml b/apollo-router-scaffold/templates/base/xtask/Cargo.toml index 137ca77f00..e175f61e76 100644 --- a/apollo-router-scaffold/templates/base/xtask/Cargo.toml +++ b/apollo-router-scaffold/templates/base/xtask/Cargo.toml @@ -13,7 +13,7 @@ apollo-router-scaffold = { path ="{{integration_test}}apollo-router-scaffold" } {{#if branch}} apollo-router-scaffold = { git="https://github.com/apollographql/router.git", branch="{{branch}}" } {{else}} -apollo-router-scaffold = { git="https://github.com/apollographql/router.git", tag="v1.4.0"} +apollo-router-scaffold = { git="https://github.com/apollographql/router.git", tag = "v1.6.0" } {{/if}} {{/if}} anyhow = "1.0.58" diff --git a/apollo-router/Cargo.toml b/apollo-router/Cargo.toml index 84f9f5569c..30c2f59cc0 100644 --- a/apollo-router/Cargo.toml +++ b/apollo-router/Cargo.toml @@ -1,14 +1,14 @@ [package] name = "apollo-router" -version = "1.5.0" +version = "1.6.0" authors = ["Apollo Graph, Inc. "] repository = "https://github.com/apollographql/router/" documentation = "https://www.apollographql.com/docs/router/" description = "A configurable, high-performance routing runtime for Apollo Federation ๐Ÿš€" -license = "LicenseRef-ELv2" +license = "Elastic-2.0" # renovate-automation: rustc version -rust-version = "1.63.0" +rust-version = "1.65.0" edition = "2021" build = "build/main.rs" @@ -42,6 +42,7 @@ access-json = "0.1.0" anyhow = "1.0.66" ansi_term = "0.12" apollo-parser = "0.4.0" +arc-swap = "1.5.1" async-compression = { version = "0.3.15", features = [ "tokio", "brotli", @@ -61,9 +62,7 @@ clap = { version = "3.2.23", default-features = false, features = [ "std", ] } console-subscriber = { version = "0.1.8", optional = true } -ctor = "0.1.26" dashmap = { version = "5.4.0", features = ["serde"] } -deadpool = { version = "0.9.5", features = ["rt_tokio_1"] } derivative = "2.2.0" derive_more = { version = "0.99.17", default-features = false, features = [ "from", @@ -82,13 +81,14 @@ http-body = "0.4.5" humantime = "2.1.0" humantime-serde = "1.1.1" hyper = { version = "0.14.23", features = ["server", "client"] } -hyper-rustls = { version = "0.23.1", features = ["http1", "http2"] } +hyper-rustls = { version = "0.23.2", features = ["http1", "http2"] } indexmap = { version = "1.9.2", features = ["serde-1"] } itertools = "0.10.5" jsonpath_lib = "0.3.0" jsonschema = { version = "0.16.1", default-features = false } lazy_static = "1.4.0" libc = "0.2.138" +linkme = "0.3.6" lru = "0.7.8" mediatype = "0.19.11" mockall = "0.11.3" @@ -110,38 +110,36 @@ once_cell = "1.16.0" # groups `^tracing` and `^opentelemetry*` dependencies together as of # https://github.com/apollographql/router/pull/1509. A comment which exists # there (and on `tracing` packages below) should be updated should this change. -opentelemetry = { version = "0.17.0", features = [ +opentelemetry = { version = "0.18.0", features = [ "rt-tokio", - "serialize", "metrics", ] } -opentelemetry-datadog = { version = "0.5.0", features = ["reqwest-client"] } -opentelemetry-http = "0.6.0" -opentelemetry-jaeger = { version = "0.16.0", features = [ +opentelemetry-datadog = { version = "0.6.0", features = ["reqwest-client"] } +opentelemetry-http = "0.7.0" +opentelemetry-jaeger = { version = "0.17.0", features = [ "collector_client", "reqwest_collector_client", "rt-tokio", ] } -opentelemetry-otlp = { version = "0.10.0", default-features = false, features = [ +opentelemetry-otlp = { version = "0.11.0", default-features = false, features = [ + "grpc-tonic", "tonic", - "tonic-build", - "prost", "tls", "http-proto", "metrics", "reqwest-client", ] } -opentelemetry-semantic-conventions = "0.9.0" -opentelemetry-zipkin = { version = "0.15.0", default-features = false, features = [ +opentelemetry-semantic-conventions = "0.10.0" +opentelemetry-zipkin = { version = "0.16.0", default-features = false, features = [ "reqwest-client", "reqwest-rustls", ] } -opentelemetry-prometheus = "0.10.0" -paste = "1.0.9" +opentelemetry-prometheus = "0.11.0" +paste = "1.0.10" pin-project-lite = "0.2.9" prometheus = "0.13" -prost = "0.9.0" -prost-types = "0.9.0" +prost = "0.11.0" +prost-types = "0.11.0" proteus = "0.5.0" rand = "0.8.5" rhai = { version = "1.11.0", features = ["sync", "serde", "internals"] } @@ -153,12 +151,12 @@ reqwest = { version = "0.11.13", default-features = false, features = [ "json", "stream", ] } -router-bridge = "0.1.11" +router-bridge = "0.1.12" rust-embed="6.4.2" schemars = { version = "0.8.11", features = ["url"] } shellexpand = "2.1.2" sha2 = "0.10.6" -serde = { version = "1.0.149", features = ["derive", "rc"] } +serde = { version = "1.0.150", features = ["derive", "rc"] } serde_json_bytes = { version = "0.2.0", features = ["preserve_order"] } serde_json = { version = "1.0.89", features = ["preserve_order"] } serde_urlencoded = "0.7.1" @@ -166,10 +164,10 @@ serde_yaml = "0.8.26" static_assertions = "1.1.0" sys-info = "0.9.1" thiserror = "1.0.37" -tokio = { version = "1.22.0", features = ["full"] } +tokio = { version = "1.23.0", features = ["full"] } tokio-stream = { version = "0.1.11", features = ["sync", "net"] } tokio-util = { version = "0.7.4", features = ["net", "codec"] } -tonic = { version = "0.6.2", features = ["transport", "tls", "tls-roots"] } +tonic = { version = "0.8.2", features = ["transport", "tls", "tls-roots"] } tower = { version = "0.4.13", features = ["full"] } tower-http = { version = "0.3.5", features = [ "trace", @@ -183,17 +181,17 @@ tower-http = { version = "0.3.5", features = [ "timeout", ] } tower-service = "0.3.2" -tracing = "0.1.35" -tracing-core = "0.1.28" +tracing = "0.1.37" +tracing-core = "0.1.30" tracing-futures = { version = "0.2.5", features = ["futures-03"] } -tracing-opentelemetry = "0.17.4" -tracing-subscriber = { version = "0.3.14", features = ["env-filter", "json"] } - +tracing-opentelemetry = "0.18.0" +tracing-subscriber = { version = "0.3.11", features = ["env-filter", "json"] } url = { version = "2.3.1", features = ["serde"] } urlencoding = "2.1.2" uuid = { version = "1.2.2", features = ["serde", "v4"] } yaml-rust = "0.4.5" askama = "0.11.1" +apollo-encoder = "0.4.0" [target.'cfg(macos)'.dependencies] uname = "0.1.1" @@ -202,7 +200,7 @@ uname = "0.1.1" uname = "0.1.1" [dev-dependencies] -insta = { version = "1.22.0", features = ["json", "redactions", "yaml"] } +insta = { version = "1.23.0", features = ["json", "redactions", "yaml"] } introspector-gadget = "0.1.0" maplit = "1.0.2" memchr = { version = "2.5.0", default-features = false } @@ -229,7 +227,7 @@ tracing-test = "0.2.2" walkdir = "2.3.2" [build-dependencies] -tonic-build = "0.6.2" +tonic-build = "0.8.2" [[test]] diff --git a/apollo-router/build/main.rs b/apollo-router/build/main.rs index d86fabcaf4..b323c668bb 100644 --- a/apollo-router/build/main.rs +++ b/apollo-router/build/main.rs @@ -1,5 +1,5 @@ -mod spaceport; +mod studio; fn main() -> Result<(), Box> { - spaceport::main() + studio::main() } diff --git a/apollo-router/build/spaceport.rs b/apollo-router/build/studio.rs similarity index 74% rename from apollo-router/build/spaceport.rs rename to apollo-router/build/studio.rs index 0d1e1ec9da..6c4024b2c3 100644 --- a/apollo-router/build/spaceport.rs +++ b/apollo-router/build/studio.rs @@ -4,12 +4,10 @@ use std::path::PathBuf; pub fn main() -> Result<(), Box> { let out_dir = PathBuf::from(std::env::var_os("OUT_DIR").unwrap()); let src = PathBuf::from(std::env::var_os("CARGO_MANIFEST_DIR").unwrap()).join("src"); - let proto_dir = src.join("spaceport").join("proto"); - let agents = proto_dir.join("agents.proto"); + let proto_dir = src.join("plugins").join("telemetry").join("proto"); let reports_src = proto_dir.join("reports.proto"); let reports_out = out_dir.join("reports.proto"); - println!("cargo:rerun-if-changed={}", agents.to_str().unwrap()); println!("cargo:rerun-if-changed={}", reports_src.to_str().unwrap()); // Process the retrieved content to: @@ -27,31 +25,30 @@ pub fn main() -> Result<(), Box> { std::fs::write(&reports_out, &content)?; // Process the proto files - let proto_files = [agents, reports_out]; + let proto_files = [reports_out]; tonic_build::configure() .field_attribute( "Trace.start_time", - "#[serde(serialize_with = \"crate::spaceport::serialize_timestamp\")]", + "#[serde(serialize_with = \"crate::plugins::telemetry::apollo_exporter::serialize_timestamp\")]", ) .field_attribute( "Trace.end_time", - "#[serde(serialize_with = \"crate::spaceport::serialize_timestamp\")]", + "#[serde(serialize_with = \"crate::plugins::telemetry::apollo_exporter::serialize_timestamp\")]", ) .field_attribute( "FetchNode.sent_time", - "#[serde(serialize_with = \"crate::spaceport::serialize_timestamp\")]", + "#[serde(serialize_with = \"crate::plugins::telemetry::apollo_exporter::serialize_timestamp\")]", ) .field_attribute( "FetchNode.received_time", - "#[serde(serialize_with = \"crate::spaceport::serialize_timestamp\")]", + "#[serde(serialize_with = \"crate::plugins::telemetry::apollo_exporter::serialize_timestamp\")]", ) .field_attribute( "Report.end_time", - "#[serde(serialize_with = \"crate::spaceport::serialize_timestamp\")]", + "#[serde(serialize_with = \"crate::plugins::telemetry::apollo_exporter::serialize_timestamp\")]", ) .type_attribute(".", "#[derive(serde::Serialize)]") .type_attribute("StatsContext", "#[derive(Eq, Hash)]") - .build_server(true) .compile(&proto_files, &[&out_dir, &proto_dir])?; Ok(()) diff --git a/apollo-router/experimental_features.json b/apollo-router/experimental_features.json new file mode 100644 index 0000000000..4b808672dc --- /dev/null +++ b/apollo-router/experimental_features.json @@ -0,0 +1,5 @@ +{ + "experimental_retry": "https://github.com/apollographql/router/discussions/2241", + "experimental_response_trace_id": "https://github.com/apollographql/router/discussions/2147", + "experimental_logging": "https://github.com/apollographql/router/discussions/1961" +} \ No newline at end of file diff --git a/apollo-router/src/axum_factory/axum_http_server_factory.rs b/apollo-router/src/axum_factory/axum_http_server_factory.rs index 53107202f8..e6a370b81c 100644 --- a/apollo-router/src/axum_factory/axum_http_server_factory.rs +++ b/apollo-router/src/axum_factory/axum_http_server_factory.rs @@ -3,6 +3,7 @@ use std::pin::Pin; use std::sync::Arc; use std::time::Duration; +use axum::extract::rejection::JsonRejection; use axum::extract::Extension; use axum::extract::Host; use axum::extract::OriginalUri; @@ -314,21 +315,16 @@ where .map(|t| t.to_string()) .unwrap_or_default(); - span.record(TRACE_ID_FIELD_NAME, &trace_id.as_str()); + span.record(TRACE_ID_FIELD_NAME, trace_id.as_str()); }) .on_response(|resp: &Response<_>, duration: Duration, span: &Span| { // Duration here is instant based - span.record("apollo_private.duration_ns", &(duration.as_nanos() as i64)); + span.record("apollo_private.duration_ns", duration.as_nanos() as i64); + // otel.status_code now has to be a string rather than enum. See opentelemetry_tracing::layer::str_to_status if resp.status() >= StatusCode::BAD_REQUEST { - span.record( - "otel.status_code", - &opentelemetry::trace::StatusCode::Error.as_str(), - ); + span.record("otel.status_code", "error"); } else { - span.record( - "otel.status_code", - &opentelemetry::trace::StatusCode::Ok.as_str(), - ); + span.record("otel.status_code", "ok"); } }), ) @@ -397,7 +393,7 @@ where .post({ move |host: Host, uri: OriginalUri, - request: Json, + request: Result, JsonRejection>, Extension(service): Extension, header_map: HeaderMap| { { diff --git a/apollo-router/src/axum_factory/handlers.rs b/apollo-router/src/axum_factory/handlers.rs index 0ea723faf7..8e48480b01 100644 --- a/apollo-router/src/axum_factory/handlers.rs +++ b/apollo-router/src/axum_factory/handlers.rs @@ -2,6 +2,7 @@ use std::str::FromStr; use axum::body::StreamBody; +use axum::extract::rejection::JsonRejection; use axum::extract::Host; use axum::extract::OriginalUri; use axum::http::header::HeaderMap; @@ -83,17 +84,38 @@ pub(super) async fn handle_get( .into_response(); } + ::tracing::error!( + monotonic_counter.apollo_router_http_requests_total = 1u64, + status = %400, + error = "missing query string", + "missing query string" + ); (StatusCode::BAD_REQUEST, "Invalid Graphql request").into_response() } pub(super) async fn handle_post( Host(host): Host, OriginalUri(uri): OriginalUri, - Json(request): Json, + request_json: Result, JsonRejection>, apq: APQLayer, service: BoxService, header_map: HeaderMap, ) -> impl IntoResponse { + let request = match request_json { + Ok(Json(req)) => req, + Err(json_err) => { + let json_err = json_err.into_response(); + ::tracing::error!( + monotonic_counter.apollo_router_http_requests_total = 1u64, + status = %json_err.status().as_u16(), + error = "failed to parse the request body as JSON", + "failed to parse the request body as JSON" + ); + + return json_err; + } + }; + let mut http_request = Request::post( Uri::from_str(&format!("http://{}{}", host, uri)) .expect("the URL is already valid because it comes from axum; qed"), @@ -124,15 +146,22 @@ where return match stream.next().await { None => { - tracing::error!("router service is not available to process request",); + tracing::error!( + monotonic_counter.apollo_router_http_requests_total = 1u64, + status = %StatusCode::SERVICE_UNAVAILABLE.as_u16(), + "router service is not available to process request" + ); ( StatusCode::SERVICE_UNAVAILABLE, "router service is not available to process request", ) .into_response() } - Some(body) => http_ext::Response::from(http::Response::from_parts(parts, body)) - .into_response(), + Some(body) => { + tracing::info!(monotonic_counter.apollo_router_http_requests_total = 1u64); + http_ext::Response::from(http::Response::from_parts(parts, body)) + .into_response() + } }; } }; diff --git a/apollo-router/src/axum_factory/snapshots/apollo_router__axum_factory__tests__defer_is_not_buffered.snap b/apollo-router/src/axum_factory/snapshots/apollo_router__axum_factory__tests__defer_is_not_buffered.snap new file mode 100644 index 0000000000..166a799aa1 --- /dev/null +++ b/apollo-router/src/axum_factory/snapshots/apollo_router__axum_factory__tests__defer_is_not_buffered.snap @@ -0,0 +1,38 @@ +--- +source: apollo-router/src/axum_factory/tests.rs +expression: "json!([{\n \"data\" :\n {\n \"topProducts\" :\n [{ \"upc\" : \"1\", \"name\" : \"Table\", \"reviews\" : null },\n { \"upc\" : \"2\", \"name\" : \"Couch\", \"reviews\" : null }]\n }, \"errors\" :\n [{\n \"message\" :\n \"couldn't find mock for query {\\\"query\\\":\\\"query TopProducts__reviews__1($representations:[_Any!]!){_entities(representations:$representations){...on Product{reviews{__typename id product{__typename upc}}}}}\\\",\\\"operationName\\\":\\\"TopProducts__reviews__1\\\",\\\"variables\\\":{\\\"representations\\\":[{\\\"__typename\\\":\\\"Product\\\",\\\"upc\\\":\\\"1\\\"},{\\\"__typename\\\":\\\"Product\\\",\\\"upc\\\":\\\"2\\\"}]}}\"\n },\n {\n \"message\" :\n \"Subgraph response from 'reviews' was missing key `_entities`\",\n \"path\" : [\"topProducts\", \"@\"]\n }], \"hasNext\" : true,\n }, { \"hasNext\" : false }])" +--- +[ + { + "data": { + "topProducts": [ + { + "upc": "1", + "name": "Table", + "reviews": null + }, + { + "upc": "2", + "name": "Couch", + "reviews": null + } + ] + }, + "errors": [ + { + "message": "couldn't find mock for query {\"query\":\"query TopProducts__reviews__1($representations:[_Any!]!){_entities(representations:$representations){...on Product{reviews{__typename id product{__typename upc}}}}}\",\"operationName\":\"TopProducts__reviews__1\",\"variables\":{\"representations\":[{\"__typename\":\"Product\",\"upc\":\"1\"},{\"__typename\":\"Product\",\"upc\":\"2\"}]}}" + }, + { + "message": "Subgraph response from 'reviews' was missing key `_entities`", + "path": [ + "topProducts", + "@" + ] + } + ], + "hasNext": true + }, + { + "hasNext": false + } +] diff --git a/apollo-router/src/axum_factory/tests.rs b/apollo-router/src/axum_factory/tests.rs index 8b41246005..d2cbed288c 100644 --- a/apollo-router/src/axum_factory/tests.rs +++ b/apollo-router/src/axum_factory/tests.rs @@ -37,6 +37,7 @@ use test_log::test; use tokio::io::AsyncRead; use tokio::io::AsyncReadExt; use tokio::io::AsyncWriteExt; +#[cfg(unix)] use tokio::io::BufReader; use tokio_util::io::StreamReader; use tower::service_fn; @@ -1576,19 +1577,19 @@ async fn deferred_response_shape() -> Result<(), ApolloRouterError> { let first = response.chunk().await.unwrap().unwrap(); assert_eq!( - std::str::from_utf8(&*first).unwrap(), + std::str::from_utf8(&first).unwrap(), "\r\n--graphql\r\ncontent-type: application/json\r\n\r\n{\"data\":{\"test\":\"hello\"},\"hasNext\":true}\r\n--graphql\r\n" ); let second = response.chunk().await.unwrap().unwrap(); assert_eq!( - std::str::from_utf8(&*second).unwrap(), + std::str::from_utf8(&second).unwrap(), "content-type: application/json\r\n\r\n{\"hasNext\":true,\"incremental\":[{\"data\":{\"other\":\"world\"},\"path\":[]}]}\r\n--graphql\r\n" ); let third = response.chunk().await.unwrap().unwrap(); assert_eq!( - std::str::from_utf8(&*third).unwrap(), + std::str::from_utf8(&third).unwrap(), "content-type: application/json\r\n\r\n{\"hasNext\":false}\r\n--graphql--\r\n" ); @@ -1639,7 +1640,7 @@ async fn multipart_response_shape_with_one_chunk() -> Result<(), ApolloRouterErr let first = response.chunk().await.unwrap().unwrap(); assert_eq!( - std::str::from_utf8(&*first).unwrap(), + std::str::from_utf8(&first).unwrap(), "\r\n--graphql\r\ncontent-type: application/json\r\n\r\n{\"data\":{\"test\":\"hello\"},\"hasNext\":false}\r\n--graphql--\r\n" ); @@ -1922,31 +1923,7 @@ async fn test_defer_is_not_buffered() { let (parts, counts): (Vec<_>, Vec<_>) = parts.map(|part| (part, counter.get())).unzip().await; let parts = serde_json::Value::Array(parts); - assert_eq!( - parts, - json!([ - { - "data": { - "topProducts": [ - {"upc": "1", "name": "Table", "reviews": null}, - {"upc": "2", "name": "Couch", "reviews": null} - ] - }, - "errors": [ - { - "message": "couldn't find mock for query {\"query\":\"query TopProducts__reviews__1($representations:[_Any!]!){_entities(representations:$representations){...on Product{reviews{__typename id product{__typename upc}}}}}\",\"operationName\":\"TopProducts__reviews__1\",\"variables\":{\"representations\":[{\"__typename\":\"Product\",\"upc\":\"1\"},{\"__typename\":\"Product\",\"upc\":\"2\"}]}}" - }, - { - "message": "Subgraph response from 'reviews' was missing key `_entities`", - "path": [ "topProducts", "@" ] - }], - "hasNext": true, - }, - {"hasNext": false} - ]), - "{}", - serde_json::to_string(&parts).unwrap() - ); + insta::assert_json_snapshot!(parts); // Non-regression test for https://github.com/apollographql/router/issues/1572 // diff --git a/apollo-router/src/axum_factory/utils.rs b/apollo-router/src/axum_factory/utils.rs index d855f1afa8..3c457d12dc 100644 --- a/apollo-router/src/axum_factory/utils.rs +++ b/apollo-router/src/axum_factory/utils.rs @@ -25,7 +25,6 @@ use mediatype::MediaType; use mediatype::MediaTypeList; use mediatype::ReadParams; use opentelemetry::global; -use opentelemetry::trace::SpanKind; use opentelemetry::trace::TraceContextExt; use tokio::io::AsyncWriteExt; use tower_http::trace::MakeSpan; @@ -102,20 +101,27 @@ pub(super) async fn decompress_request_body( "deflate" => decode_body!(ZlibDecoder, "cannot decompress (deflate) request body"), "identity" => Ok(next.run(Request::from_parts(parts, body)).await), unknown => { - tracing::error!("unknown content-encoding header value {:?}", unknown); - Err(( - StatusCode::BAD_REQUEST, - format!("unknown content-encoding header value: {unknown:?}"), - ) - .into_response()) + let message = format!("unknown content-encoding header value {:?}", unknown); + tracing::error!(message); + ::tracing::error!( + monotonic_counter.apollo_router_http_requests_total = 1u64, + status = %400u16, + error = %message, + ); + + Err((StatusCode::BAD_REQUEST, message).into_response()) } }, - Err(err) => Err(( - StatusCode::BAD_REQUEST, - format!("cannot read content-encoding header: {err}"), - ) - .into_response()), + Err(err) => { + let message = format!("cannot read content-encoding header: {err}"); + ::tracing::error!( + monotonic_counter.apollo_router_http_requests_total = 1u64, + status = %400u16, + error = %message, + ); + Err((StatusCode::BAD_REQUEST, message).into_response()) + } }, None => Ok(next.run(Request::from_parts(parts, body)).await), } @@ -134,6 +140,12 @@ pub(super) async fn check_accept_header( { Ok(next.run(req).await) } else { + ::tracing::error!( + monotonic_counter.apollo_router_http_requests_total = 1u64, + status = %406u16, + error = "accept header is wrong", + ); + Err(( StatusCode::NOT_ACCEPTABLE, format!( @@ -249,7 +261,7 @@ impl MakeSpan for PropagatingMakeSpan { "http.method" = %request.method(), "http.route" = %request.uri(), "http.flavor" = ?request.version(), - "otel.kind" = %SpanKind::Server, + "otel.kind" = "SERVER", "otel.status_code" = tracing::field::Empty, "apollo_private.duration_ns" = tracing::field::Empty, "trace_id" = tracing::field::Empty @@ -262,7 +274,7 @@ impl MakeSpan for PropagatingMakeSpan { "http.method" = %request.method(), "http.route" = %request.uri(), "http.flavor" = ?request.version(), - "otel.kind" = %SpanKind::Server, + "otel.kind" = "SERVER", "otel.status_code" = tracing::field::Empty, "apollo_private.duration_ns" = tracing::field::Empty, "trace_id" = tracing::field::Empty diff --git a/apollo-router/src/configuration/expansion.rs b/apollo-router/src/configuration/expansion.rs index 0ceed33868..bbf052ccbc 100644 --- a/apollo-router/src/configuration/expansion.rs +++ b/apollo-router/src/configuration/expansion.rs @@ -5,6 +5,8 @@ use std::env; use std::env::VarError; use std::fs; +use proteus::Parser; +use proteus::TransformBuilder; use serde_json::Value; use super::ConfigurationError; @@ -87,10 +89,39 @@ pub(crate) fn expand_env_variables( expansion: &Expansion, ) -> Result { let mut configuration = configuration.clone(); + #[cfg(not(test))] + env_defaults(&mut configuration); visit(&mut configuration, expansion)?; Ok(configuration) } +fn env_defaults(config: &mut Value) { + // Anything that needs expanding via env variable should be placed here. Don't pollute the codebase with calls to std::env. + let defaults = vec![( + "telemetry.apollo.endpoint", + "${env.APOLLO_USAGE_REPORTING_INGRESS_URL:-https://usage-reporting.api.apollographql.com/api/ingress/traces}", + )]; + let mut transformer_builder = TransformBuilder::default(); + transformer_builder = + transformer_builder.add_action(Parser::parse("", "").expect("migration must be valid")); + for (path, value) in defaults { + if jsonpath_lib::select(config, &format!("$.{}", path)) + .unwrap_or_default() + .is_empty() + { + transformer_builder = transformer_builder.add_action( + Parser::parse(&format!("const(\"{}\")", value), path) + .expect("migration must be valid"), + ); + } + } + *config = transformer_builder + .build() + .expect("failed to build config default transformer") + .apply(config) + .expect("failed to set config defaults"); +} + fn visit(value: &mut Value, expansion: &Expansion) -> Result<(), ConfigurationError> { let mut expanded: Option = None; match value { @@ -128,3 +159,20 @@ pub(crate) fn coerce(expanded: &str) -> Value { _ => Value::String(expanded.to_string()), } } + +#[cfg(test)] +mod test { + use insta::assert_yaml_snapshot; + use serde_json::json; + + use crate::configuration::expansion::env_defaults; + + #[test] + fn test_env_defaults() { + let mut value = json!({"hi": "there"}); + env_defaults(&mut value); + insta::with_settings!({sort_maps => true}, { + assert_yaml_snapshot!(value); + }) + } +} diff --git a/apollo-router/src/configuration/experimental.rs b/apollo-router/src/configuration/experimental.rs new file mode 100644 index 0000000000..f308616ae9 --- /dev/null +++ b/apollo-router/src/configuration/experimental.rs @@ -0,0 +1,93 @@ +use std::collections::HashMap; + +use serde_json::Value; + +pub(crate) fn print_all_experimental_conf() { + let available_exp_confs = serde_json::from_str::>(include_str!( + "../../experimental_features.json" + )) + .expect("cannot load the list of available experimental configurations"); + + let available_exp_confs_str: Vec = available_exp_confs + .into_iter() + .map(|(used_exp_conf, discussion_link)| format!("\t- {used_exp_conf}: {discussion_link}")) + .collect(); + println!( + "List of all experimental configurations with related GitHub discussions:\n\n{}", + available_exp_confs_str.join("\n") + ); +} + +pub(crate) fn log_used_experimental_conf(conf: &Value) { + let available_discussions = serde_json::from_str::>(include_str!( + "../../experimental_features.json" + )); + if let Ok(available_discussions) = available_discussions { + let used_experimental_conf = get_experimental_configurations(conf); + let needed_discussions: Vec = used_experimental_conf + .into_iter() + .filter_map(|used_exp_conf| { + available_discussions + .get(&used_exp_conf) + .map(|discussion_link| format!("\t- {used_exp_conf}: {discussion_link}")) + }) + .collect(); + if !needed_discussions.is_empty() { + tracing::info!( + r#"You're using some "experimental" features (configuration prefixed by "experimental_"), we may make breaking changes in future releases. +To help us design the stable version we need your feedback, here is a list of links where you can give your opinion: + +{} +"#, + needed_discussions.join("\n") + ); + } + } +} + +fn get_experimental_configurations(conf: &Value) -> Vec { + let mut experimental_fields = Vec::new(); + visit_experimental_configurations(conf, &mut experimental_fields); + + experimental_fields +} + +pub(crate) fn visit_experimental_configurations( + conf: &Value, + experimental_fields: &mut Vec, +) { + if let Value::Object(object) = conf { + object.iter().for_each(|(field_name, val)| { + if field_name.starts_with("experimental_") { + experimental_fields.push(field_name.clone()); + } + visit_experimental_configurations(val, experimental_fields); + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_get_experimental_configurations() { + let val = serde_json::json!({ + "server": "test", + "experimental_logging": { + "value": "foo", + "sub": { + "experimental_trace_id": "ok" + } + } + }); + + assert_eq!( + get_experimental_configurations(&val), + vec![ + "experimental_logging".to_string(), + "experimental_trace_id".to_string() + ] + ); + } +} diff --git a/apollo-router/src/configuration/migrations/0002-jaeger_scheduled_delay.yaml b/apollo-router/src/configuration/migrations/0002-jaeger_scheduled_delay.yaml new file mode 100644 index 0000000000..1791be3e18 --- /dev/null +++ b/apollo-router/src/configuration/migrations/0002-jaeger_scheduled_delay.yaml @@ -0,0 +1,6 @@ +description: telemetry.tracing.jaeger.scheduled_delay moved to telemetry.tracing.jaeger.batch_processor.scheduled_delay +actions: + - type: move + from: telemetry.tracing.jaeger.scheduled_delay + to: telemetry.tracing.jaeger.batch_processor.scheduled_delay + diff --git a/apollo-router/src/configuration/mod.rs b/apollo-router/src/configuration/mod.rs index a4be643455..43a7f5948a 100644 --- a/apollo-router/src/configuration/mod.rs +++ b/apollo-router/src/configuration/mod.rs @@ -2,6 +2,7 @@ // This entire file is license key functionality pub(crate) mod cors; mod expansion; +mod experimental; mod schema; #[cfg(test)] mod tests; @@ -19,6 +20,7 @@ use cors::*; use derivative::Derivative; use displaydoc::Display; use expansion::*; +pub(crate) use experimental::print_all_experimental_conf; use itertools::Itertools; pub(crate) use schema::generate_config_schema; pub(crate) use schema::generate_upgrade; @@ -422,12 +424,11 @@ impl JsonSchema for ApolloPlugins { // compile time to be picked up. let plugins = crate::plugin::plugins() - .iter() - .sorted_by_key(|(name, _)| *name) - .filter(|(name, _)| name.starts_with(APOLLO_PLUGIN_PREFIX)) - .map(|(name, factory)| { + .sorted_by_key(|factory| factory.name.clone()) + .filter(|factory| factory.name.starts_with(APOLLO_PLUGIN_PREFIX)) + .map(|factory| { ( - name[APOLLO_PLUGIN_PREFIX.len()..].to_string(), + factory.name[APOLLO_PLUGIN_PREFIX.len()..].to_string(), factory.create_schema(gen), ) }) @@ -456,10 +457,9 @@ impl JsonSchema for UserPlugins { // compile time to be picked up. let plugins = crate::plugin::plugins() - .iter() - .sorted_by_key(|(name, _)| *name) - .filter(|(name, _)| !name.starts_with(APOLLO_PLUGIN_PREFIX)) - .map(|(name, factory)| (name.to_string(), factory.create_schema(gen))) + .sorted_by_key(|factory| factory.name.clone()) + .filter(|factory| !factory.name.starts_with(APOLLO_PLUGIN_PREFIX)) + .map(|factory| (factory.name.to_string(), factory.create_schema(gen))) .collect::>(); gen_schema(plugins) } @@ -729,9 +729,11 @@ impl HealthCheck { enabled: enabled.unwrap_or_else(default_health_check), } } +} - // Used in tests - #[allow(dead_code)] +#[cfg(test)] +#[buildstructor::buildstructor] +impl HealthCheck { #[builder] pub(crate) fn fake_new(listen: Option, enabled: Option) -> Self { Self { diff --git a/apollo-router/src/configuration/schema.rs b/apollo-router/src/configuration/schema.rs index 1a737b57ec..27a92cbfe5 100644 --- a/apollo-router/src/configuration/schema.rs +++ b/apollo-router/src/configuration/schema.rs @@ -13,6 +13,7 @@ use schemars::schema::RootSchema; use super::expansion::coerce; use super::expansion::expand_env_variables; use super::expansion::Expansion; +use super::experimental::log_used_experimental_conf; use super::plugins; use super::yaml; use super::Configuration; @@ -102,6 +103,7 @@ pub(crate) fn validate_yaml_configuration( tracing::warn!("configuration could not be upgraded automatically as it had errors") } } + log_used_experimental_conf(&yaml); let expanded_yaml = expand_env_variables(&yaml, &expansion)?; if let Err(errors) = schema.validate(&expanded_yaml) { @@ -232,8 +234,7 @@ pub(crate) fn validate_yaml_configuration( // We can't do it with the `deny_unknown_fields` property on serde because we are using `flatten` let registered_plugins = plugins(); let apollo_plugin_names: Vec<&str> = registered_plugins - .keys() - .filter_map(|n| n.strip_prefix(APOLLO_PLUGIN_PREFIX)) + .filter_map(|factory| factory.name.strip_prefix(APOLLO_PLUGIN_PREFIX)) .collect(); let unknown_fields: Vec<&String> = config .apollo_plugins diff --git a/apollo-router/src/configuration/snapshots/apollo_router__configuration__expansion__test__env_defaults.snap b/apollo-router/src/configuration/snapshots/apollo_router__configuration__expansion__test__env_defaults.snap new file mode 100644 index 0000000000..578ffbd257 --- /dev/null +++ b/apollo-router/src/configuration/snapshots/apollo_router__configuration__expansion__test__env_defaults.snap @@ -0,0 +1,9 @@ +--- +source: apollo-router/src/configuration/expansion.rs +expression: value +--- +hi: there +telemetry: + apollo: + endpoint: "${env.APOLLO_USAGE_REPORTING_INGRESS_URL:-https://usage-reporting.api.apollographql.com/api/ingress/traces}" + diff --git a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap index 1b7cf9b2fb..6669243858 100644 --- a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap +++ b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap @@ -744,6 +744,46 @@ expression: "&schema" "apollo": { "type": "object", "properties": { + "batch_processor": { + "type": "object", + "properties": { + "max_concurrent_exports": { + "description": "Maximum number of concurrent exports\n\nLimits the number of spawned tasks for exports and thus memory consumed by an exporter. A value of 1 will cause exports to be performed synchronously on the BatchSpanProcessor task.", + "default": null, + "type": "integer", + "format": "uint", + "minimum": 0.0, + "nullable": true + }, + "max_export_batch_size": { + "description": "The maximum number of spans to process in a single batch. If there are more than one batch worth of spans then it processes multiple batches of spans one batch after the other without any delay. The default value is 512.", + "default": null, + "type": "integer", + "format": "uint", + "minimum": 0.0, + "nullable": true + }, + "max_export_timeout": { + "description": "The maximum duration to export a batch of data.", + "default": null, + "type": "string" + }, + "max_queue_size": { + "description": "The maximum queue size to buffer spans for delayed processing. If the queue gets full it drops the spans. The default value of is 2048.", + "default": null, + "type": "integer", + "format": "uint", + "minimum": 0.0, + "nullable": true + }, + "scheduled_delay": { + "description": "The delay interval in milliseconds between two consecutive processing of batches. The default value is 5 seconds.", + "default": null, + "type": "string" + } + }, + "nullable": true + }, "buffer_size": { "description": "The buffer size for sending traces to Apollo. Increase this if you are experiencing lost traces.", "default": 10000, @@ -765,8 +805,8 @@ expression: "&schema" }, "endpoint": { "description": "The Apollo Studio endpoint for exporting traces and metrics.", - "type": "string", - "nullable": true + "default": "https://usage-reporting.api.apollographql.com/api/ingress/traces", + "type": "string" }, "field_level_instrumentation_sampler": { "description": "Enable field level instrumentation for subgraphs via ftv1. ftv1 tracing can cause performance issues as it is transmitted in band with subgraph responses. 0.0 will result in no field level instrumentation. 1.0 will result in always instrumentation. Value MUST be less than global sampling rate", @@ -1789,6 +1829,46 @@ expression: "&schema" "endpoint" ], "properties": { + "batch_processor": { + "type": "object", + "properties": { + "max_concurrent_exports": { + "description": "Maximum number of concurrent exports\n\nLimits the number of spawned tasks for exports and thus memory consumed by an exporter. A value of 1 will cause exports to be performed synchronously on the BatchSpanProcessor task.", + "default": null, + "type": "integer", + "format": "uint", + "minimum": 0.0, + "nullable": true + }, + "max_export_batch_size": { + "description": "The maximum number of spans to process in a single batch. If there are more than one batch worth of spans then it processes multiple batches of spans one batch after the other without any delay. The default value is 512.", + "default": null, + "type": "integer", + "format": "uint", + "minimum": 0.0, + "nullable": true + }, + "max_export_timeout": { + "description": "The maximum duration to export a batch of data.", + "default": null, + "type": "string" + }, + "max_queue_size": { + "description": "The maximum queue size to buffer spans for delayed processing. If the queue gets full it drops the spans. The default value of is 2048.", + "default": null, + "type": "integer", + "format": "uint", + "minimum": 0.0, + "nullable": true + }, + "scheduled_delay": { + "description": "The delay interval in milliseconds between two consecutive processing of batches. The default value is 5 seconds.", + "default": null, + "type": "string" + } + }, + "nullable": true + }, "endpoint": { "type": "string" }, @@ -1892,6 +1972,46 @@ expression: "&schema" "datadog": { "type": "object", "properties": { + "batch_processor": { + "type": "object", + "properties": { + "max_concurrent_exports": { + "description": "Maximum number of concurrent exports\n\nLimits the number of spawned tasks for exports and thus memory consumed by an exporter. A value of 1 will cause exports to be performed synchronously on the BatchSpanProcessor task.", + "default": null, + "type": "integer", + "format": "uint", + "minimum": 0.0, + "nullable": true + }, + "max_export_batch_size": { + "description": "The maximum number of spans to process in a single batch. If there are more than one batch worth of spans then it processes multiple batches of spans one batch after the other without any delay. The default value is 512.", + "default": null, + "type": "integer", + "format": "uint", + "minimum": 0.0, + "nullable": true + }, + "max_export_timeout": { + "description": "The maximum duration to export a batch of data.", + "default": null, + "type": "string" + }, + "max_queue_size": { + "description": "The maximum queue size to buffer spans for delayed processing. If the queue gets full it drops the spans. The default value of is 2048.", + "default": null, + "type": "integer", + "format": "uint", + "minimum": 0.0, + "nullable": true + }, + "scheduled_delay": { + "description": "The delay interval in milliseconds between two consecutive processing of batches. The default value is 5 seconds.", + "default": null, + "type": "string" + } + }, + "nullable": true + }, "endpoint": { "default": "default", "type": "string" @@ -1938,8 +2058,44 @@ expression: "&schema" }, "additionalProperties": false }, - "scheduled_delay": { - "type": "string" + "batch_processor": { + "type": "object", + "properties": { + "max_concurrent_exports": { + "description": "Maximum number of concurrent exports\n\nLimits the number of spawned tasks for exports and thus memory consumed by an exporter. A value of 1 will cause exports to be performed synchronously on the BatchSpanProcessor task.", + "default": null, + "type": "integer", + "format": "uint", + "minimum": 0.0, + "nullable": true + }, + "max_export_batch_size": { + "description": "The maximum number of spans to process in a single batch. If there are more than one batch worth of spans then it processes multiple batches of spans one batch after the other without any delay. The default value is 512.", + "default": null, + "type": "integer", + "format": "uint", + "minimum": 0.0, + "nullable": true + }, + "max_export_timeout": { + "description": "The maximum duration to export a batch of data.", + "default": null, + "type": "string" + }, + "max_queue_size": { + "description": "The maximum queue size to buffer spans for delayed processing. If the queue gets full it drops the spans. The default value of is 2048.", + "default": null, + "type": "integer", + "format": "uint", + "minimum": 0.0, + "nullable": true + }, + "scheduled_delay": { + "description": "The delay interval in milliseconds between two consecutive processing of batches. The default value is 5 seconds.", + "default": null, + "type": "string" + } + } } }, "additionalProperties": false @@ -1950,6 +2106,45 @@ expression: "&schema" "collector" ], "properties": { + "batch_processor": { + "type": "object", + "properties": { + "max_concurrent_exports": { + "description": "Maximum number of concurrent exports\n\nLimits the number of spawned tasks for exports and thus memory consumed by an exporter. A value of 1 will cause exports to be performed synchronously on the BatchSpanProcessor task.", + "default": null, + "type": "integer", + "format": "uint", + "minimum": 0.0, + "nullable": true + }, + "max_export_batch_size": { + "description": "The maximum number of spans to process in a single batch. If there are more than one batch worth of spans then it processes multiple batches of spans one batch after the other without any delay. The default value is 512.", + "default": null, + "type": "integer", + "format": "uint", + "minimum": 0.0, + "nullable": true + }, + "max_export_timeout": { + "description": "The maximum duration to export a batch of data.", + "default": null, + "type": "string" + }, + "max_queue_size": { + "description": "The maximum queue size to buffer spans for delayed processing. If the queue gets full it drops the spans. The default value of is 2048.", + "default": null, + "type": "integer", + "format": "uint", + "minimum": 0.0, + "nullable": true + }, + "scheduled_delay": { + "description": "The delay interval in milliseconds between two consecutive processing of batches. The default value is 5 seconds.", + "default": null, + "type": "string" + } + } + }, "collector": { "type": "object", "required": [ @@ -1969,18 +2164,52 @@ expression: "&schema" } }, "additionalProperties": false - }, - "scheduled_delay": { - "type": "string" } }, "additionalProperties": false } ], "properties": { - "scheduled_delay": { + "batch_processor": { "default": null, - "type": "string" + "type": "object", + "properties": { + "max_concurrent_exports": { + "description": "Maximum number of concurrent exports\n\nLimits the number of spawned tasks for exports and thus memory consumed by an exporter. A value of 1 will cause exports to be performed synchronously on the BatchSpanProcessor task.", + "default": null, + "type": "integer", + "format": "uint", + "minimum": 0.0, + "nullable": true + }, + "max_export_batch_size": { + "description": "The maximum number of spans to process in a single batch. If there are more than one batch worth of spans then it processes multiple batches of spans one batch after the other without any delay. The default value is 512.", + "default": null, + "type": "integer", + "format": "uint", + "minimum": 0.0, + "nullable": true + }, + "max_export_timeout": { + "description": "The maximum duration to export a batch of data.", + "default": null, + "type": "string" + }, + "max_queue_size": { + "description": "The maximum queue size to buffer spans for delayed processing. If the queue gets full it drops the spans. The default value of is 2048.", + "default": null, + "type": "integer", + "format": "uint", + "minimum": 0.0, + "nullable": true + }, + "scheduled_delay": { + "description": "The delay interval in milliseconds between two consecutive processing of batches. The default value is 5 seconds.", + "default": null, + "type": "string" + } + }, + "nullable": true } }, "nullable": true @@ -1991,6 +2220,46 @@ expression: "&schema" "endpoint" ], "properties": { + "batch_processor": { + "type": "object", + "properties": { + "max_concurrent_exports": { + "description": "Maximum number of concurrent exports\n\nLimits the number of spawned tasks for exports and thus memory consumed by an exporter. A value of 1 will cause exports to be performed synchronously on the BatchSpanProcessor task.", + "default": null, + "type": "integer", + "format": "uint", + "minimum": 0.0, + "nullable": true + }, + "max_export_batch_size": { + "description": "The maximum number of spans to process in a single batch. If there are more than one batch worth of spans then it processes multiple batches of spans one batch after the other without any delay. The default value is 512.", + "default": null, + "type": "integer", + "format": "uint", + "minimum": 0.0, + "nullable": true + }, + "max_export_timeout": { + "description": "The maximum duration to export a batch of data.", + "default": null, + "type": "string" + }, + "max_queue_size": { + "description": "The maximum queue size to buffer spans for delayed processing. If the queue gets full it drops the spans. The default value of is 2048.", + "default": null, + "type": "integer", + "format": "uint", + "minimum": 0.0, + "nullable": true + }, + "scheduled_delay": { + "description": "The delay interval in milliseconds between two consecutive processing of batches. The default value is 5 seconds.", + "default": null, + "type": "string" + } + }, + "nullable": true + }, "endpoint": { "type": "string" }, @@ -2236,6 +2505,46 @@ expression: "&schema" "zipkin": { "type": "object", "properties": { + "batch_processor": { + "type": "object", + "properties": { + "max_concurrent_exports": { + "description": "Maximum number of concurrent exports\n\nLimits the number of spawned tasks for exports and thus memory consumed by an exporter. A value of 1 will cause exports to be performed synchronously on the BatchSpanProcessor task.", + "default": null, + "type": "integer", + "format": "uint", + "minimum": 0.0, + "nullable": true + }, + "max_export_batch_size": { + "description": "The maximum number of spans to process in a single batch. If there are more than one batch worth of spans then it processes multiple batches of spans one batch after the other without any delay. The default value is 512.", + "default": null, + "type": "integer", + "format": "uint", + "minimum": 0.0, + "nullable": true + }, + "max_export_timeout": { + "description": "The maximum duration to export a batch of data.", + "default": null, + "type": "string" + }, + "max_queue_size": { + "description": "The maximum queue size to buffer spans for delayed processing. If the queue gets full it drops the spans. The default value of is 2048.", + "default": null, + "type": "integer", + "format": "uint", + "minimum": 0.0, + "nullable": true + }, + "scheduled_delay": { + "description": "The delay interval in milliseconds between two consecutive processing of batches. The default value is 5 seconds.", + "default": null, + "type": "string" + } + }, + "nullable": true + }, "endpoint": { "default": "default", "type": "string" diff --git a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__upgrade_old_configuration@jaeger_scheduled_delay.router.yaml.snap b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__upgrade_old_configuration@jaeger_scheduled_delay.router.yaml.snap new file mode 100644 index 0000000000..3007513330 --- /dev/null +++ b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__upgrade_old_configuration@jaeger_scheduled_delay.router.yaml.snap @@ -0,0 +1,13 @@ +--- +source: apollo-router/src/configuration/tests.rs +expression: new_config +--- +--- +telemetry: + tracing: + jaeger: + agent: + endpoint: default + batch_processor: + scheduled_delay: 100ms + diff --git a/apollo-router/src/configuration/testdata/migrations/jaeger_scheduled_delay.router.yaml b/apollo-router/src/configuration/testdata/migrations/jaeger_scheduled_delay.router.yaml new file mode 100644 index 0000000000..65426f4792 --- /dev/null +++ b/apollo-router/src/configuration/testdata/migrations/jaeger_scheduled_delay.router.yaml @@ -0,0 +1,6 @@ +telemetry: + tracing: + jaeger: + scheduled_delay: 100ms + agent: + endpoint: default diff --git a/apollo-router/src/executable.rs b/apollo-router/src/executable.rs index dc807b0091..b7384222a9 100644 --- a/apollo-router/src/executable.rs +++ b/apollo-router/src/executable.rs @@ -25,6 +25,7 @@ use tracing_subscriber::EnvFilter; use url::ParseError; use url::Url; +use crate::configuration; use crate::configuration::generate_config_schema; use crate::configuration::generate_upgrade; use crate::configuration::Configuration; @@ -53,7 +54,6 @@ pub(crate) const APOLLO_ROUTER_DEV_ENV: &str = "APOLLO_ROUTER_DEV"; // Note: Constructor/Destructor functions may not play nicely with tracing, since they run after // main completes, so don't use tracing, use println!() and eprintln!().. #[cfg(feature = "dhat-heap")] -#[crate::_private::ctor::ctor] fn create_heap_profiler() { unsafe { match DHAT_HEAP_PROFILER.set(dhat::Profiler::new_heap()) { @@ -77,7 +77,6 @@ extern "C" fn drop_heap_profiler() { } #[cfg(feature = "dhat-ad-hoc")] -#[crate::_private::ctor::ctor] fn create_ad_hoc_profiler() { unsafe { match DHAT_AD_HOC_PROFILER.set(dhat::Profiler::new_ad_hoc()) { @@ -129,6 +128,8 @@ enum ConfigSubcommand { #[clap(parse(from_flag), long)] diff: bool, }, + /// List all the available experimental configurations with related GitHub discussion + Experimental, } /// Options for the router @@ -258,6 +259,12 @@ impl fmt::Display for ProjectDir { /// /// Refer to the examples if you would like to see how to run your own router with plugins. pub fn main() -> Result<()> { + #[cfg(feature = "dhat-heap")] + create_heap_profiler(); + + #[cfg(feature = "dhat-ad-hoc")] + create_ad_hoc_profiler(); + let mut builder = tokio::runtime::Builder::new_multi_thread(); builder.enable_all(); if let Some(nb) = std::env::var("APOLLO_ROUTER_NUM_CORES") @@ -373,6 +380,12 @@ impl Executable { println!("{}", output); Ok(()) } + Some(Commands::Config(ConfigSubcommandArgs { + command: ConfigSubcommand::Experimental, + })) => { + configuration::print_all_experimental_conf(); + Ok(()) + } None => { // The dispatcher we created is passed explicitly here to make sure we display the logs // in the initialization phase and in the state machine code, before a global subscriber diff --git a/apollo-router/src/json_ext.rs b/apollo-router/src/json_ext.rs index 5e27a48eae..e16d2f2e48 100644 --- a/apollo-router/src/json_ext.rs +++ b/apollo-router/src/json_ext.rs @@ -242,6 +242,7 @@ impl ValueExt for Value { .get_mut(k.as_str()) .expect("the value at that key was just inserted"); } + PathElement::Fragment(_) => {} } } @@ -319,6 +320,7 @@ impl ValueExt for Value { }) } }, + PathElement::Fragment(_) => {} } } @@ -402,8 +404,17 @@ where iterate_path(parent, &path[1..], value, f); parent.pop(); } + } else if let Value::Array(array) = data { + for (i, value) in array.iter().enumerate() { + parent.push(PathElement::Index(i)); + iterate_path(parent, path, value, f); + parent.pop(); + } } } + Some(PathElement::Fragment(_)) => { + iterate_path(parent, &path[1..], data, f); + } } } @@ -422,6 +433,10 @@ pub enum PathElement { /// An index path element. Index(usize), + /// A fragment application + #[serde(deserialize_with = "deserialize_fragment")] + Fragment(String), + /// A key path element. Key(String), } @@ -464,6 +479,37 @@ where serializer.serialize_str("@") } +fn deserialize_fragment<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + deserializer.deserialize_str(FragmentVisitor) +} + +struct FragmentVisitor; + +impl<'de> serde::de::Visitor<'de> for FragmentVisitor { + type Value = String; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + write!(formatter, "a string that begins with '... '") + } + + fn visit_str(self, s: &str) -> Result + where + E: serde::de::Error, + { + if s.starts_with("... ") { + Ok(s.to_string()) + } else { + Err(serde::de::Error::invalid_value( + serde::de::Unexpected::Str(s), + &self, + )) + } + } +} + /// A path into the result document. /// /// This can be composed of strings and numbers @@ -587,6 +633,7 @@ impl fmt::Display for Path { PathElement::Index(index) => write!(f, "{}", index)?, PathElement::Key(key) => write!(f, "{}", key)?, PathElement::Flatten => write!(f, "@")?, + PathElement::Fragment(fragment) => write!(f, "{fragment}")?, } } Ok(()) diff --git a/apollo-router/src/lib.rs b/apollo-router/src/lib.rs index 92459d1f77..11688cd0d8 100644 --- a/apollo-router/src/lib.rs +++ b/apollo-router/src/lib.rs @@ -66,7 +66,6 @@ mod response; mod router; mod router_factory; pub mod services; -mod spaceport; pub(crate) mod spec; mod state_machine; mod test_harness; @@ -91,10 +90,13 @@ pub use crate::test_harness::TestHarness; #[doc(hidden)] pub mod _private { // Reexports for macros - pub use ctor; + pub use linkme; + pub use once_cell; pub use router_bridge; pub use serde_json; + pub use crate::plugin::PluginFactory; + pub use crate::plugin::PLUGINS; // For tests pub use crate::plugins::telemetry::Telemetry as TelemetryPlugin; pub use crate::router_factory::create_test_service_factory_from_yaml; diff --git a/apollo-router/src/plugin/mod.rs b/apollo-router/src/plugin/mod.rs index 1a5484e869..ac52473fb7 100644 --- a/apollo-router/src/plugin/mod.rs +++ b/apollo-router/src/plugin/mod.rs @@ -5,7 +5,6 @@ //! Requests received by the router make their way through a processing pipeline. Each request is //! processed at: //! - router -//! - query planning //! - execution //! - subgraph (multiple in parallel if multiple subgraphs are accessed) //! stages. @@ -19,9 +18,8 @@ pub mod serde; pub mod test; use std::any::TypeId; -use std::collections::HashMap; +use std::fmt; use std::sync::Arc; -use std::sync::Mutex; use std::task::Context; use std::task::Poll; @@ -52,6 +50,10 @@ type InstanceFactory = type SchemaFactory = fn(&mut SchemaGenerator) -> schemars::schema::Schema; +/// Global list of plugins. +#[linkme::distributed_slice] +pub static PLUGINS: [Lazy] = [..]; + /// Initialise details for a plugin #[non_exhaustive] pub struct PluginInit { @@ -91,13 +93,45 @@ where /// Factories for plugin schema and configuration. #[derive(Clone)] -pub(crate) struct PluginFactory { +pub struct PluginFactory { + pub(crate) name: String, instance_factory: InstanceFactory, schema_factory: SchemaFactory, pub(crate) type_id: TypeId, } +impl fmt::Debug for PluginFactory { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PluginFactory") + .field("name", &self.name) + .field("type_id", &self.type_id) + .finish() + } +} + impl PluginFactory { + /// Create a plugin factory. + pub fn new(group: &str, name: &str) -> PluginFactory { + let plugin_factory_name = if group.is_empty() { + name.to_string() + } else { + format!("{}.{}", group, name) + }; + tracing::debug!(%plugin_factory_name, "creating plugin factory"); + PluginFactory { + name: plugin_factory_name, + instance_factory: |configuration, schema| { + Box::pin(async move { + let init = PluginInit::try_new(configuration.clone(), schema)?; + let plugin = P::new(init).await?; + Ok(Box::new(plugin) as Box) + }) + }, + schema_factory: |gen| gen.subschema_for::<

::Config>(), + type_id: TypeId::of::

(), + } + } + pub(crate) async fn create_instance( &self, configuration: &serde_json::Value, @@ -119,33 +153,10 @@ impl PluginFactory { } } -static PLUGIN_REGISTRY: Lazy>> = Lazy::new(|| { - let m = HashMap::new(); - Mutex::new(m) -}); - -/// Register a plugin factory. -pub fn register_plugin(name: String) { - let plugin_factory = PluginFactory { - instance_factory: |configuration, schema| { - Box::pin(async move { - let init = PluginInit::try_new(configuration.clone(), schema)?; - let plugin = P::new(init).await?; - Ok(Box::new(plugin) as Box) - }) - }, - schema_factory: |gen| gen.subschema_for::<

::Config>(), - type_id: TypeId::of::

(), - }; - PLUGIN_REGISTRY - .lock() - .expect("Lock poisoned") - .insert(name, plugin_factory); -} - +// If we wanted to create a custom subset of plugins, this is where we would do it /// Get a copy of the registered plugin factories. -pub(crate) fn plugins() -> HashMap { - PLUGIN_REGISTRY.lock().expect("Lock poisoned").clone() +pub(crate) fn plugins() -> impl Iterator> { + PLUGINS.iter() } /// All router plugins must implement the Plugin trait. @@ -299,16 +310,14 @@ macro_rules! register_plugin { ($group: literal, $name: literal, $plugin_type: ident) => { // Artificial scope to avoid naming collisions const _: () = { - #[$crate::_private::ctor::ctor] - fn register_plugin() { - let qualified_name = if $group == "" { - $name.to_string() - } else { - format!("{}.{}", $group, $name) - }; - - $crate::plugin::register_plugin::<$plugin_type>(qualified_name); - } + use $crate::_private::once_cell::sync::Lazy; + use $crate::_private::PluginFactory; + use $crate::_private::PLUGINS; + + #[$crate::_private::linkme::distributed_slice(PLUGINS)] + #[linkme(crate = $crate::_private::linkme)] + static REGISTER_PLUGIN: Lazy = + Lazy::new(|| $crate::plugin::PluginFactory::new::<$plugin_type>($group, $name)); }; }; } diff --git a/apollo-router/src/plugins/csrf.rs b/apollo-router/src/plugins/csrf.rs index 55ebc3e6ff..f31b7ef1c0 100644 --- a/apollo-router/src/plugins/csrf.rs +++ b/apollo-router/src/plugins/csrf.rs @@ -213,14 +213,14 @@ mod csrf_tests { #[tokio::test] async fn plugin_registered() { crate::plugin::plugins() - .get("apollo.csrf") + .find(|factory| factory.name == "apollo.csrf") .expect("Plugin not found") .create_instance_without_schema(&serde_json::json!({ "unsafe_disabled": true })) .await .unwrap(); crate::plugin::plugins() - .get("apollo.csrf") + .find(|factory| factory.name == "apollo.csrf") .expect("Plugin not found") .create_instance_without_schema(&serde_json::json!({})) .await diff --git a/apollo-router/src/plugins/expose_query_plan.rs b/apollo-router/src/plugins/expose_query_plan.rs index 98ee756b3c..e74fdd3591 100644 --- a/apollo-router/src/plugins/expose_query_plan.rs +++ b/apollo-router/src/plugins/expose_query_plan.rs @@ -190,7 +190,7 @@ mod tests { async fn get_plugin(config: &jValue) -> Box { crate::plugin::plugins() - .get("experimental.expose_query_plan") + .find(|factory| factory.name == "experimental.expose_query_plan") .expect("Plugin not found") .create_instance_without_schema(config) .await @@ -229,12 +229,12 @@ mod tests { let supergraph = build_mock_supergraph(plugin).await; execute_supergraph_test( VALID_QUERY, - &*EXPECTED_RESPONSE_WITH_QUERY_PLAN, + &EXPECTED_RESPONSE_WITH_QUERY_PLAN, supergraph.clone(), ) .await; // let's try that again - execute_supergraph_test(VALID_QUERY, &*EXPECTED_RESPONSE_WITH_QUERY_PLAN, supergraph).await; + execute_supergraph_test(VALID_QUERY, &EXPECTED_RESPONSE_WITH_QUERY_PLAN, supergraph).await; } #[tokio::test] @@ -243,7 +243,7 @@ mod tests { let supergraph = build_mock_supergraph(plugin).await; execute_supergraph_test( VALID_QUERY, - &*EXPECTED_RESPONSE_WITHOUT_QUERY_PLAN, + &EXPECTED_RESPONSE_WITHOUT_QUERY_PLAN, supergraph, ) .await; diff --git a/apollo-router/src/plugins/include_subgraph_errors.rs b/apollo-router/src/plugins/include_subgraph_errors.rs index 40b1e8f687..e17cfff077 100644 --- a/apollo-router/src/plugins/include_subgraph_errors.rs +++ b/apollo-router/src/plugins/include_subgraph_errors.rs @@ -202,7 +202,7 @@ mod test { async fn get_redacting_plugin(config: &jValue) -> Box { // Build a redacting plugin crate::plugin::plugins() - .get("apollo.include_subgraph_errors") + .find(|factory| factory.name == "apollo.include_subgraph_errors") .expect("Plugin not found") .create_instance_without_schema(config) .await @@ -214,7 +214,7 @@ mod test { // Build a redacting plugin let plugin = get_redacting_plugin(&serde_json::json!({ "all": false })).await; let router = build_mock_router(plugin).await; - execute_router_test(VALID_QUERY, &*EXPECTED_RESPONSE, router).await; + execute_router_test(VALID_QUERY, &EXPECTED_RESPONSE, router).await; } #[tokio::test] @@ -222,7 +222,7 @@ mod test { // Build a redacting plugin let plugin = get_redacting_plugin(&serde_json::json!({ "all": false })).await; let router = build_mock_router(plugin).await; - execute_router_test(ERROR_PRODUCT_QUERY, &*REDACTED_PRODUCT_RESPONSE, router).await; + execute_router_test(ERROR_PRODUCT_QUERY, &REDACTED_PRODUCT_RESPONSE, router).await; } #[tokio::test] @@ -230,7 +230,7 @@ mod test { // Build a redacting plugin let plugin = get_redacting_plugin(&serde_json::json!({})).await; let router = build_mock_router(plugin).await; - execute_router_test(ERROR_PRODUCT_QUERY, &*REDACTED_PRODUCT_RESPONSE, router).await; + execute_router_test(ERROR_PRODUCT_QUERY, &REDACTED_PRODUCT_RESPONSE, router).await; } #[tokio::test] @@ -238,7 +238,7 @@ mod test { // Build a redacting plugin let plugin = get_redacting_plugin(&serde_json::json!({ "all": true })).await; let router = build_mock_router(plugin).await; - execute_router_test(ERROR_PRODUCT_QUERY, &*UNREDACTED_PRODUCT_RESPONSE, router).await; + execute_router_test(ERROR_PRODUCT_QUERY, &UNREDACTED_PRODUCT_RESPONSE, router).await; } #[tokio::test] @@ -247,7 +247,7 @@ mod test { let plugin = get_redacting_plugin(&serde_json::json!({ "subgraphs": {"products": true }})).await; let router = build_mock_router(plugin).await; - execute_router_test(ERROR_PRODUCT_QUERY, &*UNREDACTED_PRODUCT_RESPONSE, router).await; + execute_router_test(ERROR_PRODUCT_QUERY, &UNREDACTED_PRODUCT_RESPONSE, router).await; } #[tokio::test] @@ -256,7 +256,7 @@ mod test { let plugin = get_redacting_plugin(&serde_json::json!({ "subgraphs": {"reviews": true }})).await; let router = build_mock_router(plugin).await; - execute_router_test(ERROR_PRODUCT_QUERY, &*REDACTED_PRODUCT_RESPONSE, router).await; + execute_router_test(ERROR_PRODUCT_QUERY, &REDACTED_PRODUCT_RESPONSE, router).await; } #[tokio::test] @@ -267,7 +267,7 @@ mod test { ) .await; let router = build_mock_router(plugin).await; - execute_router_test(ERROR_PRODUCT_QUERY, &*UNREDACTED_PRODUCT_RESPONSE, router).await; + execute_router_test(ERROR_PRODUCT_QUERY, &UNREDACTED_PRODUCT_RESPONSE, router).await; } #[tokio::test] @@ -278,7 +278,7 @@ mod test { ) .await; let router = build_mock_router(plugin).await; - execute_router_test(ERROR_PRODUCT_QUERY, &*REDACTED_PRODUCT_RESPONSE, router).await; + execute_router_test(ERROR_PRODUCT_QUERY, &REDACTED_PRODUCT_RESPONSE, router).await; } #[tokio::test] @@ -289,7 +289,7 @@ mod test { ) .await; let router = build_mock_router(plugin).await; - execute_router_test(ERROR_PRODUCT_QUERY, &*UNREDACTED_PRODUCT_RESPONSE, router).await; + execute_router_test(ERROR_PRODUCT_QUERY, &UNREDACTED_PRODUCT_RESPONSE, router).await; } #[tokio::test] @@ -300,6 +300,6 @@ mod test { ) .await; let router = build_mock_router(plugin).await; - execute_router_test(ERROR_ACCOUNT_QUERY, &*REDACTED_ACCOUNT_RESPONSE, router).await; + execute_router_test(ERROR_ACCOUNT_QUERY, &REDACTED_ACCOUNT_RESPONSE, router).await; } } diff --git a/apollo-router/src/plugins/override_url.rs b/apollo-router/src/plugins/override_url.rs index dadfe86f40..00f4696fb1 100644 --- a/apollo-router/src/plugins/override_url.rs +++ b/apollo-router/src/plugins/override_url.rs @@ -84,7 +84,7 @@ mod tests { }); let dyn_plugin: Box = crate::plugin::plugins() - .get("apollo.override_subgraph_url") + .find(|factory| factory.name == "apollo.override_subgraph_url") .expect("Plugin not found") .create_instance( &Value::from_str( diff --git a/apollo-router/src/plugins/rhai.rs b/apollo-router/src/plugins/rhai.rs index 28c9e39e71..340a4d26db 100644 --- a/apollo-router/src/plugins/rhai.rs +++ b/apollo-router/src/plugins/rhai.rs @@ -4,9 +4,12 @@ use std::fmt; use std::ops::ControlFlow; use std::path::PathBuf; use std::str::FromStr; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering; use std::sync::Arc; use std::sync::Mutex; +use arc_swap::ArcSwap; use futures::future::ready; use futures::stream::once; use futures::StreamExt; @@ -19,11 +22,15 @@ use http::uri::PathAndQuery; use http::HeaderMap; use http::StatusCode; use http::Uri; -use opentelemetry::trace::SpanKind; +use notify::event::ModifyKind; +use notify::EventKind; +use notify::RecursiveMode; +use notify::Watcher; use rhai::module_resolvers::FileModuleResolver; use rhai::plugin::*; use rhai::serde::from_dynamic; use rhai::serde::to_dynamic; +use rhai::Array; use rhai::Dynamic; use rhai::Engine; use rhai::EvalAltResult; @@ -355,14 +362,47 @@ mod router_plugin_mod { } } -/// Plugin which implements Rhai functionality -#[derive(Default, Clone)] -pub(crate) struct Rhai { +struct EngineBlock { ast: AST, engine: Arc, scope: Arc>>, } +impl EngineBlock { + fn try_new( + scripts: Option, + main: PathBuf, + sdl: Arc, + ) -> Result { + let engine = Arc::new(Rhai::new_rhai_engine(scripts)); + let ast = engine.compile_file(main)?; + let mut scope = Scope::new(); + scope.push_constant("apollo_sdl", sdl.to_string()); + scope.push_constant("apollo_start", Instant::now()); + + // Run the AST with our scope to put any global variables + // defined in scripts into scope. + engine.run_ast_with_scope(&mut scope, &ast)?; + + Ok(EngineBlock { + ast, + engine, + scope: Arc::new(Mutex::new(scope)), + }) + } +} + +/// Plugin which implements Rhai functionality +/// Note: We use ArcSwap here in preference to a shared RwLock. Updates to +/// the engine block will be infrequent in relation to the accesses of it. +/// We'd love to use AtomicArc if such a thing existed, but since it doesn't +/// we'll use ArcSwap to accomplish our goal. +struct Rhai { + block: Arc>, + park_flag: Arc, + watcher_handle: Option>, +} + /// Configuration for the Rhai Plugin #[derive(Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] @@ -376,6 +416,7 @@ impl Plugin for Rhai { type Config = Conf; async fn new(init: PluginInit) -> Result { + let sdl = init.supergraph_sdl.clone(); let scripts_path = match init.config.scripts { Some(path) => path, None => "./rhai".into(), @@ -387,21 +428,85 @@ impl Plugin for Rhai { }; let main = scripts_path.join(&main_file); - let sdl = init.supergraph_sdl.clone(); - let engine = Arc::new(Rhai::new_rhai_engine(Some(scripts_path))); - let ast = engine.compile_file(main)?; - let mut scope = Scope::new(); - scope.push_constant("apollo_sdl", sdl.to_string()); - scope.push_constant("apollo_start", Instant::now()); - // Run the AST with our scope to put any global variables - // defined in scripts into scope. - engine.run_ast_with_scope(&mut scope, &ast)?; + let watched_path = scripts_path.clone(); + let watched_main = main.clone(); + let watched_sdl = sdl.clone(); + + let block = Arc::new(ArcSwap::from_pointee(EngineBlock::try_new( + Some(scripts_path), + main, + sdl, + )?)); + let watched_block = block.clone(); + + let park_flag = Arc::new(AtomicBool::new(false)); + let watching_flag = park_flag.clone(); + + let watcher_handle = std::thread::spawn(move || { + let watching_path = watched_path.clone(); + let mut watcher = + notify::recommended_watcher(move |res: Result| { + match res { + Ok(event) => { + // Let's limit the events we are interested in to: + // - Modified files + // - Created/Remove files + // - with suffix "rhai" + if matches!( + event.kind, + EventKind::Modify(ModifyKind::Data(_)) + | EventKind::Create(_) + | EventKind::Remove(_) + ) { + let mut proceed = false; + for path in event.paths { + if path.extension().map_or(false, |ext| ext == "rhai") { + proceed = true; + break; + } + } + + if proceed { + match EngineBlock::try_new( + Some(watching_path.clone()), + watched_main.clone(), + watched_sdl.clone(), + ) { + Ok(eb) => { + tracing::info!("updating rhai execution engine"); + watched_block.store(Arc::new(eb)) + } + Err(e) => { + tracing::warn!( + "could not create new rhai execution engine: {}", + e + ); + } + } + } + } + } + Err(e) => tracing::error!("rhai watching event error: {:?}", e), + } + }) + .unwrap_or_else(|_| panic!("could not create watch on: {:?}", watched_path)); + watcher + .watch(&watched_path, RecursiveMode::Recursive) + .unwrap_or_else(|_| panic!("could not watch: {:?}", watched_path)); + // Park the thread until this Rhai instance is dropped (see Drop impl) + // We may actually unpark() before this code executes or exit from park() spuriously. + // Use the watching_flag to control a loop which waits from the flag to be updated + // from Drop. + while !watching_flag.load(Ordering::Acquire) { + std::thread::park(); + } + }); Ok(Self { - ast, - engine, - scope: Arc::new(Mutex::new(scope)), + block, + park_flag, + watcher_handle: Some(watcher_handle), }) } @@ -416,7 +521,7 @@ impl Plugin for Rhai { FUNCTION_NAME_SERVICE, None, ServiceStep::Supergraph(shared_service.clone()), - self.scope.clone(), + self.block.load().scope.clone(), ) { tracing::error!("service callback failed: {error}"); } @@ -434,7 +539,7 @@ impl Plugin for Rhai { FUNCTION_NAME_SERVICE, None, ServiceStep::Execution(shared_service.clone()), - self.scope.clone(), + self.block.load().scope.clone(), ) { tracing::error!("service callback failed: {error}"); } @@ -452,7 +557,7 @@ impl Plugin for Rhai { FUNCTION_NAME_SERVICE, Some(name), ServiceStep::Subgraph(shared_service.clone()), - self.scope.clone(), + self.block.load().scope.clone(), ) { tracing::error!("service callback failed: {error}"); } @@ -460,6 +565,16 @@ impl Plugin for Rhai { } } +impl Drop for Rhai { + fn drop(&mut self) { + if let Some(wh) = self.watcher_handle.take() { + self.park_flag.store(true, Ordering::Release); + wh.thread().unpark(); + wh.join().expect("rhai file watcher thread terminating"); + } + } +} + #[derive(Clone, Debug)] pub(crate) enum ServiceStep { Supergraph(SharedMut), @@ -476,7 +591,7 @@ macro_rules! gen_map_request { tracing::info_span!( "rhai plugin", "rhai service" = stringify!($base::Request), - "otel.kind" = %SpanKind::Internal + "otel.kind" = "INTERNAL" ) } } @@ -501,32 +616,26 @@ macro_rules! gen_map_request { } let shared_request = Shared::new(Mutex::new(Some(request))); let result: Result> = if $callback.is_curried() { - $callback - .call( - &$rhai_service.engine, - &$rhai_service.ast, - (shared_request.clone(),), - ) + $callback.call( + &$rhai_service.engine, + &$rhai_service.ast, + (shared_request.clone(),), + ) } else { let mut guard = $rhai_service.scope.lock().unwrap(); - $rhai_service - .engine - .call_fn( - &mut guard, - &$rhai_service.ast, - $callback.fn_name(), - (shared_request.clone(),), - ) + $rhai_service.engine.call_fn( + &mut guard, + &$rhai_service.ast, + $callback.fn_name(), + (shared_request.clone(),), + ) }; if let Err(error) = result { let error_details = process_error(error); tracing::error!("map_request callback failed: {error_details}"); let mut guard = shared_request.lock().unwrap(); let request_opt = guard.take(); - return failure_message( - request_opt.unwrap().context, - error_details, - ); + return failure_message(request_opt.unwrap().context, error_details); } let mut guard = shared_request.lock().unwrap(); let request_opt = guard.take(); @@ -547,7 +656,7 @@ macro_rules! gen_map_deferred_request { tracing::info_span!( "rhai plugin", "rhai service" = stringify!($request), - "otel.kind" = %SpanKind::Internal + "otel.kind" = "INTERNAL" ) } } @@ -577,10 +686,7 @@ macro_rules! gen_map_deferred_request { let error_details = process_error(error); let mut guard = shared_request.lock().unwrap(); let request_opt = guard.take(); - return failure_message( - request_opt.unwrap().context, - error_details - ); + return failure_message(request_opt.unwrap().context, error_details); } let mut guard = shared_request.lock().unwrap(); let request_opt = guard.take(); @@ -1068,11 +1174,12 @@ impl Rhai { service: ServiceStep, scope: Arc>>, ) -> Result<(), String> { + let block = self.block.load(); let rhai_service = RhaiService { scope: scope.clone(), service, - engine: self.engine.clone(), - ast: self.ast.clone(), + engine: block.engine.clone(), + ast: block.ast.clone(), }; let mut guard = scope.lock().unwrap(); // Note: We don't use `process_error()` here, because this code executes in the context of @@ -1082,18 +1189,20 @@ impl Rhai { // change and one that requires more thought in the future. match subgraph { Some(name) => { - self.engine + block + .engine .call_fn( &mut guard, - &self.ast, + &block.ast, function_name, (rhai_service, name.to_string()), ) .map_err(|err| err.to_string())?; } None => { - self.engine - .call_fn(&mut guard, &self.ast, function_name, (rhai_service,)) + block + .engine + .call_fn(&mut guard, &block.ast, function_name, (rhai_service,)) .map_err(|err| err.to_string())?; } } @@ -1196,6 +1305,39 @@ impl Rhai { ); Ok(()) }) + // Register an additional getter which allows us to get multiple values for the same + // key. + // Note: We can't register this as an indexer, because that would simply override the + // existing one, which would break code. When router 2.0 is released, we should replace + // the existing indexer_get for HeaderMap with this function and mark it as an + // incompatible change. + .register_fn("values", + |x: &mut HeaderMap, key: &str| -> Result> { + let search_name = + HeaderName::from_str(key).map_err(|e: InvalidHeaderName| e.to_string())?; + let mut response = Array::new(); + for value in x.get_all(search_name).iter() { + response.push(value + .to_str() + .map_err(|e| e.to_string())? + .to_string() + .into()) + } + Ok(response) + } + ) + // Register an additional setter which allows us to set multiple values for the same + // key. + .register_indexer_set(|x: &mut HeaderMap, key: &str, value: Array| { + let h_key = HeaderName::from_str(key).map_err(|e| e.to_string())?; + for v in value { + x.append( + h_key.clone(), + HeaderValue::from_str(&v.into_string()?).map_err(|e| e.to_string())?, + ); + } + Ok(()) + }) // Register a Context indexer so we can get/set context .register_indexer_get( |x: &mut Context, key: &str| -> Result> { @@ -1473,7 +1615,11 @@ impl Rhai { } fn ast_has_function(&self, name: &str) -> bool { - self.ast.iter_fn_def().any(|fn_def| fn_def.name == name) + self.block + .load() + .ast + .iter_fn_def() + .any(|fn_def| fn_def.name == name) } } @@ -1511,7 +1657,7 @@ mod tests { }); let dyn_plugin: Box = crate::plugin::plugins() - .get("apollo.rhai") + .find(|factory| factory.name == "apollo.rhai") .expect("Plugin not found") .create_instance_without_schema( &Value::from_str(r#"{"scripts":"tests/fixtures", "main":"test.rhai"}"#).unwrap(), @@ -1562,7 +1708,7 @@ mod tests { }); let dyn_plugin: Box = crate::plugin::plugins() - .get("apollo.rhai") + .find(|factory| factory.name == "apollo.rhai") .expect("Plugin not found") .create_instance_without_schema( &Value::from_str(r#"{"scripts":"tests/fixtures", "main":"test.rhai"}"#).unwrap(), @@ -1680,7 +1826,7 @@ mod tests { #[tokio::test] async fn it_can_access_sdl_constant() { let dyn_plugin: Box = crate::plugin::plugins() - .get("apollo.rhai") + .find(|factory| factory.name == "apollo.rhai") .expect("Plugin not found") .create_instance_without_schema( &Value::from_str(r#"{"scripts":"tests/fixtures", "main":"test.rhai"}"#).unwrap(), @@ -1692,15 +1838,17 @@ mod tests { let it: &dyn std::any::Any = dyn_plugin.as_any(); let rhai_instance: &Rhai = it.downcast_ref::().expect("downcast"); + let block = rhai_instance.block.load(); + // Get a scope to use for our test - let scope = rhai_instance.scope.clone(); + let scope = block.scope.clone(); let mut guard = scope.lock().unwrap(); // Call our function to make sure we can access the sdl - let sdl: String = rhai_instance + let sdl: String = block .engine - .call_fn(&mut guard, &rhai_instance.ast, "get_sdl", ()) + .call_fn(&mut guard, &block.ast, "get_sdl", ()) .expect("can get sdl"); assert_eq!(sdl.as_str(), ""); } @@ -1730,7 +1878,7 @@ mod tests { macro_rules! gen_request_test { ($base: ident, $fn_name: literal) => { let dyn_plugin: Box = crate::plugin::plugins() - .get("apollo.rhai") + .find(|factory| factory.name == "apollo.rhai") .expect("Plugin not found") .create_instance_without_schema( &Value::from_str( @@ -1745,8 +1893,10 @@ mod tests { let it: &dyn std::any::Any = dyn_plugin.as_any(); let rhai_instance: &Rhai = it.downcast_ref::().expect("downcast"); + let block = rhai_instance.block.load(); + // Get a scope to use for our test - let scope = rhai_instance.scope.clone(); + let scope = block.scope.clone(); let mut guard = scope.lock().unwrap(); @@ -1756,9 +1906,9 @@ mod tests { // Call our rhai test function. If it return an error, the test failed. let result: Result<(), Box> = - rhai_instance + block .engine - .call_fn(&mut guard, &rhai_instance.ast, $fn_name, (request,)); + .call_fn(&mut guard, &block.ast, $fn_name, (request,)); result.expect("test failed"); }; } @@ -1766,7 +1916,7 @@ mod tests { macro_rules! gen_response_test { ($base: ident, $fn_name: literal) => { let dyn_plugin: Box = crate::plugin::plugins() - .get("apollo.rhai") + .find(|factory| factory.name == "apollo.rhai") .expect("Plugin not found") .create_instance_without_schema( &Value::from_str( @@ -1781,8 +1931,10 @@ mod tests { let it: &dyn std::any::Any = dyn_plugin.as_any(); let rhai_instance: &Rhai = it.downcast_ref::().expect("downcast"); + let block = rhai_instance.block.load(); + // Get a scope to use for our test - let scope = rhai_instance.scope.clone(); + let scope = block.scope.clone(); let mut guard = scope.lock().unwrap(); @@ -1792,9 +1944,9 @@ mod tests { // Call our rhai test function. If it return an error, the test failed. let result: Result<(), Box> = - rhai_instance + block .engine - .call_fn(&mut guard, &rhai_instance.ast, $fn_name, (response,)); + .call_fn(&mut guard, &block.ast, $fn_name, (response,)); result.expect("test failed"); }; } @@ -1802,7 +1954,7 @@ mod tests { #[tokio::test] async fn it_can_process_supergraph_request() { let dyn_plugin: Box = crate::plugin::plugins() - .get("apollo.rhai") + .find(|factory| factory.name == "apollo.rhai") .expect("Plugin not found") .create_instance_without_schema( &Value::from_str( @@ -1817,8 +1969,10 @@ mod tests { let it: &dyn std::any::Any = dyn_plugin.as_any(); let rhai_instance: &Rhai = it.downcast_ref::().expect("downcast"); + let block = rhai_instance.block.load(); + // Get a scope to use for our test - let scope = rhai_instance.scope.clone(); + let scope = block.scope.clone(); let mut guard = scope.lock().unwrap(); @@ -1832,9 +1986,9 @@ mod tests { ))); // Call our rhai test function. If it return an error, the test failed. - let result: Result<(), Box> = rhai_instance.engine.call_fn( + let result: Result<(), Box> = block.engine.call_fn( &mut guard, - &rhai_instance.ast, + &block.ast, "process_supergraph_request", (request,), ); @@ -1877,7 +2031,7 @@ mod tests { #[tokio::test] async fn it_can_process_subgraph_response() { let dyn_plugin: Box = crate::plugin::plugins() - .get("apollo.rhai") + .find(|factory| factory.name == "apollo.rhai") .expect("Plugin not found") .create_instance_without_schema( &Value::from_str( @@ -1892,8 +2046,9 @@ mod tests { let it: &dyn std::any::Any = dyn_plugin.as_any(); let rhai_instance: &Rhai = it.downcast_ref::().expect("downcast"); + let block = rhai_instance.block.load(); // Get a scope to use for our test - let scope = rhai_instance.scope.clone(); + let scope = block.scope.clone(); let mut guard = scope.lock().unwrap(); @@ -1902,9 +2057,9 @@ mod tests { let response = Arc::new(Mutex::new(Some(subgraph::Response::fake_builder().build()))); // Call our rhai test function. If it return an error, the test failed. - let result: Result<(), Box> = rhai_instance.engine.call_fn( + let result: Result<(), Box> = block.engine.call_fn( &mut guard, - &rhai_instance.ast, + &block.ast, "process_subgraph_response", (response,), ); @@ -1931,7 +2086,7 @@ mod tests { async fn base_process_function(fn_name: &str) -> Result<(), Box> { let dyn_plugin: Box = crate::plugin::plugins() - .get("apollo.rhai") + .find(|factory| factory.name == "apollo.rhai") .expect("Plugin not found") .create_instance_without_schema( &Value::from_str( @@ -1946,8 +2101,10 @@ mod tests { let it: &dyn std::any::Any = dyn_plugin.as_any(); let rhai_instance: &Rhai = it.downcast_ref::().expect("downcast"); + let block = rhai_instance.block.load(); + // Get a scope to use for our test - let scope = rhai_instance.scope.clone(); + let scope = block.scope.clone(); let mut guard = scope.lock().unwrap(); @@ -1956,9 +2113,9 @@ mod tests { let response = Arc::new(Mutex::new(Some(subgraph::Response::fake_builder().build()))); // Call our rhai test function. If it doesn't return an error, the test failed. - rhai_instance + block .engine - .call_fn(&mut guard, &rhai_instance.ast, fn_name, (response,)) + .call_fn(&mut guard, &block.ast, fn_name, (response,)) } #[tokio::test] diff --git a/apollo-router/src/plugins/telemetry/apollo.rs b/apollo-router/src/plugins/telemetry/apollo.rs index b38757fd91..12af73f488 100644 --- a/apollo-router/src/plugins/telemetry/apollo.rs +++ b/apollo-router/src/plugins/telemetry/apollo.rs @@ -19,11 +19,15 @@ use super::metrics::apollo::studio::SingleStatsReport; use super::tracing::apollo::TracesReport; use crate::plugin::serde::deserialize_header_name; use crate::plugin::serde::deserialize_vec_header_name; +use crate::plugins::telemetry::apollo_exporter::proto::ReferencedFieldsForType; +use crate::plugins::telemetry::apollo_exporter::proto::ReportHeader; +use crate::plugins::telemetry::apollo_exporter::proto::StatsContext; +use crate::plugins::telemetry::apollo_exporter::proto::Trace; use crate::plugins::telemetry::config::SamplerOption; -use crate::spaceport::ReferencedFieldsForType; -use crate::spaceport::ReportHeader; -use crate::spaceport::StatsContext; -use crate::spaceport::Trace; +use crate::plugins::telemetry::tracing::BatchProcessorConfig; + +pub(crate) const ENDPOINT_DEFAULT: &str = + "https://usage-reporting.api.apollographql.com/api/ingress/traces"; #[derive(Derivative)] #[derivative(Debug)] @@ -31,8 +35,9 @@ use crate::spaceport::Trace; #[serde(deny_unknown_fields)] pub(crate) struct Config { /// The Apollo Studio endpoint for exporting traces and metrics. - #[schemars(with = "Option")] - pub(crate) endpoint: Option, + #[schemars(with = "String", default = "endpoint_default")] + #[serde(default = "endpoint_default")] + pub(crate) endpoint: Url, /// The Apollo Studio API key. #[schemars(skip)] @@ -85,16 +90,36 @@ pub(crate) struct Config { #[schemars(skip)] #[serde(skip)] pub(crate) expose_trace_id: ExposeTraceId, + + pub(crate) batch_processor: Option, +} + +#[cfg(test)] +fn apollo_key() -> Option { + // During tests we don't want env variables to affect defaults + None } +#[cfg(not(test))] fn apollo_key() -> Option { std::env::var("APOLLO_KEY").ok() } +#[cfg(test)] +fn apollo_graph_reference() -> Option { + // During tests we don't want env variables to affect defaults + None +} + +#[cfg(not(test))] fn apollo_graph_reference() -> Option { std::env::var("APOLLO_GRAPH_REF").ok() } +fn endpoint_default() -> Url { + Url::parse(ENDPOINT_DEFAULT).expect("must be valid url") +} + const fn client_name_header_default_str() -> &'static str { "apollographql-client-name" } @@ -118,7 +143,7 @@ pub(crate) const fn default_buffer_size() -> usize { impl Default for Config { fn default() -> Self { Self { - endpoint: None, + endpoint: Url::parse(ENDPOINT_DEFAULT).expect("default endpoint URL must be parseable"), apollo_key: None, apollo_graph_ref: None, client_name_header: client_name_header_default(), @@ -129,6 +154,7 @@ impl Default for Config { send_headers: ForwardHeaders::None, send_variable_values: ForwardValues::None, expose_trace_id: ExposeTraceId::default(), + batch_processor: Some(BatchProcessorConfig::default()), } } } @@ -189,8 +215,11 @@ impl Report { aggregated_report } - pub(crate) fn into_report(self, header: ReportHeader) -> crate::spaceport::Report { - let mut report = crate::spaceport::Report { + pub(crate) fn into_report( + self, + header: ReportHeader, + ) -> crate::plugins::telemetry::apollo_exporter::proto::Report { + let mut report = crate::plugins::telemetry::apollo_exporter::proto::Report { header: Some(header), end_time: Some(SystemTime::now().into()), operation_count: self.operation_count, @@ -244,7 +273,7 @@ pub(crate) struct TracesAndStats { pub(crate) referenced_fields_by_type: HashMap, } -impl From for crate::spaceport::TracesAndStats { +impl From for crate::plugins::telemetry::apollo_exporter::proto::TracesAndStats { fn from(stats: TracesAndStats) -> Self { Self { stats_with_context: stats.stats_with_context.into_values().map_into().collect(), diff --git a/apollo-router/src/plugins/telemetry/apollo_exporter.rs b/apollo-router/src/plugins/telemetry/apollo_exporter.rs index 2028b667c1..ebd90be67f 100644 --- a/apollo-router/src/plugins/telemetry/apollo_exporter.rs +++ b/apollo-router/src/plugins/telemetry/apollo_exporter.rs @@ -1,49 +1,71 @@ //! Configuration for apollo telemetry exporter. -#[cfg(test)] +// This entire file is license key functionality +use std::error::Error; +use std::fmt::Debug; +use std::io::Write; +use std::str::FromStr; use std::sync::Arc; -#[cfg(test)] use std::sync::Mutex; -// This entire file is license key functionality use std::time::Duration; -use async_trait::async_trait; -use deadpool::managed; -use deadpool::managed::Pool; -use deadpool::managed::RecycleError; -use deadpool::Runtime; +use bytes::BytesMut; +use flate2::write::GzEncoder; +use flate2::Compression; use futures::channel::mpsc; use futures::stream::StreamExt; +use http::header::ACCEPT; +use http::header::CONTENT_ENCODING; +use http::header::CONTENT_TYPE; +use http::header::USER_AGENT; +use opentelemetry::ExportError; +pub(crate) use prost::*; +use reqwest::Client; +use serde::ser::SerializeStruct; +use serde_json::Value; use sys_info::hostname; +use tokio::task::JoinError; +use tonic::codegen::http::uri::InvalidUri; use tower::BoxError; use url::Url; use super::apollo::Report; use super::apollo::SingleReport; -use crate::spaceport::ReportHeader; -use crate::spaceport::Reporter; -use crate::spaceport::ReporterError; -// use crate::plugins::telemetry::apollo::ReportBuilder; const DEFAULT_QUEUE_SIZE: usize = 65_536; -const DEADPOOL_SIZE: usize = 128; // Do not set to 5 secs because it's also the default value for the BatchSpanProcesser of tracing. // It's less error prone to set a different value to let us compute traces and metrics pub(crate) const EXPORTER_TIMEOUT_DURATION: Duration = Duration::from_secs(6); -pub(crate) const POOL_TIMEOUT: Duration = Duration::from_secs(5); +const BACKOFF_INCREMENT: Duration = Duration::from_millis(50); + +#[derive(thiserror::Error, Debug)] +pub(crate) enum ApolloExportError { + #[error("Apollo exporter server error: {0}")] + ServerError(String), + + #[error("Apollo exporter client error: {0}")] + ClientError(String), + + #[error("Apollo exporter unavailable error: {0}")] + Unavailable(String), +} + +impl ExportError for ApolloExportError { + fn exporter_name(&self) -> &'static str { + "ApolloExporter" + } +} #[derive(Clone)] pub(crate) enum Sender { Noop, - Spaceport(mpsc::Sender), - #[cfg(test)] - InMemory(Arc>>), + Apollo(mpsc::Sender), } impl Sender { pub(crate) fn send(&self, report: SingleReport) { match &self { Sender::Noop => {} - Sender::Spaceport(channel) => { + Sender::Apollo(channel) => { if let Err(err) = channel.to_owned().try_send(report) { tracing::warn!( "could not send metrics to spaceport, metric will be dropped: {}", @@ -51,10 +73,6 @@ impl Sender { ); } } - #[cfg(test)] - Sender::InMemory(storage) => { - storage.lock().expect("mutex poisoned").push(report); - } } } } @@ -65,8 +83,16 @@ impl Default for Sender { } } +/// The Apollo exporter is responsible for attaching report header information for individual requests +/// Retrying when sending fails. +/// Sending periodically (in the case of metrics). +#[derive(Clone)] pub(crate) struct ApolloExporter { - tx: mpsc::Sender, + endpoint: Url, + apollo_key: String, + header: crate::plugins::telemetry::apollo_exporter::proto::ReportHeader, + client: Client, + strip_traces: Arc>, } impl ApolloExporter { @@ -76,15 +102,7 @@ impl ApolloExporter { apollo_graph_ref: &str, schema_id: &str, ) -> Result { - let apollo_key = apollo_key.to_string(); - // Desired behavior: - // * Metrics are batched with a timeout. - // * If we cannot connect to spaceport metrics are discarded and a warning raised. - // * When the stream of metrics finishes we terminate the task. - // * If the exporter is dropped the remaining records are flushed. - let (tx, mut rx) = mpsc::channel::(DEFAULT_QUEUE_SIZE); - - let header = crate::spaceport::ReportHeader { + let header = crate::plugins::telemetry::apollo_exporter::proto::ReportHeader { graph_ref: apollo_graph_ref.to_string(), hostname: hostname()?, agent_version: format!( @@ -98,35 +116,19 @@ impl ApolloExporter { ..Default::default() }; - // Pool Sizing: by default Deadpool will configure a maximum - // pool size based on the number of physical CPUs: - // `cpu_count * 4` ignoring any logical CPUs (Hyper-Threading). - // This is going to be very low in containerised environments - // For example, in my k8s testing I get max_size: 16 - // - // Since we know we can support large numbers of connections to the - // data ingestion endpoint, I'm going to manually set this to - // be [`DEADPOOL_SIZE`] which should be plenty. I'm not setting - // it to be a very high number (e.g.: 100000), because resources - // are consumed and conserving them is important. - - // - // Deadpool gives us connection pooling to spaceport - // It also significantly simplifies initialisation of the connection and gives us options in the future for configuring timeouts. - let pool = deadpool::managed::Pool::::builder(ReporterManager { + tracing::debug!("creating apollo exporter {}", endpoint); + + Ok(ApolloExporter { endpoint: endpoint.clone(), + apollo_key: apollo_key.to_string(), + client: reqwest::Client::default(), + header, + strip_traces: Default::default(), }) - .max_size(DEADPOOL_SIZE) - .create_timeout(Some(POOL_TIMEOUT)) - .recycle_timeout(Some(POOL_TIMEOUT)) - .wait_timeout(Some(POOL_TIMEOUT)) - .runtime(Runtime::Tokio1) - .build() - .unwrap(); - - let spaceport_endpoint = endpoint.clone(); - tracing::info!(%spaceport_endpoint, "creating apollo exporter"); + } + pub(crate) fn start(self) -> Sender { + let (tx, mut rx) = mpsc::channel::(DEFAULT_QUEUE_SIZE); // This is the task that actually sends metrics tokio::spawn(async move { let timeout = tokio::time::interval(EXPORTER_TIMEOUT_DURATION); @@ -140,80 +142,131 @@ impl ApolloExporter { if let Some(r) = single_report { report += r; } else { - tracing::info!(%spaceport_endpoint, "terminating apollo exporter"); + tracing::debug!("terminating apollo exporter"); break; } }, _ = timeout.tick() => { - Self::send_report(&pool, &apollo_key, &header, std::mem::take(&mut report)).await; + if let Err(e) = self.submit_report(std::mem::take(&mut report)).await { + tracing::error!("failed to submit Apollo report: {}", e) + } } }; } - Self::send_report(&pool, &apollo_key, &header, report).await; + if let Err(e) = self.submit_report(std::mem::take(&mut report)).await { + tracing::error!("failed to submit Apollo report: {}", e) + } }); - Ok(ApolloExporter { tx }) + Sender::Apollo(tx) } - pub(crate) fn provider(&self) -> Sender { - Sender::Spaceport(self.tx.clone()) - } - - async fn send_report( - pool: &Pool, - apollo_key: &str, - header: &ReportHeader, - report: Report, - ) { - if report.operation_count == 0 && report.traces_per_query.is_empty() { - return; + pub(crate) async fn submit_report(&self, report: Report) -> Result<(), ApolloExportError> { + if report.operation_count == 0 { + return Ok(()); } + tracing::debug!("submitting report: {:?}", report); + // Protobuf encode message + let mut content = BytesMut::new(); + let mut report = report.into_report(self.header.clone()); + prost::Message::encode(&report, &mut content) + .map_err(|e| ApolloExportError::ClientError(e.to_string()))?; + // Create a gzip encoder + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + // Write our content to our encoder + encoder + .write_all(&content) + .map_err(|e| ApolloExportError::ClientError(e.to_string()))?; + // Finish encoding and retrieve content + let compressed_content = encoder + .finish() + .map_err(|e| ApolloExportError::ClientError(e.to_string()))?; + let mut backoff = Duration::from_millis(0); - match pool.get().await { - Ok(mut reporter) => { - let report = report.into_report(header.clone()); - match reporter - .submit(crate::spaceport::ReporterRequest { - apollo_key: apollo_key.to_string(), - report: Some(report), - }) - .await - { - Ok(_) => {} - Err(e) => { - tracing::warn!("failed to submit stats to spaceport: {}", e); - } - }; - } - Err(err) => { - tracing::warn!( - "stats discarded as unable to get connection to spaceport: {}", - err - ); - } - }; - } -} - -#[derive(Debug)] -pub(crate) struct ReporterManager { - endpoint: Url, -} + let req = self + .client + .post(self.endpoint.clone()) + .body(compressed_content) + .header("X-Api-Key", self.apollo_key.clone()) + .header(CONTENT_ENCODING, "gzip") + .header(CONTENT_TYPE, "application/protobuf") + .header(ACCEPT, "application/json") + .header( + USER_AGENT, + format!( + "{} / {} usage reporting", + std::env!("CARGO_PKG_NAME"), + std::env!("CARGO_PKG_VERSION") + ), + ) + .build() + .map_err(|e| ApolloExportError::Unavailable(e.to_string()))?; -#[async_trait] -impl managed::Manager for ReporterManager { - type Type = Reporter; - type Error = ReporterError; + let mut msg = "default error message".to_string(); + let mut has_traces = false; - async fn create(&self) -> Result { - tracing::debug!("creating reporter: {:?}", self.endpoint); - let url = self.endpoint.to_string(); - Reporter::try_new(url).await - } + for (_, traces_and_stats) in report.traces_per_query.iter_mut() { + if !traces_and_stats.trace.is_empty() + || !traces_and_stats + .internal_traces_contributing_to_stats + .is_empty() + { + has_traces = true; + if *self.strip_traces.lock().expect("lock poisoned") { + traces_and_stats.trace.clear(); + traces_and_stats + .internal_traces_contributing_to_stats + .clear(); + } + } + } - async fn recycle(&self, r: &mut Reporter) -> managed::RecycleResult { - tracing::debug!("recycling reporter: {:?}", r); - r.reconnect().await.map_err(RecycleError::Backend) + for i in 0..5 { + // We know these requests can be cloned + let task_req = req.try_clone().expect("requests must be clone-able"); + match self.client.execute(task_req).await { + Ok(v) => { + let status = v.status(); + let data = v + .text() + .await + .map_err(|e| ApolloExportError::ServerError(e.to_string()))?; + // Handle various kinds of status: + // - if client error, terminate immediately + // - if server error, it may be transient so treat as retry-able + // - if ok, return ok + if status.is_client_error() { + tracing::error!("client error reported at ingress: {}", data); + return Err(ApolloExportError::ClientError(data)); + } else if status.is_server_error() { + tracing::warn!("attempt: {}, could not transfer: {}", i + 1, data); + msg = data; + } else { + tracing::debug!("ingress response text: {:?}", data); + if has_traces && !*self.strip_traces.lock().expect("lock poisoned") { + // If we had traces then maybe disable sending traces from this exporter based on the response. + if let Ok(response) = serde_json::Value::from_str(&data) { + if let Some(Value::Bool(true)) = response.get("tracesIgnored") { + tracing::warn!("traces will not be sent to Apollo as this account is on a free plan"); + *self.strip_traces.lock().expect("lock poisoned") = true; + } + } + } + return Ok(()); + } + } + Err(e) => { + // TODO: Ultimately need more sophisticated handling here. For example + // a redirect should not be treated the same way as a connect or a + // type builder error... + tracing::warn!("attempt: {}, could not transfer: {}", i + 1, e); + msg = e.to_string(); + } + } + backoff += BACKOFF_INCREMENT; + tokio::time::sleep(backoff).await; + } + Err(ApolloExportError::Unavailable(msg)) } } @@ -239,3 +292,105 @@ pub(crate) fn get_uname() -> Result { sysname, nodename, release, version, machine )) } + +#[allow(unreachable_pub)] +pub(crate) mod proto { + #![allow(clippy::derive_partial_eq_without_eq)] + tonic::include_proto!("report"); +} + +/// Reporting Error type +#[derive(Debug)] +pub(crate) struct ReporterError { + source: Box, + msg: String, +} + +impl std::error::Error for ReporterError {} + +impl From for ReporterError { + fn from(error: InvalidUri) -> Self { + ReporterError { + msg: error.to_string(), + source: Box::new(error), + } + } +} + +impl From for ReporterError { + fn from(error: tonic::transport::Error) -> Self { + ReporterError { + msg: error.to_string(), + source: Box::new(error), + } + } +} + +impl From for ReporterError { + fn from(error: std::io::Error) -> Self { + ReporterError { + msg: error.to_string(), + source: Box::new(error), + } + } +} + +impl From for ReporterError { + fn from(error: sys_info::Error) -> Self { + ReporterError { + msg: error.to_string(), + source: Box::new(error), + } + } +} + +impl From for ReporterError { + fn from(error: JoinError) -> Self { + ReporterError { + msg: error.to_string(), + source: Box::new(error), + } + } +} + +impl std::fmt::Display for ReporterError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "ReporterError: source: {}, message: {}", + self.source, self.msg + ) + } +} + +pub(crate) fn serialize_timestamp( + timestamp: &Option, + serializer: S, +) -> Result +where + S: serde::Serializer, +{ + match timestamp { + Some(ts) => { + let mut ts_strukt = serializer.serialize_struct("Timestamp", 2)?; + ts_strukt.serialize_field("seconds", &ts.seconds)?; + ts_strukt.serialize_field("nanos", &ts.nanos)?; + ts_strukt.end() + } + None => serializer.serialize_none(), + } +} + +#[cfg(not(windows))] // git checkout converts \n to \r\n, making == below fail +#[test] +fn check_reports_proto_is_up_to_date() { + let proto_url = "https://usage-reporting.api.apollographql.com/proto/reports.proto"; + let response = reqwest::blocking::get(proto_url).unwrap(); + let content = response.text().unwrap(); + // Not using assert_eq! as printing the entire file would be too verbose + assert!( + content == include_str!("proto/reports.proto"), + "Protobuf file is out of date. Run this command to update it:\n\n \ + curl -f {proto_url} > apollo-router/src/plugins/telemetry/proto/reports.proto\n\n" + ); +} diff --git a/apollo-router/src/plugins/telemetry/config.rs b/apollo-router/src/plugins/telemetry/config.rs index fb810649f3..bc6a40a014 100644 --- a/apollo-router/src/plugins/telemetry/config.rs +++ b/apollo-router/src/plugins/telemetry/config.rs @@ -1,5 +1,4 @@ //! Configuration for the telemetry plugin. -use std::borrow::Cow; use std::collections::BTreeMap; use axum::headers::HeaderName; @@ -323,7 +322,7 @@ impl From for opentelemetry::Value { AttributeValue::Bool(v) => Value::Bool(v), AttributeValue::I64(v) => Value::I64(v), AttributeValue::F64(v) => Value::F64(v), - AttributeValue::String(v) => Value::String(Cow::from(v)), + AttributeValue::String(v) => Value::String(v.into()), AttributeValue::Array(v) => Value::Array(v.into()), } } @@ -339,7 +338,7 @@ pub(crate) enum AttributeArray { /// Array of floats F64(Vec), /// Array of strings - String(Vec>), + String(Vec), } impl From for opentelemetry::Array { @@ -348,7 +347,7 @@ impl From for opentelemetry::Array { AttributeArray::Bool(v) => Array::Bool(v), AttributeArray::I64(v) => Array::I64(v), AttributeArray::F64(v) => Array::F64(v), - AttributeArray::String(v) => Array::String(v), + AttributeArray::String(v) => Array::String(v.into_iter().map(|v| v.into()).collect()), } } } diff --git a/apollo-router/src/plugins/telemetry/formatters/mod.rs b/apollo-router/src/plugins/telemetry/formatters/mod.rs index b480a4b17b..e96002c12f 100644 --- a/apollo-router/src/plugins/telemetry/formatters/mod.rs +++ b/apollo-router/src/plugins/telemetry/formatters/mod.rs @@ -2,7 +2,72 @@ pub(crate) mod json; pub(crate) mod text; -pub(crate) use json::JsonFields; -pub(crate) use text::TextFormatter; +use std::fmt; + +use tracing::Subscriber; +use tracing_subscriber::fmt::format::Writer; +use tracing_subscriber::fmt::FormatEvent; +use tracing_subscriber::fmt::FormatFields; +use tracing_subscriber::registry::LookupSpan; + +use super::metrics::METRIC_PREFIX_COUNTER; +use super::metrics::METRIC_PREFIX_HISTOGRAM; +use super::metrics::METRIC_PREFIX_MONOTONIC_COUNTER; pub(crate) const TRACE_ID_FIELD_NAME: &str = "trace_id"; + +/// `FilteringFormatter` is useful if you want to not filter the entire event but only want to not display it +/// ```ignore +/// use tracing_core::Event; +/// use tracing_subscriber::fmt::format::{Format}; +/// tracing_subscriber::fmt::fmt() +/// .event_format(FilteringFormatter::new( +/// Format::default().pretty(), +/// // Do not display the event if an attribute name starts with "counter" +/// |event: &Event| !event.metadata().fields().iter().any(|f| f.name().starts_with("counter")), +/// )) +/// .finish(); +/// ``` +pub(crate) struct FilteringFormatter { + inner: T, + filter_fn: F, +} + +impl FilteringFormatter +where + F: Fn(&tracing::Event<'_>) -> bool, +{ + pub(crate) fn new(inner: T, filter_fn: F) -> Self { + Self { inner, filter_fn } + } +} + +impl FormatEvent for FilteringFormatter +where + T: FormatEvent, + F: Fn(&tracing::Event<'_>) -> bool, + S: Subscriber + for<'a> LookupSpan<'a>, + N: for<'a> FormatFields<'a> + 'static, +{ + fn format_event( + &self, + ctx: &tracing_subscriber::fmt::FmtContext<'_, S, N>, + writer: Writer<'_>, + event: &tracing::Event<'_>, + ) -> fmt::Result { + if (self.filter_fn)(event) { + self.inner.format_event(ctx, writer, event) + } else { + Ok(()) + } + } +} + +// Function to filter metric event for the filter formatter +pub(crate) fn filter_metric_events(event: &tracing::Event<'_>) -> bool { + !event.metadata().fields().iter().any(|f| { + f.name().starts_with(METRIC_PREFIX_COUNTER) + || f.name().starts_with(METRIC_PREFIX_HISTOGRAM) + || f.name().starts_with(METRIC_PREFIX_MONOTONIC_COUNTER) + }) +} diff --git a/apollo-router/src/plugins/telemetry/metrics/aggregation.rs b/apollo-router/src/plugins/telemetry/metrics/aggregation.rs new file mode 100644 index 0000000000..10d321af87 --- /dev/null +++ b/apollo-router/src/plugins/telemetry/metrics/aggregation.rs @@ -0,0 +1,240 @@ +use std::sync::Arc; + +use itertools::Itertools; +use opentelemetry::metrics::AsyncCounter; +use opentelemetry::metrics::AsyncGauge; +use opentelemetry::metrics::AsyncUpDownCounter; +use opentelemetry::metrics::Counter; +use opentelemetry::metrics::Histogram; +use opentelemetry::metrics::InstrumentProvider; +use opentelemetry::metrics::Meter; +use opentelemetry::metrics::MeterProvider; +use opentelemetry::metrics::ObservableCounter; +use opentelemetry::metrics::ObservableGauge; +use opentelemetry::metrics::ObservableUpDownCounter; +use opentelemetry::metrics::SyncCounter; +use opentelemetry::metrics::SyncHistogram; +use opentelemetry::metrics::SyncUpDownCounter; +use opentelemetry::metrics::Unit; +use opentelemetry::metrics::UpDownCounter; +use opentelemetry::Context; +use opentelemetry::InstrumentationLibrary; +use opentelemetry::KeyValue; + +#[derive(Clone, Default)] +pub(crate) struct AggregateMeterProvider { + providers: Vec>, +} +impl AggregateMeterProvider { + pub(crate) fn new( + providers: Vec>, + ) -> AggregateMeterProvider { + AggregateMeterProvider { providers } + } +} + +impl MeterProvider for AggregateMeterProvider { + fn versioned_meter( + &self, + name: &'static str, + version: Option<&'static str>, + schema_url: Option<&'static str>, + ) -> Meter { + Meter::new( + InstrumentationLibrary::new(name, version, schema_url), + Arc::new(AggregateInstrumentProvider { + meters: self + .providers + .iter() + .map(|p| p.versioned_meter(name, version, schema_url)) + .collect(), + }), + ) + } +} + +pub(crate) struct AggregateInstrumentProvider { + meters: Vec, +} + +pub(crate) struct AggregateCounter { + delegates: Vec>, +} + +impl SyncCounter for AggregateCounter { + fn add(&self, cx: &Context, value: T, attributes: &[KeyValue]) { + for counter in &self.delegates { + counter.add(cx, value, attributes) + } + } +} + +pub(crate) struct AggregateObservableCounter { + delegates: Vec>, +} + +impl AsyncCounter for AggregateObservableCounter { + fn observe(&self, cx: &Context, value: T, attributes: &[KeyValue]) { + for counter in &self.delegates { + counter.observe(cx, value, attributes) + } + } +} + +pub(crate) struct AggregateHistogram { + delegates: Vec>, +} + +impl SyncHistogram for AggregateHistogram { + fn record(&self, cx: &Context, value: T, attributes: &[KeyValue]) { + for histogram in &self.delegates { + histogram.record(cx, value, attributes) + } + } +} + +pub(crate) struct AggregateUpDownCounter { + delegates: Vec>, +} + +impl SyncUpDownCounter for AggregateUpDownCounter { + fn add(&self, cx: &Context, value: T, attributes: &[KeyValue]) { + for counter in &self.delegates { + counter.add(cx, value, attributes) + } + } +} + +pub(crate) struct AggregateObservableUpDownCounter { + delegates: Vec>, +} + +impl AsyncUpDownCounter for AggregateObservableUpDownCounter { + fn observe(&self, cx: &Context, value: T, attributes: &[KeyValue]) { + for counter in &self.delegates { + counter.observe(cx, value, attributes) + } + } +} + +pub(crate) struct AggregateObservableGauge { + delegates: Vec>, +} + +impl AsyncGauge for AggregateObservableGauge { + fn observe(&self, cx: &Context, value: T, attributes: &[KeyValue]) { + for gauge in &self.delegates { + gauge.observe(cx, value, attributes) + } + } +} + +macro_rules! aggregate_meter_fn { + ($name:ident, $ty:ty, $wrapper:ident, $implementation:ident) => { + fn $name( + &self, + name: String, + description: Option, + unit: Option, + ) -> opentelemetry::metrics::Result<$wrapper<$ty>> { + let delegates = self + .meters + .iter() + .map(|p| { + let mut b = p.$name(name.clone()); + if let Some(description) = &description { + b = b.with_description(description); + } + if let Some(unit) = &unit { + b = b.with_unit(unit.clone()); + } + b.try_init() + }) + .try_collect()?; + Ok($wrapper::new(Arc::new($implementation { delegates }))) + } + }; +} + +impl InstrumentProvider for AggregateInstrumentProvider { + aggregate_meter_fn!(u64_counter, u64, Counter, AggregateCounter); + aggregate_meter_fn!(f64_counter, f64, Counter, AggregateCounter); + + aggregate_meter_fn!( + f64_observable_counter, + f64, + ObservableCounter, + AggregateObservableCounter + ); + aggregate_meter_fn!( + u64_observable_counter, + u64, + ObservableCounter, + AggregateObservableCounter + ); + + aggregate_meter_fn!(u64_histogram, u64, Histogram, AggregateHistogram); + aggregate_meter_fn!(f64_histogram, f64, Histogram, AggregateHistogram); + aggregate_meter_fn!(i64_histogram, i64, Histogram, AggregateHistogram); + + aggregate_meter_fn!( + i64_up_down_counter, + i64, + UpDownCounter, + AggregateUpDownCounter + ); + aggregate_meter_fn!( + f64_up_down_counter, + f64, + UpDownCounter, + AggregateUpDownCounter + ); + + aggregate_meter_fn!( + i64_observable_up_down_counter, + i64, + ObservableUpDownCounter, + AggregateObservableUpDownCounter + ); + aggregate_meter_fn!( + f64_observable_up_down_counter, + f64, + ObservableUpDownCounter, + AggregateObservableUpDownCounter + ); + + aggregate_meter_fn!( + f64_observable_gauge, + f64, + ObservableGauge, + AggregateObservableGauge + ); + aggregate_meter_fn!( + i64_observable_gauge, + i64, + ObservableGauge, + AggregateObservableGauge + ); + aggregate_meter_fn!( + u64_observable_gauge, + u64, + ObservableGauge, + AggregateObservableGauge + ); + + fn register_callback( + &self, + callback: Box, + ) -> opentelemetry::metrics::Result<()> { + // The reason that this is OK is that calling observe outside of a callback is a no-op. + // So the callback is called, an observable is updated, but only the observable associated with the correct meter will take effect + + let callback = Arc::new(callback); + for meter in &self.meters { + let callback = callback.clone(); + // If this fails there is no recovery as some callbacks may be registered + meter.register_callback(move |c| callback(c))? + } + Ok(()) + } +} diff --git a/apollo-router/src/plugins/telemetry/metrics/apollo.rs b/apollo-router/src/plugins/telemetry/metrics/apollo.rs index df97556df6..45dd2af220 100644 --- a/apollo-router/src/plugins/telemetry/metrics/apollo.rs +++ b/apollo-router/src/plugins/telemetry/metrics/apollo.rs @@ -24,7 +24,7 @@ impl MetricsConfigurator for Config { static ENABLED: AtomicBool = AtomicBool::new(false); Ok(match self { Config { - endpoint: Some(endpoint), + endpoint, apollo_key: Some(key), apollo_graph_ref: Some(reference), schema_id, @@ -36,9 +36,7 @@ impl MetricsConfigurator for Config { tracing::debug!("creating metrics exporter"); let exporter = ApolloExporter::new(endpoint, key, reference, schema_id)?; - builder - .with_apollo_metrics_collector(exporter.provider()) - .with_exporter(exporter) + builder.with_apollo_metrics_collector(exporter.start()) } _ => { ENABLED.swap(false, Ordering::Relaxed); @@ -56,6 +54,7 @@ mod test { use futures::stream::StreamExt; use http::header::HeaderName; use tower::ServiceExt; + use url::Url; use super::super::super::config; use super::studio::SingleStatsReport; @@ -64,6 +63,7 @@ mod test { use crate::plugin::PluginInit; use crate::plugins::telemetry::apollo; use crate::plugins::telemetry::apollo::default_buffer_size; + use crate::plugins::telemetry::apollo::ENDPOINT_DEFAULT; use crate::plugins::telemetry::apollo_exporter::Sender; use crate::plugins::telemetry::Telemetry; use crate::plugins::telemetry::STUDIO_EXCLUDE; @@ -74,7 +74,7 @@ mod test { #[tokio::test] async fn apollo_metrics_disabled() -> Result<(), BoxError> { let plugin = create_plugin_with_apollo_config(super::super::apollo::Config { - endpoint: None, + endpoint: Url::parse("http://example.com")?, apollo_key: None, apollo_graph_ref: None, client_name_header: HeaderName::from_static("name_header"), @@ -91,7 +91,7 @@ mod test { #[tokio::test(flavor = "multi_thread")] async fn apollo_metrics_enabled() -> Result<(), BoxError> { let plugin = create_plugin().await?; - assert!(matches!(plugin.apollo_metrics_sender, Sender::Spaceport(_))); + assert!(matches!(plugin.apollo_metrics_sender, Sender::Apollo(_))); Ok(()) } @@ -184,7 +184,7 @@ mod test { let mut plugin = create_plugin().await?; // Replace the apollo metrics sender so we can test metrics collection. let (tx, rx) = futures::channel::mpsc::channel(100); - plugin.apollo_metrics_sender = Sender::Spaceport(tx); + plugin.apollo_metrics_sender = Sender::Apollo(tx); TestHarness::builder() .extra_plugin(plugin) .build() @@ -224,7 +224,7 @@ mod test { fn create_plugin() -> impl Future> { create_plugin_with_apollo_config(apollo::Config { - endpoint: None, + endpoint: Url::parse(ENDPOINT_DEFAULT).expect("default endpoint must be parseable"), apollo_key: Some("key".to_string()), apollo_graph_ref: Some("ref".to_string()), client_name_header: HeaderName::from_static("name_header"), diff --git a/apollo-router/src/plugins/telemetry/metrics/apollo/studio.rs b/apollo-router/src/plugins/telemetry/metrics/apollo/studio.rs index 9f7ec75b5a..046c086d70 100644 --- a/apollo-router/src/plugins/telemetry/metrics/apollo/studio.rs +++ b/apollo-router/src/plugins/telemetry/metrics/apollo/studio.rs @@ -7,8 +7,8 @@ use serde::Serialize; use uuid::Uuid; use super::duration_histogram::DurationHistogram; -use crate::spaceport::ReferencedFieldsForType; -use crate::spaceport::StatsContext; +use crate::plugins::telemetry::apollo_exporter::proto::ReferencedFieldsForType; +use crate::plugins::telemetry::apollo_exporter::proto::StatsContext; #[derive(Default, Debug, Serialize)] pub(crate) struct SingleStatsReport { @@ -211,7 +211,9 @@ impl AddAssign for FieldStat { } } -impl From for crate::spaceport::ContextualizedStats { +impl From + for crate::plugins::telemetry::apollo_exporter::proto::ContextualizedStats +{ fn from(stats: ContextualizedStats) -> Self { Self { per_type_stat: stats @@ -225,7 +227,9 @@ impl From for crate::spaceport::ContextualizedStats { } } -impl From for crate::spaceport::QueryLatencyStats { +impl From + for crate::plugins::telemetry::apollo_exporter::proto::QueryLatencyStats +{ fn from(stats: QueryLatencyStats) -> Self { Self { latency_count: stats.request_latencies.buckets, @@ -245,7 +249,7 @@ impl From for crate::spaceport::QueryLatencyStats { } } -impl From for crate::spaceport::PathErrorStats { +impl From for crate::plugins::telemetry::apollo_exporter::proto::PathErrorStats { fn from(stats: PathErrorStats) -> Self { Self { children: stats @@ -259,7 +263,7 @@ impl From for crate::spaceport::PathErrorStats { } } -impl From for crate::spaceport::TypeStat { +impl From for crate::plugins::telemetry::apollo_exporter::proto::TypeStat { fn from(stat: TypeStat) -> Self { Self { per_field_stat: stat @@ -271,7 +275,7 @@ impl From for crate::spaceport::TypeStat { } } -impl From for crate::spaceport::FieldStat { +impl From for crate::plugins::telemetry::apollo_exporter::proto::FieldStat { fn from(stat: FieldStat) -> Self { Self { return_type: stat.return_type, @@ -291,7 +295,6 @@ mod test { use super::*; use crate::plugins::telemetry::apollo::Report; - use crate::spaceport::ReferencedFieldsForType; #[test] fn test_aggregation() { diff --git a/apollo-router/src/plugins/telemetry/metrics/layer.rs b/apollo-router/src/plugins/telemetry/metrics/layer.rs new file mode 100644 index 0000000000..27dca8e050 --- /dev/null +++ b/apollo-router/src/plugins/telemetry/metrics/layer.rs @@ -0,0 +1,259 @@ +use std::collections::HashMap; +use std::fmt; +use std::sync::RwLock; + +use opentelemetry::metrics::Counter; +use opentelemetry::metrics::Histogram; +use opentelemetry::metrics::Meter; +use opentelemetry::metrics::MeterProvider; +use opentelemetry::metrics::UpDownCounter; +use opentelemetry::Context as OtelContext; +use opentelemetry::Key; +use opentelemetry::KeyValue; +use opentelemetry::Value; +use tracing::field::Visit; +use tracing::Subscriber; +use tracing_core::Field; +use tracing_subscriber::layer::Context; +use tracing_subscriber::registry::LookupSpan; +use tracing_subscriber::Layer; + +use super::METRIC_PREFIX_COUNTER; +use super::METRIC_PREFIX_HISTOGRAM; +use super::METRIC_PREFIX_MONOTONIC_COUNTER; + +const I64_MAX: u64 = i64::MAX as u64; + +#[derive(Default)] +pub(crate) struct Instruments { + u64_counter: MetricsMap>, + f64_counter: MetricsMap>, + i64_up_down_counter: MetricsMap>, + f64_up_down_counter: MetricsMap>, + u64_histogram: MetricsMap>, + i64_histogram: MetricsMap>, + f64_histogram: MetricsMap>, +} + +type MetricsMap = RwLock>; + +#[derive(Copy, Clone, Debug)] +pub(crate) enum InstrumentType { + CounterU64(u64), + CounterF64(f64), + UpDownCounterI64(i64), + UpDownCounterF64(f64), + HistogramU64(u64), + HistogramI64(i64), + HistogramF64(f64), +} + +impl Instruments { + pub(crate) fn update_metric( + &self, + cx: &OtelContext, + meter: &Meter, + instrument_type: InstrumentType, + metric_name: &'static str, + custom_attributes: &[KeyValue], + ) { + fn update_or_insert( + map: &MetricsMap, + name: &'static str, + insert: impl FnOnce() -> T, + update: impl FnOnce(&T), + ) { + { + let lock = map.read().unwrap(); + if let Some(metric) = lock.get(name) { + update(metric); + return; + } + } + + // that metric did not already exist, so we have to acquire a write lock to + // create it. + let mut lock = map.write().unwrap(); + + // handle the case where the entry was created while we were waiting to + // acquire the write lock + let metric = lock.entry(name).or_insert_with(insert); + update(metric) + } + + match instrument_type { + InstrumentType::CounterU64(value) => { + update_or_insert( + &self.u64_counter, + metric_name, + || meter.u64_counter(metric_name).init(), + |ctr| ctr.add(cx, value, custom_attributes), + ); + } + InstrumentType::CounterF64(value) => { + update_or_insert( + &self.f64_counter, + metric_name, + || meter.f64_counter(metric_name).init(), + |ctr| ctr.add(cx, value, custom_attributes), + ); + } + InstrumentType::UpDownCounterI64(value) => { + update_or_insert( + &self.i64_up_down_counter, + metric_name, + || meter.i64_up_down_counter(metric_name).init(), + |ctr| ctr.add(cx, value, custom_attributes), + ); + } + InstrumentType::UpDownCounterF64(value) => { + update_or_insert( + &self.f64_up_down_counter, + metric_name, + || meter.f64_up_down_counter(metric_name).init(), + |ctr| ctr.add(cx, value, custom_attributes), + ); + } + InstrumentType::HistogramU64(value) => { + update_or_insert( + &self.u64_histogram, + metric_name, + || meter.u64_histogram(metric_name).init(), + |rec| rec.record(cx, value, custom_attributes), + ); + } + InstrumentType::HistogramI64(value) => { + update_or_insert( + &self.i64_histogram, + metric_name, + || meter.i64_histogram(metric_name).init(), + |rec| rec.record(cx, value, custom_attributes), + ); + } + InstrumentType::HistogramF64(value) => { + update_or_insert( + &self.f64_histogram, + metric_name, + || meter.f64_histogram(metric_name).init(), + |rec| rec.record(cx, value, custom_attributes), + ); + } + }; + } +} + +pub(crate) struct MetricVisitor<'a> { + pub(crate) instruments: &'a Instruments, + pub(crate) metric: Option<(&'static str, InstrumentType)>, + pub(crate) custom_attributes: Vec, + pub(crate) meter: &'a Meter, +} + +impl<'a> Visit for MetricVisitor<'a> { + fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) { + // Do not display the log content + if field.name() != "message" { + self.custom_attributes.push(KeyValue::new( + Key::from_static_str(field.name()), + Value::from(format!("{value:?}")), + )); + } + } + + fn record_str(&mut self, field: &Field, value: &str) { + self.custom_attributes.push(KeyValue::new( + Key::from_static_str(field.name()), + Value::from(value.to_string()), + )); + } + + fn record_u64(&mut self, field: &Field, value: u64) { + if let Some(metric_name) = field.name().strip_prefix(METRIC_PREFIX_MONOTONIC_COUNTER) { + self.metric = Some((metric_name, InstrumentType::CounterU64(value))); + } else if let Some(metric_name) = field.name().strip_prefix(METRIC_PREFIX_COUNTER) { + if value <= I64_MAX { + self.metric = Some((metric_name, InstrumentType::UpDownCounterI64(value as i64))); + } else { + eprintln!( + "[tracing-opentelemetry]: Received Counter metric, but \ + provided u64: {} is greater than i64::MAX. Ignoring \ + this metric.", + value + ); + } + } else if let Some(metric_name) = field.name().strip_prefix(METRIC_PREFIX_HISTOGRAM) { + self.metric = Some((metric_name, InstrumentType::HistogramU64(value))); + } else { + self.record_debug(field, &value); + } + } + + fn record_f64(&mut self, field: &Field, value: f64) { + if let Some(metric_name) = field.name().strip_prefix(METRIC_PREFIX_MONOTONIC_COUNTER) { + self.metric = Some((metric_name, InstrumentType::CounterF64(value))); + } else if let Some(metric_name) = field.name().strip_prefix(METRIC_PREFIX_COUNTER) { + self.metric = Some((metric_name, InstrumentType::UpDownCounterF64(value))); + } else if let Some(metric_name) = field.name().strip_prefix(METRIC_PREFIX_HISTOGRAM) { + self.metric = Some((metric_name, InstrumentType::HistogramF64(value))); + } else { + self.record_debug(field, &value); + } + } + + fn record_i64(&mut self, field: &Field, value: i64) { + if let Some(metric_name) = field.name().strip_prefix(METRIC_PREFIX_MONOTONIC_COUNTER) { + self.metric = Some((metric_name, InstrumentType::CounterU64(value as u64))); + } else if let Some(metric_name) = field.name().strip_prefix(METRIC_PREFIX_COUNTER) { + self.metric = Some((metric_name, InstrumentType::UpDownCounterI64(value))); + } else if let Some(metric_name) = field.name().strip_prefix(METRIC_PREFIX_HISTOGRAM) { + self.metric = Some((metric_name, InstrumentType::HistogramI64(value))); + } else { + self.record_debug(field, &value); + } + } +} + +impl<'a> MetricVisitor<'a> { + fn finish(self) { + if let Some((metric_name, instrument_type)) = self.metric { + let cx = OtelContext::current(); + self.instruments.update_metric( + &cx, + self.meter, + instrument_type, + metric_name, + &self.custom_attributes, + ); + } + } +} + +pub(crate) struct MetricsLayer { + meter: Meter, + instruments: Instruments, +} + +impl Default for MetricsLayer { + fn default() -> Self { + Self { + meter: opentelemetry::global::meter_provider().meter("apollo/router"), + instruments: Default::default(), + } + } +} + +impl Layer for MetricsLayer +where + S: Subscriber + for<'span> LookupSpan<'span>, +{ + fn on_event(&self, event: &tracing::Event<'_>, _ctx: Context<'_, S>) { + let mut metric_visitor = MetricVisitor { + instruments: &self.instruments, + meter: &self.meter, + metric: None, + custom_attributes: Vec::new(), + }; + event.record(&mut metric_visitor); + metric_visitor.finish(); + } +} diff --git a/apollo-router/src/plugins/telemetry/metrics/mod.rs b/apollo-router/src/plugins/telemetry/metrics/mod.rs index 97fa39ca13..126f60fd2a 100644 --- a/apollo-router/src/plugins/telemetry/metrics/mod.rs +++ b/apollo-router/src/plugins/telemetry/metrics/mod.rs @@ -9,11 +9,8 @@ use http::response::Parts; use http::HeaderMap; use multimap::MultiMap; use opentelemetry::metrics::Counter; -use opentelemetry::metrics::Meter; +use opentelemetry::metrics::Histogram; use opentelemetry::metrics::MeterProvider; -use opentelemetry::metrics::Number; -use opentelemetry::metrics::ValueRecorder; -use opentelemetry::KeyValue; use regex::Regex; use schemars::JsonSchema; use serde::Serialize; @@ -28,13 +25,21 @@ use crate::plugin::serde::deserialize_json_query; use crate::plugin::serde::deserialize_regex; use crate::plugins::telemetry::apollo_exporter::Sender; use crate::plugins::telemetry::config::MetricsCommon; +use crate::plugins::telemetry::metrics::aggregation::AggregateMeterProvider; use crate::router_factory::Endpoint; use crate::Context; use crate::ListenAddr; +mod aggregation; pub(crate) mod apollo; +pub(crate) mod layer; pub(crate) mod otlp; pub(crate) mod prometheus; +pub(crate) mod span_metrics_exporter; + +pub(crate) const METRIC_PREFIX_MONOTONIC_COUNTER: &str = "monotonic_counter."; +pub(crate) const METRIC_PREFIX_COUNTER: &str = "counter."; +pub(crate) const METRIC_PREFIX_HISTOGRAM: &str = "histogram."; pub(crate) type MetricsExporterHandle = Box; @@ -508,97 +513,22 @@ pub(crate) trait MetricsConfigurator { #[derive(Clone)] pub(crate) struct BasicMetrics { - pub(crate) http_requests_total: AggregateCounter, - pub(crate) http_requests_error_total: AggregateCounter, - pub(crate) http_requests_duration: AggregateValueRecorder, + pub(crate) http_requests_total: Counter, + pub(crate) http_requests_duration: Histogram, } -impl BasicMetrics { - pub(crate) fn new(meter_provider: &AggregateMeterProvider) -> BasicMetrics { - let meter = meter_provider.meter("apollo/router", None); +impl Default for BasicMetrics { + fn default() -> BasicMetrics { + let meter = opentelemetry::global::meter_provider().meter("apollo/router"); BasicMetrics { - http_requests_total: meter.build_counter(|m| { - m.u64_counter("apollo_router_http_requests_total") - .with_description("Total number of HTTP requests made.") - .init() - }), - http_requests_error_total: meter.build_counter(|m| { - m.u64_counter("apollo_router_http_requests_error_total") - .with_description("Total number of HTTP requests in error made.") - .init() - }), - http_requests_duration: meter.build_value_recorder(|m| { - m.f64_value_recorder("apollo_router_http_request_duration_seconds") - .with_description("Total number of HTTP requests made.") - .init() - }), - } - } -} - -#[derive(Clone, Default)] -pub(crate) struct AggregateMeterProvider(Vec>); -impl AggregateMeterProvider { - pub(crate) fn new( - meters: Vec>, - ) -> AggregateMeterProvider { - AggregateMeterProvider(meters) - } - - pub(crate) fn meter( - &self, - instrumentation_name: &'static str, - instrumentation_version: Option<&'static str>, - ) -> AggregateMeter { - AggregateMeter( - self.0 - .iter() - .map(|p| Arc::new(p.meter(instrumentation_name, instrumentation_version))) - .collect(), - ) - } -} - -#[derive(Clone)] -pub(crate) struct AggregateMeter(Vec>); -impl AggregateMeter { - pub(crate) fn build_counter + Copy>( - &self, - build: fn(&Meter) -> Counter, - ) -> AggregateCounter { - AggregateCounter(self.0.iter().map(|m| build(m)).collect()) - } - - pub(crate) fn build_value_recorder + Copy>( - &self, - build: fn(&Meter) -> ValueRecorder, - ) -> AggregateValueRecorder { - AggregateValueRecorder(self.0.iter().map(|m| build(m)).collect()) - } -} - -#[derive(Clone)] -pub(crate) struct AggregateCounter + Copy>(Vec>); -impl AggregateCounter -where - T: Into + Copy, -{ - pub(crate) fn add(&self, value: T, attributes: &[KeyValue]) { - for counter in &self.0 { - counter.add(value, attributes) - } - } -} - -#[derive(Clone)] -pub(crate) struct AggregateValueRecorder + Copy>(Vec>); -impl AggregateValueRecorder -where - T: Into + Copy, -{ - pub(crate) fn record(&self, value: T, attributes: &[KeyValue]) { - for value_recorder in &self.0 { - value_recorder.record(value, attributes) + http_requests_total: meter + .u64_counter("apollo_router_http_requests_total") + .with_description("Total number of HTTP requests made.") + .init(), + http_requests_duration: meter + .f64_histogram("apollo_router_http_request_duration_seconds") + .with_description("Total number of HTTP requests made.") + .init(), } } } diff --git a/apollo-router/src/plugins/telemetry/metrics/otlp.rs b/apollo-router/src/plugins/telemetry/metrics/otlp.rs index 5bb5eebe1b..3d2aef22db 100644 --- a/apollo-router/src/plugins/telemetry/metrics/otlp.rs +++ b/apollo-router/src/plugins/telemetry/metrics/otlp.rs @@ -1,9 +1,5 @@ -use std::time::Duration; - -use futures::Stream; -use futures::StreamExt; -use opentelemetry::sdk::metrics::selectors; -use opentelemetry::util::tokio_interval_stream; +use opentelemetry::sdk::export::metrics::aggregation; +use opentelemetry::sdk::Resource; use opentelemetry::KeyValue; use opentelemetry_otlp::HttpExporterBuilder; use opentelemetry_otlp::TonicExporterBuilder; @@ -44,18 +40,21 @@ impl MetricsConfigurator for super::super::otlp::Config { match exporter.exporter { Some(exporter) => { let exporter = opentelemetry_otlp::new_pipeline() - .metrics(tokio::spawn, delayed_interval) + .metrics( + opentelemetry::sdk::metrics::selectors::simple::inexpensive(), + aggregation::stateless_temporality_selector(), + opentelemetry::runtime::Tokio, + ) .with_exporter(exporter) - .with_aggregator_selector(selectors::simple::Selector::Exact) - .with_resource( + .with_resource(Resource::new( metrics_config .resources .clone() .into_iter() .map(|(k, v)| KeyValue::new(k, v)), - ) + )) .build()?; - builder = builder.with_meter_provider(exporter.provider()); + builder = builder.with_meter_provider(exporter.clone()); builder = builder.with_exporter(exporter); Ok(builder) } @@ -63,7 +62,3 @@ impl MetricsConfigurator for super::super::otlp::Config { } } } - -fn delayed_interval(duration: Duration) -> impl Stream { - tokio_interval_stream(duration).skip(1) -} diff --git a/apollo-router/src/plugins/telemetry/metrics/prometheus.rs b/apollo-router/src/plugins/telemetry/metrics/prometheus.rs index 35c794d89f..0bc2dcce60 100644 --- a/apollo-router/src/plugins/telemetry/metrics/prometheus.rs +++ b/apollo-router/src/plugins/telemetry/metrics/prometheus.rs @@ -3,6 +3,10 @@ use std::task::Poll; use futures::future::BoxFuture; use http::StatusCode; +use opentelemetry::sdk::export::metrics::aggregation; +use opentelemetry::sdk::metrics::controllers; +use opentelemetry::sdk::metrics::processors; +use opentelemetry::sdk::metrics::selectors; use opentelemetry::sdk::Resource; use opentelemetry::KeyValue; use prometheus::Encoder; @@ -56,18 +60,29 @@ impl MetricsConfigurator for Config { metrics_config: &MetricsCommon, ) -> Result { if self.enabled { - let exporter = opentelemetry_prometheus::exporter() - .with_default_histogram_boundaries(vec![ - 0.001, 0.005, 0.015, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 1.0, 5.0, 10.0, - ]) - .with_resource(Resource::new( - metrics_config - .resources - .clone() - .into_iter() - .map(|(k, v)| KeyValue::new(k, v)), - )) - .try_init()?; + tracing::info!( + "prometheus endpoint exposed at {}{}", + self.listen, + self.path + ); + let controller = controllers::basic( + processors::factory( + selectors::simple::histogram([ + 0.001, 0.005, 0.015, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 1.0, 5.0, 10.0, + ]), + aggregation::stateless_temporality_selector(), + ) + .with_memory(true), + ) + .with_resource(Resource::new( + metrics_config + .resources + .clone() + .into_iter() + .map(|(k, v)| KeyValue::new(k, v)), + )) + .build(); + let exporter = opentelemetry_prometheus::exporter(controller).try_init()?; builder = builder.with_custom_endpoint( self.listen.clone(), @@ -79,7 +94,7 @@ impl MetricsConfigurator for Config { .boxed(), ), ); - builder = builder.with_meter_provider(exporter.provider()?); + builder = builder.with_meter_provider(exporter.meter_provider()?); builder = builder.with_exporter(exporter); } Ok(builder) diff --git a/apollo-router/src/plugins/telemetry/metrics/span_metrics_exporter.rs b/apollo-router/src/plugins/telemetry/metrics/span_metrics_exporter.rs new file mode 100644 index 0000000000..4c070337c4 --- /dev/null +++ b/apollo-router/src/plugins/telemetry/metrics/span_metrics_exporter.rs @@ -0,0 +1,81 @@ +use async_trait::async_trait; +use futures::future::BoxFuture; +use futures::FutureExt; +use opentelemetry::sdk::export::trace::ExportResult; +use opentelemetry::sdk::export::trace::SpanData; +use opentelemetry::sdk::export::trace::SpanExporter; +use opentelemetry::Key; +use opentelemetry::Value; + +use crate::axum_factory::utils::REQUEST_SPAN_NAME; +use crate::plugins::telemetry::EXECUTION_SPAN_NAME; +use crate::plugins::telemetry::SUBGRAPH_SPAN_NAME; +use crate::plugins::telemetry::SUPERGRAPH_SPAN_NAME; +use crate::services::QUERY_PLANNING_SPAN_NAME; + +const SPAN_NAMES: &[&str] = &[ + REQUEST_SPAN_NAME, + SUPERGRAPH_SPAN_NAME, + SUBGRAPH_SPAN_NAME, + QUERY_PLANNING_SPAN_NAME, + EXECUTION_SPAN_NAME, +]; + +const BUSY_NS_ATTRIBUTE_NAME: Key = Key::from_static_str("busy_ns"); +const IDLE_NS_ATTRIBUTE_NAME: Key = Key::from_static_str("idle_ns"); +const SUBGRAPH_ATTRIBUTE_NAME: Key = Key::from_static_str("apollo.subgraph.name"); + +#[derive(Debug, Default)] +pub(crate) struct Exporter {} +#[async_trait] +impl SpanExporter for Exporter { + /// Export spans metrics to real metrics + fn export(&mut self, batch: Vec) -> BoxFuture<'static, ExportResult> { + for span in batch + .into_iter() + .filter(|s| SPAN_NAMES.contains(&s.name.as_ref())) + { + let busy = span + .attributes + .get(&BUSY_NS_ATTRIBUTE_NAME) + .and_then(|attr| match attr { + Value::I64(v) => Some(*v), + _ => None, + }) + .unwrap_or_default(); + let idle = span + .attributes + .get(&IDLE_NS_ATTRIBUTE_NAME) + .and_then(|attr| match attr { + Value::I64(v) => Some(*v), + _ => None, + }) + .unwrap_or_default(); + let duration = span + .end_time + .duration_since(span.start_time) + .unwrap_or_default() + .as_secs_f64() as f64; + + // Convert it in seconds + let idle: f64 = idle as f64 / 1_000_000_000_f64; + let busy: f64 = busy as f64 / 1_000_000_000_f64; + if span.name == SUBGRAPH_SPAN_NAME { + let subgraph_name = span + .attributes + .get(&SUBGRAPH_ATTRIBUTE_NAME) + .map(|name| name.as_str()) + .unwrap_or_default(); + ::tracing::info!(histogram.apollo_router_span = duration, kind = %"duration", span = %span.name, subgraph = %subgraph_name); + ::tracing::info!(histogram.apollo_router_span = idle, kind = %"idle", span = %span.name, subgraph = %subgraph_name); + ::tracing::info!(histogram.apollo_router_span = busy, kind = %"busy", span = %span.name, subgraph = %subgraph_name); + } else { + ::tracing::info!(histogram.apollo_router_span = duration, kind = %"duration", span = %span.name); + ::tracing::info!(histogram.apollo_router_span = idle, kind = %"idle", span = %span.name); + ::tracing::info!(histogram.apollo_router_span = busy, kind = %"busy", span = %span.name); + } + } + + async { Ok(()) }.boxed() + } +} diff --git a/apollo-router/src/plugins/telemetry/mod.rs b/apollo-router/src/plugins/telemetry/mod.rs index 7aae5c34e9..0f3e0be2c6 100644 --- a/apollo-router/src/plugins/telemetry/mod.rs +++ b/apollo-router/src/plugins/telemetry/mod.rs @@ -2,10 +2,10 @@ // With regards to ELv2 licensing, this entire file is license key functionality use std::collections::BTreeMap; use std::collections::HashMap; -use std::error::Error as Errors; use std::fmt; use std::sync::atomic::AtomicU8; use std::sync::atomic::Ordering; +use std::sync::mpsc; use std::sync::Arc; use std::time::Duration; use std::time::Instant; @@ -27,7 +27,6 @@ use http::HeaderMap; use http::HeaderValue; use multimap::MultiMap; use once_cell::sync::OnceCell; -use opentelemetry::global; use opentelemetry::propagation::text_map_propagator::FieldIter; use opentelemetry::propagation::Extractor; use opentelemetry::propagation::Injector; @@ -38,7 +37,6 @@ use opentelemetry::sdk::propagation::TraceContextPropagator; use opentelemetry::sdk::trace::Builder; use opentelemetry::trace::SpanContext; use opentelemetry::trace::SpanId; -use opentelemetry::trace::SpanKind; use opentelemetry::trace::TraceContextExt; use opentelemetry::trace::TraceFlags; use opentelemetry::trace::TraceState; @@ -53,12 +51,13 @@ use tower::BoxError; use tower::ServiceBuilder; use tower::ServiceExt; use tracing_opentelemetry::OpenTelemetrySpanExt; +#[cfg(not(feature = "console"))] +use tracing_subscriber::fmt::format::JsonFields; use tracing_subscriber::prelude::__tracing_subscriber_SubscriberExt; use tracing_subscriber::registry::LookupSpan; #[cfg(not(feature = "console"))] use tracing_subscriber::EnvFilter; use tracing_subscriber::Registry; -use url::Url; use self::apollo::ForwardValues; use self::apollo::SingleReport; @@ -72,16 +71,24 @@ use crate::layers::ServiceBuilderExt; use crate::plugin::Plugin; use crate::plugin::PluginInit; use crate::plugins::telemetry::apollo::ForwardHeaders; +use crate::plugins::telemetry::apollo_exporter::proto::StatsContext; +#[cfg(not(feature = "console"))] use crate::plugins::telemetry::config::default_display_filename; +#[cfg(not(feature = "console"))] use crate::plugins::telemetry::config::default_display_line_number; use crate::plugins::telemetry::config::MetricsCommon; use crate::plugins::telemetry::config::Trace; -use crate::plugins::telemetry::formatters::JsonFields; +#[cfg(not(feature = "console"))] +use crate::plugins::telemetry::formatters::filter_metric_events; +#[cfg(not(feature = "console"))] +use crate::plugins::telemetry::formatters::text::TextFormatter; +#[cfg(not(feature = "console"))] +use crate::plugins::telemetry::formatters::FilteringFormatter; use crate::plugins::telemetry::metrics::apollo::studio::SingleContextualizedStats; use crate::plugins::telemetry::metrics::apollo::studio::SingleQueryLatencyStats; use crate::plugins::telemetry::metrics::apollo::studio::SingleStats; use crate::plugins::telemetry::metrics::apollo::studio::SingleStatsReport; -use crate::plugins::telemetry::metrics::AggregateMeterProvider; +use crate::plugins::telemetry::metrics::layer::MetricsLayer; use crate::plugins::telemetry::metrics::BasicMetrics; use crate::plugins::telemetry::metrics::MetricsBuilder; use crate::plugins::telemetry::metrics::MetricsConfigurator; @@ -94,8 +101,6 @@ use crate::router_factory::Endpoint; use crate::services::execution; use crate::services::subgraph; use crate::services::supergraph; -use crate::spaceport::server::ReportSpaceport; -use crate::spaceport::StatsContext; use crate::subgraph::Request; use crate::subgraph::Response; use crate::tracer::TraceId; @@ -106,7 +111,6 @@ use crate::SubgraphRequest; use crate::SubgraphResponse; use crate::SupergraphRequest; use crate::SupergraphResponse; - pub(crate) mod apollo; pub(crate) mod apollo_exporter; pub(crate) mod config; @@ -117,6 +121,7 @@ mod tracing; // Tracing consts pub(crate) const SUPERGRAPH_SPAN_NAME: &str = "supergraph"; pub(crate) const SUBGRAPH_SPAN_NAME: &str = "subgraph"; +pub(crate) const EXECUTION_SPAN_NAME: &str = "execution"; const CLIENT_NAME: &str = "apollo_telemetry::client_name"; const CLIENT_VERSION: &str = "apollo_telemetry::client_version"; const ATTRIBUTES: &str = "apollo_telemetry::metrics_attributes"; @@ -134,11 +139,11 @@ static TELEMETRY_REFCOUNT: AtomicU8 = AtomicU8::new(0); #[doc(hidden)] // Only public for integration tests pub struct Telemetry { config: config::Conf, + metrics: BasicMetrics, // Do not remove _metrics_exporters. Metrics will not be exported if it is removed. // Typically the handles are a PushController but may be something else. Dropping the handle will // shutdown exporter. _metrics_exporters: Vec, - meter_provider: AggregateMeterProvider, custom_endpoints: MultiMap, apollo_metrics_sender: apollo_exporter::Sender, field_level_instrumentation_ratio: f64, @@ -177,14 +182,34 @@ fn setup_metrics_exporter( Ok(builder) } +fn run_with_timeout(f: F, timeout: Duration) -> Result +where + F: FnOnce() -> T + Send + 'static, + T: Send + 'static, +{ + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || tx.send(f())); + + rx.recv_timeout(timeout) +} + +const TRACER_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5); + impl Drop for Telemetry { fn drop(&mut self) { ::tracing::debug!("dropping telemetry..."); let count = TELEMETRY_REFCOUNT.fetch_sub(1, Ordering::Relaxed); if count < 2 { - std::thread::spawn(|| { - opentelemetry::global::shutdown_tracer_provider(); - }); + // We don't want telemetry to drop until the shutdown completes, + // but we also don't want to wait forever. Let's allow 5 seconds + // for now. + // We log errors as warnings + if let Err(e) = run_with_timeout( + opentelemetry::global::shutdown_tracer_provider, + TRACER_SHUTDOWN_TIMEOUT, + ) { + ::tracing::warn!("tracer shutdown failed: {:?}", e); + } } } } @@ -199,7 +224,7 @@ impl Plugin for Telemetry { fn supergraph_service(&self, service: supergraph::BoxService) -> supergraph::BoxService { let metrics_sender = self.apollo_metrics_sender.clone(); - let metrics = BasicMetrics::new(&self.meter_provider); + let metrics = self.metrics.clone(); let config = Arc::new(self.config.clone()); let config_map_res_first = config.clone(); let config_map_res = config.clone(); @@ -216,7 +241,7 @@ impl Plugin for Telemetry { // Record the operation signature on the router span Span::current().record( APOLLO_PRIVATE_OPERATION_SIGNATURE.as_str(), - &usage_reporting.stats_report_key.as_str(), + usage_reporting.stats_report_key.as_str(), ); } // To expose trace_id or not @@ -280,16 +305,14 @@ impl Plugin for Telemetry { fn execution_service(&self, service: execution::BoxService) -> execution::BoxService { ServiceBuilder::new() .instrument(move |_req: &ExecutionRequest| { - info_span!("execution", - "otel.kind" = %SpanKind::Internal, - ) + info_span!("execution", "otel.kind" = "INTERNAL",) }) .service(service) .boxed() } fn subgraph_service(&self, name: &str, service: subgraph::BoxService) -> subgraph::BoxService { - let metrics = BasicMetrics::new(&self.meter_provider); + let metrics = self.metrics.clone(); let subgraph_attribute = KeyValue::new("subgraph", name.to_string()); let subgraph_metrics_conf_req = self.create_subgraph_metrics_conf(name); let subgraph_metrics_conf_resp = subgraph_metrics_conf_req.clone(); @@ -310,11 +333,12 @@ impl Plugin for Telemetry { .clone() .unwrap_or_default(); - info_span!(SUBGRAPH_SPAN_NAME, + info_span!( + SUBGRAPH_SPAN_NAME, "apollo.subgraph.name" = name.as_str(), graphql.document = query.as_str(), graphql.operation.name = operation_name.as_str(), - "otel.kind" = %SpanKind::Internal, + "otel.kind" = "INTERNAL", "apollo_private.ftv1" = field::Empty ) }) @@ -371,49 +395,19 @@ impl Telemetry { /// This method can be used instead of `Plugin::new` to override the subscriber async fn new_common( - mut config: ::Config, + config: ::Config, #[cfg_attr(feature = "console", allow(unused_variables))] subscriber: Option, ) -> Result where S: Subscriber + Send + Sync + for<'span> LookupSpan<'span>, { - // Apollo config is special because we enable tracing if some env variables are present. - let apollo = config - .apollo - .as_mut() - .expect("telemetry apollo config must be present"); - if let Some(tracing_conf) = &config.tracing { - apollo.expose_trace_id = tracing_conf.response_trace_id.clone(); - } - - // If we have key and graph ref but no endpoint we start embedded spaceport - let spaceport = match apollo { - apollo::Config { - apollo_key: Some(_), - apollo_graph_ref: Some(_), - endpoint: None, - .. - } => { - ::tracing::debug!("starting Spaceport"); - let report_spaceport = ReportSpaceport::new("127.0.0.1:0".parse()?).await?; - // Now that the port is known update the config - apollo.endpoint = Some(Url::parse(&format!( - "https://{}", - report_spaceport.address() - ))?); - Some(report_spaceport) - } - _ => None, - }; - if let Some(logging_conf) = &config.logging { logging_conf.validate()?; } // Setup metrics // The act of setting up metrics will overwrite a global meter. However it is essential that // we use the aggregate meter provider that is created below. It enables us to support - // sending metrics to multiple providers at once, of which hopefully Apollo Studio will - // eventually be one. + // sending metrics to multiple providers at once, of which hopefully Apollo Studio is one. let mut builder = Self::create_metrics_exporters(&config)?; // the global tracer and subscriber initialization step must be performed only once @@ -428,10 +422,12 @@ impl Telemetry { None, ); - global::set_tracer_provider(tracer_provider); - global::set_error_handler(handle_error) + opentelemetry::global::set_tracer_provider(tracer_provider); + opentelemetry::global::set_error_handler(handle_error) .expect("otel error handler lock poisoned, fatal"); - global::set_text_map_propagator(Self::create_propagator(&config)); + opentelemetry::global::set_text_map_propagator(Self::create_propagator(&config)); + // Set the meter provider + opentelemetry::global::set_meter_provider(builder.meter_provider()); #[cfg(feature = "console")] { @@ -446,6 +442,8 @@ impl Telemetry { #[cfg(not(feature = "console"))] { + // let otel_metrics = builder.layers(); + let otel_metrics = MetricsLayer::default(); let log_level = GLOBAL_ENV_FILTER .get() .map(|s| s.as_str()) @@ -473,7 +471,7 @@ impl Telemetry { if let Some(sub) = subscriber { let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); - let subscriber = sub.with(telemetry); + let subscriber = sub.with(telemetry).with(otel_metrics); if let Err(e) = set_global_default(subscriber) { ::tracing::error!("cannot set global subscriber: {:?}", e); } @@ -488,9 +486,13 @@ impl Telemetry { let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); let subscriber = sub_builder - .event_format(formatters::TextFormatter::new()) + .event_format(FilteringFormatter::new( + TextFormatter::new(), + filter_metric_events, + )) .finish() - .with(telemetry); + .with(telemetry) + .with(otel_metrics); if let Err(e) = set_global_default(subscriber) { ::tracing::error!("cannot set global subscriber: {:?}", e); } @@ -500,19 +502,23 @@ impl Telemetry { let subscriber = sub_builder .map_event_format(|e| { - e.json() - .with_current_span(true) - .with_span_list(true) - .flatten_event(true) + FilteringFormatter::new( + e.json() + .with_current_span(true) + .with_span_list(true) + .flatten_event(true), + filter_metric_events, + ) }) .map_fmt_fields(|_f| JsonFields::new()) .finish() - .with(telemetry); + .with(telemetry) + .with(otel_metrics); if let Err(e) = set_global_default(subscriber) { ::tracing::error!("cannot set global subscriber: {:?}", e); } } - } + }; } } @@ -525,34 +531,12 @@ impl Telemetry { let plugin = Ok(Telemetry { custom_endpoints: builder.custom_endpoints(), _metrics_exporters: builder.exporters(), - meter_provider: builder.meter_provider(), + metrics: BasicMetrics::default(), apollo_metrics_sender: builder.apollo_metrics_provider(), field_level_instrumentation_ratio, config, }); - // We're now safe for shutdown. - // Start spaceport - if let Some(spaceport) = spaceport { - tokio::spawn(async move { - ::tracing::debug!("serving spaceport"); - match spaceport.serve().await { - Ok(v) => { - ::tracing::debug!("spaceport terminated normally: {:?}", v); - } - Err(e) => match e.source() { - Some(source) => { - ::tracing::warn!("spaceport did not terminate normally: {}", source); - } - None => { - ::tracing::warn!("spaceport did not terminate normally: {}", e); - } - }, - } - ::tracing::debug!("stopped serving spaceport"); - }); - } - let _ = TELEMETRY_REFCOUNT.fetch_add(1, Ordering::Relaxed); plugin } @@ -607,6 +591,8 @@ impl Telemetry { builder = setup_tracing(builder, &tracing_config.datadog, trace_config)?; builder = setup_tracing(builder, &tracing_config.otlp, trace_config)?; builder = setup_tracing(builder, &config.apollo, trace_config)?; + // For metrics + builder = builder.with_simple_exporter(metrics::span_metrics_exporter::Exporter::default()); let tracer_provider = builder.build(); Ok(tracer_provider) @@ -677,8 +663,9 @@ impl Telemetry { graphql.operation.name = operation_name.as_str(), client.name = client_name.to_str().unwrap_or_default(), client.version = client_version.to_str().unwrap_or_default(), - otel.kind = %SpanKind::Internal, - apollo_private.field_level_instrumentation_ratio = field_level_instrumentation_ratio, + otel.kind = "INTERNAL", + apollo_private.field_level_instrumentation_ratio = + field_level_instrumentation_ratio, apollo_private.operation_signature = field::Empty, apollo_private.graphql.variables = field::Empty, apollo_private.http.request_headers = field::Empty @@ -687,7 +674,7 @@ impl Telemetry { if is_span_sampled() { span.record( "apollo_private.graphql.variables", - &Self::filter_variables_values( + Self::filter_variables_values( &request.supergraph_request.body().variables, &config.send_variable_values, ) @@ -695,7 +682,7 @@ impl Telemetry { ); span.record( "apollo_private.http.request_headers", - &Self::filter_headers( + Self::filter_headers( request.supergraph_request.headers(), &config.send_headers, ) @@ -833,7 +820,6 @@ impl Telemetry { if !parts.status.is_success() { metric_attrs.push(KeyValue::new("error", parts.status.to_string())); - metrics.http_requests_error_total.add(1, &metric_attrs); } let response = http::Response::from_parts( parts, @@ -848,11 +834,15 @@ impl Telemetry { }; // http_requests_total - the total number of HTTP requests received - metrics.http_requests_total.add(1, &metric_attrs); - metrics - .http_requests_duration - .record(request_duration.as_secs_f64(), &metric_attrs); + .http_requests_total + .add(&opentelemetry::Context::current(), 1, &metric_attrs); + + metrics.http_requests_duration.record( + &opentelemetry::Context::current(), + request_duration.as_secs_f64(), + &metric_attrs, + ); res } @@ -1070,7 +1060,11 @@ impl Telemetry { ); } - metrics.http_requests_total.add(1, &metric_attrs); + metrics.http_requests_total.add( + &opentelemetry::Context::current(), + 1, + &metric_attrs, + ); } Err(err) => { // Fill attributes from error @@ -1083,12 +1077,18 @@ impl Telemetry { ); } - metrics.http_requests_error_total.add(1, &metric_attrs); + metrics.http_requests_total.add( + &opentelemetry::Context::current(), + 1, + &metric_attrs, + ); } } - metrics - .http_requests_duration - .record(now.elapsed().as_secs_f64(), &metric_attrs); + metrics.http_requests_duration.record( + &opentelemetry::Context::current(), + now.elapsed().as_secs_f64(), + &metric_attrs, + ); } #[allow(clippy::too_many_arguments)] @@ -1122,7 +1122,11 @@ impl Telemetry { ); } - metrics.http_requests_error_total.add(1, &metric_attrs); + metrics.http_requests_total.add( + &opentelemetry::Context::current(), + 1, + &metric_attrs, + ); Err(e) } @@ -1249,8 +1253,8 @@ fn operation_count(stats_report_key: &str) -> u64 { fn convert( referenced_fields: router_bridge::planner::ReferencedFieldsForType, -) -> crate::spaceport::ReferencedFieldsForType { - crate::spaceport::ReferencedFieldsForType { +) -> crate::plugins::telemetry::apollo_exporter::proto::ReferencedFieldsForType { + crate::plugins::telemetry::apollo_exporter::proto::ReferencedFieldsForType { field_names: referenced_fields.field_names, is_interface: referenced_fields.is_interface, } @@ -1261,6 +1265,9 @@ fn handle_error>(err: T) { opentelemetry::global::Error::Trace(err) => { ::tracing::error!("OpenTelemetry trace error occurred: {}", err) } + opentelemetry::global::Error::Metric(err_msg) => { + ::tracing::error!("OpenTelemetry metric error occurred: {}", err_msg) + } opentelemetry::global::Error::Other(err_msg) => { ::tracing::error!("OpenTelemetry error occurred: {}", err_msg) } @@ -1305,7 +1312,7 @@ impl ApolloFtv1Handler { resp.response.body().extensions.get("ftv1") { // Record the ftv1 trace for processing later - Span::current().record("apollo_private.ftv1", &ftv1.as_str()); + Span::current().record("apollo_private.ftv1", ftv1.as_str()); } } resp @@ -1390,6 +1397,8 @@ mod tests { use std::str::FromStr; use http::StatusCode; + use insta::assert_snapshot; + use itertools::Itertools; use serde_json::Value; use serde_json_bytes::json; use serde_json_bytes::ByteString; @@ -1413,7 +1422,7 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn plugin_registered() { crate::plugin::plugins() - .get("apollo.telemetry") + .find(|factory| factory.name == "apollo.telemetry") .expect("Plugin not found") .create_instance( &serde_json::json!({"apollo": {"schema_id":"abc"}, "tracing": {}}), @@ -1426,7 +1435,7 @@ mod tests { #[tokio::test(flavor = "multi_thread")] async fn attribute_serialization() { crate::plugin::plugins() - .get("apollo.telemetry") + .find(|factory| factory.name == "apollo.telemetry") .expect("Plugin not found") .create_instance( &serde_json::json!({ @@ -1636,7 +1645,7 @@ mod tests { }); let dyn_plugin: Box = crate::plugin::plugins() - .get("apollo.telemetry") + .find(|factory| factory.name == "apollo.telemetry") .expect("Plugin not found") .create_instance( &Value::from_str( @@ -1840,26 +1849,12 @@ mod tests { let mut resp = web_endpoint.oneshot(http_req_prom).await.unwrap(); assert_eq!(resp.status(), StatusCode::OK); let body = hyper::body::to_bytes(resp.body_mut()).await.unwrap(); - let prom_metrics = String::from_utf8_lossy(&body); - assert!(prom_metrics.contains(r#"apollo_router_http_requests_error_total{message="cannot contact the subgraph",service_name="apollo-router",subgraph="my_subgraph_name_error",subgraph_error_extended_type="SubrequestHttpError"} 1"#)); - assert!(prom_metrics.contains(r#"apollo_router_http_requests_total{another_test="my_default_value",my_value="2",myname="label_value",renamed_value="my_value_set",service_name="apollo-router",status="200",x_custom="coming_from_header"} 1"#)); - assert!(prom_metrics.contains(r#"apollo_router_http_request_duration_seconds_count{another_test="my_default_value",my_value="2",myname="label_value",renamed_value="my_value_set",service_name="apollo-router",status="200",x_custom="coming_from_header"}"#)); - assert!(prom_metrics.contains(r#"apollo_router_http_request_duration_seconds_bucket{another_test="my_default_value",my_value="2",myname="label_value",renamed_value="my_value_set",service_name="apollo-router",status="200",x_custom="coming_from_header",le="0.001"}"#)); - assert!(prom_metrics.contains(r#"apollo_router_http_request_duration_seconds_bucket{another_test="my_default_value",my_value="2",myname="label_value",renamed_value="my_value_set",service_name="apollo-router",status="200",x_custom="coming_from_header",le="0.005"}"#)); - assert!(prom_metrics.contains(r#"apollo_router_http_request_duration_seconds_bucket{another_test="my_default_value",my_value="2",myname="label_value",renamed_value="my_value_set",service_name="apollo-router",status="200",x_custom="coming_from_header",le="0.015"}"#)); - assert!(prom_metrics.contains(r#"apollo_router_http_request_duration_seconds_bucket{another_test="my_default_value",my_value="2",myname="label_value",renamed_value="my_value_set",service_name="apollo-router",status="200",x_custom="coming_from_header",le="0.05"}"#)); - assert!(prom_metrics.contains(r#"apollo_router_http_request_duration_seconds_bucket{another_test="my_default_value",my_value="2",myname="label_value",renamed_value="my_value_set",service_name="apollo-router",status="200",x_custom="coming_from_header",le="0.3"}"#)); - assert!(prom_metrics.contains(r#"apollo_router_http_request_duration_seconds_bucket{another_test="my_default_value",my_value="2",myname="label_value",renamed_value="my_value_set",service_name="apollo-router",status="200",x_custom="coming_from_header",le="0.4"}"#)); - assert!(prom_metrics.contains(r#"apollo_router_http_request_duration_seconds_bucket{another_test="my_default_value",my_value="2",myname="label_value",renamed_value="my_value_set",service_name="apollo-router",status="200",x_custom="coming_from_header",le="0.5"}"#)); - assert!(prom_metrics.contains(r#"apollo_router_http_request_duration_seconds_bucket{another_test="my_default_value",my_value="2",myname="label_value",renamed_value="my_value_set",service_name="apollo-router",status="200",x_custom="coming_from_header",le="1"}"#)); - assert!(prom_metrics.contains(r#"apollo_router_http_request_duration_seconds_bucket{another_test="my_default_value",my_value="2",myname="label_value",renamed_value="my_value_set",service_name="apollo-router",status="200",x_custom="coming_from_header",le="5"}"#)); - assert!(prom_metrics.contains(r#"apollo_router_http_request_duration_seconds_bucket{another_test="my_default_value",my_value="2",myname="label_value",renamed_value="my_value_set",service_name="apollo-router",status="200",x_custom="coming_from_header",le="10"}"#)); - assert!(prom_metrics.contains(r#"apollo_router_http_request_duration_seconds_bucket{another_test="my_default_value",my_value="2",myname="label_value",renamed_value="my_value_set",service_name="apollo-router",status="200",x_custom="coming_from_header",le="+Inf"}"#)); - assert!(prom_metrics.contains(r#"apollo_router_http_request_duration_seconds_count{another_test="my_default_value",my_value="2",myname="label_value",renamed_value="my_value_set",service_name="apollo-router",status="200",x_custom="coming_from_header"}"#)); - assert!(prom_metrics.contains(r#"apollo_router_http_request_duration_seconds_sum{another_test="my_default_value",my_value="2",myname="label_value",renamed_value="my_value_set",service_name="apollo-router",status="200",x_custom="coming_from_header"}"#)); - assert!(prom_metrics.contains(r#"apollo_router_http_request_duration_seconds_bucket{error="INTERNAL_SERVER_ERROR",my_key="my_custom_attribute_from_context",query_from_request="query { test }",service_name="apollo-router",status="200",subgraph="my_subgraph_name",unknown_data="default_value",le="1"}"#)); - assert!(prom_metrics.contains(r#"apollo_router_http_requests_total{error="INTERNAL_SERVER_ERROR",my_key="my_custom_attribute_from_context",query_from_request="query { test }",service_name="apollo-router",status="200",subgraph="my_subgraph_name",unknown_data="default_value"} 1"#)); - assert!(prom_metrics.contains(r#"apollo_router_http_requests_total{another_test="my_default_value",error="400 Bad Request",myname="label_value",renamed_value="my_value_set",service_name="apollo-router",status="400"} 1"#)); - assert!(prom_metrics.contains(r#"apollo_router_http_requests_error_total{another_test="my_default_value",error="400 Bad Request",myname="label_value",renamed_value="my_value_set",service_name="apollo-router",status="400"} 1"#)) + let prom_metrics = String::from_utf8_lossy(&body) + .to_string() + .split('\n') + .filter(|l| l.contains("_count") && !l.contains("apollo_router_span_count")) + .sorted() + .join("\n"); + assert_snapshot!(prom_metrics); } } diff --git a/apollo-router/src/plugins/telemetry/otlp.rs b/apollo-router/src/plugins/telemetry/otlp.rs index 29801f56d2..a90c483e7b 100644 --- a/apollo-router/src/plugins/telemetry/otlp.rs +++ b/apollo-router/src/plugins/telemetry/otlp.rs @@ -19,6 +19,7 @@ use url::Url; use crate::plugins::telemetry::config::GenericWith; use crate::plugins::telemetry::tracing::parse_url_for_endpoint; +use crate::plugins::telemetry::tracing::BatchProcessorConfig; #[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] #[serde(deny_unknown_fields)] @@ -33,6 +34,8 @@ pub(crate) struct Config { pub(crate) timeout: Option, pub(crate) grpc: Option, pub(crate) http: Option, + + pub(crate) batch_processor: Option, } impl Config { diff --git a/apollo-router/src/spaceport/proto/reports.proto b/apollo-router/src/plugins/telemetry/proto/reports.proto similarity index 96% rename from apollo-router/src/spaceport/proto/reports.proto rename to apollo-router/src/plugins/telemetry/proto/reports.proto index 26fe614cc1..bb32e48f6e 100644 --- a/apollo-router/src/spaceport/proto/reports.proto +++ b/apollo-router/src/plugins/telemetry/proto/reports.proto @@ -436,6 +436,13 @@ message Report { // Total number of operations processed during this period. uint64 operation_count = 6; + + // If this is set to true, the stats in TracesWithStats.stats_with_context + // represent all of the operations described from this report, and the + // traces in TracesWithStats.trace are a sampling of some of the same + // operations. If this is false, each operation is described in precisely + // one of those two fields. + bool traces_pre_aggregated = 7; } message ContextualizedStats { @@ -447,8 +454,11 @@ message ContextualizedStats { } -// A sequence of traces and stats. An individual operation should either be described as a trace -// or as part of stats, but not both. +// A sequence of traces and stats. If Report.traces_pre_aggregated (at the top +// level of the report) is false, an individual operation should either be +// described as a trace or as part of stats, but not both. If that flag +// is true, then all operations are described as stats and some are also +// described as traces. message TracesAndStats { repeated Trace trace = 1 [(js_preEncoded)=true]; repeated ContextualizedStats stats_with_context = 2 [(js_use_toArray)=true]; diff --git a/apollo-router/src/plugins/telemetry/snapshots/apollo_router__plugins__telemetry__tests__it_test_prometheus_metrics.snap b/apollo-router/src/plugins/telemetry/snapshots/apollo_router__plugins__telemetry__tests__it_test_prometheus_metrics.snap new file mode 100644 index 0000000000..3f2c4713ba --- /dev/null +++ b/apollo-router/src/plugins/telemetry/snapshots/apollo_router__plugins__telemetry__tests__it_test_prometheus_metrics.snap @@ -0,0 +1,8 @@ +--- +source: apollo-router/src/plugins/telemetry/mod.rs +expression: prom_metrics +--- +apollo_router_http_request_duration_seconds_count{another_test="my_default_value",error="400 Bad Request",myname="label_value",renamed_value="my_value_set",service_name="apollo-router",status="400"} 1 +apollo_router_http_request_duration_seconds_count{another_test="my_default_value",my_value="2",myname="label_value",renamed_value="my_value_set",service_name="apollo-router",status="200",x_custom="coming_from_header"} 1 +apollo_router_http_request_duration_seconds_count{error="INTERNAL_SERVER_ERROR",my_key="my_custom_attribute_from_context",query_from_request="query { test }",service_name="apollo-router",status="200",subgraph="my_subgraph_name",unknown_data="default_value"} 1 +apollo_router_http_request_duration_seconds_count{message="cannot contact the subgraph",service_name="apollo-router",subgraph="my_subgraph_name_error",subgraph_error_extended_type="SubrequestHttpError"} 1 diff --git a/apollo-router/src/plugins/telemetry/tracing/apollo.rs b/apollo-router/src/plugins/telemetry/tracing/apollo.rs index 46ddbb785a..632542ef6b 100644 --- a/apollo-router/src/plugins/telemetry/tracing/apollo.rs +++ b/apollo-router/src/plugins/telemetry/tracing/apollo.rs @@ -1,21 +1,22 @@ //! Tracing configuration for apollo telemetry. // With regards to ELv2 licensing, this entire file is license key functionality +use opentelemetry::sdk::trace::BatchSpanProcessor; use opentelemetry::sdk::trace::Builder; use serde::Serialize; use tower::BoxError; use crate::plugins::telemetry::apollo::Config; +use crate::plugins::telemetry::apollo_exporter::proto::Trace; use crate::plugins::telemetry::config; use crate::plugins::telemetry::tracing::apollo_telemetry; use crate::plugins::telemetry::tracing::TracingConfigurator; -use crate::spaceport::Trace; impl TracingConfigurator for Config { fn apply(&self, builder: Builder, _trace_config: &config::Trace) -> Result { tracing::debug!("configuring Apollo tracing"); Ok(match self { Config { - endpoint: Some(endpoint), + endpoint, apollo_key: Some(key), apollo_graph_ref: Some(reference), schema_id, @@ -35,7 +36,17 @@ impl TracingConfigurator for Config { .buffer_size(*buffer_size) .and_field_execution_sampler(field_level_instrumentation_sampler.clone()) .build()?; - builder.with_batch_exporter(exporter, opentelemetry::runtime::Tokio) + builder.with_span_processor( + BatchSpanProcessor::builder(exporter, opentelemetry::runtime::Tokio) + .with_batch_config( + self.batch_processor + .as_ref() + .cloned() + .unwrap_or_default() + .into(), + ) + .build(), + ) } _ => builder, }) diff --git a/apollo-router/src/plugins/telemetry/tracing/apollo_telemetry.rs b/apollo-router/src/plugins/telemetry/tracing/apollo_telemetry.rs index 9ed26f161a..b4ad1f1874 100644 --- a/apollo-router/src/plugins/telemetry/tracing/apollo_telemetry.rs +++ b/apollo-router/src/plugins/telemetry/tracing/apollo_telemetry.rs @@ -1,26 +1,52 @@ use std::collections::HashMap; use std::io::Cursor; +use std::sync::Arc; +#[cfg(test)] +use std::sync::Mutex; use std::time::SystemTimeError; use async_trait::async_trait; use derivative::Derivative; +use futures::future::BoxFuture; +use futures::FutureExt; +use futures::TryFutureExt; use itertools::Itertools; use lru::LruCache; use opentelemetry::sdk::export::trace::ExportResult; use opentelemetry::sdk::export::trace::SpanData; use opentelemetry::sdk::export::trace::SpanExporter; use opentelemetry::trace::SpanId; +use opentelemetry::trace::TraceError; use opentelemetry::Key; use opentelemetry::Value; use opentelemetry_semantic_conventions::trace::HTTP_METHOD; +use prost::Message; use serde::de::DeserializeOwned; use thiserror::Error; use url::Url; use crate::axum_factory::utils::REQUEST_SPAN_NAME; +use crate::plugins::telemetry; +use crate::plugins::telemetry::apollo::Report; use crate::plugins::telemetry::apollo::SingleReport; +use crate::plugins::telemetry::apollo_exporter::proto; +use crate::plugins::telemetry::apollo_exporter::proto::trace::http::Values; +use crate::plugins::telemetry::apollo_exporter::proto::trace::query_plan_node::ConditionNode; +use crate::plugins::telemetry::apollo_exporter::proto::trace::query_plan_node::DeferNode; +use crate::plugins::telemetry::apollo_exporter::proto::trace::query_plan_node::DeferNodePrimary; +use crate::plugins::telemetry::apollo_exporter::proto::trace::query_plan_node::DeferredNode; +use crate::plugins::telemetry::apollo_exporter::proto::trace::query_plan_node::DeferredNodeDepends; +use crate::plugins::telemetry::apollo_exporter::proto::trace::query_plan_node::FetchNode; +use crate::plugins::telemetry::apollo_exporter::proto::trace::query_plan_node::FlattenNode; +use crate::plugins::telemetry::apollo_exporter::proto::trace::query_plan_node::Node; +use crate::plugins::telemetry::apollo_exporter::proto::trace::query_plan_node::ParallelNode; +use crate::plugins::telemetry::apollo_exporter::proto::trace::query_plan_node::ResponsePathElement; +use crate::plugins::telemetry::apollo_exporter::proto::trace::query_plan_node::SequenceNode; +use crate::plugins::telemetry::apollo_exporter::proto::trace::Details; +use crate::plugins::telemetry::apollo_exporter::proto::trace::Http; +use crate::plugins::telemetry::apollo_exporter::proto::trace::QueryPlanNode; +use crate::plugins::telemetry::apollo_exporter::ApolloExportError; use crate::plugins::telemetry::apollo_exporter::ApolloExporter; -use crate::plugins::telemetry::apollo_exporter::Sender; use crate::plugins::telemetry::config; use crate::plugins::telemetry::config::ExposeTraceId; use crate::plugins::telemetry::config::Sampler; @@ -39,21 +65,6 @@ use crate::query_planner::FETCH_SPAN_NAME; use crate::query_planner::FLATTEN_SPAN_NAME; use crate::query_planner::PARALLEL_SPAN_NAME; use crate::query_planner::SEQUENCE_SPAN_NAME; -use crate::spaceport::trace::http::Values; -use crate::spaceport::trace::query_plan_node::ConditionNode; -use crate::spaceport::trace::query_plan_node::DeferNode; -use crate::spaceport::trace::query_plan_node::DeferNodePrimary; -use crate::spaceport::trace::query_plan_node::DeferredNode; -use crate::spaceport::trace::query_plan_node::DeferredNodeDepends; -use crate::spaceport::trace::query_plan_node::FetchNode; -use crate::spaceport::trace::query_plan_node::FlattenNode; -use crate::spaceport::trace::query_plan_node::ParallelNode; -use crate::spaceport::trace::query_plan_node::ResponsePathElement; -use crate::spaceport::trace::query_plan_node::SequenceNode; -use crate::spaceport::trace::Details; -use crate::spaceport::trace::Http; -use crate::spaceport::trace::QueryPlanNode; -use crate::spaceport::Message; const APOLLO_PRIVATE_DURATION_NS: Key = Key::from_static_str("apollo_private.duration_ns"); const APOLLO_PRIVATE_SENT_TIME_OFFSET: Key = @@ -78,7 +89,7 @@ pub(crate) const DEFAULT_TRACE_ID_HEADER_NAME: &str = "apollo-trace-id"; #[derive(Error, Debug)] pub(crate) enum Error { #[error("subgraph protobuf decode error")] - ProtobufDecode(#[from] crate::spaceport::DecodeError), + ProtobufDecode(#[from] crate::plugins::telemetry::apollo_exporter::DecodeError), #[error("subgraph trace payload was not base64")] Base64Decode(#[from] base64::DecodeError), @@ -96,21 +107,21 @@ pub(crate) enum Error { /// A [`SpanExporter`] that writes to [`Reporter`]. /// /// [`SpanExporter`]: super::SpanExporter -/// [`Reporter`]: crate::spaceport::Reporter +/// [`Reporter`]: crate::plugins::telemetry::Reporter #[derive(Derivative)] #[derivative(Debug)] pub(crate) struct Exporter { expose_trace_id_config: config::ExposeTraceId, spans_by_parent_id: LruCache>, #[derivative(Debug = "ignore")] - apollo_sender: Sender, + report_exporter: ReportExporter, field_execution_weight: f64, } enum TreeData { - Request(Result, Error>), + Request(Result, Error>), Supergraph { - http: Http, + http: Box, client_name: Option, client_version: Option, operation_signature: String, @@ -122,7 +133,7 @@ enum TreeData { DeferDeferred(DeferredNode), ConditionIf(Option), ConditionElse(Option), - Trace(Option, Error>>), + Trace(Option, Error>>), } #[buildstructor::buildstructor] @@ -138,12 +149,15 @@ impl Exporter { field_execution_sampler: Option, ) -> Result { tracing::debug!("creating studio exporter"); - let apollo_exporter = - ApolloExporter::new(&endpoint, &apollo_key, &apollo_graph_ref, &schema_id)?; Ok(Self { expose_trace_id_config, spans_by_parent_id: LruCache::new(buffer_size), - apollo_sender: apollo_exporter.provider(), + report_exporter: ReportExporter::Apollo(Arc::new(ApolloExporter::new( + &endpoint, + &apollo_key, + &apollo_graph_ref, + &schema_id, + )?)), field_execution_weight: match field_execution_sampler { Some(SamplerOption::Always(Sampler::AlwaysOn)) => 1.0, Some(SamplerOption::Always(Sampler::AlwaysOff)) => 0.0, @@ -157,10 +171,9 @@ impl Exporter { &mut self, span: &SpanData, child_nodes: Vec, - ) -> Result, Error> { + ) -> Result, Error> { let http = extract_http_data(span, &self.expose_trace_id_config); - - let mut root_trace = crate::spaceport::Trace { + let mut root_trace = proto::Trace { start_time: Some(span.start_time.into()), end_time: Some(span.end_time.into()), duration_ns: span @@ -209,8 +222,8 @@ impl Exporter { Ok(Box::new(root_trace)) } - fn extract_trace(&mut self, span: SpanData) -> Result, Error> { - self.extract_data_from_spans(&span, &span)? + fn extract_trace(&mut self, span: SpanData) -> Result, Error> { + self.extract_data_from_spans(&span)? .pop() .and_then(|node| { if let TreeData::Request(trace) = node { @@ -222,18 +235,14 @@ impl Exporter { .expect("root trace must exist because it is constructed on the request span, qed") } - fn extract_data_from_spans( - &mut self, - root_span: &SpanData, - span: &SpanData, - ) -> Result, Error> { + fn extract_data_from_spans(&mut self, span: &SpanData) -> Result, Error> { let (mut child_nodes, errors) = self .spans_by_parent_id .pop_entry(&span.span_context.span_id()) .map(|(_, spans)| spans) .unwrap_or_default() .into_iter() - .map(|span| self.extract_data_from_spans(root_span, &span)) + .map(|span| self.extract_data_from_spans(&span)) .fold((Vec::new(), Vec::new()), |(mut oks, mut errors), next| { match next { Ok(mut children) => oks.append(&mut children), @@ -247,14 +256,14 @@ impl Exporter { Ok(match span.name.as_ref() { PARALLEL_SPAN_NAME => vec![TreeData::QueryPlanNode(QueryPlanNode { - node: Some(crate::spaceport::trace::query_plan_node::Node::Parallel( + node: Some(proto::trace::query_plan_node::Node::Parallel( ParallelNode { nodes: child_nodes.remove_query_plan_nodes(), }, )), })], SEQUENCE_SPAN_NAME => vec![TreeData::QueryPlanNode(QueryPlanNode { - node: Some(crate::spaceport::trace::query_plan_node::Node::Sequence( + node: Some(proto::trace::query_plan_node::Node::Sequence( SequenceNode { nodes: child_nodes.remove_query_plan_nodes(), }, @@ -274,8 +283,8 @@ impl Exporter { .as_str()) .to_string(); vec![TreeData::QueryPlanNode(QueryPlanNode { - node: Some(crate::spaceport::trace::query_plan_node::Node::Fetch( - Box::new(FetchNode { + node: Some(proto::trace::query_plan_node::Node::Fetch(Box::new( + FetchNode { service_name, trace_parsing_failed, trace, @@ -287,22 +296,22 @@ impl Exporter { .unwrap_or_default(), sent_time: Some(span.start_time.into()), received_time: Some(span.end_time.into()), - }), - )), + }, + ))), })] } FLATTEN_SPAN_NAME => { vec![TreeData::QueryPlanNode(QueryPlanNode { - node: Some(crate::spaceport::trace::query_plan_node::Node::Flatten( - Box::new(FlattenNode { + node: Some(proto::trace::query_plan_node::Node::Flatten(Box::new( + FlattenNode { response_path: span .attributes .get(&PATH) .map(extract_path) .unwrap_or_default(), node: child_nodes.remove_first_query_plan_node().map(Box::new), - }), - )), + }, + ))), })] } SUBGRAPH_SPAN_NAME => { @@ -315,7 +324,7 @@ impl Exporter { SUPERGRAPH_SPAN_NAME => { //Currently some data is in the supergraph span as we don't have the a request hook in plugin. child_nodes.push(TreeData::Supergraph { - http: extract_http_data(span, &self.expose_trace_id_config), + http: Box::new(extract_http_data(span, &self.expose_trace_id_config)), client_name: span.attributes.get(&CLIENT_NAME).and_then(extract_string), client_version: span .attributes @@ -346,12 +355,10 @@ impl Exporter { } DEFER_SPAN_NAME => { vec![TreeData::QueryPlanNode(QueryPlanNode { - node: Some(crate::spaceport::trace::query_plan_node::Node::Defer( - Box::new(DeferNode { - primary: child_nodes.remove_first_defer_primary_node().map(Box::new), - deferred: child_nodes.remove_defer_deferred_nodes(), - }), - )), + node: Some(Node::Defer(Box::new(DeferNode { + primary: child_nodes.remove_first_defer_primary_node().map(Box::new), + deferred: child_nodes.remove_defer_deferred_nodes(), + }))), })] } DEFER_PRIMARY_SPAN_NAME => { @@ -389,19 +396,15 @@ impl Exporter { CONDITION_SPAN_NAME => { vec![TreeData::QueryPlanNode(QueryPlanNode { - node: Some(crate::spaceport::trace::query_plan_node::Node::Condition( - Box::new(ConditionNode { - condition: span - .attributes - .get(&CONDITION) - .and_then(extract_string) - .unwrap_or_default(), - if_clause: child_nodes.remove_first_condition_if_node().map(Box::new), - else_clause: child_nodes - .remove_first_condition_else_node() - .map(Box::new), - }), - )), + node: Some(Node::Condition(Box::new(ConditionNode { + condition: span + .attributes + .get(&CONDITION) + .and_then(extract_string) + .unwrap_or_default(), + if_clause: child_nodes.remove_first_condition_if_node().map(Box::new), + else_clause: child_nodes.remove_first_condition_else_node().map(Box::new), + }))), })] } CONDITION_IF_SPAN_NAME => { @@ -437,14 +440,30 @@ fn extract_string(v: &Value) -> Option { fn extract_path(v: &Value) -> Vec { extract_string(v) .map(|v| { - v.split('/').filter(|v|!v.is_empty() && *v != "@").map(|v| { - if let Ok(index) = v.parse::() { - ResponsePathElement { id: Some(crate::spaceport::trace::query_plan_node::response_path_element::Id::Index(index))} - } else { - ResponsePathElement { id: Some(crate::spaceport::trace::query_plan_node::response_path_element::Id::FieldName(v.to_string())) } - } - }).collect() - }).unwrap_or_default() + v.split('/') + .filter(|v| !v.is_empty() && *v != "@") + .map(|v| { + if let Ok(index) = v.parse::() { + ResponsePathElement { + id: Some( + proto::trace::query_plan_node::response_path_element::Id::Index( + index, + ), + ), + } + } else { + ResponsePathElement { + id: Some( + proto::trace::query_plan_node::response_path_element::Id::FieldName( + v.to_string(), + ), + ), + } + } + }) + .collect() + }) + .unwrap_or_default() } fn extract_i64(v: &Value) -> Option { @@ -455,10 +474,10 @@ fn extract_i64(v: &Value) -> Option { } } -fn extract_ftv1_trace(v: &Value) -> Option, Error>> { +fn extract_ftv1_trace(v: &Value) -> Option, Error>> { if let Some(v) = extract_string(v) { if let Ok(v) = base64::decode(v) { - if let Ok(t) = crate::spaceport::Trace::decode(Cursor::new(v)) { + if let Ok(t) = proto::Trace::decode(Cursor::new(v)) { return Some(Ok(Box::new(t))); } } @@ -476,16 +495,16 @@ fn extract_http_data(span: &SpanData, expose_trace_id_config: &ExposeTraceId) -> .unwrap_or_default() .as_ref() { - "OPTIONS" => crate::spaceport::trace::http::Method::Options, - "GET" => crate::spaceport::trace::http::Method::Get, - "HEAD" => crate::spaceport::trace::http::Method::Head, - "POST" => crate::spaceport::trace::http::Method::Post, - "PUT" => crate::spaceport::trace::http::Method::Put, - "DELETE" => crate::spaceport::trace::http::Method::Delete, - "TRACE" => crate::spaceport::trace::http::Method::Trace, - "CONNECT" => crate::spaceport::trace::http::Method::Connect, - "PATCH" => crate::spaceport::trace::http::Method::Patch, - _ => crate::spaceport::trace::http::Method::Unknown, + "OPTIONS" => proto::trace::http::Method::Options, + "GET" => proto::trace::http::Method::Get, + "HEAD" => proto::trace::http::Method::Head, + "POST" => proto::trace::http::Method::Post, + "PUT" => proto::trace::http::Method::Put, + "DELETE" => proto::trace::http::Method::Delete, + "TRACE" => proto::trace::http::Method::Trace, + "CONNECT" => proto::trace::http::Method::Connect, + "PATCH" => proto::trace::http::Method::Patch, + _ => proto::trace::http::Method::Unknown, }; let request_headers = span .attributes @@ -525,13 +544,13 @@ fn extract_http_data(span: &SpanData, expose_trace_id_config: &ExposeTraceId) -> #[async_trait] impl SpanExporter for Exporter { /// Export spans to apollo telemetry - async fn export(&mut self, batch: Vec) -> ExportResult { + fn export(&mut self, batch: Vec) -> BoxFuture<'static, ExportResult> { // Exporting to apollo means that we must have complete trace as the entire trace must be built. // We do what we can, and if there are any traces that are not complete then we keep them for the next export event. // We may get spans that simply don't complete. These need to be cleaned up after a period. It's the price of using ftv1. // Note that apollo-tracing won't really work with defer/stream/live queries. In this situation it's difficult to know when a request has actually finished. - let mut traces: Vec<(String, crate::spaceport::Trace)> = Vec::new(); + let mut traces: Vec<(String, proto::Trace)> = Vec::new(); for span in batch { if span.name == REQUEST_SPAN_NAME { // Write spans for testing @@ -576,10 +595,16 @@ impl SpanExporter for Exporter { .push(span); } } - self.apollo_sender - .send(SingleReport::Traces(TracesReport { traces })); - - return ExportResult::Ok(()); + let mut report = telemetry::apollo::Report::default(); + report += SingleReport::Traces(TracesReport { traces }); + let exporter = self.report_exporter.clone(); + let fut = async move { + exporter + .submit_report(report) + .map_err(|e| TraceError::ExportFailed(Box::new(e))) + .await + }; + fut.boxed() } } @@ -670,6 +695,26 @@ impl ChildNodes for Vec { } } +#[derive(Clone)] +enum ReportExporter { + Apollo(Arc), + #[cfg(test)] + InMemory(Arc>>), +} + +impl ReportExporter { + async fn submit_report(self, report: Report) -> Result<(), ApolloExportError> { + match self { + ReportExporter::Apollo(apollo) => apollo.submit_report(report).await, + #[cfg(test)] + ReportExporter::InMemory(store) => { + store.lock().expect("poisoned").push(report); + Ok(()) + } + } + } +} + #[buildstructor::buildstructor] #[cfg(test)] impl Exporter { @@ -678,7 +723,7 @@ impl Exporter { Exporter { expose_trace_id_config: expose_trace_id_config.unwrap_or_default(), spans_by_parent_id: LruCache::unbounded(), - apollo_sender: Sender::InMemory(Default::default()), + report_exporter: ReportExporter::InMemory(Default::default()), field_execution_weight: 1.0, } } @@ -686,41 +731,138 @@ impl Exporter { #[cfg(test)] mod test { - use std::borrow::Cow; - use http::header::HeaderName; - use opentelemetry::sdk::export::trace::SpanExporter; - use opentelemetry::Value; + use opentelemetry::sdk::export::trace::{SpanData, SpanExporter}; + use opentelemetry::sdk::trace::{EvictedHashMap, EvictedQueue}; + use opentelemetry::trace::{SpanContext, SpanId, SpanKind, TraceId}; + use opentelemetry::{Key, KeyValue, Value}; use prost::Message; use serde_json::json; + use std::str::FromStr; + use std::time::SystemTime; - use crate::plugins::telemetry::apollo::SingleReport; - use crate::plugins::telemetry::apollo_exporter::Sender; + use crate::plugins::telemetry::apollo::{Report}; + use crate::plugins::telemetry::apollo_exporter::proto::Trace; + use crate::plugins::telemetry::apollo_exporter::proto::trace::query_plan_node::{DeferNodePrimary, DeferredNode, ResponsePathElement}; + use crate::plugins::telemetry::apollo_exporter::proto::trace::QueryPlanNode; use crate::plugins::telemetry::config::ExposeTraceId; - use crate::plugins::telemetry::tracing::apollo_telemetry::extract_ftv1_trace; - use crate::plugins::telemetry::tracing::apollo_telemetry::extract_i64; - use crate::plugins::telemetry::tracing::apollo_telemetry::extract_json; - use crate::plugins::telemetry::tracing::apollo_telemetry::extract_path; - use crate::plugins::telemetry::tracing::apollo_telemetry::extract_string; - use crate::plugins::telemetry::tracing::apollo_telemetry::ChildNodes; - use crate::plugins::telemetry::tracing::apollo_telemetry::Exporter; - use crate::plugins::telemetry::tracing::apollo_telemetry::TreeData; - use crate::spaceport; - use crate::spaceport::trace::query_plan_node::response_path_element::Id; - use crate::spaceport::trace::query_plan_node::DeferNodePrimary; - use crate::spaceport::trace::query_plan_node::DeferredNode; - use crate::spaceport::trace::query_plan_node::ResponsePathElement; - use crate::spaceport::trace::QueryPlanNode; - - async fn report(mut exporter: Exporter, spandata: &str) -> SingleReport { - let spandata = serde_yaml::from_str(spandata).expect("test spans must be parsable"); + use crate::plugins::telemetry::tracing::apollo_telemetry::proto::trace::query_plan_node::response_path_element::Id; + use crate::plugins::telemetry::tracing::apollo_telemetry::{ChildNodes, Exporter, extract_ftv1_trace, extract_i64, extract_json, extract_path, extract_string, ReportExporter, TreeData}; + + fn load_span_data(spandata: &str) -> Vec { + // Serde support was removed from otel 0.18 + let value: Vec = + serde_yaml::from_str(spandata).expect("test spans must be parsable"); + value + .iter() + .map(|v| { + let span_data = v.as_mapping().expect("expected mapping"); + let span_context = span_data + .get(&serde_yaml::Value::String("span_context".into())) + .expect("expected span_context") + .as_mapping() + .expect("expected mapping"); + let mut attributes = EvictedHashMap::new(256, 256); + for (key, value) in span_data + .get(&serde_yaml::Value::String("attributes".into())) + .expect("expected attributes") + .as_mapping() + .expect("expected mapping") + .get(&serde_yaml::Value::String("map".into())) + .expect("expected map") + .as_mapping() + .expect("expected mapping") + { + let value: Value = match value + .as_mapping() + .expect("expected mapping") + .iter() + .map(|(k, v)| (k.as_str().expect("expected str"), v)) + .next() + .expect("expected value") + { + ("Bool", serde_yaml::Value::Bool(b)) => Value::Bool(*b), + ("I64", serde_yaml::Value::Number(n)) if n.is_i64() => { + Value::I64(n.as_i64().expect("qed")) + } + ("F64", serde_yaml::Value::Number(n)) if n.is_f64() => { + Value::F64(n.as_f64().expect("qed")) + } + ("String", serde_yaml::Value::String(s)) => s.clone().into(), + _ => panic!("unexpected value type {:?}", value), + }; + attributes.insert(KeyValue::new( + Key::from(key.as_str().expect("expected str").to_string()), + value, + )); + } + SpanData { + span_context: SpanContext::new( + TraceId::from_bytes( + u128::from_str( + span_context + .get(&serde_yaml::Value::String("trace_id".into())) + .expect("expected trace_id") + .as_str() + .expect("expected str"), + ) + .expect("expected u128 parse") + .to_be_bytes(), + ), + SpanId::from_bytes( + span_context + .get(&serde_yaml::Value::String("span_id".into())) + .expect("expected span_id") + .as_u64() + .expect("expected u64") + .to_be_bytes(), + ), + Default::default(), + false, + Default::default(), + ), + parent_span_id: SpanId::from_bytes( + span_data + .get(&serde_yaml::Value::String("parent_span_id".into())) + .cloned() + .unwrap_or_else(|| serde_yaml::Value::Number(0.into())) + .as_u64() + .expect("expected u64") + .to_be_bytes(), + ), + span_kind: SpanKind::Client, + name: span_data + .get(&serde_yaml::Value::String("name".into())) + .expect("name") + .as_str() + .expect("expected str") + .to_string() + .into(), + start_time: SystemTime::now(), + end_time: SystemTime::now(), + attributes, + events: EvictedQueue::new(100), + links: EvictedQueue::new(100), + status: Default::default(), + resource: Default::default(), + instrumentation_lib: Default::default(), + } + }) + .collect() + } + + async fn report(mut exporter: Exporter, spandata: &str) -> Report { + let spandata = load_span_data(spandata); exporter .export(spandata) .await .expect("span export must succeed"); - assert!(matches!(exporter.apollo_sender, Sender::InMemory(_))); - if let Sender::InMemory(storage) = exporter.apollo_sender { + assert!(matches!( + exporter.report_exporter, + ReportExporter::InMemory(_) + )); + if let ReportExporter::InMemory(storage) = exporter.report_exporter { return storage .lock() .expect("lock poisoned") @@ -896,7 +1038,7 @@ mod test { fn test_extract_json() { let val = json!({"hi": "there"}); assert_eq!( - extract_json::(&Value::String(Cow::Owned(val.to_string()))), + extract_json::(&Value::String(val.to_string().into())), Some(val) ); } @@ -904,7 +1046,7 @@ mod test { #[test] fn test_extract_string() { assert_eq!( - extract_string(&Value::String(Cow::Owned("hi".to_string()))), + extract_string(&Value::String("hi".into())), Some("hi".to_string()) ); } @@ -912,7 +1054,7 @@ mod test { #[test] fn test_extract_path() { assert_eq!( - extract_path(&Value::String(Cow::Owned("/hi/3/there".to_string()))), + extract_path(&Value::String("/hi/3/there".into())), vec![ ResponsePathElement { id: Some(Id::FieldName("hi".to_string())), @@ -934,10 +1076,10 @@ mod test { #[test] fn test_extract_ftv1_trace() { - let trace = spaceport::Trace::default(); + let trace = Trace::default(); let encoded = base64::encode(trace.encode_to_vec()); assert_eq!( - *extract_ftv1_trace(&Value::String(Cow::Owned(encoded))) + *extract_ftv1_trace(&Value::String(encoded.into())) .expect("there was a trace here") .expect("the trace must be decoded"), trace diff --git a/apollo-router/src/plugins/telemetry/tracing/datadog.rs b/apollo-router/src/plugins/telemetry/tracing/datadog.rs index 285d3dce68..466dcdbe51 100644 --- a/apollo-router/src/plugins/telemetry/tracing/datadog.rs +++ b/apollo-router/src/plugins/telemetry/tracing/datadog.rs @@ -10,6 +10,7 @@ use super::deser_endpoint; use super::AgentEndpoint; use crate::plugins::telemetry::config::GenericWith; use crate::plugins::telemetry::config::Trace; +use crate::plugins::telemetry::tracing::BatchProcessorConfig; use crate::plugins::telemetry::tracing::SpanProcessorExt; use crate::plugins::telemetry::tracing::TracingConfigurator; @@ -19,6 +20,8 @@ pub(crate) struct Config { #[serde(deserialize_with = "deser_endpoint")] #[schemars(with = "String", default = "default_agent_endpoint")] pub(crate) endpoint: AgentEndpoint, + + pub(crate) batch_processor: Option, } const fn default_agent_endpoint() -> &'static str { "default" @@ -26,7 +29,10 @@ const fn default_agent_endpoint() -> &'static str { impl TracingConfigurator for Config { fn apply(&self, builder: Builder, trace_config: &Trace) -> Result { - tracing::debug!("configuring Datadog tracing"); + tracing::info!( + "configuring Datadog tracing: {}", + self.batch_processor.as_ref().cloned().unwrap_or_default() + ); let url = match &self.endpoint { AgentEndpoint::Default(_) => None, AgentEndpoint::Url(s) => Some(s), @@ -40,6 +46,13 @@ impl TracingConfigurator for Config { .build_exporter()?; Ok(builder.with_span_processor( BatchSpanProcessor::builder(exporter, opentelemetry::runtime::Tokio) + .with_batch_config( + self.batch_processor + .as_ref() + .cloned() + .unwrap_or_default() + .into(), + ) .build() .filtered(), )) diff --git a/apollo-router/src/plugins/telemetry/tracing/jaeger.rs b/apollo-router/src/plugins/telemetry/tracing/jaeger.rs index b0246e056f..1e24563ce3 100644 --- a/apollo-router/src/plugins/telemetry/tracing/jaeger.rs +++ b/apollo-router/src/plugins/telemetry/tracing/jaeger.rs @@ -1,8 +1,14 @@ //! Configuration for jaeger tracing. -use std::time::Duration; +use std::fmt::Debug; +use opentelemetry::sdk::export::trace::SpanData; use opentelemetry::sdk::trace::BatchSpanProcessor; use opentelemetry::sdk::trace::Builder; +use opentelemetry::sdk::trace::Span; +use opentelemetry::sdk::trace::SpanProcessor; +use opentelemetry::sdk::trace::TracerProvider; +use opentelemetry::trace::TraceResult; +use opentelemetry::Context; use schemars::gen::SchemaGenerator; use schemars::schema::Schema; use schemars::schema::SchemaObject; @@ -16,6 +22,7 @@ use super::deser_endpoint; use super::AgentEndpoint; use crate::plugins::telemetry::config::GenericWith; use crate::plugins::telemetry::config::Trace; +use crate::plugins::telemetry::tracing::BatchProcessorConfig; use crate::plugins::telemetry::tracing::SpanProcessorExt; use crate::plugins::telemetry::tracing::TracingConfigurator; @@ -26,9 +33,9 @@ pub(crate) struct Config { #[schemars(schema_with = "endpoint_schema")] pub(crate) endpoint: Endpoint, - #[serde(deserialize_with = "humantime_serde::deserialize", default)] - #[schemars(with = "String", default)] - pub(crate) scheduled_delay: Option, + #[serde(default)] + #[schemars(default)] + pub(crate) batch_processor: Option, } // This is needed because of the use of flatten. @@ -45,11 +52,10 @@ fn endpoint_schema(gen: &mut SchemaGenerator) -> Schema { .iter_mut() .for_each(|s| { if let Schema::Object(o) = s { - o.object - .as_mut() - .unwrap() - .properties - .insert("scheduled_delay".to_string(), String::json_schema(gen)); + o.object.as_mut().unwrap().properties.insert( + "batch_processor".to_string(), + BatchProcessorConfig::json_schema(gen), + ); } }); @@ -77,8 +83,11 @@ fn default_agent_endpoint() -> &'static str { impl TracingConfigurator for Config { fn apply(&self, builder: Builder, trace_config: &Trace) -> Result { - tracing::debug!("configuring Jaeger tracing"); - let exporter = match &self.endpoint { + tracing::info!( + "configuring Jaeger tracing: {}", + self.batch_processor.as_ref().cloned().unwrap_or_default() + ); + match &self.endpoint { Endpoint::Agent { endpoint } => { let socket = match endpoint { AgentEndpoint::Default(_) => None, @@ -89,30 +98,73 @@ impl TracingConfigurator for Config { Some(socket_addr) } }; - opentelemetry_jaeger::new_pipeline() + let exporter = opentelemetry_jaeger::new_agent_pipeline() .with_trace_config(trace_config.into()) .with(&trace_config.service_name, |b, n| b.with_service_name(n)) - .with(&socket, |b, s| b.with_agent_endpoint(s)) - .init_async_exporter(opentelemetry::runtime::Tokio)? + .with(&socket, |b, s| b.with_endpoint(s)) + .build_async_agent_exporter(opentelemetry::runtime::Tokio)?; + Ok(builder.with_span_processor( + BatchSpanProcessor::builder(exporter, opentelemetry::runtime::Tokio) + .with_batch_config( + self.batch_processor + .as_ref() + .cloned() + .unwrap_or_default() + .into(), + ) + .build() + .filtered(), + )) } Endpoint::Collector { endpoint, username, password, - } => opentelemetry_jaeger::new_pipeline() - .with_trace_config(trace_config.into()) - .with(&trace_config.service_name, |b, n| b.with_service_name(n)) - .with(username, |b, u| b.with_collector_username(u)) - .with(password, |b, p| b.with_collector_password(p)) - .with_collector_endpoint(&endpoint.to_string()) - .init_async_exporter(opentelemetry::runtime::Tokio)?, - }; + .. + } => { + // We are waiting for a release of https://github.com/open-telemetry/opentelemetry-rust/issues/894 + // Until that time we need to wrap a tracer provider with Jeager in. + let tracer_provider = opentelemetry_jaeger::new_collector_pipeline() + .with_trace_config(trace_config.into()) + .with(&trace_config.service_name, |b, n| b.with_service_name(n)) + .with(username, |b, u| b.with_username(u)) + .with(password, |b, p| b.with_password(p)) + .with_endpoint(&endpoint.to_string()) + .with_reqwest() + .with_batch_processor_config( + self.batch_processor + .as_ref() + .cloned() + .unwrap_or_default() + .into(), + ) + .build_batch(opentelemetry::runtime::Tokio)?; + Ok(builder.with_span_processor(DelegateSpanProcessor { tracer_provider })) + } + } + } +} + +#[derive(Debug)] +struct DelegateSpanProcessor { + tracer_provider: TracerProvider, +} + +impl SpanProcessor for DelegateSpanProcessor { + fn on_start(&self, span: &mut Span, cx: &Context) { + self.tracer_provider.span_processors()[0].on_start(span, cx) + } + + fn on_end(&self, span: SpanData) { + self.tracer_provider.span_processors()[0].on_end(span) + } + + fn force_flush(&self) -> TraceResult<()> { + self.tracer_provider.span_processors()[0].force_flush() + } - Ok(builder.with_span_processor( - BatchSpanProcessor::builder(exporter, opentelemetry::runtime::Tokio) - .with(&self.scheduled_delay, |b, d| b.with_scheduled_delay(*d)) - .build() - .filtered(), - )) + fn shutdown(&mut self) -> TraceResult<()> { + // It's safe to not call shutdown as dropping tracer_provider will cause shutdown to happen separately. + Ok(()) } } diff --git a/apollo-router/src/plugins/telemetry/tracing/mod.rs b/apollo-router/src/plugins/telemetry/tracing/mod.rs index 04e47f7654..cd80d6b883 100644 --- a/apollo-router/src/plugins/telemetry/tracing/mod.rs +++ b/apollo-router/src/plugins/telemetry/tracing/mod.rs @@ -1,4 +1,9 @@ +use std::fmt::Display; +use std::fmt::Formatter; +use std::time::Duration; + use opentelemetry::sdk::export::trace::SpanData; +use opentelemetry::sdk::trace::BatchConfig; use opentelemetry::sdk::trace::Builder; use opentelemetry::sdk::trace::EvictedHashMap; use opentelemetry::sdk::trace::Span; @@ -142,3 +147,71 @@ where ApolloFilterSpanProcessor { delegate: self } } } + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] +pub(crate) struct BatchProcessorConfig { + #[serde(deserialize_with = "humantime_serde::deserialize", default)] + #[schemars(with = "String", default)] + /// The delay interval in milliseconds between two consecutive processing + /// of batches. The default value is 5 seconds. + scheduled_delay: Option, + + /// The maximum queue size to buffer spans for delayed processing. If the + /// queue gets full it drops the spans. The default value of is 2048. + #[schemars(default)] + #[serde(default)] + max_queue_size: Option, + + /// The maximum number of spans to process in a single batch. If there are + /// more than one batch worth of spans then it processes multiple batches + /// of spans one batch after the other without any delay. The default value + /// is 512. + #[schemars(default)] + #[serde(default)] + max_export_batch_size: Option, + + #[serde(deserialize_with = "humantime_serde::deserialize", default)] + #[schemars(with = "String", default)] + /// The maximum duration to export a batch of data. + max_export_timeout: Option, + + /// Maximum number of concurrent exports + /// + /// Limits the number of spawned tasks for exports and thus memory consumed + /// by an exporter. A value of 1 will cause exports to be performed + /// synchronously on the BatchSpanProcessor task. + #[schemars(default)] + #[serde(default)] + max_concurrent_exports: Option, +} + +impl From for BatchConfig { + fn from(config: BatchProcessorConfig) -> Self { + let mut default = BatchConfig::default(); + if let Some(scheduled_delay) = config.scheduled_delay { + default = default.with_scheduled_delay(scheduled_delay); + } + if let Some(max_queue_size) = config.max_queue_size { + default = default.with_max_queue_size(max_queue_size); + } + if let Some(max_export_batch_size) = config.max_export_batch_size { + default = default.with_max_export_batch_size(max_export_batch_size); + } + if let Some(max_export_timeout) = config.max_export_timeout { + default = default.with_max_export_timeout(max_export_timeout); + } + if let Some(max_concurrent_exports) = config.max_concurrent_exports { + default = default.with_max_concurrent_exports(max_concurrent_exports); + } + default + } +} + +impl Display for BatchProcessorConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let batch_config: BatchConfig = self.clone().into(); + let debug_str = format!("{:?}", batch_config); + // Yes horrible, but there is no other way to get at the actual configured values. + f.write_str(&debug_str["BatchConfig { ".len()..debug_str.len() - 1]) + } +} diff --git a/apollo-router/src/plugins/telemetry/tracing/otlp.rs b/apollo-router/src/plugins/telemetry/tracing/otlp.rs index 656d8f3970..7c6ad4f945 100644 --- a/apollo-router/src/plugins/telemetry/tracing/otlp.rs +++ b/apollo-router/src/plugins/telemetry/tracing/otlp.rs @@ -12,13 +12,23 @@ use crate::plugins::telemetry::tracing::TracingConfigurator; impl TracingConfigurator for super::super::otlp::Config { fn apply(&self, builder: Builder, _trace_config: &Trace) -> Result { - tracing::debug!("configuring Otlp tracing"); + tracing::info!( + "configuring Otlp tracing: {}", + self.batch_processor.as_ref().cloned().unwrap_or_default() + ); let exporter: SpanExporterBuilder = self.exporter()?; Ok(builder.with_span_processor( BatchSpanProcessor::builder( exporter.build_span_exporter()?, opentelemetry::runtime::Tokio, ) + .with_batch_config( + self.batch_processor + .as_ref() + .cloned() + .unwrap_or_default() + .into(), + ) .build() .filtered(), )) diff --git a/apollo-router/src/plugins/telemetry/tracing/snapshots/apollo_router__plugins__telemetry__tracing__apollo_telemetry__test__condition_else.snap b/apollo-router/src/plugins/telemetry/tracing/snapshots/apollo_router__plugins__telemetry__tracing__apollo_telemetry__test__condition_else.snap index b7fc231720..5dd95f672c 100644 --- a/apollo-router/src/plugins/telemetry/tracing/snapshots/apollo_router__plugins__telemetry__tracing__apollo_telemetry__test__condition_else.snap +++ b/apollo-router/src/plugins/telemetry/tracing/snapshots/apollo_router__plugins__telemetry__tracing__apollo_telemetry__test__condition_else.snap @@ -2,9 +2,9 @@ source: apollo-router/src/plugins/telemetry/tracing/apollo_telemetry.rs expression: report --- -Traces: - traces: - - - "# -\nquery($if:Boolean!){topProducts{name...@defer(if:$if){reviews{author{name}}reviews{author{name}}}}}" +traces_per_query: + "# -\nquery($if:Boolean!){topProducts{name...@defer(if:$if){reviews{author{name}}reviews{author{name}}}}}": + traces: - start_time: seconds: "[seconds]" nanos: "[nanos]" @@ -590,4 +590,7 @@ Traces: registered_operation: false forbidden_operation: false field_execution_weight: 1 + stats_with_context: [] + referenced_fields_by_type: {} +operation_count: 1 diff --git a/apollo-router/src/plugins/telemetry/tracing/snapshots/apollo_router__plugins__telemetry__tracing__apollo_telemetry__test__condition_if.snap b/apollo-router/src/plugins/telemetry/tracing/snapshots/apollo_router__plugins__telemetry__tracing__apollo_telemetry__test__condition_if.snap index 72ca9af214..f20b561f4a 100644 --- a/apollo-router/src/plugins/telemetry/tracing/snapshots/apollo_router__plugins__telemetry__tracing__apollo_telemetry__test__condition_if.snap +++ b/apollo-router/src/plugins/telemetry/tracing/snapshots/apollo_router__plugins__telemetry__tracing__apollo_telemetry__test__condition_if.snap @@ -2,9 +2,9 @@ source: apollo-router/src/plugins/telemetry/tracing/apollo_telemetry.rs expression: report --- -Traces: - traces: - - - "# -\nquery($if:Boolean!){topProducts{name...@defer(if:$if){reviews{author{name}}reviews{author{name}}}}}" +traces_per_query: + "# -\nquery($if:Boolean!){topProducts{name...@defer(if:$if){reviews{author{name}}reviews{author{name}}}}}": + traces: - start_time: seconds: "[seconds]" nanos: "[nanos]" @@ -603,4 +603,7 @@ Traces: registered_operation: false forbidden_operation: false field_execution_weight: 1 + stats_with_context: [] + referenced_fields_by_type: {} +operation_count: 1 diff --git a/apollo-router/src/plugins/telemetry/tracing/snapshots/apollo_router__plugins__telemetry__tracing__apollo_telemetry__test__trace_id.snap b/apollo-router/src/plugins/telemetry/tracing/snapshots/apollo_router__plugins__telemetry__tracing__apollo_telemetry__test__trace_id.snap index 706a60cea1..240bdb604e 100644 --- a/apollo-router/src/plugins/telemetry/tracing/snapshots/apollo_router__plugins__telemetry__tracing__apollo_telemetry__test__trace_id.snap +++ b/apollo-router/src/plugins/telemetry/tracing/snapshots/apollo_router__plugins__telemetry__tracing__apollo_telemetry__test__trace_id.snap @@ -2,9 +2,9 @@ source: apollo-router/src/plugins/telemetry/tracing/apollo_telemetry.rs expression: report --- -Traces: - traces: - - - "# -\nquery($if:Boolean!){topProducts{name...@defer(if:$if){reviews{author{name}}reviews{author{name}}}}}" +traces_per_query: + "# -\nquery($if:Boolean!){topProducts{name...@defer(if:$if){reviews{author{name}}reviews{author{name}}}}}": + traces: - start_time: seconds: "[seconds]" nanos: "[nanos]" @@ -606,4 +606,7 @@ Traces: registered_operation: false forbidden_operation: false field_execution_weight: 1 + stats_with_context: [] + referenced_fields_by_type: {} +operation_count: 1 diff --git a/apollo-router/src/plugins/telemetry/tracing/testdata/condition_else_spandata.yaml b/apollo-router/src/plugins/telemetry/tracing/testdata/condition_else_spandata.yaml index 158ec9527b..6d5cfd5e32 100644 --- a/apollo-router/src/plugins/telemetry/tracing/testdata/condition_else_spandata.yaml +++ b/apollo-router/src/plugins/telemetry/tracing/testdata/condition_else_spandata.yaml @@ -1,6 +1,6 @@ --- - span_context: - trace_id: 122960190946202642409915189222685399094 + trace_id: "122960190946202642409915189222685399094" span_id: 14287822546131581520 trace_flags: 1 is_remote: false @@ -85,7 +85,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 122960190946202642409915189222685399094 + trace_id: "122960190946202642409915189222685399094" span_id: 15237674702119070751 trace_flags: 1 is_remote: false @@ -152,7 +152,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 122960190946202642409915189222685399094 + trace_id: "122960190946202642409915189222685399094" span_id: 13865784024556574003 trace_flags: 1 is_remote: false @@ -213,7 +213,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 122960190946202642409915189222685399094 + trace_id: "122960190946202642409915189222685399094" span_id: 6782401025275156233 trace_flags: 1 is_remote: false @@ -277,7 +277,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 122960190946202642409915189222685399094 + trace_id: "122960190946202642409915189222685399094" span_id: 8824695332656836540 trace_flags: 1 is_remote: false @@ -338,7 +338,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 122960190946202642409915189222685399094 + trace_id: "122960190946202642409915189222685399094" span_id: 16743763821444799861 trace_flags: 1 is_remote: false @@ -399,7 +399,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 122960190946202642409915189222685399094 + trace_id: "122960190946202642409915189222685399094" span_id: 17407667985756513925 trace_flags: 1 is_remote: false @@ -466,7 +466,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 122960190946202642409915189222685399094 + trace_id: "122960190946202642409915189222685399094" span_id: 7936167443054682587 trace_flags: 1 is_remote: false @@ -530,7 +530,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 122960190946202642409915189222685399094 + trace_id: "122960190946202642409915189222685399094" span_id: 16253144712595734830 trace_flags: 1 is_remote: false @@ -594,7 +594,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 122960190946202642409915189222685399094 + trace_id: "122960190946202642409915189222685399094" span_id: 15278772753518055685 trace_flags: 1 is_remote: false @@ -661,7 +661,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 122960190946202642409915189222685399094 + trace_id: "122960190946202642409915189222685399094" span_id: 11170049527526553842 trace_flags: 1 is_remote: false @@ -734,7 +734,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 122960190946202642409915189222685399094 + trace_id: "122960190946202642409915189222685399094" span_id: 12193731640433132388 trace_flags: 1 is_remote: false @@ -810,7 +810,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 122960190946202642409915189222685399094 + trace_id: "122960190946202642409915189222685399094" span_id: 17716600377366594972 trace_flags: 1 is_remote: false @@ -877,7 +877,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 122960190946202642409915189222685399094 + trace_id: "122960190946202642409915189222685399094" span_id: 12581790234036960487 trace_flags: 1 is_remote: false @@ -950,7 +950,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 122960190946202642409915189222685399094 + trace_id: "122960190946202642409915189222685399094" span_id: 13059514760148115210 trace_flags: 1 is_remote: false @@ -1026,7 +1026,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 122960190946202642409915189222685399094 + trace_id: "122960190946202642409915189222685399094" span_id: 12520246751970884479 trace_flags: 1 is_remote: false @@ -1099,7 +1099,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 122960190946202642409915189222685399094 + trace_id: "122960190946202642409915189222685399094" span_id: 3398738514204614748 trace_flags: 1 is_remote: false @@ -1175,7 +1175,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 122960190946202642409915189222685399094 + trace_id: "122960190946202642409915189222685399094" span_id: 17691584597033290046 trace_flags: 1 is_remote: false diff --git a/apollo-router/src/plugins/telemetry/tracing/testdata/condition_if_spandata.yaml b/apollo-router/src/plugins/telemetry/tracing/testdata/condition_if_spandata.yaml index a9ced9409c..0dd3f4b1cd 100644 --- a/apollo-router/src/plugins/telemetry/tracing/testdata/condition_if_spandata.yaml +++ b/apollo-router/src/plugins/telemetry/tracing/testdata/condition_if_spandata.yaml @@ -1,6 +1,6 @@ --- - span_context: - trace_id: 48179106241482103848812575548686436429 + trace_id: "48179106241482103848812575548686436429" span_id: 2226108203856432052 trace_flags: 1 is_remote: false @@ -85,7 +85,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 48179106241482103848812575548686436429 + trace_id: "48179106241482103848812575548686436429" span_id: 7005602647906255377 trace_flags: 1 is_remote: false @@ -152,7 +152,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 48179106241482103848812575548686436429 + trace_id: "48179106241482103848812575548686436429" span_id: 5496672787700345629 trace_flags: 1 is_remote: false @@ -213,7 +213,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 48179106241482103848812575548686436429 + trace_id: "48179106241482103848812575548686436429" span_id: 8052379820523592996 trace_flags: 1 is_remote: false @@ -277,7 +277,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 48179106241482103848812575548686436429 + trace_id: "48179106241482103848812575548686436429" span_id: 14511206469911177014 trace_flags: 1 is_remote: false @@ -338,7 +338,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 48179106241482103848812575548686436429 + trace_id: "48179106241482103848812575548686436429" span_id: 12847835672039219100 trace_flags: 1 is_remote: false @@ -399,7 +399,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 48179106241482103848812575548686436429 + trace_id: "48179106241482103848812575548686436429" span_id: 4412163015592220412 trace_flags: 1 is_remote: false @@ -460,7 +460,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 48179106241482103848812575548686436429 + trace_id: "48179106241482103848812575548686436429" span_id: 1343706171599318485 trace_flags: 1 is_remote: false @@ -527,7 +527,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 48179106241482103848812575548686436429 + trace_id: "48179106241482103848812575548686436429" span_id: 1615094331109834215 trace_flags: 1 is_remote: false @@ -588,7 +588,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 48179106241482103848812575548686436429 + trace_id: "48179106241482103848812575548686436429" span_id: 9487579781210749637 trace_flags: 1 is_remote: false @@ -652,7 +652,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 48179106241482103848812575548686436429 + trace_id: "48179106241482103848812575548686436429" span_id: 4988545802786443624 trace_flags: 1 is_remote: false @@ -716,7 +716,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 48179106241482103848812575548686436429 + trace_id: "48179106241482103848812575548686436429" span_id: 4477074139770004131 trace_flags: 1 is_remote: false @@ -783,7 +783,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 48179106241482103848812575548686436429 + trace_id: "48179106241482103848812575548686436429" span_id: 4307400006762176495 trace_flags: 1 is_remote: false @@ -856,7 +856,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 48179106241482103848812575548686436429 + trace_id: "48179106241482103848812575548686436429" span_id: 15682297860951097268 trace_flags: 1 is_remote: false @@ -932,7 +932,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 48179106241482103848812575548686436429 + trace_id: "48179106241482103848812575548686436429" span_id: 374273098096638287 trace_flags: 1 is_remote: false @@ -999,7 +999,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 48179106241482103848812575548686436429 + trace_id: "48179106241482103848812575548686436429" span_id: 12633132268468348696 trace_flags: 1 is_remote: false @@ -1072,7 +1072,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 48179106241482103848812575548686436429 + trace_id: "48179106241482103848812575548686436429" span_id: 11564887256002176974 trace_flags: 1 is_remote: false @@ -1148,7 +1148,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 48179106241482103848812575548686436429 + trace_id: "48179106241482103848812575548686436429" span_id: 13334624373206909001 trace_flags: 1 is_remote: false @@ -1215,7 +1215,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 48179106241482103848812575548686436429 + trace_id: "48179106241482103848812575548686436429" span_id: 357853883512773358 trace_flags: 1 is_remote: false @@ -1288,7 +1288,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 48179106241482103848812575548686436429 + trace_id: "48179106241482103848812575548686436429" span_id: 18275235823533906477 trace_flags: 1 is_remote: false @@ -1364,7 +1364,7 @@ service.version: String: 1.4.0 - span_context: - trace_id: 48179106241482103848812575548686436429 + trace_id: "48179106241482103848812575548686436429" span_id: 6681943477281517192 trace_flags: 1 is_remote: false diff --git a/apollo-router/src/plugins/telemetry/tracing/zipkin.rs b/apollo-router/src/plugins/telemetry/tracing/zipkin.rs index 1b44b169df..7a0471b636 100644 --- a/apollo-router/src/plugins/telemetry/tracing/zipkin.rs +++ b/apollo-router/src/plugins/telemetry/tracing/zipkin.rs @@ -12,6 +12,7 @@ use super::AgentDefault; use super::AgentEndpoint; use crate::plugins::telemetry::config::GenericWith; use crate::plugins::telemetry::config::Trace; +use crate::plugins::telemetry::tracing::BatchProcessorConfig; use crate::plugins::telemetry::tracing::SpanProcessorExt; use crate::plugins::telemetry::tracing::TracingConfigurator; @@ -21,6 +22,8 @@ pub(crate) struct Config { #[schemars(with = "String", default = "default_agent_endpoint")] #[serde(deserialize_with = "deser_endpoint")] pub(crate) endpoint: AgentEndpoint, + + pub(crate) batch_processor: Option, } const fn default_agent_endpoint() -> &'static str { @@ -49,7 +52,10 @@ where impl TracingConfigurator for Config { fn apply(&self, builder: Builder, trace_config: &Trace) -> Result { - tracing::debug!("configuring Zipkin tracing"); + tracing::info!( + "configuring Zipkin tracing: {}", + self.batch_processor.as_ref().cloned().unwrap_or_default() + ); let collector_endpoint = match &self.endpoint { AgentEndpoint::Default(_) => None, AgentEndpoint::Url(url) => Some(url), @@ -65,6 +71,13 @@ impl TracingConfigurator for Config { Ok(builder.with_span_processor( BatchSpanProcessor::builder(exporter, opentelemetry::runtime::Tokio) + .with_batch_config( + self.batch_processor + .as_ref() + .cloned() + .unwrap_or_default() + .into(), + ) .build() .filtered(), )) diff --git a/apollo-router/src/plugins/traffic_shaping/mod.rs b/apollo-router/src/plugins/traffic_shaping/mod.rs index bd0d277355..8c58863f92 100644 --- a/apollo-router/src/plugins/traffic_shaping/mod.rs +++ b/apollo-router/src/plugins/traffic_shaping/mod.rs @@ -485,7 +485,7 @@ mod test { async fn get_traffic_shaping_plugin(config: &serde_json::Value) -> Box { // Build a traffic shaping plugin crate::plugin::plugins() - .get(APOLLO_TRAFFIC_SHAPING) + .find(|factory| factory.name == APOLLO_TRAFFIC_SHAPING) .expect("Plugin not found") .create_instance_without_schema(config) .await @@ -503,7 +503,7 @@ mod test { // Build a traffic shaping plugin let plugin = get_traffic_shaping_plugin(&config).await; let router = build_mock_router_with_variable_dedup_optimization(plugin).await; - execute_router_test(VALID_QUERY, &*EXPECTED_RESPONSE, router).await; + execute_router_test(VALID_QUERY, &EXPECTED_RESPONSE, router).await; } #[tokio::test] @@ -579,7 +579,7 @@ mod test { ); } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn it_rate_limit_subgraph_requests() { let config = serde_yaml::from_str::( r#" @@ -587,7 +587,7 @@ mod test { test: global_rate_limit: capacity: 1 - interval: 300ms + interval: 100ms timeout: 500ms "#, ) @@ -636,7 +636,7 @@ mod test { .unwrap(); } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn it_rate_limit_router_requests() { let config = serde_yaml::from_str::( r#" diff --git a/apollo-router/src/query_planner/bridge_query_planner.rs b/apollo-router/src/query_planner/bridge_query_planner.rs index 8fd072e311..6e3323bc9b 100644 --- a/apollo-router/src/query_planner/bridge_query_planner.rs +++ b/apollo-router/src/query_planner/bridge_query_planner.rs @@ -5,7 +5,6 @@ use std::fmt::Debug; use std::sync::Arc; use futures::future::BoxFuture; -use opentelemetry::trace::SpanKind; use router_bridge::planner::IncrementalDeliverySupport; use router_bridge::planner::PlanSuccess; use router_bridge::planner::Planner; @@ -73,7 +72,7 @@ impl BridgeQueryPlanner { let configuration = self.configuration.clone(); let query_parsing_future = tokio::task::spawn_blocking(move || Query::parse(query, &schema, &configuration)) - .instrument(tracing::info_span!("parse_query", "otel.kind" = %SpanKind::Internal)); + .instrument(tracing::info_span!("parse_query", "otel.kind" = "INTERNAL")); match query_parsing_future.await { Ok(res) => res.map_err(QueryPlannerError::from), Err(err) => { @@ -122,7 +121,7 @@ impl BridgeQueryPlanner { }, usage_reporting, } => { - let subselections = node.parse_subselections(&*self.schema)?; + let subselections = node.parse_subselections(&self.schema)?; selections.subselections = subselections; Ok(QueryPlannerContent::Plan { plan: Arc::new(query_planner::QueryPlan { diff --git a/apollo-router/src/query_planner/caching_query_planner.rs b/apollo-router/src/query_planner/caching_query_planner.rs index 7f4d2861be..a2e263a8ed 100644 --- a/apollo-router/src/query_planner/caching_query_planner.rs +++ b/apollo-router/src/query_planner/caching_query_planner.rs @@ -94,7 +94,7 @@ where } if let Some(QueryPlannerContent::Plan { plan, .. }) = &content { - match (&plan.usage_reporting).serialize(Serializer) { + match (plan.usage_reporting).serialize(Serializer) { Ok(v) => { context.insert_json_value(USAGE_REPORTING, v); } @@ -127,7 +127,7 @@ where match res { Ok(content) => { if let QueryPlannerContent::Plan { plan, .. } = &content { - match (&plan.usage_reporting).serialize(Serializer) { + match (plan.usage_reporting).serialize(Serializer) { Ok(v) => { context.insert_json_value(USAGE_REPORTING, v); } diff --git a/apollo-router/src/query_planner/execution.rs b/apollo-router/src/query_planner/execution.rs index 363115b02b..14f05fd764 100644 --- a/apollo-router/src/query_planner/execution.rs +++ b/apollo-router/src/query_planner/execution.rs @@ -3,7 +3,6 @@ use std::sync::Arc; use futures::future::join_all; use futures::prelude::*; -use opentelemetry::trace::SpanKind; use tokio::sync::broadcast::Sender; use tokio_stream::wrappers::BroadcastStream; use tracing::Instrument; @@ -129,9 +128,10 @@ impl PlanNode { subselection = subselect; } } - .instrument( - tracing::info_span!(SEQUENCE_SPAN_NAME, "otel.kind" = %SpanKind::Internal), - ) + .instrument(tracing::info_span!( + SEQUENCE_SPAN_NAME, + "otel.kind" = "INTERNAL" + )) .await } PlanNode::Parallel { nodes } => { @@ -157,9 +157,10 @@ impl PlanNode { errors.extend(err.into_iter()); } } - .instrument( - tracing::info_span!(PARALLEL_SPAN_NAME, "otel.kind" = %SpanKind::Internal), - ) + .instrument(tracing::info_span!( + PARALLEL_SPAN_NAME, + "otel.kind" = "INTERNAL" + )) .await } PlanNode::Flatten(FlattenNode { path, node }) => { @@ -173,7 +174,7 @@ impl PlanNode { parent_value, sender, ) - .instrument(tracing::info_span!(FLATTEN_SPAN_NAME, "graphql.path" = %current_dir, "otel.kind" = %SpanKind::Internal)) + .instrument(tracing::info_span!(FLATTEN_SPAN_NAME, "graphql.path" = %current_dir, "otel.kind" = "INTERNAL")) .await; value = v; @@ -187,7 +188,7 @@ impl PlanNode { .fetch_node(parameters, parent_value, current_dir) .instrument(tracing::info_span!( FETCH_SPAN_NAME, - "otel.kind" = %SpanKind::Internal, + "otel.kind" = "INTERNAL", "apollo.subgraph.name" = fetch_node.service_name.as_str(), "apollo_private.sent_time_offset" = fetch_time_offset )) @@ -220,7 +221,8 @@ impl PlanNode { HashMap::new(); let mut futures = Vec::new(); - let (primary_sender, _) = tokio::sync::broadcast::channel::(1); + let (primary_sender, _) = + tokio::sync::broadcast::channel::<(Value, Vec)>(1); for deferred_node in deferred { let fut = deferred_node @@ -256,19 +258,25 @@ impl PlanNode { &value, sender, ) - .instrument(tracing::info_span!(DEFER_PRIMARY_SPAN_NAME, "otel.kind" = %SpanKind::Internal)) + .instrument(tracing::info_span!( + DEFER_PRIMARY_SPAN_NAME, + "otel.kind" = "INTERNAL" + )) .await; value.deep_merge(v); errors.extend(err.into_iter()); subselection = primary_subselection.clone(); - let _ = primary_sender.send(value.clone()); + let _ = primary_sender.send((value.clone(), errors.clone())); } else { subselection = primary_subselection.clone(); - let _ = primary_sender.send(value.clone()); + let _ = primary_sender.send((value.clone(), errors.clone())); } } - .instrument(tracing::info_span!(DEFER_SPAN_NAME, "otel.kind" = %SpanKind::Internal)) + .instrument(tracing::info_span!( + DEFER_SPAN_NAME, + "otel.kind" = "INTERNAL" + )) .await } PlanNode::Condition { @@ -303,7 +311,10 @@ impl PlanNode { parent_value, sender.clone(), ) - .instrument(tracing::info_span!(CONDITION_IF_SPAN_NAME, "otel.kind" = %SpanKind::Internal)) + .instrument(tracing::info_span!( + CONDITION_IF_SPAN_NAME, + "otel.kind" = "INTERNAL" + )) .await; value.deep_merge(v); errors.extend(err.into_iter()); @@ -317,7 +328,10 @@ impl PlanNode { parent_value, sender.clone(), ) - .instrument(tracing::info_span!(CONDITION_ELSE_SPAN_NAME, "otel.kind" = %SpanKind::Internal)) + .instrument(tracing::info_span!( + CONDITION_ELSE_SPAN_NAME, + "otel.kind" = "INTERNAL" + )) .await; value.deep_merge(v); errors.extend(err.into_iter()); @@ -327,7 +341,7 @@ impl PlanNode { .instrument(tracing::info_span!( CONDITION_SPAN_NAME, "graphql.condition" = condition, - "otel.kind" = %SpanKind::Internal + "otel.kind" = "INTERNAL" )) .await } @@ -344,7 +358,7 @@ impl DeferredNode { parameters: &'a ExecutionParameters<'a, SF>, parent_value: &Value, sender: futures::channel::mpsc::Sender, - primary_sender: &Sender, + primary_sender: &Sender<(Value, Vec)>, deferred_fetches: &mut HashMap)>>, ) -> impl Future where @@ -376,7 +390,7 @@ impl DeferredNode { let mut stream: stream::FuturesUnordered<_> = deferred_receivers.into_iter().collect(); //FIXME/ is there a solution without cloning the entire node? Maybe it could be moved instead? let deferred_inner = self.node.clone(); - let deferred_path = self.path.clone(); + let deferred_path = self.query_path.clone(); let subselection = self.subselection(); let label = self.label.clone(); let mut tx = sender; @@ -393,8 +407,10 @@ impl DeferredNode { let mut errors = Vec::new(); if is_depends_empty { - let primary_value = primary_receiver.recv().await.unwrap_or_default(); + let (primary_value, primary_errors) = + primary_receiver.recv().await.unwrap_or_default(); value.deep_merge(primary_value); + errors.extend(primary_errors.into_iter()) } else { while let Some((v, _remaining)) = stream.next().await { // a Err(RecvError) means either that the fetch was not performed and the @@ -431,13 +447,15 @@ impl DeferredNode { "graphql.label" = label, "graphql.depends" = depends_json, "graphql.path" = deferred_path.to_string(), - "otel.kind" = %SpanKind::Internal + "otel.kind" = "INTERNAL" )) .await; if !is_depends_empty { - let primary_value = primary_receiver.recv().await.unwrap_or_default(); + let (primary_value, primary_errors) = + primary_receiver.recv().await.unwrap_or_default(); v.deep_merge(primary_value); + errors.extend(primary_errors.into_iter()) } if let Err(e) = tx @@ -460,8 +478,10 @@ impl DeferredNode { }; tx.disconnect(); } else { - let primary_value = primary_receiver.recv().await.unwrap_or_default(); + let (primary_value, primary_errors) = + primary_receiver.recv().await.unwrap_or_default(); value.deep_merge(primary_value); + errors.extend(primary_errors.into_iter()); if let Err(e) = tx .send( diff --git a/apollo-router/src/query_planner/fetch.rs b/apollo-router/src/query_planner/fetch.rs index 67adce27d8..4a95f2463b 100644 --- a/apollo-router/src/query_planner/fetch.rs +++ b/apollo-router/src/query_planner/fetch.rs @@ -228,7 +228,7 @@ impl FetchNode { parameters .schema .subgraphs() - .find_map(|(name, url)| (name == service_name).then(|| url)) + .find_map(|(name, url)| (name == service_name).then_some(url)) .unwrap_or_else(|| { panic!( "schema uri for subgraph '{}' should already have been checked", diff --git a/apollo-router/src/query_planner/plan.rs b/apollo-router/src/query_planner/plan.rs index 2635d081b9..699e3a0f58 100644 --- a/apollo-router/src/query_planner/plan.rs +++ b/apollo-router/src/query_planner/plan.rs @@ -183,6 +183,7 @@ impl PlanNode { // re-create full query with the right path // parse the subselection let mut subselections = HashMap::new(); + let operation_kind = if self.contains_mutations() { OperationKind::Mutation } else { @@ -225,6 +226,7 @@ impl PlanNode { let primary_path = initial_path.join(&primary.path.clone().unwrap_or_default()); if let Some(primary_subselection) = &primary.subselection { let query = reconstruct_full_query(&primary_path, kind, primary_subselection); + // ----------------------- Parse --------------------------------- let sub_selection = Query::parse(&query, schema, &Default::default())?; // ----------------------- END Parse --------------------------------- @@ -240,14 +242,15 @@ impl PlanNode { deferred.iter().try_fold(subselections, |subs, current| { if let Some(subselection) = ¤t.subselection { - let query = reconstruct_full_query(¤t.path, kind, subselection); + let query = reconstruct_full_query(¤t.query_path, kind, subselection); + // ----------------------- Parse --------------------------------- let sub_selection = Query::parse(&query, schema, &Default::default())?; // ----------------------- END Parse --------------------------------- subs.insert( SubSelection { - path: current.path.clone(), + path: current.query_path.clone(), subselection: subselection.clone(), }, sub_selection, @@ -256,7 +259,7 @@ impl PlanNode { if let Some(current_node) = ¤t.node { current_node.collect_subselections( schema, - &initial_path.join(¤t.path), + &initial_path.join(¤t.query_path), kind, subs, )?; @@ -342,6 +345,11 @@ fn reconstruct_full_query(path: &Path, kind: &OperationKind, subselection: &str) .expect("writing to a String should not fail because it can reallocate"); len += 1; } + json_ext::PathElement::Fragment(fragment) => { + write!(&mut query, "{{ {fragment}") + .expect("writing to a String should not fail because it can reallocate"); + len += 1; + } } } @@ -392,7 +400,7 @@ pub(crate) struct DeferredNode { /// The optional defer label. pub(crate) label: Option, /// Path to the @defer this correspond to. `subselection` start at that `path`. - pub(crate) path: Path, + pub(crate) query_path: Path, /// The part of the original query that "selects" the data to send /// in that deferred response (once the plan in `node` completes). /// Will be set _unless_ `node` is a `DeferNode` itself. diff --git a/apollo-router/src/query_planner/testdata/defer_clause_plan.json b/apollo-router/src/query_planner/testdata/defer_clause_plan.json index 62de983cdf..f483b0e3b3 100644 --- a/apollo-router/src/query_planner/testdata/defer_clause_plan.json +++ b/apollo-router/src/query_planner/testdata/defer_clause_plan.json @@ -25,7 +25,7 @@ } ], "label": null, - "path": ["me"], + "queryPath": ["me"], "subselection": "{ ... on User { name username } }", "node": { "kind": "Flatten", diff --git a/apollo-router/src/query_planner/tests.rs b/apollo-router/src/query_planner/tests.rs index 7d372f9e9e..acbface5f7 100644 --- a/apollo-router/src/query_planner/tests.rs +++ b/apollo-router/src/query_planner/tests.rs @@ -242,7 +242,7 @@ async fn defer() { defer_label: None, }], label: None, - path: Path(vec![PathElement::Key("t".to_string())]), + query_path: Path(vec![PathElement::Key("t".to_string())]), subselection: Some("{ y }".to_string()), node: Some(Arc::new(PlanNode::Flatten(FlattenNode { path: Path(vec![PathElement::Key("t".to_string())]), diff --git a/apollo-router/src/router.rs b/apollo-router/src/router.rs index 7e0d741ac0..7a4a70438e 100644 --- a/apollo-router/src/router.rs +++ b/apollo-router/src/router.rs @@ -113,7 +113,7 @@ pub enum ApolloRouterError { /// no valid schema was supplied NoSchema, - /// could not create the HTTP pipeline: {0} + /// could not create router: {0} ServiceCreationError(BoxError), /// could not create the HTTP server: {0} diff --git a/apollo-router/src/router_factory.rs b/apollo-router/src/router_factory.rs index 333aae551a..bfbf736085 100644 --- a/apollo-router/src/router_factory.rs +++ b/apollo-router/src/router_factory.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use axum::response::IntoResponse; use http::StatusCode; use multimap::MultiMap; +use once_cell::sync::Lazy; use serde_json::Map; use serde_json::Value; use tower::service_fn; @@ -16,6 +17,7 @@ use crate::configuration::Configuration; use crate::configuration::ConfigurationError; use crate::plugin::DynPlugin; use crate::plugin::Handler; +use crate::plugin::PluginFactory; use crate::plugins::traffic_shaping::TrafficShaping; use crate::plugins::traffic_shaping::APOLLO_TRAFFIC_SHAPING; use crate::services::new_service::NewService; @@ -127,7 +129,7 @@ impl SupergraphServiceConfigurator for YamlSupergraphServiceFactory { let subgraph_service = match plugins .iter() .find(|i| i.0.as_str() == APOLLO_TRAFFIC_SHAPING) - .and_then(|plugin| (&*plugin.1).as_any().downcast_ref::()) + .and_then(|plugin| (*plugin.1).as_any().downcast_ref::()) { Some(shaping) => { Either::A(shaping.subgraph_service_internal(name, SubgraphService::new(name))) @@ -184,7 +186,7 @@ async fn create_plugins( ]; let mut errors = Vec::new(); - let plugin_registry = crate::plugin::plugins(); + let plugin_registry: Vec<&'static Lazy> = crate::plugin::plugins().collect(); let mut plugin_instances = Vec::new(); let extra = extra_plugins.unwrap_or_default(); @@ -194,7 +196,7 @@ async fn create_plugins( continue; } - match plugin_registry.get(name.as_str()) { + match plugin_registry.iter().find(|factory| factory.name == name) { Some(factory) => { tracing::debug!( "creating plugin: '{}' with configuration:\n{:#}", @@ -204,7 +206,6 @@ async fn create_plugins( if name == "apollo.telemetry" { inject_schema_id(schema, &mut configuration); } - // expand any env variables in the config before processing. match factory .create_instance(&configuration, schema.as_string().clone()) .await @@ -241,7 +242,10 @@ async fn create_plugins( } None => { // Didn't find it, insert - match plugin_registry.get(*name) { + match plugin_registry + .iter() + .find(|factory| factory.name == **name) + { // Create an instance Some(factory) => { // Create default (empty) config @@ -290,13 +294,10 @@ async fn create_plugins( tracing::error!("{:#}", error); } - Err(BoxError::from( - errors - .into_iter() - .map(|e| e.to_string()) - .collect::>() - .join("\n"), - )) + Err(BoxError::from(format!( + "there were {} configuration errors", + errors.len() + ))) } else { Ok(plugin_instances) } @@ -308,7 +309,6 @@ fn inject_schema_id(schema: &Schema, configuration: &mut Value) { telemetry.insert("apollo".to_string(), Value::Object(Default::default())); } } - if let (Some(schema_id), Some(apollo)) = ( &schema.api_schema().schema_id, configuration.get_mut("apollo"), diff --git a/apollo-router/src/services/execution_service.rs b/apollo-router/src/services/execution_service.rs index 3b3fa67ede..05088b5487 100644 --- a/apollo-router/src/services/execution_service.rs +++ b/apollo-router/src/services/execution_service.rs @@ -11,6 +11,7 @@ use futures::future::BoxFuture; use futures::stream::once; use futures::SinkExt; use futures::StreamExt; +use serde_json_bytes::Value; use tower::BoxError; use tower::ServiceBuilder; use tower::ServiceExt; @@ -23,6 +24,9 @@ use super::subgraph_service::SubgraphServiceFactory; use super::Plugins; use crate::graphql::IncrementalResponse; use crate::graphql::Response; +use crate::json_ext::Object; +use crate::json_ext::Path; +use crate::json_ext::PathElement; use crate::json_ext::ValueExt; use crate::services::execution; use crate::ExecutionRequest; @@ -90,18 +94,33 @@ where }; let schema = this.schema.clone(); + let mut nullified_paths: Vec = vec![]; let stream = stream - .map(move |mut response: Response| { + .filter_map(move |mut response: Response| { + // responses that would fall under a path that was previously nullified are not sent + if nullified_paths.iter().any(|path| match &response.path { + None => false, + Some(response_path) => response_path.starts_with(path), + }) { + if response.has_next == Some(false) { + return ready(Some(Response::builder().has_next(false).build())); + } else { + return ready(None); + } + } + let has_next = response.has_next.unwrap_or(true); tracing::debug_span!("format_response").in_scope(|| { - query.format_response( + let paths = query.format_response( &mut response, operation_name.as_deref(), is_deferred, variables.clone(), schema.api_schema(), - ) + ); + + nullified_paths.extend(paths.into_iter()); }); match (response.path.as_ref(), response.data.as_ref()) { @@ -110,7 +129,11 @@ where response.has_next = Some(has_next); } - response + response.errors.retain(|error| match &error.path { + None => true, + Some(error_path) => query.contains_error_path(operation_name.as_deref(), response.subselection.as_deref(), response.path.as_ref(), error_path), + }); + ready(Some(response)) } // if the deferred response specified a path, we must extract the // values matched by that path and create a separate response for @@ -125,35 +148,108 @@ where (Some(response_path), Some(response_data)) => { let mut sub_responses = Vec::new(); response_data.select_values_and_paths(response_path, |path, value| { - sub_responses.push((path.clone(), value.clone())); + // if the deferred path points to an array, split it into multiple subresponses + // because the root must be an object + if let Value::Array(array) = value { + let mut parent = path.clone(); + for (i, value) in array.iter().enumerate() { + parent.push(PathElement::Index(i)); + sub_responses.push((parent.clone(), value.clone())); + parent.pop(); + } + } else { + sub_responses.push((path.clone(), value.clone())); + } }); - Response::builder() - .has_next(has_next) - .incremental( - sub_responses - .into_iter() - .map(move |(path, data)| { - let errors = response - .errors - .iter() - .filter(|error| match &error.path { - None => false, - Some(err_path) => err_path.starts_with(&path), - }) - .cloned() - .collect::>(); + let query = query.clone(); + let operation_name = operation_name.clone(); + + let incremental = sub_responses + .into_iter() + .filter_map(move |(path, data)| { + // filter errors that match the path of this incremental response + let errors = response + .errors + .iter() + .filter(|error| match &error.path { + None => false, + Some(error_path) =>query.contains_error_path(operation_name.as_deref(), response.subselection.as_deref(), response.path.as_ref(), error_path) && error_path.starts_with(&path), + + }) + .cloned() + .collect::>(); + + let extensions: Object = response + .extensions + .iter() + .map(|(key, value)| { + if key.as_str() == "valueCompletion" { + let value = match value.as_array() { + None => Value::Null, + Some(v) => Value::Array( + v.iter() + .filter(|ext| { + match ext + .as_object() + .as_ref() + .and_then(|ext| { + ext.get("path") + }) + .and_then(|v| { + let p:Option = serde_json_bytes::from_value(v.clone()).ok(); + p + }) { + None => false, + Some(ext_path) => { + ext_path + .starts_with( + &path, + ) + } + } + }) + .cloned() + .collect(), + ), + }; + + (key.clone(), value) + } else { + (key.clone(), value.clone()) + } + }) + .collect(); + + // an empty response should not be sent + // still, if there's an error or extension to show, we should + // send it + if !data.is_null() + || !errors.is_empty() + || !extensions.is_empty() + { + Some( IncrementalResponse::builder() .and_label(response.label.clone()) .data(data) .path(path) .errors(errors) - .extensions(response.extensions.clone()) - .build() - }) - .collect(), - ) - .build() + .extensions(extensions) + .build(), + ) + } else { + None + } + }) + .collect(); + + ready(Some( + Response::builder() + .has_next(has_next) + .incremental(incremental) + .build(), + )) + } } }) diff --git a/apollo-router/src/services/layers/ensure_query_presence.rs b/apollo-router/src/services/layers/ensure_query_presence.rs index 142aa8050b..5a8fb9e052 100644 --- a/apollo-router/src/services/layers/ensure_query_presence.rs +++ b/apollo-router/src/services/layers/ensure_query_presence.rs @@ -37,6 +37,12 @@ where message: "Must provide query string.".to_string(), ..Default::default() }]; + tracing::error!( + monotonic_counter.apollo_router_http_requests_total = 1u64, + status = %StatusCode::BAD_REQUEST.as_u16(), + error = "Must provide query string", + "Must provide query string" + ); //We do not copy headers from the request to the response as this may lead to leakable of sensitive data let res = SupergraphResponse::builder() diff --git a/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__deferred_fragment_bounds_nullability-2.snap b/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__deferred_fragment_bounds_nullability-2.snap index acd2170b0b..524ed427a0 100644 --- a/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__deferred_fragment_bounds_nullability-2.snap +++ b/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__deferred_fragment_bounds_nullability-2.snap @@ -23,24 +23,6 @@ expression: stream.next_response().await.unwrap() "suborga", 0 ] - }, - { - "message": "Cannot return null for non-nullable field Organization.nonNullId", - "path": [ - "currentUser", - "activeOrganization", - "suborga", - 1 - ] - }, - { - "message": "Cannot return null for non-nullable field Organization.nonNullId", - "path": [ - "currentUser", - "activeOrganization", - "suborga", - 2 - ] } ] } @@ -55,15 +37,6 @@ expression: stream.next_response().await.unwrap() ], "extensions": { "valueCompletion": [ - { - "message": "Cannot return null for non-nullable field Organization.nonNullId", - "path": [ - "currentUser", - "activeOrganization", - "suborga", - 0 - ] - }, { "message": "Cannot return null for non-nullable field Organization.nonNullId", "path": [ @@ -72,15 +45,6 @@ expression: stream.next_response().await.unwrap() "suborga", 1 ] - }, - { - "message": "Cannot return null for non-nullable field Organization.nonNullId", - "path": [ - "currentUser", - "activeOrganization", - "suborga", - 2 - ] } ] } @@ -95,24 +59,6 @@ expression: stream.next_response().await.unwrap() ], "extensions": { "valueCompletion": [ - { - "message": "Cannot return null for non-nullable field Organization.nonNullId", - "path": [ - "currentUser", - "activeOrganization", - "suborga", - 0 - ] - }, - { - "message": "Cannot return null for non-nullable field Organization.nonNullId", - "path": [ - "currentUser", - "activeOrganization", - "suborga", - 1 - ] - }, { "message": "Cannot return null for non-nullable field Organization.nonNullId", "path": [ diff --git a/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__errors_from_primary_on_deferred_responses-2.snap b/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__errors_from_primary_on_deferred_responses-2.snap new file mode 100644 index 0000000000..54d405aa47 --- /dev/null +++ b/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__errors_from_primary_on_deferred_responses-2.snap @@ -0,0 +1,32 @@ +--- +source: apollo-router/src/services/supergraph_service.rs +expression: stream.next_response().await.unwrap() +--- +{ + "hasNext": false, + "incremental": [ + { + "data": { + "errorField": null + }, + "path": [ + "computer" + ], + "errors": [ + { + "message": "Error field", + "locations": [ + { + "line": 1, + "column": 93 + } + ], + "path": [ + "computer", + "errorField" + ] + } + ] + } + ] +} diff --git a/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__errors_from_primary_on_deferred_responses.snap b/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__errors_from_primary_on_deferred_responses.snap new file mode 100644 index 0000000000..c444dc838c --- /dev/null +++ b/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__errors_from_primary_on_deferred_responses.snap @@ -0,0 +1,12 @@ +--- +source: apollo-router/src/services/supergraph_service.rs +expression: stream.next_response().await.unwrap() +--- +{ + "data": { + "computer": { + "id": "Computer1" + } + }, + "hasNext": true +} diff --git a/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__filter_nullified_deferred_responses-2.snap b/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__filter_nullified_deferred_responses-2.snap new file mode 100644 index 0000000000..ec73eb4353 --- /dev/null +++ b/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__filter_nullified_deferred_responses-2.snap @@ -0,0 +1,29 @@ +--- +source: apollo-router/src/services/supergraph_service/tests.rs +expression: deferred +--- +{ + "hasNext": true, + "incremental": [ + { + "data": { + "activeOrganization": null + }, + "path": [ + "currentUser" + ], + "extensions": { + "valueCompletion": [ + { + "message": "Cannot return null for non-nullable field Organization.nonNullId", + "path": [ + "currentUser", + "activeOrganization", + "nonNullId" + ] + } + ] + } + } + ] +} diff --git a/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__filter_nullified_deferred_responses-3.snap b/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__filter_nullified_deferred_responses-3.snap new file mode 100644 index 0000000000..085171f371 --- /dev/null +++ b/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__filter_nullified_deferred_responses-3.snap @@ -0,0 +1,7 @@ +--- +source: apollo-router/src/services/supergraph_service.rs +expression: last +--- +{ + "hasNext": false +} diff --git a/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__filter_nullified_deferred_responses.snap b/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__filter_nullified_deferred_responses.snap new file mode 100644 index 0000000000..db42a89b63 --- /dev/null +++ b/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__filter_nullified_deferred_responses.snap @@ -0,0 +1,12 @@ +--- +source: apollo-router/src/services/supergraph_service/tests.rs +expression: primary +--- +{ + "data": { + "currentUser": { + "name": "Ada" + } + }, + "hasNext": true +} diff --git a/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__reconstruct_deferred_query_under_interface-2.snap b/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__reconstruct_deferred_query_under_interface-2.snap new file mode 100644 index 0000000000..b40d41ef1a --- /dev/null +++ b/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__reconstruct_deferred_query_under_interface-2.snap @@ -0,0 +1,20 @@ +--- +source: apollo-router/src/services/supergraph_service.rs +expression: stream.next_response().await.unwrap() +--- +{ + "hasNext": false, + "incremental": [ + { + "data": { + "name": "B" + }, + "path": [ + "me", + "memberships", + 0, + "account" + ] + } + ] +} diff --git a/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__reconstruct_deferred_query_under_interface.snap b/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__reconstruct_deferred_query_under_interface.snap new file mode 100644 index 0000000000..b9fea0b168 --- /dev/null +++ b/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__reconstruct_deferred_query_under_interface.snap @@ -0,0 +1,18 @@ +--- +source: apollo-router/src/services/supergraph_service.rs +expression: stream.next_response().await.unwrap() +--- +{ + "data": { + "me": { + "id": 0, + "fullName": "A", + "memberships": [ + { + "permission": "USER" + } + ] + } + }, + "hasNext": true +} diff --git a/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__root_typename_with_defer-2.snap b/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__root_typename_with_defer-2.snap new file mode 100644 index 0000000000..5862863f46 --- /dev/null +++ b/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__root_typename_with_defer-2.snap @@ -0,0 +1,42 @@ +--- +source: apollo-router/src/services/supergraph_service.rs +expression: stream.next_response().await.unwrap() +--- +{ + "hasNext": false, + "incremental": [ + { + "data": { + "name": null + }, + "path": [ + "currentUser", + "activeOrganization", + "suborga", + 0 + ] + }, + { + "data": { + "name": "A" + }, + "path": [ + "currentUser", + "activeOrganization", + "suborga", + 1 + ] + }, + { + "data": { + "name": null + }, + "path": [ + "currentUser", + "activeOrganization", + "suborga", + 2 + ] + } + ] +} diff --git a/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__root_typename_with_defer.snap b/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__root_typename_with_defer.snap new file mode 100644 index 0000000000..3640cf4d26 --- /dev/null +++ b/apollo-router/src/services/snapshots/apollo_router__services__supergraph_service__tests__root_typename_with_defer.snap @@ -0,0 +1,26 @@ +--- +source: apollo-router/src/services/supergraph_service.rs +expression: res +--- +{ + "data": { + "__typename": "Query", + "currentUser": { + "activeOrganization": { + "id": "0", + "suborga": [ + { + "id": "1" + }, + { + "id": "2" + }, + { + "id": "3" + } + ] + } + } + }, + "hasNext": true +} diff --git a/apollo-router/src/services/subgraph_service.rs b/apollo-router/src/services/subgraph_service.rs index e9137ce55b..d2f8a98c89 100644 --- a/apollo-router/src/services/subgraph_service.rs +++ b/apollo-router/src/services/subgraph_service.rs @@ -20,7 +20,6 @@ use http::HeaderValue; use hyper::client::HttpConnector; use hyper_rustls::HttpsConnector; use opentelemetry::global; -use opentelemetry::trace::SpanKind; use schemars::JsonSchema; use tokio::io::AsyncWriteExt; use tower::util::BoxService; @@ -171,7 +170,7 @@ impl tower::Service for SubgraphService { let response = client .call(request) .instrument(tracing::info_span!("subgraph_request", - "otel.kind" = %SpanKind::Client, + "otel.kind" = "CLIENT", "net.peer.name" = &display(host), "net.peer.port" = &display(port), "http.route" = &display(path), @@ -212,7 +211,7 @@ impl tower::Service for SubgraphService { } else { Err(BoxError::from(FetchError::SubrequestHttpError { service: service_name.clone(), - reason: format!("subgraph didn't return JSON (expected content-type: application/json or content-type: application/graphql+json; found content-type: {content_type:?})"), + reason: format!("subgraph didn't return JSON (expected content-type: {APPLICATION_JSON_HEADER_VALUE} or content-type: {GRAPHQL_JSON_RESPONSE_HEADER_VALUE}; found content-type: {content_type:?})"), })) }; } @@ -549,7 +548,7 @@ mod tests { .unwrap_err(); assert_eq!( err.to_string(), - "HTTP fetch failed from 'test': subgraph didn't return JSON (expected content-type: application/json or content-type: application/graphql+json; found content-type: \"text/html\")" + "HTTP fetch failed from 'test': subgraph didn't return JSON (expected content-type: application/json or content-type: application/graphql-response+json; found content-type: \"text/html\")" ); } diff --git a/apollo-router/src/services/supergraph_service.rs b/apollo-router/src/services/supergraph_service.rs index da3c5c9840..9c816ed9cd 100644 --- a/apollo-router/src/services/supergraph_service.rs +++ b/apollo-router/src/services/supergraph_service.rs @@ -9,7 +9,6 @@ use futures::TryFutureExt; use http::StatusCode; use indexmap::IndexMap; use multimap::MultiMap; -use opentelemetry::trace::SpanKind; use tower::util::BoxService; use tower::util::Either; use tower::BoxError; @@ -49,6 +48,8 @@ use crate::Schema; use crate::SupergraphRequest; use crate::SupergraphResponse; +pub(crate) const QUERY_PLANNING_SPAN_NAME: &str = "query_planning"; + /// An [`IndexMap`] of available plugins. pub(crate) type Plugins = IndexMap>; @@ -244,10 +245,15 @@ async fn plan_query( .context(context) .build(), ) - .instrument(tracing::info_span!("query_planning", - graphql.document = body.query.clone().expect("the query presence was already checked by a plugin").as_str(), + .instrument(tracing::info_span!( + QUERY_PLANNING_SPAN_NAME, + graphql.document = body + .query + .clone() + .expect("the query presence was already checked by a plugin") + .as_str(), graphql.operation.name = body.operation_name.clone().unwrap_or_default().as_str(), - "otel.kind" = %SpanKind::Internal + "otel.kind" = "INTERNAL" )) .await } @@ -609,6 +615,112 @@ mod tests { insta::assert_json_snapshot!(stream.next_response().await.unwrap()); } + #[tokio::test] + async fn errors_from_primary_on_deferred_responses() { + let schema = r#" + schema + @link(url: "https://specs.apollo.dev/link/v1.0") + @link(url: "https://specs.apollo.dev/join/v0.2", for: EXECUTION) + { + query: Query + } + + directive @join__field(graph: join__Graph!, requires: join__FieldSet, provides: join__FieldSet, type: String, external: Boolean, override: String, usedOverridden: Boolean) repeatable on FIELD_DEFINITION | INPUT_FIELD_DEFINITION + directive @join__graph(name: String!, url: String!) on ENUM_VALUE + directive @join__implements(graph: join__Graph!, interface: String!) repeatable on OBJECT | INTERFACE + directive @join__type(graph: join__Graph!, key: join__FieldSet, extension: Boolean! = false, resolvable: Boolean! = true) repeatable on OBJECT | INTERFACE | UNION | ENUM | INPUT_OBJECT | SCALAR + directive @link(url: String, as: String, for: link__Purpose, import: [link__Import]) repeatable on SCHEMA + + scalar link__Import + enum link__Purpose { + SECURITY + EXECUTION + } + + type Computer + @join__type(graph: COMPUTERS) + { + id: ID! + errorField: String + nonNullErrorField: String! + } + + scalar join__FieldSet + + enum join__Graph { + COMPUTERS @join__graph(name: "computers", url: "http://localhost:4001/") + } + + + type Query + @join__type(graph: COMPUTERS) + { + computer(id: ID!): Computer + }"#; + + let subgraphs = MockedSubgraphs([ + ("computers", MockSubgraph::builder().with_json( + serde_json::json!{{"query":"{currentUser{__typename id}}"}}, + serde_json::json!{{"data": {"currentUser": { "__typename": "User", "id": "0" }}}} + ) + .with_json( + serde_json::json!{{ + "query":"{computer(id:\"Computer1\"){id errorField}}", + }}, + serde_json::json!{{ + "data": { + "computer": { + "id": "Computer1" + } + }, + "errors": [ + { + "message": "Error field", + "locations": [ + { + "line": 1, + "column": 93 + } + ], + "path": ["computer","errorField"], + } + ] + }} + ).build()), + ].into_iter().collect()); + + let service = TestHarness::builder() + .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true } })) + .unwrap() + .schema(schema) + .extra_plugin(subgraphs) + .build() + .await + .unwrap(); + + let request = supergraph::Request::fake_builder() + .header("Accept", "multipart/mixed; deferSpec=20220824") + .query( + r#"query { + computer(id: "Computer1") { + id + ...ComputerErrorField @defer + } + } + fragment ComputerErrorField on Computer { + errorField + }"#, + ) + .build() + .unwrap(); + + let mut stream = service.oneshot(request).await.unwrap(); + + insta::assert_json_snapshot!(stream.next_response().await.unwrap()); + + insta::assert_json_snapshot!(stream.next_response().await.unwrap()); + } + #[tokio::test] async fn deferred_fragment_bounds_nullability() { let subgraphs = MockedSubgraphs([ @@ -773,6 +885,140 @@ mod tests { insta::assert_json_snapshot!(stream.next_response().await.unwrap()); } + #[tokio::test] + async fn root_typename_with_defer() { + let subgraphs = MockedSubgraphs([ + ("user", MockSubgraph::builder().with_json( + serde_json::json!{{"query":"{currentUser{activeOrganization{__typename id}}}"}}, + serde_json::json!{{"data": {"currentUser": { "activeOrganization": { "__typename": "Organization", "id": "0" } }}}} + ).build()), + ("orga", MockSubgraph::builder().with_json( + serde_json::json!{{ + "query":"query($representations:[_Any!]!){_entities(representations:$representations){...on Organization{suborga{__typename id}}}}", + "variables": { + "representations":[{"__typename": "Organization", "id":"0"}] + } + }}, + serde_json::json!{{ + "data": { + "_entities": [{ "suborga": [ + { "__typename": "Organization", "id": "1"}, + { "__typename": "Organization", "id": "2"}, + { "__typename": "Organization", "id": "3"}, + ] }] + }, + }} + ) + .with_json( + serde_json::json!{{ + "query":"query($representations:[_Any!]!){_entities(representations:$representations){...on Organization{name}}}", + "variables": { + "representations":[ + {"__typename": "Organization", "id":"1"}, + {"__typename": "Organization", "id":"2"}, + {"__typename": "Organization", "id":"3"} + + ] + } + }}, + serde_json::json!{{ + "data": { + "_entities": [ + { "__typename": "Organization", "id": "1"}, + { "__typename": "Organization", "id": "2", "name": "A"}, + { "__typename": "Organization", "id": "3"}, + ] + } + }} + ).build()) + ].into_iter().collect()); + + let service = TestHarness::builder() + .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true } })) + .unwrap() + .schema(SCHEMA) + .extra_plugin(subgraphs) + .build() + .await + .unwrap(); + + let request = supergraph::Request::fake_builder() + .header("Accept", "multipart/mixed; deferSpec=20220824") + .query( + "query { __typename currentUser { activeOrganization { id suborga { id ...@defer { name } } } } }", + ) + .build() + .unwrap(); + + let mut stream = service.oneshot(request).await.unwrap(); + let res = stream.next_response().await.unwrap(); + assert_eq!( + res.data.as_ref().unwrap().get("__typename"), + Some(&serde_json_bytes::Value::String("Query".into())) + ); + insta::assert_json_snapshot!(res); + + insta::assert_json_snapshot!(stream.next_response().await.unwrap()); + } + + #[tokio::test] + async fn root_typename_with_defer_in_defer() { + let subgraphs = MockedSubgraphs([ + ("user", MockSubgraph::builder().with_json( + serde_json::json!{{"query":"{currentUser{activeOrganization{__typename id}}}"}}, + serde_json::json!{{"data": {"currentUser": { "activeOrganization": { "__typename": "Organization", "id": "0" } }}}} + ).build()), + ("orga", MockSubgraph::builder().with_json( + serde_json::json!{{ + "query":"query($representations:[_Any!]!){_entities(representations:$representations){...on Organization{suborga{__typename id name}}}}", + "variables": { + "representations":[{"__typename": "Organization", "id":"0"}] + } + }}, + serde_json::json!{{ + "data": { + "_entities": [{ "suborga": [ + { "__typename": "Organization", "id": "1"}, + { "__typename": "Organization", "id": "2", "name": "A"}, + { "__typename": "Organization", "id": "3"}, + ] }] + }, + }} + ).build()) + ].into_iter().collect()); + + let service = TestHarness::builder() + .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true } })) + .unwrap() + .schema(SCHEMA) + .extra_plugin(subgraphs) + .build() + .await + .unwrap(); + + let request = supergraph::Request::fake_builder() + .header("Accept", "multipart/mixed; deferSpec=20220824") + .query( + "query { ...@defer { __typename currentUser { activeOrganization { id suborga { id name } } } } }", + ) + .build() + .unwrap(); + + let mut stream = service.oneshot(request).await.unwrap(); + let _res = stream.next_response().await.unwrap(); + let res = stream.next_response().await.unwrap(); + assert_eq!( + res.incremental + .get(0) + .unwrap() + .data + .as_ref() + .unwrap() + .get("__typename"), + Some(&serde_json_bytes::Value::String("Query".into())) + ); + } + #[tokio::test] async fn query_reconstruction() { let schema = r#"schema @@ -841,6 +1087,8 @@ mod tests { } "#; + // this test does not need to generate a valid response, it is only here to check + // that the router does not panic when reconstructing the query for the deferred part let service = TestHarness::builder() .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true } })) .unwrap() @@ -870,4 +1118,275 @@ mod tests { insta::assert_json_snapshot!(stream.next_response().await.unwrap()); } + + // if a deferred response falls under a path that was nullified in the primary response, + // the deferred response must not be sent + #[tokio::test] + async fn filter_nullified_deferred_responses() { + let subgraphs = MockedSubgraphs([ + ("user", MockSubgraph::builder() + .with_json( + serde_json::json!{{"query":"{currentUser{__typename name id}}"}}, + serde_json::json!{{"data": {"currentUser": { "__typename": "User", "name": "Ada", "id": "1" }}}} + ) + .with_json( + serde_json::json!{{ + "query":"query($representations:[_Any!]!){_entities(representations:$representations){...on User{activeOrganization{__typename id}}}}", + "variables": { + "representations":[{"__typename": "User", "id":"1"}] + } + }}, + serde_json::json!{{ + "data": { + "_entities": [ + { + "activeOrganization": { + "__typename": "Organization", "id": "2" + } + } + ] + } + }}) + .with_json( + serde_json::json!{{ + "query":"query($representations:[_Any!]!){_entities(representations:$representations){...on User{name}}}", + "variables": { + "representations":[{"__typename": "User", "id":"3"}] + } + }}, + serde_json::json!{{ + "data": { + "_entities": [ + { + "name": "A" + } + ] + } + }}) + .build()), + ("orga", MockSubgraph::builder() + .with_json( + serde_json::json!{{ + "query":"query($representations:[_Any!]!){_entities(representations:$representations){...on Organization{creatorUser{__typename id}}}}", + "variables": { + "representations":[{"__typename": "Organization", "id":"2"}] + } + }}, + serde_json::json!{{ + "data": { + "_entities": [ + { + "creatorUser": { + "__typename": "User", "id": "3" + } + } + ] + } + }}) + .with_json( + serde_json::json!{{ + "query":"query($representations:[_Any!]!){_entities(representations:$representations){...on Organization{nonNullId}}}", + "variables": { + "representations":[{"__typename": "Organization", "id":"2"}] + } + }}, + serde_json::json!{{ + "data": { + "_entities": [ + { + "nonNullId": null + } + ] + } + }}).build()) + ].into_iter().collect()); + + let service = TestHarness::builder() + .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true } })) + .unwrap() + .schema(SCHEMA) + .extra_plugin(subgraphs) + .build() + .await + .unwrap(); + + let request = supergraph::Request::fake_builder() + .query( + r#"query { + currentUser { + name + ... @defer { + activeOrganization { + id + nonNullId + ... @defer { + creatorUser { + name + } + } + } + } + } + }"#, + ) + .header("Accept", "multipart/mixed; deferSpec=20220824") + .build() + .unwrap(); + let mut response = service.oneshot(request).await.unwrap(); + + let primary = response.next_response().await.unwrap(); + insta::assert_json_snapshot!(primary); + + let deferred = response.next_response().await.unwrap(); + insta::assert_json_snapshot!(deferred); + + // the last deferred response was replace with an empty response, + // to still have one containing has_next = false + let last = response.next_response().await.unwrap(); + insta::assert_json_snapshot!(last); + } + + #[tokio::test] + async fn reconstruct_deferred_query_under_interface() { + let schema = r#"schema + @link(url: "https://specs.apollo.dev/link/v1.0") + @link(url: "https://specs.apollo.dev/join/v0.2", for: EXECUTION) + @link(url: "https://specs.apollo.dev/tag/v0.2") + @link(url: "https://specs.apollo.dev/inaccessible/v0.2") + { + query: Query + } + + directive @inaccessible on FIELD_DEFINITION | OBJECT | INTERFACE | UNION | ARGUMENT_DEFINITION | SCALAR | ENUM | ENUM_VALUE | INPUT_OBJECT | INPUT_FIELD_DEFINITION + directive @join__field(graph: join__Graph!, requires: join__FieldSet, provides: join__FieldSet, type: String, external: Boolean, override: String, usedOverridden: Boolean) repeatable on FIELD_DEFINITION | INPUT_FIELD_DEFINITION + directive @join__graph(name: String!, url: String!) on ENUM_VALUE + directive @join__implements(graph: join__Graph!, interface: String!) repeatable on OBJECT | INTERFACE + directive @join__type(graph: join__Graph!, key: join__FieldSet, extension: Boolean! = false, resolvable: Boolean! = true) repeatable on OBJECT | INTERFACE | UNION | ENUM | INPUT_OBJECT | SCALAR + directive @link(url: String, as: String, for: link__Purpose, import: [link__Import]) repeatable on SCHEMA + directive @tag(name: String!) repeatable on FIELD_DEFINITION | OBJECT | INTERFACE | UNION | ARGUMENT_DEFINITION | SCALAR | ENUM | ENUM_VALUE | INPUT_OBJECT | INPUT_FIELD_DEFINITION + + scalar join__FieldSet + enum join__Graph { + USER @join__graph(name: "user", url: "http://localhost:4000/graphql") + } + scalar link__Import + enum link__Purpose { + SECURITY + EXECUTION + } + type Query + @join__type(graph: USER) + { + me: Identity @join__field(graph: USER) + } + interface Identity + @join__type(graph: USER) + { + id: ID! + name: String! + } + + type User implements Identity + @join__implements(graph: USER, interface: "Identity") + @join__type(graph: USER, key: "id") + { + fullName: String! @join__field(graph: USER) + id: ID! + memberships: [UserMembership!]! @join__field(graph: USER) + name: String! @join__field(graph: USER) + } + type UserMembership + @join__type(graph: USER) + @tag(name: "platform-api") + { + """The organization that the user belongs to.""" + account: Account! + """The user's permission level within the organization.""" + permission: UserPermission! + } + enum UserPermission + @join__type(graph: USER) + { + USER + ADMIN + } + type Account + @join__type(graph: USER, key: "id") + { + id: ID! @join__field(graph: USER) + name: String! @join__field(graph: USER) + }"#; + + let subgraphs = MockedSubgraphs([ + ("user", MockSubgraph::builder().with_json( + serde_json::json!{{"query":"{me{__typename ...on User{id fullName memberships{permission account{__typename id}}}}}"}}, + serde_json::json!{{"data": {"me": { + "__typename": "User", + "id": 0, + "fullName": "A", + "memberships": [ + { + "permission": "USER", + "account": { + "__typename": "Account", + "id": 1 + } + } + ] + }}}} + ) .with_json( + serde_json::json!{{ + "query":"query($representations:[_Any!]!){_entities(representations:$representations){...on Account{name}}}", + "variables": { + "representations":[ + {"__typename": "Account", "id": 1} + ] + } + }}, + serde_json::json!{{ + "data": { + "_entities": [ + { "__typename": "Account", "id": 1, "name": "B"} + ] + } + }}).build()), + ].into_iter().collect()); + + let service = TestHarness::builder() + .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true } })) + .unwrap() + .schema(schema) + .extra_plugin(subgraphs) + .build() + .await + .unwrap(); + + let request = supergraph::Request::fake_builder() + .header("Accept", "multipart/mixed; deferSpec=20220824") + .query( + r#"query { + me { + ... on User { + id + fullName + memberships { + permission + account { + ... on Account @defer { + name + } + } + } + } + } + }"#, + ) + .build() + .unwrap(); + + let mut stream = service.oneshot(request).await.unwrap(); + + insta::assert_json_snapshot!(stream.next_response().await.unwrap()); + insta::assert_json_snapshot!(stream.next_response().await.unwrap()); + } } diff --git a/apollo-router/src/spaceport/README.md b/apollo-router/src/spaceport/README.md deleted file mode 100644 index 810cc823af..0000000000 --- a/apollo-router/src/spaceport/README.md +++ /dev/null @@ -1,73 +0,0 @@ -# Apollo Telemetry -Transfer statistics (e.g., operation usage) to Apollo Studio's ingress - -## Authentication - -The server must be authenticated to submit messages to the Apollo ingress. -Furthermore, the statistics must be submitted "to" a particular graph. In -the existing products this is accomplished using [environment variables](https://www.apollographql.com/docs/federation/managed-federation/setup/#4-connect-the-gateway-to-studio). - -In the router, we have a configuration file which can be dynamically -re-loaded, so it makes more sense to include this configuration here. There -is a new optional section that looks like this: - -``` -telemetry: - # Optional Apollo telemetry configuration. - apollo: - - # Optional external Spaceport URL. - # If not specified an in-process spaceport is used. - endpoint: "https://my-spaceport" - - # Optional Apollo key. If not specified the env variable APOLLO_KEY will be used. - apollo_key: "${APOLLO_KEY}" - - # Optional graphs reference. If not specified the env variable APOLLO_GRAPH_REF will be used. - apollo_graph_ref: "${APOLLO_GRAPH_REF}" -``` - -## Design - -There are two main components: - - Apollo Telemetry - - Apollo Spaceport - -### Configuration - -The telemetry statistics are internally delivered via gRPC service to a spaceport -which then buffers data before finally delivering statistics to the Apollo -ingress. That spaceport can be internal, which is the default, or external. - -The spaceport is configured from a new optional configuration section which looks -like this: - -``` -telemetry: - # Optional Apollo telemetry configuration. - apollo: - - # Optional external Spaceport URL. - # If not specified an in-process spaceport is used. - endpoint: "https://my-spaceport" -``` - -### Components - -#### ApolloTelemetry - -An open telemetry collector which processes spans and extracts data to -create "Reports" which are then submited over gRPC to either an -in-process or an out of process spaceport. - -#### Spaceport - -A gRPC server which accepts "Reports" and regularly (every 5 seconds) -submits the collected Reports to the Apollo Reporting ingress. If the -quantity of Reports exceeds a specified limit, then a transfer will -be triggered early, so a very busy Spaceport will deliver more frequently -than every 5 seconds. - -Delivery to the ingress is on a "best efforts" basis and the spaceport -will attempt to deliver the data 5 times before discarding. - diff --git a/apollo-router/src/spaceport/main.rs b/apollo-router/src/spaceport/main.rs deleted file mode 100644 index e1e7e8ca4b..0000000000 --- a/apollo-router/src/spaceport/main.rs +++ /dev/null @@ -1,40 +0,0 @@ -//! Main entry point for CLI command to start spaceport. -// With regards to ELv2 licensing, this entire file is license key functionality -use std::net::SocketAddr; - -use crate::spaceport::server::ReportSpaceport; -use clap::Parser; -use tracing_subscriber::filter::EnvFilter; - -const DEFAULT_LISTEN: &str = "127.0.0.1:50051"; - -#[derive(Parser, Debug)] -#[clap(author, version, about, long_about = None)] -struct Args { - /// Address to serve - #[clap(short, long, default_value = DEFAULT_LISTEN)] - address: SocketAddr, -} - -#[tokio::main] -async fn main() -> Result<(), Box> { - let args = Args::parse(); - - // By default, tracing will give us a filter which filters - // at level ERROR. That's not what we want, so if we don't - // have a filter specification, let's create one set at - // level INFO. - let filter = match EnvFilter::try_from_default_env() { - Ok(f) => f, - Err(_e) => EnvFilter::new("info"), - }; - tracing_subscriber::fmt::fmt() - .with_env_filter(filter) - .json() - .init(); - tracing::info!("spaceport starting"); - let spaceport = ReportSpaceport::new(args.address, None).await?; - spaceport.serve().await?; - - Ok(()) -} diff --git a/apollo-router/src/spaceport/mod.rs b/apollo-router/src/spaceport/mod.rs deleted file mode 100644 index a08fcd280c..0000000000 --- a/apollo-router/src/spaceport/mod.rs +++ /dev/null @@ -1,172 +0,0 @@ -// With regards to ELv2 licensing, this entire file is license key functionality -// tonic does not derive `Eq` for the gRPC message types, which causes a warning from Clippy. The -// current suggestion is to explicitly allow the lint in the module that imports the protos. -// Read more: https://github.com/hyperium/tonic/issues/1056 -#![allow(clippy::derive_partial_eq_without_eq)] - -#[allow(unreachable_pub)] -mod report { - tonic::include_proto!("report"); -} - -#[allow(unreachable_pub)] -mod agent { - tonic::include_proto!("agent"); -} - -/// The server module contains the server components -pub(crate) mod server; - -use std::error::Error; - -use agent::reporter_client::ReporterClient; -pub(crate) use agent::*; -pub(crate) use prost::*; -pub(crate) use report::*; -use serde::ser::SerializeStruct; -use tokio::task::JoinError; -use tonic::codegen::http::uri::InvalidUri; -use tonic::transport::Channel; -use tonic::transport::Endpoint; -use tonic::Request; -use tonic::Response; -use tonic::Status; - -/// Reporting Error type -#[derive(Debug)] -pub(crate) struct ReporterError { - source: Box, - msg: String, -} - -impl std::error::Error for ReporterError {} - -impl From for ReporterError { - fn from(error: InvalidUri) -> Self { - ReporterError { - msg: error.to_string(), - source: Box::new(error), - } - } -} - -impl From for ReporterError { - fn from(error: tonic::transport::Error) -> Self { - ReporterError { - msg: error.to_string(), - source: Box::new(error), - } - } -} - -impl From for ReporterError { - fn from(error: std::io::Error) -> Self { - ReporterError { - msg: error.to_string(), - source: Box::new(error), - } - } -} - -impl From for ReporterError { - fn from(error: sys_info::Error) -> Self { - ReporterError { - msg: error.to_string(), - source: Box::new(error), - } - } -} - -impl From for ReporterError { - fn from(error: JoinError) -> Self { - ReporterError { - msg: error.to_string(), - source: Box::new(error), - } - } -} - -impl std::fmt::Display for ReporterError { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!( - f, - "ReporterError: source: {}, message: {}", - self.source, self.msg - ) - } -} - -/// The Reporter accepts requests from clients to transfer statistics -/// and traces to the Apollo spaceport. -#[derive(Debug)] -pub(crate) struct Reporter { - client: ReporterClient, - ep: Endpoint, -} - -impl Reporter { - /// Try to create a new reporter which will communicate with the supplied address. - /// - /// This can fail if: - /// - the address cannot be parsed - /// - the reporter can't connect to the address - pub(crate) async fn try_new>(addr: T) -> Result - where - prost::bytes::Bytes: From, - { - let ep = Endpoint::from_shared(addr)?; - let client = ReporterClient::connect(ep.clone()).await?; - Ok(Self { client, ep }) - } - - /// Try to re-connect a reporter. - /// - /// This can fail if: - /// - the reporter can't connect to the address - pub(crate) async fn reconnect(&mut self) -> Result<(), ReporterError> { - self.client = ReporterClient::connect(self.ep.clone()).await?; - Ok(()) - } - - /// Submit a report onto the spaceport for eventual processing. - /// - /// The spaceport will buffer reports, transferring them when convenient. - pub(crate) async fn submit( - &mut self, - request: ReporterRequest, - ) -> Result, Status> { - self.client.add(Request::new(request)).await - } -} - -pub(crate) fn serialize_timestamp( - timestamp: &Option, - serializer: S, -) -> Result -where - S: serde::Serializer, -{ - match timestamp { - Some(ts) => { - let mut ts_strukt = serializer.serialize_struct("Timestamp", 2)?; - ts_strukt.serialize_field("seconds", &ts.seconds)?; - ts_strukt.serialize_field("nanos", &ts.nanos)?; - ts_strukt.end() - } - None => serializer.serialize_none(), - } -} - -#[cfg(not(windows))] // git checkout converts \n to \r\n, making == below fail -#[test] -fn check_reports_proto_is_up_to_date() { - let proto_url = "https://usage-reporting.api.apollographql.com/proto/reports.proto"; - let response = reqwest::blocking::get(proto_url).unwrap(); - let content = response.text().unwrap(); - // Not using assert_eq! as printing the entire file would be too verbose - assert!( - content == include_str!("proto/reports.proto"), - "Protobuf file is out of date. Run this command to update it:\n\n \ - curl -f {proto_url} > apollo-router/src/spaceport/proto/reports.proto\n\n" - ); -} diff --git a/apollo-router/src/spaceport/proto/agents.proto b/apollo-router/src/spaceport/proto/agents.proto deleted file mode 100644 index 9e9d85d3f3..0000000000 --- a/apollo-router/src/spaceport/proto/agents.proto +++ /dev/null @@ -1,21 +0,0 @@ -// With regards to ELv2 licensing, this entire file is license key functionality -syntax = "proto3"; - -import "reports.proto"; - -package Agent; - -service Reporter { - rpc Add(ReporterRequest) returns (ReporterResponse) {} -} - -message ReporterResponse { - string message = 1; -} -// This is just a wrapper around Report that supplies the Apollo key. -// Spaceport will use this when sending data to Apollo. -message ReporterRequest { - string apollo_key = 1; - Report.Report report = 2; -} - diff --git a/apollo-router/src/spaceport/server.rs b/apollo-router/src/spaceport/server.rs deleted file mode 100644 index f9f90f95b4..0000000000 --- a/apollo-router/src/spaceport/server.rs +++ /dev/null @@ -1,206 +0,0 @@ -// This entire file is license key functionality -use std::io::Write; -use std::net::SocketAddr; - -use bytes::BytesMut; -use flate2::write::GzEncoder; -use flate2::Compression; -use prost::Message; -use reqwest::header::CONTENT_TYPE; -use reqwest::Client; -use tokio::net::TcpListener; -use tokio::sync::mpsc::error::TrySendError; -use tokio::sync::mpsc::Sender; -use tokio::time::Duration; -use tokio_stream::wrappers::TcpListenerStream; -use tonic::transport::Error; -use tonic::transport::Server; -use tonic::Request; -use tonic::Response; -use tonic::Status; - -use crate::spaceport::agent::reporter_server::Reporter; -use crate::spaceport::agent::reporter_server::ReporterServer; -use crate::spaceport::agent::ReporterRequest; -use crate::spaceport::agent::ReporterResponse; -use crate::spaceport::report::Report; - -static DEFAULT_APOLLO_USAGE_REPORTING_INGRESS_URL: &str = - "https://usage-reporting.api.apollographql.com/api/ingress/traces"; -const BACKOFF_INCREMENT: Duration = Duration::from_millis(50); - -/// Accept Traces and Stats from clients and transfer to an Apollo Ingress -pub(crate) struct ReportSpaceport { - listener: Option, - addr: SocketAddr, - tx: Sender, -} - -impl ReportSpaceport { - /// Create a new ReportSpaceport which is configured to serve requests at the - /// supplied address - /// - /// The spaceport will transfer reports to the Apollo Ingress. - /// - /// The spaceport will attempt to make the transfer 5 times before failing. If - /// the spaceport fails, the data is discarded. - pub(crate) async fn new(addr: SocketAddr) -> Result { - let listener = TcpListener::bind(addr).await?; - let addr = listener.local_addr()?; - - // Spawn a task which will transmit reports - let (tx, mut rx) = tokio::sync::mpsc::channel::(1024); - - tokio::task::spawn(async move { - let client = Client::new(); - while let Some(report) = rx.recv().await { - if let Some(report_to_send) = report.report { - match ReportSpaceport::submit_report(&client, report.apollo_key, report_to_send) - .await - { - Ok(v) => tracing::debug!("report submission succeeded: {:?}", v), - Err(e) => tracing::error!("report submission failed: {}", e), - } - } - } - }); - Ok(Self { - listener: Some(listener), - addr, - tx, - }) - } - - pub(crate) fn address(&self) -> &SocketAddr { - &self.addr - } - - /// Start serving requests. - pub(crate) async fn serve(mut self) -> Result<(), Error> { - let listener = self - .listener - .take() - .expect("should have allocated listener"); - Server::builder() - .add_service(ReporterServer::new(self)) - .serve_with_incoming(TcpListenerStream::new(listener)) - .await - } - - async fn submit_report( - client: &Client, - key: String, - report: Report, - ) -> Result, Status> { - tracing::debug!("submitting report: {:?}", report); - // Protobuf encode message - let mut content = BytesMut::new(); - report - .encode(&mut content) - .map_err(|e| Status::invalid_argument(e.to_string()))?; - // Create a gzip encoder - let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); - // Write our content to our encoder - encoder - .write_all(&content) - .map_err(|e| Status::internal(e.to_string()))?; - // Finish encoding and retrieve content - let compressed_content = encoder - .finish() - .map_err(|e| Status::internal(e.to_string()))?; - let mut backoff = Duration::from_millis(0); - let ingress = match std::env::var("APOLLO_USAGE_REPORTING_INGRESS_URL") { - Ok(v) => v, - Err(_e) => DEFAULT_APOLLO_USAGE_REPORTING_INGRESS_URL.to_string(), - }; - let req = client - .post(ingress) - .body(compressed_content) - .header("X-Api-Key", key) - .header("Content-Encoding", "gzip") - .header(CONTENT_TYPE, "application/protobuf") - .header("Accept", "application/json") - .header( - "User-Agent", - format!( - "{} / {} usage reporting", - std::env!("CARGO_PKG_NAME"), - std::env!("CARGO_PKG_VERSION") - ), - ) - .build() - .map_err(|e| Status::unavailable(e.to_string()))?; - - let mut msg = "default error message".to_string(); - for i in 0..5 { - // We know these requests can be cloned - let task_req = req.try_clone().expect("requests must be clone-able"); - match client.execute(task_req).await { - Ok(v) => { - let status = v.status(); - let data = v - .text() - .await - .map_err(|e| Status::internal(e.to_string()))?; - // Handle various kinds of status: - // - if client error, terminate immediately - // - if server error, it may be transient so treat as retry-able - // - if ok, return ok - if status.is_client_error() { - tracing::error!("client error reported at ingress: {}", data); - return Err(Status::invalid_argument(data)); - } else if status.is_server_error() { - tracing::warn!("attempt: {}, could not transfer: {}", i + 1, data); - msg = data; - } else { - tracing::debug!("ingress response text: {:?}", data); - let response = ReporterResponse { - message: "Report accepted".to_string(), - }; - return Ok(Response::new(response)); - } - } - Err(e) => { - // TODO: Ultimately need more sophisticated handling here. For example - // a redirect should not be treated the same way as a connect or a - // type builder error... - tracing::warn!("attempt: {}, could not transfer: {}", i + 1, e); - msg = e.to_string(); - } - } - backoff += BACKOFF_INCREMENT; - tokio::time::sleep(backoff).await; - } - Err(Status::unavailable(msg)) - } -} - -#[tonic::async_trait] -impl Reporter for ReportSpaceport { - async fn add( - &self, - request: Request, - ) -> Result, Status> { - tracing::debug!("received request: {:?}", request); - let msg = request.into_inner(); - self.add_report(msg).await - } -} - -impl ReportSpaceport { - async fn add_report( - &self, - report: ReporterRequest, - ) -> Result, Status> { - match self.tx.try_send(report) { - Ok(()) => { - let response = ReporterResponse { - message: "Report accepted".to_string(), - }; - Ok(Response::new(response)) - } - Err(TrySendError::Closed(_)) => Err(Status::internal("channel closed")), - Err(TrySendError::Full(_)) => Err(Status::resource_exhausted("channel full")), - } - } -} diff --git a/apollo-router/src/spec/query.rs b/apollo-router/src/spec/query.rs index 956ebd8070..3d08e23ec0 100644 --- a/apollo-router/src/spec/query.rs +++ b/apollo-router/src/spec/query.rs @@ -103,10 +103,10 @@ impl Query { is_deferred: bool, variables: Object, schema: &Schema, - ) { + ) -> Vec { let data = std::mem::take(&mut response.data); if let Some(Value::Object(mut input)) = data { - let operation = self.operation(operation_name); + let original_operation = self.operation(operation_name); if is_deferred { if let Some(subselection) = &response.subselection { // Get subselection from hashmap @@ -121,7 +121,21 @@ impl Query { variables: &variables, schema, errors: Vec::new(), + nullified: Vec::new(), }; + // Detect if root __typename is asked in the original query (the qp doesn't put root __typename in subselections) + // cf https://github.com/apollographql/router/issues/1677 + let operation_kind_if_root_typename = + original_operation.and_then(|op| { + op.selection_set + .iter() + .any(|f| f.is_typename_field()) + .then(|| *op.kind()) + }); + if let Some(operation_kind) = operation_kind_if_root_typename { + output.insert(TYPENAME, operation_kind.as_str().into()); + } + response.data = Some( match self.apply_root_selection_set( operation, @@ -141,16 +155,16 @@ impl Query { } } - return; + return parameters.nullified; } None => failfast_debug!("can't find subselection for {:?}", subselection), } // the primary query was empty, we return an empty object } else { response.data = Some(Value::Object(Object::default())); - return; + return vec![]; } - } else if let Some(operation) = operation { + } else if let Some(operation) = original_operation { let mut output = Object::default(); let all_variables = if operation.variables.is_empty() { @@ -171,6 +185,7 @@ impl Query { variables: &all_variables, schema, errors: Vec::new(), + nullified: Vec::new(), }; response.data = Some( @@ -191,7 +206,7 @@ impl Query { } } - return; + return parameters.nullified; } else { failfast_debug!("can't find operation for {:?}", operation_name); } @@ -200,6 +215,8 @@ impl Query { } response.data = Some(Value::default()); + + vec![] } pub(crate) fn parse( @@ -207,9 +224,8 @@ impl Query { schema: &Schema, configuration: &Configuration, ) -> Result { - let string = query.into(); - - let parser = apollo_parser::Parser::new(string.as_str()) + let query = query.into(); + let parser = apollo_parser::Parser::new(query.as_str()) .recursion_limit(configuration.server.experimental_parser_recursion_limit); let tree = parser.parse(); @@ -244,7 +260,7 @@ impl Query { .collect::, SpecError>>()?; Ok(Query { - string, + string: query, fragments, operations, subselections: HashMap::new(), @@ -338,6 +354,7 @@ impl Query { res }) { Err(InvalidValue) => { + parameters.nullified.push(path.clone()); *output = Value::Null; Ok(()) } @@ -377,6 +394,7 @@ impl Query { input_object.get(TYPENAME).and_then(|val| val.as_str()) { if !parameters.schema.object_types.contains_key(input_type) { + parameters.nullified.push(path.clone()); *output = Value::Null; return Ok(()); } @@ -398,12 +416,14 @@ impl Query { ) .is_err() { + parameters.nullified.push(path.clone()); *output = Value::Null; } Ok(()) } _ => { + parameters.nullified.push(path.clone()); *output = Value::Null; Ok(()) } @@ -948,12 +968,46 @@ impl Query { None => self.operations.get(0), } } + + pub(crate) fn contains_error_path( + &self, + operation_name: Option<&str>, + subselection: Option<&str>, + response_path: Option<&Path>, + path: &Path, + ) -> bool { + println!( + "Query::contains_error_path: path = {path}, query: {}", + self.string, + ); + let operation = if let Some(subselection) = subselection { + // Get subselection from hashmap + match self.subselections.get(&SubSelection { + path: response_path.cloned().unwrap_or_default(), + subselection: subselection.to_string(), + }) { + Some(subselection_query) => &subselection_query.operations[0], + None => return false, + } + } else { + match self.operation(operation_name) { + None => return false, + Some(op) => op, + } + }; + + operation + .selection_set + .iter() + .any(|selection| selection.contains_error_path(&path.0, &self.fragments)) + } } /// Intermediate structure for arguments passed through the entire formatting struct FormatParameters<'a> { variables: &'a Object, errors: Vec, + nullified: Vec, schema: &'a Schema, } @@ -1057,7 +1111,7 @@ impl Operation { && self .selection_set .get(0) - .map(|s| matches!(s, Selection::Field {name, ..} if name.as_str() == TYPENAME)) + .map(|s| s.is_typename_field()) .unwrap_or_default() } diff --git a/apollo-router/src/spec/selection.rs b/apollo-router/src/spec/selection.rs index 11e67e36db..ddc57ceba0 100644 --- a/apollo-router/src/spec/selection.rs +++ b/apollo-router/src/spec/selection.rs @@ -4,7 +4,9 @@ use serde::Deserialize; use serde::Serialize; use serde_json_bytes::ByteString; +use super::Fragments; use crate::json_ext::Object; +use crate::json_ext::PathElement; use crate::spec::TYPENAME; use crate::FieldType; use crate::Schema; @@ -314,6 +316,85 @@ impl Selection { Ok(selection) } + + pub(crate) fn is_typename_field(&self) -> bool { + matches!(self, Selection::Field {name, ..} if name.as_str() == TYPENAME) + } + + pub(crate) fn contains_error_path(&self, path: &[PathElement], fragments: &Fragments) -> bool { + match (path.get(0), self) { + (None, _) => true, + ( + Some(PathElement::Key(key)), + Selection::Field { + name, + alias, + selection_set, + .. + }, + ) => { + if alias.as_ref().unwrap_or(name).as_str() == key.as_str() { + match selection_set { + // if we don't select after that field, the path should stop there + None => path.len() == 1, + Some(set) => set + .iter() + .any(|selection| selection.contains_error_path(&path[1..], fragments)), + } + } else { + false + } + } + ( + Some(PathElement::Fragment(fragment)), + Selection::InlineFragment { + type_condition, + selection_set, + .. + }, + ) => { + if fragment.as_str().strip_prefix("... on ") == Some(type_condition.as_str()) { + selection_set + .iter() + .any(|selection| selection.contains_error_path(&path[1..], fragments)) + } else { + false + } + } + (Some(PathElement::Fragment(fragment)), Self::FragmentSpread { name, .. }) => { + if let Some(f) = fragments.get(name) { + if fragment.as_str().strip_prefix("... on ") == Some(f.type_condition.as_str()) + { + f.selection_set + .iter() + .any(|selection| selection.contains_error_path(&path[1..], fragments)) + } else { + false + } + } else { + false + } + } + (_, Self::FragmentSpread { name, .. }) => { + if let Some(f) = fragments.get(name) { + f.selection_set + .iter() + .any(|selection| selection.contains_error_path(path, fragments)) + } else { + false + } + } + (Some(PathElement::Index(_)), _) | (Some(PathElement::Flatten), _) => { + self.contains_error_path(&path[1..], fragments) + } + (Some(PathElement::Key(_)), Selection::InlineFragment { selection_set, .. }) => { + selection_set + .iter() + .any(|selection| selection.contains_error_path(&path[1..], fragments)) + } + (Some(PathElement::Fragment(_)), Selection::Field { .. }) => false, + } + } } pub(crate) fn parse_skip(directive: &ast::Directive) -> Option { diff --git a/apollo-router/src/state_machine.rs b/apollo-router/src/state_machine.rs index 80abdfb5f3..714fb0e623 100644 --- a/apollo-router/src/state_machine.rs +++ b/apollo-router/src/state_machine.rs @@ -316,10 +316,7 @@ where .router_configurator .create(configuration.clone(), schema.clone(), None, None) .await - .map_err(|err| { - tracing::error!("cannot create the router: {}", err); - Errored(ApolloRouterError::ServiceCreationError(err)) - })?; + .map_err(|err| Errored(ApolloRouterError::ServiceCreationError(err)))?; let web_endpoints = router_factory.web_endpoints(); diff --git a/apollo-router/src/test_harness.rs b/apollo-router/src/test_harness.rs index f22450d91b..bb64a452fa 100644 --- a/apollo-router/src/test_harness.rs +++ b/apollo-router/src/test_harness.rs @@ -122,11 +122,8 @@ impl<'a> TestHarness<'a> { /// These extra plugins are added after plugins specified in configuration. pub fn extra_plugin(mut self, plugin: P) -> Self { let type_id = std::any::TypeId::of::

(); - let name = match crate::plugin::plugins() - .iter() - .find(|(_name, factory)| factory.type_id == type_id) - { - Some((name, _factory)) => name.clone(), + let name = match crate::plugin::plugins().find(|factory| factory.type_id == type_id) { + Some(factory) => factory.name.clone(), None => format!( "extra_plugins.{}.{}", self.extra_plugins.len(), diff --git a/apollo-router/src/testdata/jaeger.router.yaml b/apollo-router/src/testdata/jaeger.router.yaml index fcc9a7b0e6..e5a4508d1a 100644 --- a/apollo-router/src/testdata/jaeger.router.yaml +++ b/apollo-router/src/testdata/jaeger.router.yaml @@ -8,7 +8,8 @@ telemetry: trace_config: service_name: router jaeger: - scheduled_delay: 100ms + batch_processor: + scheduled_delay: 100ms agent: endpoint: default experimental_logging: diff --git a/apollo-router/tests/jaeger_test.rs b/apollo-router/tests/jaeger_test.rs index e352a829ad..18a8828237 100644 --- a/apollo-router/tests/jaeger_test.rs +++ b/apollo-router/tests/jaeger_test.rs @@ -29,7 +29,7 @@ use crate::common::ValueExt; #[tokio::test(flavor = "multi_thread")] async fn test_jaeger_tracing() -> Result<(), BoxError> { - let tracer = opentelemetry_jaeger::new_pipeline() + let tracer = opentelemetry_jaeger::new_agent_pipeline() .with_service_name("my_app") .install_simple()?; @@ -232,7 +232,7 @@ fn parent_span<'a>(trace: &'a Value, span: &'a Value) -> Option<&'a Value> { async fn subgraph() { async fn handle(request: Request) -> Result, Infallible> { // create the opentelemetry-jaeger tracing infrastructure - let tracer_provider = opentelemetry_jaeger::new_pipeline() + let tracer_provider = opentelemetry_jaeger::new_agent_pipeline() .with_service_name("products") .build_simple() .unwrap(); diff --git a/apollo-router/tests/snapshots/integration_tests__traced_basic_composition.snap b/apollo-router/tests/snapshots/integration_tests__traced_basic_composition.snap index 0cb443e284..af31a12cb6 100644 --- a/apollo-router/tests/snapshots/integration_tests__traced_basic_composition.snap +++ b/apollo-router/tests/snapshots/integration_tests__traced_basic_composition.snap @@ -39,7 +39,7 @@ expression: get_spans() ], [ "otel.kind", - "internal" + "INTERNAL" ], [ "apollo_private.field_level_instrumentation_ratio", @@ -93,7 +93,7 @@ expression: get_spans() ], [ "otel.kind", - "internal" + "INTERNAL" ] ], "metadata": { @@ -117,7 +117,7 @@ expression: get_spans() "entries": [ [ "otel.kind", - "internal" + "INTERNAL" ] ], "metadata": { @@ -142,7 +142,7 @@ expression: get_spans() "entries": [ [ "otel.kind", - "internal" + "INTERNAL" ], [ "message", @@ -168,7 +168,7 @@ expression: get_spans() "entries": [ [ "otel.kind", - "internal" + "INTERNAL" ] ], "metadata": { @@ -190,7 +190,7 @@ expression: get_spans() "entries": [ [ "otel.kind", - "internal" + "INTERNAL" ], [ "apollo.subgraph.name", @@ -250,7 +250,7 @@ expression: get_spans() ], [ "otel.kind", - "internal" + "INTERNAL" ] ], "metadata": { @@ -292,7 +292,7 @@ expression: get_spans() "entries": [ [ "otel.kind", - "client" + "CLIENT" ], [ "net.peer.name", @@ -396,7 +396,7 @@ expression: get_spans() ], [ "otel.kind", - "internal" + "INTERNAL" ] ], "metadata": { @@ -419,7 +419,7 @@ expression: get_spans() "entries": [ [ "otel.kind", - "internal" + "INTERNAL" ], [ "apollo.subgraph.name", @@ -479,7 +479,7 @@ expression: get_spans() ], [ "otel.kind", - "internal" + "INTERNAL" ] ], "metadata": { @@ -521,7 +521,7 @@ expression: get_spans() "entries": [ [ "otel.kind", - "client" + "CLIENT" ], [ "net.peer.name", @@ -623,7 +623,7 @@ expression: get_spans() "entries": [ [ "otel.kind", - "internal" + "INTERNAL" ] ], "metadata": { @@ -649,7 +649,7 @@ expression: get_spans() ], [ "otel.kind", - "internal" + "INTERNAL" ] ], "metadata": { @@ -672,7 +672,7 @@ expression: get_spans() "entries": [ [ "otel.kind", - "internal" + "INTERNAL" ], [ "apollo.subgraph.name", @@ -732,7 +732,7 @@ expression: get_spans() ], [ "otel.kind", - "internal" + "INTERNAL" ] ], "metadata": { @@ -774,7 +774,7 @@ expression: get_spans() "entries": [ [ "otel.kind", - "client" + "CLIENT" ], [ "net.peer.name", @@ -880,7 +880,7 @@ expression: get_spans() ], [ "otel.kind", - "internal" + "INTERNAL" ] ], "metadata": { @@ -903,7 +903,7 @@ expression: get_spans() "entries": [ [ "otel.kind", - "internal" + "INTERNAL" ], [ "apollo.subgraph.name", @@ -963,7 +963,7 @@ expression: get_spans() ], [ "otel.kind", - "internal" + "INTERNAL" ] ], "metadata": { @@ -1005,7 +1005,7 @@ expression: get_spans() "entries": [ [ "otel.kind", - "client" + "CLIENT" ], [ "net.peer.name", diff --git a/apollo-router/tests/snapshots/integration_tests__traced_basic_request.snap b/apollo-router/tests/snapshots/integration_tests__traced_basic_request.snap index bc33af8a03..1a1c2edb19 100644 --- a/apollo-router/tests/snapshots/integration_tests__traced_basic_request.snap +++ b/apollo-router/tests/snapshots/integration_tests__traced_basic_request.snap @@ -39,7 +39,7 @@ expression: get_spans() ], [ "otel.kind", - "internal" + "INTERNAL" ], [ "apollo_private.field_level_instrumentation_ratio", @@ -93,7 +93,7 @@ expression: get_spans() ], [ "otel.kind", - "internal" + "INTERNAL" ] ], "metadata": { @@ -117,7 +117,7 @@ expression: get_spans() "entries": [ [ "otel.kind", - "internal" + "INTERNAL" ] ], "metadata": { @@ -142,7 +142,7 @@ expression: get_spans() "entries": [ [ "otel.kind", - "internal" + "INTERNAL" ], [ "message", @@ -168,7 +168,7 @@ expression: get_spans() "entries": [ [ "otel.kind", - "internal" + "INTERNAL" ], [ "apollo.subgraph.name", @@ -228,7 +228,7 @@ expression: get_spans() ], [ "otel.kind", - "internal" + "INTERNAL" ] ], "metadata": { @@ -270,7 +270,7 @@ expression: get_spans() "entries": [ [ "otel.kind", - "client" + "CLIENT" ], [ "net.peer.name", diff --git a/deny.toml b/deny.toml index 03c8787c83..8b68376f36 100644 --- a/deny.toml +++ b/deny.toml @@ -44,7 +44,7 @@ allow = [ "LicenseRef-ring", "MIT", "MPL-2.0", - "LicenseRef-ELv2", + "Elastic-2.0", "Unicode-DFS-2016" ] copyleft = "warn" diff --git a/dev-docs/yaml-design-guidance.md b/dev-docs/yaml-design-guidance.md new file mode 100644 index 0000000000..a0feb885de --- /dev/null +++ b/dev-docs/yaml-design-guidance.md @@ -0,0 +1,337 @@ +# Yaml config design + +The router uses yaml configuration, and when creating new features or extending existing features you'll likely need to think about how configuration is exposed. + +In general users should have a pretty good idea of what a configuration option does without referring to the documentation. + +## Migrations + +We won't always get things right, and sometimes we'll need to provide [migrations](apollo-router/src/configuration/migrations/README.md) from old config to new config. + +Make sure you: +1. Mention the change in the changelog +2. Update docs +3. Update any test configuration +4. Create a migration test as detailed in [migrations](apollo-router/src/configuration/migrations/README.md) +5. In your migration description tell the users what they have to update. + +## Process +It should be obvious to the user what they are configuring and how it will affect Router behaviour. It's tricky for us as developers to know when something isn't obvious to users as often we are too close to the domain. + +Complex configuration changes should be discussed with the team before starting the implementation, since they will drive the code's design. The process is as follows: +1. In the github issue put the proposed config in. +2. List any concerns. +3. Notify the team that you are looking for request for comment. +4. Ask users what they think. +5. If you are an Apollo Router team member then schedule a meeting to discuss. (This is important, often design considerations will fall out of conversation) +6. If it is not completely clear what the direction should be: +7. Wait a few days, often people will have ideas later even if they didn't in the meeting. +8. Make your changes. + +Note that these are not hard and fast rules, and if your config is really obviously correct then by all means make the change and be prepared to deal with comments at the review stage. + +## Design patterns + +Use the following as a rule of thumb, also look at existing config for inspiration. +The most important goal is usability, so do break the rules if it makes sense, but it's worth bringing the discussion to the team in such circumstances. + +1. [Avoid empty config](#avoid-empty-config). +2. [Use `#[serde(default)]`](#use-serdedefault). +3. [Do use `#[serde(deny_unknown_fields)]`](#use-serdedenyunknownfields). +4. [Don't use `#[serde(flatten)]`](#dont-use-serdeflatten). +5. [Use consistent terminology](#use-consistent-terminology). +6. [Don't use negative options](#dont-use-negative-options). +7. [Document your configuration options](#document-your-configuration-options). +8. [Plan for the future](#plan-for-the-future). + +### Avoid empty config + +In Rust you can use `Option` to say that config is optional, however this can give a bad experience if the type is complex and all fields are optional. + +#### GOOD +```rust +#[serde(deny_unknown_fields)] +struct Export { + url: Url // url is required +} +``` +```yaml +export: + url: http://example.com +``` + +#### GOOD +```rust +enum ExportUrl { + Default, + Url(Url) +} + +#[serde(deny_unknown_fields)] +struct Export { + url: ExportUrl // Url is required but user may specify `default` +} +``` +```yaml +export: + url: default +``` + +#### GOOD +In the case where you genuinely have no config or all sub-options have obvious defaults then use an `enabled: bool` flag. +```rust +#[serde(deny_unknown_fields)] +struct Export { + enabled: bool, + #[serde(default = "default_resource")] + url: Url // url is optional, see also but see advice on defaults. +} +``` +```yaml +export: + enabled: true +``` + +#### BAD +```rust +#[serde(deny_unknown_fields)] +struct Export { + url: Url +} +``` +```yaml +export: # The user is not aware that url was defaulted. +``` + +### Use `#[serde(default)]` +`#[serde(default="default_value_fn")` can be used to give fields defaults, and using this means that a generated json schema will also contain those defaults. The result of a default fn should be static. + +#### GOOD +```rust +#[serde(deny_unknown_fields)] +struct Export { + #[serde(default="default_url_fn") + url: Url +} +``` + +#### BAD +This could leak a password into a generated schema. +```rust +#[serde(deny_unknown_fields)] +struct Export { + #[serde(default="password_from_env_fn") + password: String +} +``` +Take a look at `env_defaults` in `expansion.rs` to see how env variables should be defaulted. + +### Use `#[serde(deny_unknown_fields)]` +Every container that takes part in config should be annotated with `#[serde(deny_unknown_fields)]`. If not the user can make mistakes on their config and they they won't get errors. + +#### GOOD +```rust +#[serde(deny_unknown_fields)] +struct Export { + url: Url +} +``` +```yaml +export: + url: http://example.com + backup: http://example2.com # The user will receive an error for this +``` + +#### BAD +```rust +struct Export { + url: Url +} +``` +```yaml +export: + url: http://example.com + backup: http://example2.com # The user will NOT receive an error for this +``` + +### Don't use `#[serde(flatten)]` +Serde flatten is tempting to use where you have identified common functionality, but creates a bad user experience as it is incompatible with `#[serde(deny_unknown_fields)]`. There isn't a great solution to this, but nesting config can sometimes help. + +See [serde documentation](https://serde.rs/field-attrs.html#flatten) for more details. + +#### MAYBE +```rust +#[serde(deny_unknown_fields)] +struct Export { + url: Url, + backup: Url +} +#[serde(deny_unknown_fields)] +struct Telemetry { + export: Export +} +#[serde(deny_unknown_fields)] +struct Metrics { + export: Export +} +``` +```yaml +telemetry: + export: + url: http://example.com + backup: http://example2.com +metrics: + export: + url: http://example.com + backup: http://example2.com +``` + +#### BAD +```rust +#[serde(deny_unknown_fields)] +struct Export { + url: Url, + backup: Url +} +struct Telemetry { + export: Export +} +``` +```yaml +telemetry: + url: http://example.com + backup: http://example2.com + unknown: sadness # The user will NOT receive an error for this +``` + +### Use consistent terminology +Be consistent with the rust API terminology. +* request - functionality that modifies the request or retrieves data from the request of a service. +* response - functionality that modifies the response or retrieves data from the response of a service. +* supergraph - functionality within Plugin::supergraph_service +* execution - functionality within Plugin::execution_service +* subgraph(s) - functionality within Plugin::subgraph_service + +If you use the above terminology then chances are you are doing something that will take place on every request. In this case make sure to include an `action` verb so the user know what the config is doing. + +#### GOOD +```yaml +headers: + subgraphs: # Modifies the subgraph service + products: + request: # Retrieves data from the request + - propagate: # The action. + named: foo +``` + +#### BAD +```yaml +headers: + named: foo # From where, what are we doing, when is it happening? +``` + +### Don't use negative options + +Router config uses positive options with defaults, this way users don't have to do the negation when reading the config. + +#### GOOD +```yaml +homepage: + enabled: true + log_headers: true +``` + +#### BAD +```yaml +my_plugin: + disabled: false + redact_headers: false +``` + + +### Document your configuration options +If your config is well documented in Rust then it will be well documented in the generated JSON Schema. This means that when users are modifying their config either in their IDE or in Apollo GraphOS, documentation is available. + +Example configuration should be included on all containers. + +#### GOOD +```rust +/// Export the data to the metrics endpoint +/// Example configuration: +/// ```yaml +/// export: +/// url: http://example.com +/// ``` +#[serde(deny_unknown_fields)] +struct Export { + /// The url to export metrics to. + url: Url +} +``` + +#### BAD +```rust +#[serde(deny_unknown_fields)] +struct Export { + url: Url +} +``` + +In addition, make sure to update the published documentation in the `docs/` folder. + +### Don't leak config +There are exceptions, but in general config should not be leaked from plugins. By reaching into a plugin config from outside of a plugin, there is leakage of functionality outside of compilation units. + +For Routers where the `Plugin` trait does not yet have `http_service` there will be leakage of config. The addition of the `http_service` to `Plugin` should eliminate the need to leak config. + +### Plan for the future + +Often configuration will be limited initially as a feature will be developed over time. It's important to consider what may be added in future. + +Examples of things that typically require extending later: +* Connection info to other systems. +* An action that retrieves information from a domain object e.g. `request.body`, `request.header` + +Often adding container objects can help. + +#### GOOD +```rust +#[serde(deny_unknown_fields)] +struct Export { + url: Url + // Future export options may be added here +} +#[serde(deny_unknown_fields)] +struct Telemetry { + export: Export +} +``` +```yaml +telemetry: + export: + url: http://example.com +``` +#### BAD +```rust +#[serde(deny_unknown_fields)] +struct Telemetry { + url: Url +} +``` +```yaml +telemetry: + url: http://example.com # Url for what? +``` +#### BAD +```rust +#[serde(deny_unknown_fields)] +struct Telemetry { + export_url: Url // export_url is not extendable. You can't add things like auth. +} +``` +```yaml +telemetry: + export_url: http://example.com # How do I specify auth +``` + + diff --git a/dockerfiles/diy/dockerfiles/Dockerfile.repo b/dockerfiles/diy/dockerfiles/Dockerfile.repo index 4203c3ca12..26cbba55e0 100644 --- a/dockerfiles/diy/dockerfiles/Dockerfile.repo +++ b/dockerfiles/diy/dockerfiles/Dockerfile.repo @@ -1,6 +1,6 @@ # Use the rust build image from docker as our base # renovate-automation: rustc version -FROM rust:1.63.0 as build +FROM rust:1.65.0 as build # Set our working directory for the build WORKDIR /usr/src/router @@ -9,7 +9,8 @@ WORKDIR /usr/src/router RUN apt-get update RUN apt-get -y install \ npm \ - nodejs + nodejs \ + protobuf-compiler # Add rustfmt since build requires it RUN rustup component add rustfmt diff --git a/dockerfiles/fed2-demo-gateway/package.json b/dockerfiles/fed2-demo-gateway/package.json index 48e53c6bd6..c6082c02da 100644 --- a/dockerfiles/fed2-demo-gateway/package.json +++ b/dockerfiles/fed2-demo-gateway/package.json @@ -7,8 +7,8 @@ "start": "node gateway.js" }, "dependencies": { - "@apollo/server": "4.2.2", - "@apollo/gateway": "2.2.1", + "@apollo/server": "4.3.0", + "@apollo/gateway": "2.2.2", "supergraph-demo-opentelemetry": "0.2.4", "graphql": "16.6.0" }, diff --git a/dockerfiles/tracing/datadog-subgraph/package-lock.json b/dockerfiles/tracing/datadog-subgraph/package-lock.json index 5f0625b622..5b74965973 100644 --- a/dockerfiles/tracing/datadog-subgraph/package-lock.json +++ b/dockerfiles/tracing/datadog-subgraph/package-lock.json @@ -9,7 +9,7 @@ "version": "1.0.0", "license": "ISC", "dependencies": { - "@apollo/federation": "^0.37.0", + "@apollo/federation": "^0.38.0", "apollo-server-core": "^3.7.0", "apollo-server-express": "^3.7.0", "dd-trace": "^3.0.0", @@ -17,7 +17,7 @@ "graphql": "^16.5.0" }, "devDependencies": { - "typescript": "4.9.3" + "typescript": "4.9.4" } }, "node_modules/@apollo/cache-control-types": { @@ -29,16 +29,16 @@ } }, "node_modules/@apollo/federation": { - "version": "0.37.1", - "resolved": "https://registry.npmjs.org/@apollo/federation/-/federation-0.37.1.tgz", - "integrity": "sha512-cLoBrBLt2dUEUmfISvGJ9YevnRGWhj+bVVJ8pP0bBrLfy1GWRYrsV8Jd87U2YeMEp7wuYM6M2PjE4Oy6PBMf2w==", + "version": "0.38.1", + "resolved": "https://registry.npmjs.org/@apollo/federation/-/federation-0.38.1.tgz", + "integrity": "sha512-miifyAEsFgiYKeM3lUHFH6+vKa2vm9dXKSyWVpX6oeJiPblFLe2/iByN3psZQO2sRdVqO1OKYrGXdgKc74XDKw==", "dependencies": { - "@apollo/subgraph": "^0.5.1", + "@apollo/subgraph": "^0.6.1", "apollo-server-types": "^3.0.2", "lodash.xorby": "^4.7.0" }, "engines": { - "node": ">=12.13.0 <18.0" + "node": ">=12.13.0" }, "peerDependencies": { "graphql": "^15.8.0 || ^16.0.0" @@ -70,14 +70,14 @@ } }, "node_modules/@apollo/subgraph": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/@apollo/subgraph/-/subgraph-0.5.1.tgz", - "integrity": "sha512-pj+igKgdpmTfgUmscTNuVdLip8WZ8jFKS5FGb/tD2hj4xPwaQ+MfszLsuNfoytp7d63PdLorIndxcHCW+rb7Dg==", + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/@apollo/subgraph/-/subgraph-0.6.1.tgz", + "integrity": "sha512-w/6FoubSxuzXSx8uvLE1wEuHZVHRXFyfHPKdM76wX5U/xw82zlUKseVO7wTuVODTcnUzEA30udYeCApUoC3/Xw==", "dependencies": { "@apollo/cache-control-types": "^1.0.2" }, "engines": { - "node": ">=12.13.0 <18.0" + "node": ">=12.13.0" }, "peerDependencies": { "graphql": "^15.8.0 || ^16.0.0" @@ -1718,9 +1718,9 @@ } }, "node_modules/typescript": { - "version": "4.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.3.tgz", - "integrity": "sha512-CIfGzTelbKNEnLpLdGFgdyKhG23CKdKgQPOBc+OUNrkJ2vr+KSzsSV5kq5iWhEQbok+quxgGzrAtGWCyU7tHnA==", + "version": "4.9.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.4.tgz", + "integrity": "sha512-Uz+dTXYzxXXbsFpM86Wh3dKCxrQqUcVMxwU54orwlJjOpO3ao8L7j5lH+dWfTwgCwIuM9GQ2kvVotzYJMXTBZg==", "dev": true, "bin": { "tsc": "bin/tsc", @@ -1837,11 +1837,11 @@ "requires": {} }, "@apollo/federation": { - "version": "0.37.1", - "resolved": "https://registry.npmjs.org/@apollo/federation/-/federation-0.37.1.tgz", - "integrity": "sha512-cLoBrBLt2dUEUmfISvGJ9YevnRGWhj+bVVJ8pP0bBrLfy1GWRYrsV8Jd87U2YeMEp7wuYM6M2PjE4Oy6PBMf2w==", + "version": "0.38.1", + "resolved": "https://registry.npmjs.org/@apollo/federation/-/federation-0.38.1.tgz", + "integrity": "sha512-miifyAEsFgiYKeM3lUHFH6+vKa2vm9dXKSyWVpX6oeJiPblFLe2/iByN3psZQO2sRdVqO1OKYrGXdgKc74XDKw==", "requires": { - "@apollo/subgraph": "^0.5.1", + "@apollo/subgraph": "^0.6.1", "apollo-server-types": "^3.0.2", "lodash.xorby": "^4.7.0" } @@ -1867,9 +1867,9 @@ } }, "@apollo/subgraph": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/@apollo/subgraph/-/subgraph-0.5.1.tgz", - "integrity": "sha512-pj+igKgdpmTfgUmscTNuVdLip8WZ8jFKS5FGb/tD2hj4xPwaQ+MfszLsuNfoytp7d63PdLorIndxcHCW+rb7Dg==", + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/@apollo/subgraph/-/subgraph-0.6.1.tgz", + "integrity": "sha512-w/6FoubSxuzXSx8uvLE1wEuHZVHRXFyfHPKdM76wX5U/xw82zlUKseVO7wTuVODTcnUzEA30udYeCApUoC3/Xw==", "requires": { "@apollo/cache-control-types": "^1.0.2" } @@ -3119,9 +3119,9 @@ } }, "typescript": { - "version": "4.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.3.tgz", - "integrity": "sha512-CIfGzTelbKNEnLpLdGFgdyKhG23CKdKgQPOBc+OUNrkJ2vr+KSzsSV5kq5iWhEQbok+quxgGzrAtGWCyU7tHnA==", + "version": "4.9.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.4.tgz", + "integrity": "sha512-Uz+dTXYzxXXbsFpM86Wh3dKCxrQqUcVMxwU54orwlJjOpO3ao8L7j5lH+dWfTwgCwIuM9GQ2kvVotzYJMXTBZg==", "dev": true }, "unpipe": { diff --git a/dockerfiles/tracing/datadog-subgraph/package.json b/dockerfiles/tracing/datadog-subgraph/package.json index 133c1ea381..8f8e225d50 100644 --- a/dockerfiles/tracing/datadog-subgraph/package.json +++ b/dockerfiles/tracing/datadog-subgraph/package.json @@ -10,7 +10,7 @@ "author": "", "license": "ISC", "dependencies": { - "@apollo/federation": "^0.37.0", + "@apollo/federation": "^0.38.0", "apollo-server-core": "^3.7.0", "apollo-server-express": "^3.7.0", "dd-trace": "^3.0.0", @@ -18,6 +18,6 @@ "graphql": "^16.5.0" }, "devDependencies": { - "typescript": "4.9.3" + "typescript": "4.9.4" } } diff --git a/dockerfiles/tracing/docker-compose.datadog.yml b/dockerfiles/tracing/docker-compose.datadog.yml index c2fe3f7100..ea355eb84f 100644 --- a/dockerfiles/tracing/docker-compose.datadog.yml +++ b/dockerfiles/tracing/docker-compose.datadog.yml @@ -3,7 +3,7 @@ services: apollo-router: container_name: apollo-router - image: ghcr.io/apollographql/router:v1.5.0 + image: ghcr.io/apollographql/router:v1.6.0 volumes: - ./supergraph.graphql:/etc/config/supergraph.graphql - ./router/datadog.router.yaml:/etc/config/configuration.yaml diff --git a/dockerfiles/tracing/docker-compose.jaeger.yml b/dockerfiles/tracing/docker-compose.jaeger.yml index 4ff6a379e8..6b59c6f04b 100644 --- a/dockerfiles/tracing/docker-compose.jaeger.yml +++ b/dockerfiles/tracing/docker-compose.jaeger.yml @@ -4,7 +4,7 @@ services: apollo-router: container_name: apollo-router #build: ./router - image: ghcr.io/apollographql/router:v1.5.0 + image: ghcr.io/apollographql/router:v1.6.0 volumes: - ./supergraph.graphql:/etc/config/supergraph.graphql - ./router/jaeger.router.yaml:/etc/config/configuration.yaml diff --git a/dockerfiles/tracing/docker-compose.zipkin.yml b/dockerfiles/tracing/docker-compose.zipkin.yml index 098a9f43a3..e0f5b0afb3 100644 --- a/dockerfiles/tracing/docker-compose.zipkin.yml +++ b/dockerfiles/tracing/docker-compose.zipkin.yml @@ -4,7 +4,7 @@ services: apollo-router: container_name: apollo-router build: ./router - image: ghcr.io/apollographql/router:v1.5.0 + image: ghcr.io/apollographql/router:v1.6.0 volumes: - ./supergraph.graphql:/etc/config/supergraph.graphql - ./router/zipkin.router.yaml:/etc/config/configuration.yaml diff --git a/dockerfiles/tracing/jaeger-subgraph/package-lock.json b/dockerfiles/tracing/jaeger-subgraph/package-lock.json index 343b0c96f8..8a207be001 100644 --- a/dockerfiles/tracing/jaeger-subgraph/package-lock.json +++ b/dockerfiles/tracing/jaeger-subgraph/package-lock.json @@ -9,7 +9,7 @@ "version": "1.0.0", "license": "ISC", "dependencies": { - "@apollo/federation": "^0.37.0", + "@apollo/federation": "^0.38.0", "apollo-server-core": "^3.7.0", "apollo-server-express": "^3.7.0", "express": "^4.18.1", @@ -18,7 +18,7 @@ "opentracing": "^0.14.7" }, "devDependencies": { - "typescript": "4.9.3" + "typescript": "4.9.4" } }, "node_modules/@apollo/cache-control-types": { @@ -30,16 +30,16 @@ } }, "node_modules/@apollo/federation": { - "version": "0.37.1", - "resolved": "https://registry.npmjs.org/@apollo/federation/-/federation-0.37.1.tgz", - "integrity": "sha512-cLoBrBLt2dUEUmfISvGJ9YevnRGWhj+bVVJ8pP0bBrLfy1GWRYrsV8Jd87U2YeMEp7wuYM6M2PjE4Oy6PBMf2w==", + "version": "0.38.1", + "resolved": "https://registry.npmjs.org/@apollo/federation/-/federation-0.38.1.tgz", + "integrity": "sha512-miifyAEsFgiYKeM3lUHFH6+vKa2vm9dXKSyWVpX6oeJiPblFLe2/iByN3psZQO2sRdVqO1OKYrGXdgKc74XDKw==", "dependencies": { - "@apollo/subgraph": "^0.5.1", + "@apollo/subgraph": "^0.6.1", "apollo-server-types": "^3.0.2", "lodash.xorby": "^4.7.0" }, "engines": { - "node": ">=12.13.0 <18.0" + "node": ">=12.13.0" }, "peerDependencies": { "graphql": "^15.8.0 || ^16.0.0" @@ -71,14 +71,14 @@ } }, "node_modules/@apollo/subgraph": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/@apollo/subgraph/-/subgraph-0.5.1.tgz", - "integrity": "sha512-pj+igKgdpmTfgUmscTNuVdLip8WZ8jFKS5FGb/tD2hj4xPwaQ+MfszLsuNfoytp7d63PdLorIndxcHCW+rb7Dg==", + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/@apollo/subgraph/-/subgraph-0.6.1.tgz", + "integrity": "sha512-w/6FoubSxuzXSx8uvLE1wEuHZVHRXFyfHPKdM76wX5U/xw82zlUKseVO7wTuVODTcnUzEA30udYeCApUoC3/Xw==", "dependencies": { "@apollo/cache-control-types": "^1.0.2" }, "engines": { - "node": ">=12.13.0 <18.0" + "node": ">=12.13.0" }, "peerDependencies": { "graphql": "^15.8.0 || ^16.0.0" @@ -1354,9 +1354,9 @@ } }, "node_modules/typescript": { - "version": "4.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.3.tgz", - "integrity": "sha512-CIfGzTelbKNEnLpLdGFgdyKhG23CKdKgQPOBc+OUNrkJ2vr+KSzsSV5kq5iWhEQbok+quxgGzrAtGWCyU7tHnA==", + "version": "4.9.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.4.tgz", + "integrity": "sha512-Uz+dTXYzxXXbsFpM86Wh3dKCxrQqUcVMxwU54orwlJjOpO3ao8L7j5lH+dWfTwgCwIuM9GQ2kvVotzYJMXTBZg==", "dev": true, "bin": { "tsc": "bin/tsc", @@ -1470,11 +1470,11 @@ "requires": {} }, "@apollo/federation": { - "version": "0.37.1", - "resolved": "https://registry.npmjs.org/@apollo/federation/-/federation-0.37.1.tgz", - "integrity": "sha512-cLoBrBLt2dUEUmfISvGJ9YevnRGWhj+bVVJ8pP0bBrLfy1GWRYrsV8Jd87U2YeMEp7wuYM6M2PjE4Oy6PBMf2w==", + "version": "0.38.1", + "resolved": "https://registry.npmjs.org/@apollo/federation/-/federation-0.38.1.tgz", + "integrity": "sha512-miifyAEsFgiYKeM3lUHFH6+vKa2vm9dXKSyWVpX6oeJiPblFLe2/iByN3psZQO2sRdVqO1OKYrGXdgKc74XDKw==", "requires": { - "@apollo/subgraph": "^0.5.1", + "@apollo/subgraph": "^0.6.1", "apollo-server-types": "^3.0.2", "lodash.xorby": "^4.7.0" } @@ -1500,9 +1500,9 @@ } }, "@apollo/subgraph": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/@apollo/subgraph/-/subgraph-0.5.1.tgz", - "integrity": "sha512-pj+igKgdpmTfgUmscTNuVdLip8WZ8jFKS5FGb/tD2hj4xPwaQ+MfszLsuNfoytp7d63PdLorIndxcHCW+rb7Dg==", + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/@apollo/subgraph/-/subgraph-0.6.1.tgz", + "integrity": "sha512-w/6FoubSxuzXSx8uvLE1wEuHZVHRXFyfHPKdM76wX5U/xw82zlUKseVO7wTuVODTcnUzEA30udYeCApUoC3/Xw==", "requires": { "@apollo/cache-control-types": "^1.0.2" } @@ -2477,9 +2477,9 @@ } }, "typescript": { - "version": "4.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.3.tgz", - "integrity": "sha512-CIfGzTelbKNEnLpLdGFgdyKhG23CKdKgQPOBc+OUNrkJ2vr+KSzsSV5kq5iWhEQbok+quxgGzrAtGWCyU7tHnA==", + "version": "4.9.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.4.tgz", + "integrity": "sha512-Uz+dTXYzxXXbsFpM86Wh3dKCxrQqUcVMxwU54orwlJjOpO3ao8L7j5lH+dWfTwgCwIuM9GQ2kvVotzYJMXTBZg==", "dev": true }, "unpipe": { diff --git a/dockerfiles/tracing/jaeger-subgraph/package.json b/dockerfiles/tracing/jaeger-subgraph/package.json index 92ed871e06..f0a98cef2e 100644 --- a/dockerfiles/tracing/jaeger-subgraph/package.json +++ b/dockerfiles/tracing/jaeger-subgraph/package.json @@ -10,7 +10,7 @@ "author": "", "license": "ISC", "dependencies": { - "@apollo/federation": "^0.37.0", + "@apollo/federation": "^0.38.0", "apollo-server-core": "^3.7.0", "apollo-server-express": "^3.7.0", "express": "^4.18.1", @@ -19,6 +19,6 @@ "opentracing": "^0.14.7" }, "devDependencies": { - "typescript": "4.9.3" + "typescript": "4.9.4" } } diff --git a/dockerfiles/tracing/zipkin-subgraph/package-lock.json b/dockerfiles/tracing/zipkin-subgraph/package-lock.json index c8c49d6cb5..4a01dd106c 100644 --- a/dockerfiles/tracing/zipkin-subgraph/package-lock.json +++ b/dockerfiles/tracing/zipkin-subgraph/package-lock.json @@ -9,7 +9,7 @@ "version": "1.0.0", "license": "ISC", "dependencies": { - "@apollo/federation": "^0.37.0", + "@apollo/federation": "^0.38.0", "apollo-server-core": "^3.7.0", "apollo-server-express": "^3.7.0", "express": "^4.18.1", @@ -19,7 +19,7 @@ "zipkin-javascript-opentracing": "^3.0.0" }, "devDependencies": { - "typescript": "4.9.3" + "typescript": "4.9.4" } }, "node_modules/@apollo/cache-control-types": { @@ -31,16 +31,16 @@ } }, "node_modules/@apollo/federation": { - "version": "0.37.1", - "resolved": "https://registry.npmjs.org/@apollo/federation/-/federation-0.37.1.tgz", - "integrity": "sha512-cLoBrBLt2dUEUmfISvGJ9YevnRGWhj+bVVJ8pP0bBrLfy1GWRYrsV8Jd87U2YeMEp7wuYM6M2PjE4Oy6PBMf2w==", + "version": "0.38.1", + "resolved": "https://registry.npmjs.org/@apollo/federation/-/federation-0.38.1.tgz", + "integrity": "sha512-miifyAEsFgiYKeM3lUHFH6+vKa2vm9dXKSyWVpX6oeJiPblFLe2/iByN3psZQO2sRdVqO1OKYrGXdgKc74XDKw==", "dependencies": { - "@apollo/subgraph": "^0.5.1", + "@apollo/subgraph": "^0.6.1", "apollo-server-types": "^3.0.2", "lodash.xorby": "^4.7.0" }, "engines": { - "node": ">=12.13.0 <18.0" + "node": ">=12.13.0" }, "peerDependencies": { "graphql": "^15.8.0 || ^16.0.0" @@ -72,14 +72,14 @@ } }, "node_modules/@apollo/subgraph": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/@apollo/subgraph/-/subgraph-0.5.1.tgz", - "integrity": "sha512-pj+igKgdpmTfgUmscTNuVdLip8WZ8jFKS5FGb/tD2hj4xPwaQ+MfszLsuNfoytp7d63PdLorIndxcHCW+rb7Dg==", + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/@apollo/subgraph/-/subgraph-0.6.1.tgz", + "integrity": "sha512-w/6FoubSxuzXSx8uvLE1wEuHZVHRXFyfHPKdM76wX5U/xw82zlUKseVO7wTuVODTcnUzEA30udYeCApUoC3/Xw==", "dependencies": { "@apollo/cache-control-types": "^1.0.2" }, "engines": { - "node": ">=12.13.0 <18.0" + "node": ">=12.13.0" }, "peerDependencies": { "graphql": "^15.8.0 || ^16.0.0" @@ -1381,9 +1381,9 @@ } }, "node_modules/typescript": { - "version": "4.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.3.tgz", - "integrity": "sha512-CIfGzTelbKNEnLpLdGFgdyKhG23CKdKgQPOBc+OUNrkJ2vr+KSzsSV5kq5iWhEQbok+quxgGzrAtGWCyU7tHnA==", + "version": "4.9.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.4.tgz", + "integrity": "sha512-Uz+dTXYzxXXbsFpM86Wh3dKCxrQqUcVMxwU54orwlJjOpO3ao8L7j5lH+dWfTwgCwIuM9GQ2kvVotzYJMXTBZg==", "dev": true, "bin": { "tsc": "bin/tsc", @@ -1529,11 +1529,11 @@ "requires": {} }, "@apollo/federation": { - "version": "0.37.1", - "resolved": "https://registry.npmjs.org/@apollo/federation/-/federation-0.37.1.tgz", - "integrity": "sha512-cLoBrBLt2dUEUmfISvGJ9YevnRGWhj+bVVJ8pP0bBrLfy1GWRYrsV8Jd87U2YeMEp7wuYM6M2PjE4Oy6PBMf2w==", + "version": "0.38.1", + "resolved": "https://registry.npmjs.org/@apollo/federation/-/federation-0.38.1.tgz", + "integrity": "sha512-miifyAEsFgiYKeM3lUHFH6+vKa2vm9dXKSyWVpX6oeJiPblFLe2/iByN3psZQO2sRdVqO1OKYrGXdgKc74XDKw==", "requires": { - "@apollo/subgraph": "^0.5.1", + "@apollo/subgraph": "^0.6.1", "apollo-server-types": "^3.0.2", "lodash.xorby": "^4.7.0" } @@ -1559,9 +1559,9 @@ } }, "@apollo/subgraph": { - "version": "0.5.1", - "resolved": "https://registry.npmjs.org/@apollo/subgraph/-/subgraph-0.5.1.tgz", - "integrity": "sha512-pj+igKgdpmTfgUmscTNuVdLip8WZ8jFKS5FGb/tD2hj4xPwaQ+MfszLsuNfoytp7d63PdLorIndxcHCW+rb7Dg==", + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/@apollo/subgraph/-/subgraph-0.6.1.tgz", + "integrity": "sha512-w/6FoubSxuzXSx8uvLE1wEuHZVHRXFyfHPKdM76wX5U/xw82zlUKseVO7wTuVODTcnUzEA30udYeCApUoC3/Xw==", "requires": { "@apollo/cache-control-types": "^1.0.2" } @@ -2548,9 +2548,9 @@ } }, "typescript": { - "version": "4.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.3.tgz", - "integrity": "sha512-CIfGzTelbKNEnLpLdGFgdyKhG23CKdKgQPOBc+OUNrkJ2vr+KSzsSV5kq5iWhEQbok+quxgGzrAtGWCyU7tHnA==", + "version": "4.9.4", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-4.9.4.tgz", + "integrity": "sha512-Uz+dTXYzxXXbsFpM86Wh3dKCxrQqUcVMxwU54orwlJjOpO3ao8L7j5lH+dWfTwgCwIuM9GQ2kvVotzYJMXTBZg==", "dev": true }, "unpipe": { diff --git a/dockerfiles/tracing/zipkin-subgraph/package.json b/dockerfiles/tracing/zipkin-subgraph/package.json index 0dd058f839..82da09fd25 100644 --- a/dockerfiles/tracing/zipkin-subgraph/package.json +++ b/dockerfiles/tracing/zipkin-subgraph/package.json @@ -10,7 +10,7 @@ "author": "", "license": "ISC", "dependencies": { - "@apollo/federation": "^0.37.0", + "@apollo/federation": "^0.38.0", "apollo-server-core": "^3.7.0", "apollo-server-express": "^3.7.0", "express": "^4.18.1", @@ -20,6 +20,6 @@ "zipkin-javascript-opentracing": "^3.0.0" }, "devDependencies": { - "typescript": "4.9.3" + "typescript": "4.9.4" } } diff --git a/docs/source/configuration/metrics.mdx b/docs/source/configuration/metrics.mdx index 80b466d9d9..edc13dbecb 100644 --- a/docs/source/configuration/metrics.mdx +++ b/docs/source/configuration/metrics.mdx @@ -61,7 +61,6 @@ The following metrics are available using Prometheus: - HTTP router request duration (`apollo_router_http_request_duration_seconds_bucket`) - HTTP request duration by subgraph (`apollo_router_http_request_duration_seconds_bucket` with attribute `subgraph`) - Total number of HTTP requests by HTTP Status (`apollo_router_http_requests_total`) -- Total number of HTTP requests in error (`apollo_router_http_requests_error_total`) ## Using OpenTelemetry Collector diff --git a/docs/source/configuration/subgraph-error-inclusion.mdx b/docs/source/configuration/subgraph-error-inclusion.mdx index 5ee656bff4..cd349913e5 100644 --- a/docs/source/configuration/subgraph-error-inclusion.mdx +++ b/docs/source/configuration/subgraph-error-inclusion.mdx @@ -16,7 +16,7 @@ To configure subgraph error inclusion, add the `include_subgraph_errors` plugin ```yaml title="router.yaml" include_subgraph_errors: - all: true # Propagate errors from all subraphs + all: true # Propagate errors from all subgraphs subgraphs: products: false # Do not propagate errors from the products subgraph ``` diff --git a/docs/source/configuration/tracing.mdx b/docs/source/configuration/tracing.mdx index 8a3a7972fc..df62ac570a 100644 --- a/docs/source/configuration/tracing.mdx +++ b/docs/source/configuration/tracing.mdx @@ -88,7 +88,7 @@ telemetry: Specifying explicit propagation is generally only required if you're using an exporter that supports multiple trace ID formats (e.g., OpenTelemetry Collector, Jaeger, or OpenTracing compatible exporters). -### Trace ID +## Trace ID > This is part of an experimental feature, it means any time until it's stabilized (without the prefix `experimental_`) we might change the configuration shape or adding/removing features. > If you want to give feedback or participate in that feature feel free to join [this discussion on GitHub](https://github.com/apollographql/router/discussions/2147). @@ -105,6 +105,46 @@ telemetry: Using this configuration you will have a response header called `my-trace-id` containing the trace ID. It could help you to debug a specific query if you want to grep your log with this trace id to have more context. +## Batch Processor + +All trace exporters (apollo|datadog|zipkin|jaeger|otlp) have batch span processor configuration, it will be necessary to tune this if you see the following in your logs: + +`OpenTelemetry trace error occurred: cannot send span to the batch span processor because the channel is full` + +* **scheduled_delay** The delay from receiving the first span to the batch being sent. +* **max_concurrent_exports** The maximum number of overlapping export requests. For instance if ingest is taking a long time to respond there may be several overlapping export requests. +* **max_export_batch_size** The number of spans to include in a batch. Your ingest may have max message size limits. +* **max_export_timeout** The timeout for sending spans before dropping the data. +* **max_queue_size** The maximum number of spans to be buffered before dropping span data. + +```yaml title="router.yaml" +telemetry: + # Apollo tracing + apollo: + batch_processor: + scheduled_delay: 100ms + max_concurrent_exports: 1000 + max_export_batch_size: 10000 + max_export_timeout: 100s + max_queue_size: 10000 + + tracing: + # Datadog + datadog: + batch_processor: + scheduled_delay: 100ms + max_concurrent_exports: 1000 + max_export_batch_size: 10000 + max_export_timeout: 100s + max_queue_size: 10000 + endpoint: default + # Jaeger + # Otlp + # Zipkin +``` + +You will need to experiment to find the setting that are appropriate for your use case. + ## Using Datadog The Apollo Router can be configured to connect to either the default agent address or a URL. diff --git a/docs/source/containerization/docker.mdx b/docs/source/containerization/docker.mdx index 9314399b5c..c23310dd84 100644 --- a/docs/source/containerization/docker.mdx +++ b/docs/source/containerization/docker.mdx @@ -11,7 +11,7 @@ The default behaviour of the router images is suitable for a quickstart or devel Note: The [docker documentation](https://docs.docker.com/engine/reference/run/) for the run command may be helpful when reading through the examples. -Note: The exact image version to use is your choice depending on which release you wish to use. In the following examples, replace `` with your chosen version. e.g.: `v1.5.0` +Note: The exact image version to use is your choice depending on which release you wish to use. In the following examples, replace `` with your chosen version. e.g.: `v1.6.0` ## Override the configuration diff --git a/docs/source/containerization/kubernetes.mdx b/docs/source/containerization/kubernetes.mdx index 537015a552..1026c0f7a6 100644 --- a/docs/source/containerization/kubernetes.mdx +++ b/docs/source/containerization/kubernetes.mdx @@ -13,7 +13,7 @@ import { Link } from 'gatsby'; [Helm](https://helm.sh) is the package manager for kubernetes. -There is a complete [helm chart definition](https://github.com/apollographql/router/tree/v1.5.0/helm/chart/router) in the repo which illustrates how to use helm to deploy the router in kubernetes. +There is a complete [helm chart definition](https://github.com/apollographql/router/tree/v1.6.0/helm/chart/router) in the repo which illustrates how to use helm to deploy the router in kubernetes. In both the following examples, we are using helm to install the router: - into namespace "router-deploy" (create namespace if it doesn't exist) @@ -64,10 +64,10 @@ kind: ServiceAccount metadata: name: release-name-router labels: - helm.sh/chart: router-1.0.0-rc.8 + helm.sh/chart: router-1.0.0-rc.10 app.kubernetes.io/name: router app.kubernetes.io/instance: release-name - app.kubernetes.io/version: "v1.4.0" + app.kubernetes.io/version: "v1.6.0" app.kubernetes.io/managed-by: Helm --- # Source: router/templates/secret.yaml @@ -76,10 +76,10 @@ kind: Secret metadata: name: "release-name-router" labels: - helm.sh/chart: router-1.0.0-rc.8 + helm.sh/chart: router-1.0.0-rc.10 app.kubernetes.io/name: router app.kubernetes.io/instance: release-name - app.kubernetes.io/version: "v1.4.0" + app.kubernetes.io/version: "v1.6.0" app.kubernetes.io/managed-by: Helm data: managedFederationApiKey: "IlJFREFDVEVEIg==" @@ -90,10 +90,10 @@ kind: ConfigMap metadata: name: release-name-router labels: - helm.sh/chart: router-1.0.0-rc.8 + helm.sh/chart: router-1.0.0-rc.10 app.kubernetes.io/name: router app.kubernetes.io/instance: release-name - app.kubernetes.io/version: "v1.4.0" + app.kubernetes.io/version: "v1.6.0" app.kubernetes.io/managed-by: Helm data: configuration.yaml: | @@ -117,10 +117,10 @@ kind: Service metadata: name: release-name-router labels: - helm.sh/chart: router-1.0.0-rc.8 + helm.sh/chart: router-1.0.0-rc.10 app.kubernetes.io/name: router app.kubernetes.io/instance: release-name - app.kubernetes.io/version: "v1.4.0" + app.kubernetes.io/version: "v1.6.0" app.kubernetes.io/managed-by: Helm spec: type: ClusterIP @@ -143,10 +143,10 @@ kind: Deployment metadata: name: release-name-router labels: - helm.sh/chart: router-1.0.0-rc.8 + helm.sh/chart: router-1.0.0-rc.10 app.kubernetes.io/name: router app.kubernetes.io/instance: release-name - app.kubernetes.io/version: "v1.4.0" + app.kubernetes.io/version: "v1.6.0" app.kubernetes.io/managed-by: Helm annotations: @@ -172,7 +172,7 @@ spec: - name: router securityContext: {} - image: "ghcr.io/apollographql/router:v1.4.0" + image: "ghcr.io/apollographql/router:v1.6.0" imagePullPolicy: IfNotPresent args: - --hot-reload @@ -220,10 +220,10 @@ kind: Pod metadata: name: "release-name-router-test-connection" labels: - helm.sh/chart: router-1.0.0-rc.8 + helm.sh/chart: router-1.0.0-rc.10 app.kubernetes.io/name: router app.kubernetes.io/instance: release-name - app.kubernetes.io/version: "v1.4.0" + app.kubernetes.io/version: "v1.6.0" app.kubernetes.io/managed-by: Helm annotations: "helm.sh/hook": test diff --git a/docs/source/customizations/rhai-api.mdx b/docs/source/customizations/rhai-api.mdx index 201a616262..e2f264e9bf 100644 --- a/docs/source/customizations/rhai-api.mdx +++ b/docs/source/customizations/rhai-api.mdx @@ -165,6 +165,14 @@ fn supergraph_service(service) { } ``` +### Headers with multiple values + +The simple get/set api for dealing with single value headers is sufficient for most use cases. If you wish to set multiple values on a key then you should do this by supplying an array of values. + +If you wish to get multiple values for a header key, then you must use the `values()` fn, NOT the indexed accessor. If you do use the indexed accessor, it will only return the first value (as a string) associated with the key. + +Look at the examples to see how this works in practice. + ## `Request` interface All callback functions registered via `map_request` are passed a `request` object that represents the request sent by the client. This object provides the following fields, any of which a callback can modify in-place (read-write): @@ -250,6 +258,15 @@ request.headers["x-my-new-header"] = 42.to_string(); // Inserts a new header "x- print(`${request.headers["x-my-new-header"]}`); // Writes "42" into the router log at info level // Rhai also supports extended dot notation for indexed variables, so this is equivalent request.headers.x-my-new-header = 42.to_string(); +// You can also set an header value from an array. Useful with the "set-cookie" header, +// Note: It's probably more useful to do this on response headers. Simply illustrating the syntax here. +request.headers["set-cookie"] = [ + "foo=bar; Domain=localhost; Path=/; Expires=Wed, 04 Jan 2023 17:25:27 GMT; HttpOnly; Secure; SameSite=None", + "foo2=bar2; Domain=localhost; Path=/; Expires=Wed, 04 Jan 2023 17:25:27 GMT; HttpOnly; Secure; SameSite=None", +]; +// You can also get multiple header values for a header using the values() fn +// Note: It's probably more useful to do this on response headers. Simply illustrating the syntax here. +print(`${request.headers.values("set-cookie")}`); ``` ### `request.body.query` diff --git a/docs/source/customizations/rhai.mdx b/docs/source/customizations/rhai.mdx index ed652422d3..9a1fa34193 100644 --- a/docs/source/customizations/rhai.mdx +++ b/docs/source/customizations/rhai.mdx @@ -48,6 +48,17 @@ To use Rhai scripts with the Apollo Router, you must do the following: * By default, the Apollo Router looks for `main.rhai` in your Rhai script directory. * You can override this default with the `main` key (see above). +## Hot reloading + +The router will "watch" your "rhai.scripts" directory for changes and prompt an interpreter re-load if changes are detected. Changes are defined as: + + * creating a new file with a ".rhai" suffix + * modifying or removing an existing file with a ".rhai" suffix + +The watch is recursive, so files in sub-directories of the "rhai.scripts" directory are also watched. + +The router attempts to identify errors in scripts before applying the changes. If errors are detected, these will be logged and the changes will not be applied to the runtime. Not all classes of error can be reliably detected, so check the log output of your router to make sure that changes have been applied. + ## The main file Your Rhai script's main file defines whichever combination of supported entry point hooks you want to use. Here's a skeleton `main.rhai` file that includes all available hooks and also registers all available [callbacks](#service-callbacks): diff --git a/docs/source/index.mdx b/docs/source/index.mdx index 727f9c75a5..22f3a85809 100644 --- a/docs/source/index.mdx +++ b/docs/source/index.mdx @@ -20,8 +20,6 @@ flowchart BT; clients -.- gateway; class clients secondary; ``` - - The Apollo Router is [implemented in Rust](https://github.com/apollographql/router), which provides [performance benefits](https://www.apollographql.com/blog/announcement/backend/apollo-router-our-graphql-federation-runtime-in-rust/) over the Node.js `@apollo/gateway` library. If you have an existing supergraph that currently uses `@apollo/gateway`, you can move to the Apollo Router without changing any other part of your supergraph. diff --git a/examples/telemetry/README.md b/examples/telemetry/README.md index 12d137ae1d..702cd6d398 100644 --- a/examples/telemetry/README.md +++ b/examples/telemetry/README.md @@ -2,10 +2,10 @@ Demonstrates configuring of the router for: -- OpenTelemetry - - Jaeger - - OpenTelemetry Collector -- Spaceport (Apollo Studio) +* OpenTelemetry + * Jaeger + * OpenTelemetry Collector + ## OpenTelemetry diff --git a/helm/chart/router/Chart.yaml b/helm/chart/router/Chart.yaml index 91cd86e321..d0ec9a7dd3 100644 --- a/helm/chart/router/Chart.yaml +++ b/helm/chart/router/Chart.yaml @@ -19,10 +19,10 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.0.0-rc.9 +version: 1.0.0-rc.10 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "v1.5.0" +appVersion: "v1.6.0" diff --git a/helm/chart/router/README.md b/helm/chart/router/README.md index e07074daf9..327eb04833 100644 --- a/helm/chart/router/README.md +++ b/helm/chart/router/README.md @@ -2,7 +2,7 @@ [router](https://github.com/apollographql/router) Rust Graph Routing runtime for Apollo Federation -![Version: 1.0.0-rc.8](https://img.shields.io/badge/Version-1.0.0--rc.8-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.4.0](https://img.shields.io/badge/AppVersion-v1.4.0-informational?style=flat-square) +![Version: 1.0.0-rc.10](https://img.shields.io/badge/Version-1.0.0--rc.10-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v1.6.0](https://img.shields.io/badge/AppVersion-v1.6.0-informational?style=flat-square) ## Prerequisites @@ -11,7 +11,7 @@ ## Get Repo Info ```console -helm pull oci://ghcr.io/apollographql/helm-charts/router --version 1.0.0-rc.8 +helm pull oci://ghcr.io/apollographql/helm-charts/router --version 1.0.0-rc.10 ``` ## Install Chart @@ -19,7 +19,7 @@ helm pull oci://ghcr.io/apollographql/helm-charts/router --version 1.0.0-rc.8 **Important:** only helm3 is supported ```console -helm upgrade --install [RELEASE_NAME] oci://ghcr.io/apollographql/helm-charts/router --version 1.0.0-rc.8 --values my-values.yaml +helm upgrade --install [RELEASE_NAME] oci://ghcr.io/apollographql/helm-charts/router --version 1.0.0-rc.10 --values my-values.yaml ``` _See [configuration](#configuration) below._ diff --git a/helm/chart/router/templates/deployment.yaml b/helm/chart/router/templates/deployment.yaml index 05ec4abc0a..9ebc162d2b 100644 --- a/helm/chart/router/templates/deployment.yaml +++ b/helm/chart/router/templates/deployment.yaml @@ -20,9 +20,14 @@ spec: {{- include "router.selectorLabels" . | nindent 6 }} template: metadata: - {{- with .Values.podAnnotations }} + {{- if or .Values.podAnnotations .Values.supergraphFile }} annotations: - {{- toYaml . | nindent 8 }} + {{- if .Values.supergraphFile }} + checksum/supergraph-config-map: {{ .Values.supergraphFile | sha256sum }} + {{- end }} + {{- if .Values.podAnnotations }} + {{- toYaml .Values.podAnnotations | nindent 8 }} + {{- end }} {{- end }} labels: {{- include "router.selectorLabels" . | nindent 8 }} diff --git a/licenses.html b/licenses.html index 49cc81fd3d..7f27d0842e 100644 --- a/licenses.html +++ b/licenses.html @@ -44,10 +44,11 @@

Third Party Licenses

Overview of licenses:

    -
  • MIT License (76)
    • -
  • Apache License 2.0 (54)
    • +
  • MIT License (77)
    • +
  • Apache License 2.0 (53)
  • ISC License (12)
  • BSD 3-Clause "New" or "Revised" License (6)
    • +
  • Elastic License 2.0 (2)
  • Mozilla Public License 2.0 (2)
  • BSD 2-Clause "Simplified" License (1)
  • Creative Commons Zero v1.0 Universal (1)
    • @@ -1292,10 +1293,13 @@

    Used by:

  • opentelemetry-http
  • opentelemetry-jaeger
  • opentelemetry-otlp
    • -
  • opentelemetry-otlp
  • opentelemetry-prometheus
    • +
  • opentelemetry-proto
    • +
  • opentelemetry-proto
  • opentelemetry-semantic-conventions
  • opentelemetry-zipkin
    • +
  • opentelemetry_api
    • +
  • opentelemetry_sdk
  • os_str_bytes
  • ryu
  • structopt
    • @@ -1712,216 +1716,6 @@

    Used by:

    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - - -
  • -

    Apache License 2.0

    -

    Used by:

    - - 
                                     Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "{}"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright 2019 Michael P. Jung
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-
  • @@ -4176,7 +3970,6 @@

    Apache License 2.0

    Used by:

                                  Apache License
                         Version 2.0, January 2004
@@ -6065,6 +5858,7 @@ 
    Used by:
    
                     
  • ahash
    • 
                     
  • anyhow
    • 
                     
  • arbitrary
    • 
+                    
  • arc-swap
    • 
                     
  • async-channel
    • 
                     
  • async-compression
    • 
                     
  • async-trait
    • 
@@ -6128,6 +5922,7 @@ 
    Used by:
    
                     
  • libgit2-sys
    • 
                     
  • libm
    • 
                     
  • libz-sys
    • 
+                    
  • linkme
    • 
                     
  • lock_api
    • 
                     
  • mime
    • 
                     
  • mockall
    • 
@@ -6157,9 +5952,13 @@ 
    Used by:
    
                     
  • pest_meta
    • 
                     
  • petgraph
    • 
                     
  • pkg-config
    • 
+                    
  • prettyplease
    • 
                     
  • proc-macro-hack
    • 
                     
  • proc-macro2
    • 
                     
  • prost
    • 
+                    
  • prost-build
    • 
+                    
  • prost-derive
    • 
+                    
  • prost-types
    • 
                     
  • proteus
    • 
                     
  • quote
    • 
                     
  • regex
    • 
@@ -6169,17 +5968,14 @@ 
    Used by:
    
                     
  • rustc-hash
    • 
                     
  • rustc_version
    • 
                     
  • rustc_version
    • 
-                    
  • rustls
    • 
                     
  • rustls
    • 
                     
  • rustls-native-certs
    • 
-                    
  • rustls-native-certs
    • 
                     
  • rustls-pemfile
    • 
                     
  • rustversion
    • 
                     
  • salsa
    • 
                     
  • salsa-macros
    • 
                     
  • scopeguard
    • 
                     
  • sct
    • 
-                    
  • sct
    • 
                     
  • security-framework
    • 
                     
  • security-framework-sys
    • 
                     
  • semver
    • 
@@ -9798,18 +9594,14 @@ 
    Used by:
    
                     
  • apollo-parser
    • 
                     
  • apollo-smith
    • 
                     
  • askama_shared
    • 
-                    
  • buildstructor
    • 
-                    
  • deadpool-runtime
    • 
                     
  • graphql-introspection-query
    • 
                     
  • graphql_client
    • 
                     
  • graphql_client_codegen
    • 
                     
  • graphql_query_derive
    • 
                     
  • libssh2-sys
    • 
+                    
  • linkme-impl
    • 
                     
  • md5
    • 
                     
  • num-cmp
    • 
-                    
  • prost-build
    • 
-                    
  • prost-derive
    • 
-                    
  • prost-types
    • 
                     
  • rhai_codegen
    • 
                     
  • thrift
    • 
                     
  • tinyvec_macros
    • 
@@ -10173,6 +9965,7 @@ 
    Apache License 2.0
    
                 
    Used by:
    
                 
                 
    Copyright [2022] [Bryn Cooke]
 
@@ -10220,7 +10013,7 @@ 
    Used by:
    
                 
-                
    Copyright (c) <year> <owner> All rights reserved.
+                
    Copyright (c) <year> <owner> 
 
 Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
 
@@ -10369,7 +10162,7 @@ 
    Used by:
    
                 
-                
    Copyright (c) <year> <owner>. All rights reserved.
+                
    Copyright (c) <year> <owner>. 
 
 Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
 
@@ -10544,6 +10337,211 @@ 
    Used by:
    
  d. Affirmer understands and acknowledges that Creative Commons is not a
     party to this document and has no duty or obligation with respect to
     this CC0 or use of the Work.
+
    
+
    • +
  • +

    Elastic License 2.0

    +

    Used by:

    + + 
    Copyright 2021 Apollo Graph, Inc.
+
+Elastic License 2.0
+
+## Acceptance
+
+By using the software, you agree to all of the terms and conditions below.
+
+## Copyright License
+
+The licensor grants you a non-exclusive, royalty-free, worldwide,
+non-sublicensable, non-transferable license to use, copy, distribute, make
+available, and prepare derivative works of the software, in each case subject to
+the limitations and conditions below.
+
+## Limitations
+
+You may not provide the software to third parties as a hosted or managed
+service, where the service provides users with access to any substantial set of
+the features or functionality of the software.
+
+You may not move, change, disable, or circumvent the license key functionality
+in the software, and you may not remove or obscure any functionality in the
+software that is protected by the license key.
+
+You may not alter, remove, or obscure any licensing, copyright, or other notices
+of the licensor in the software. Any use of the licensorโ€™s trademarks is subject
+to applicable law.
+
+## Patents
+
+The licensor grants you a license, under any patent claims the licensor can
+license, or becomes able to license, to make, have made, use, sell, offer for
+sale, import and have imported the software, in each case subject to the
+limitations and conditions in this license. This license does not cover any
+patent claims that you cause to be infringed by modifications or additions to
+the software. If you or your company make any written claim that the software
+infringes or contributes to infringement of any patent, your patent license for
+the software granted under these terms ends immediately. If your company makes
+such a claim, your patent license ends immediately for work on behalf of your
+company.
+
+## Notices
+
+You must ensure that anyone who gets a copy of any part of the software from you
+also gets a copy of these terms.
+
+If you modify the software, you must include in any modified copies of the
+software prominent notices stating that you have modified the software.
+
+## No Other Rights
+
+These terms do not imply any licenses other than those expressly granted in
+these terms.
+
+## Termination
+
+If you use the software in violation of these terms, such use is not licensed,
+and your licenses will automatically terminate. If the licensor provides you
+with a notice of your violation, and you cease all violation of this license no
+later than 30 days after you receive that notice, your licenses will be
+reinstated retroactively. However, if you violate these terms after such
+reinstatement, any additional violation of these terms will cause your licenses
+to terminate automatically and permanently.
+
+## No Liability
+
+*As far as the law allows, the software comes as is, without any warranty or
+condition, and the licensor will not be liable to you for any damages arising
+out of these terms or the use or nature of the software, under any kind of
+legal claim.*
+
+## Definitions
+
+The **licensor** is the entity offering these terms, and the **software** is the
+software the licensor makes available under these terms, including any portion
+of it.
+
+**you** refers to the individual or entity agreeing to these terms.
+
+**your company** is any legal entity, sole proprietorship, or other kind of
+organization that you work for, plus all organizations that have control over,
+are under the control of, or are under common control with that
+organization. **control** means ownership of substantially all the assets of an
+entity, or the power to direct its management and policies by vote, contract, or
+otherwise. Control can be direct or indirect.
+
+**your licenses** are all the licenses granted to you for the software under
+these terms.
+
+**use** means anything you do with the software requiring one of your licenses.
+
+**trademark** means trademarks, service marks, and similar rights.
+
+--------------------------------------------------------------------------------
    +
    • +
  • +

    Elastic License 2.0

    +

    Used by:

    + + 
    Copyright 2021 Apollo Graph, Inc.
+
+Elastic License 2.0
+
+## Acceptance
+
+By using the software, you agree to all of the terms and conditions below.
+
+## Copyright License
+
+The licensor grants you a non-exclusive, royalty-free, worldwide,
+non-sublicensable, non-transferable license to use, copy, distribute, make
+available, and prepare derivative works of the software, in each case subject to
+the limitations and conditions below.
+
+## Limitations
+
+You may not provide the software to third parties as a hosted or managed
+service, where the service provides users with access to any substantial set of
+the features or functionality of the software.
+
+You may not move, change, disable, or circumvent the license key functionality
+in the software, and you may not remove or obscure any functionality in the
+software that is protected by the license key.
+
+You may not alter, remove, or obscure any licensing, copyright, or other notices
+of the licensor in the software. Any use of the licensorโ€™s trademarks is subject
+to applicable law.
+
+## Patents
+
+The licensor grants you a license, under any patent claims the licensor can
+license, or becomes able to license, to make, have made, use, sell, offer for
+sale, import and have imported the software, in each case subject to the
+limitations and conditions in this license. This license does not cover any
+patent claims that you cause to be infringed by modifications or additions to
+the software. If you or your company make any written claim that the software
+infringes or contributes to infringement of any patent, your patent license for
+the software granted under these terms ends immediately. If your company makes
+such a claim, your patent license ends immediately for work on behalf of your
+company.
+
+## Notices
+
+You must ensure that anyone who gets a copy of any part of the software from you
+also gets a copy of these terms.
+
+If you modify the software, you must include in any modified copies of the
+software prominent notices stating that you have modified the software.
+
+## No Other Rights
+
+These terms do not imply any licenses other than those expressly granted in
+these terms.
+
+## Termination
+
+If you use the software in violation of these terms, such use is not licensed,
+and your licenses will automatically terminate. If the licensor provides you
+with a notice of your violation, and you cease all violation of this license no
+later than 30 days after you receive that notice, your licenses will be
+reinstated retroactively. However, if you violate these terms after such
+reinstatement, any additional violation of these terms will cause your licenses
+to terminate automatically and permanently.
+
+## No Liability
+
+*As far as the law allows, the software comes as is, without any warranty or
+condition, and the licensor will not be liable to you for any damages arising
+out of these terms or the use or nature of the software, under any kind of
+legal claim.*
+
+## Definitions
+
+The **licensor** is the entity offering these terms, and the **software** is the
+software the licensor makes available under these terms, including any portion
+of it.
+
+**you** refers to the individual or entity agreeing to these terms.
+
+**your company** is any legal entity, sole proprietorship, or other kind of
+organization that you work for, plus all organizations that have control over,
+are under the control of, or are under common control with that
+organization. **control** means ownership of substantially all the assets of an
+entity, or the power to direct its management and policies by vote, contract, or
+otherwise. Control can be direct or indirect.
+
+**your licenses** are all the licenses granted to you for the software under
+these terms.
+
+**use** means anything you do with the software requiring one of your licenses.
+
+**trademark** means trademarks, service marks, and similar rights.
+
+--------------------------------------------------------------------------------
  • @@ -10592,7 +10590,6 @@

    ISC License

    Used by:

    // Copyright 2015 The Chromium Authors. All rights reserved.
 //
@@ -10690,7 +10687,6 @@ 
    ISC License
    
                 
    Used by:
    
                 
                 
    Except as otherwise noted, this project is licensed under the following
 (ISC-style) terms:
@@ -11795,33 +11791,28 @@ 
    Used by:
    
                 
    MIT License
    
                 
    Used by:
    
                 
-                
    Copyright (c) 2021 Tokio Contributors
+                
    Copyright (c) 2020 Lucio Franco
 
-Permission is hereby granted, free of charge, to any
-person obtaining a copy of this software and associated
-documentation files (the "Software"), to deal in the
-Software without restriction, including without
-limitation the rights to use, copy, modify, merge,
-publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software
-is furnished to do so, subject to the following
-conditions:
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
 
-The above copyright notice and this permission notice
-shall be included in all copies or substantial portions
-of the Software.
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
-ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
-TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
-PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
-SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
-IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
  • @@ -12080,7 +12071,6 @@

    MIT License

    Used by:

    MIT License
 
@@ -12295,8 +12285,6 @@ 
    Used by:
    
                     
  • jsonschema
    • 
                     
  • matchit
    • 
                     
  • serde_v8
    • 
-                    
  • tonic
    • 
-                    
  • tonic-build
    • 
                     
  • v8
    • 
                     
  • valuable
    • 
                     
  • void
    • 
@@ -12338,6 +12326,34 @@ 
    Used by:
    
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
    +
    • +
  • +

    MIT License

    +

    Used by:

    + + 
    MIT License
+
+Copyright (c) 2019 Daniel Augusto Rizzi Salvadori
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
  • @@ -12390,42 +12406,40 @@

    Used by:

    MIT License

    Used by:

    - 
    Permission is hereby granted, free of charge, to any
-person obtaining a copy of this software and associated
-documentation files (the "Software"), to deal in the
-Software without restriction, including without
-limitation the rights to use, copy, modify, merge,
-publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software
-is furnished to do so, subject to the following
-conditions:
+                
    The MIT License
 
-The above copyright notice and this permission notice
-shall be included in all copies or substantial portions
-of the Software.
+Copyright 2015 The Fancy Regex Authors.
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF
-ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED
-TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
-PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
-SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
-CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
-IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

  • MIT License

    Used by:

    - 
    The MIT License
+                
    The MIT License (MIT)
 
-Copyright 2015 The Fancy Regex Authors.
+Copyright (c) 2014 Benjamin Sago
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -12434,27 +12448,28 @@ 
    Used by:
    
 copies of the Software, and to permit persons to whom the Software is
 furnished to do so, subject to the following conditions:
 
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
 
 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

  • MIT License

    Used by:

    The MIT License (MIT)
 
 Copyright (c) 2014 Benjamin Sago
+Copyright (c) 2021-2022 The Nushell Project Developers
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
diff --git a/rust-toolchain.toml b/rust-toolchain.toml
index edee2099f9..8bfb9f2b1c 100644
--- a/rust-toolchain.toml
+++ b/rust-toolchain.toml
@@ -1,4 +1,4 @@
 [toolchain]
 # renovate-automation: rustc version
-channel = "1.63.0"
+channel = "1.65.0"
 components = [ "rustfmt", "clippy" ]
diff --git a/scripts/install.sh b/scripts/install.sh
index 297d3e3713..a99fa55d0d 100755
--- a/scripts/install.sh
+++ b/scripts/install.sh
@@ -11,7 +11,7 @@ BINARY_DOWNLOAD_PREFIX="https://github.com/apollographql/router/releases/downloa
 
 # Router version defined in apollo-router's Cargo.toml
 # Note: Change this line manually during the release steps.
-PACKAGE_VERSION="v1.5.0"
+PACKAGE_VERSION="v1.6.0"
 
 download_binary() {
     downloader --check
diff --git a/xtask/Cargo.lock b/xtask/Cargo.lock
index d7cb5a23fb..7044c87d9a 100644
--- a/xtask/Cargo.lock
+++ b/xtask/Cargo.lock
@@ -254,9 +254,9 @@ dependencies = [
 
 [[package]]
 name = "cxx"
-version = "1.0.82"
+version = "1.0.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d4a41a86530d0fe7f5d9ea779916b7cadd2d4f9add748b99c2c029cbbdfaf453"
+checksum = "bdf07d07d6531bfcdbe9b8b739b104610c6508dcc4d63b410585faf338241daf"
 dependencies = [
  "cc",
  "cxxbridge-flags",
@@ -266,9 +266,9 @@ dependencies = [
 
 [[package]]
 name = "cxx-build"
-version = "1.0.82"
+version = "1.0.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06416d667ff3e3ad2df1cd8cd8afae5da26cf9cec4d0825040f88b5ca659a2f0"
+checksum = "d2eb5b96ecdc99f72657332953d4d9c50135af1bac34277801cc3937906ebd39"
 dependencies = [
  "cc",
  "codespan-reporting",
@@ -281,15 +281,15 @@ dependencies = [
 
 [[package]]
 name = "cxxbridge-flags"
-version = "1.0.82"
+version = "1.0.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "820a9a2af1669deeef27cb271f476ffd196a2c4b6731336011e0ba63e2c7cf71"
+checksum = "ac040a39517fd1674e0f32177648334b0f4074625b5588a64519804ba0553b12"
 
 [[package]]
 name = "cxxbridge-macro"
-version = "1.0.82"
+version = "1.0.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a08a6e2fcc370a089ad3b4aaf54db3b1b4cee38ddabce5896b33eb693275f470"
+checksum = "1362b0ddcfc4eb0a1f57b68bd77dd99f0e826958a96abd0ae9bd092e114ffed6"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -338,9 +338,9 @@ dependencies = [
 
 [[package]]
 name = "filetime"
-version = "0.2.18"
+version = "0.2.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b9663d381d07ae25dc88dbdf27df458faa83a9b25336bcac83d5e452b5fc9d3"
+checksum = "4e884668cd0c7480504233e951174ddc3b382f7c2666e3b7310b5c4e7b0c37f9"
 dependencies = [
  "cfg-if",
  "libc",
@@ -602,9 +602,9 @@ dependencies = [
 
 [[package]]
 name = "hyper-rustls"
-version = "0.23.1"
+version = "0.23.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59df7c4e19c950e6e0e868dcc0a300b09a9b88e9ec55bd879ca819087a77355d"
+checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c"
 dependencies = [
  "http",
  "hyper",
@@ -698,9 +698,9 @@ dependencies = [
 
 [[package]]
 name = "ipnet"
-version = "2.5.1"
+version = "2.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f88c5561171189e69df9d98bcf18fd5f9558300f7ea7b801eb8a0fd748bd8745"
+checksum = "11b0d96e660696543b251e58030cf9787df56da39dab19ad60eae7353040917e"
 
 [[package]]
 name = "itertools"
@@ -728,9 +728,9 @@ dependencies = [
 
 [[package]]
 name = "jsonwebtoken"
-version = "8.1.1"
+version = "8.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1aa4b4af834c6cfd35d8763d359661b90f2e45d8f750a0849156c7f4671af09c"
+checksum = "09f4f04699947111ec1733e71778d763555737579e44b85844cae8e1940a1828"
 dependencies = [
  "base64",
  "pem",
@@ -754,9 +754,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.137"
+version = "0.2.138"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89"
+checksum = "db6d7e329c562c5dfab7a46a2afabc8b987ab9a4834c9d1ca04dc54c1546cef8"
 
 [[package]]
 name = "link-cplusplus"
@@ -932,9 +932,9 @@ checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860"
 
 [[package]]
 name = "openssl"
-version = "0.10.43"
+version = "0.10.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "020433887e44c27ff16365eaa2d380547a94544ad509aff6eb5b6e3e0b27b376"
+checksum = "29d971fd5722fec23977260f6e81aa67d2f22cadbdc2aa049f1022d9a3be1566"
 dependencies = [
  "bitflags",
  "cfg-if",
@@ -964,9 +964,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.78"
+version = "0.9.79"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07d5c8cb6e57b3a3612064d7b18b117912b4ce70955c2504d4b741c9e244b132"
+checksum = "5454462c0eced1e97f2ec09036abc8da362e66802f66fd20f86854d9d8cbcbc4"
 dependencies = [
  "autocfg",
  "cc",
@@ -1446,18 +1446,18 @@ dependencies = [
 
 [[package]]
 name = "serde"
-version = "1.0.148"
+version = "1.0.150"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e53f64bb4ba0191d6d0676e1b141ca55047d83b74f5607e6d8eb88126c52c2dc"
+checksum = "e326c9ec8042f1b5da33252c8a37e9ffbd2c9bef0155215b6e6c80c790e05f91"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.148"
+version = "1.0.150"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a55492425aa53521babf6137309e7d34c20bbfbbfcfe2c7f3a047fd1f6b92c0c"
+checksum = "42a3df25b0713732468deadad63ab9da1f1fd75a48a15024b50363f128db627e"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1753,9 +1753,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
 
 [[package]]
 name = "tokio"
-version = "1.22.0"
+version = "1.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d76ce4a75fb488c605c54bf610f221cea8b0dafb53333c1a67e8ee199dcd2ae3"
+checksum = "eab6d665857cc6ca78d6e80303a02cea7a7851e85dfbd77cbdc09bd129f1ef46"
 dependencies = [
  "autocfg",
  "bytes",
@@ -1768,7 +1768,7 @@ dependencies = [
  "signal-hook-registry",
  "socket2",
  "tokio-macros",
- "winapi",
+ "windows-sys 0.42.0",
 ]
 
 [[package]]
@@ -1900,9 +1900,9 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
 
 [[package]]
 name = "typenum"
-version = "1.15.0"
+version = "1.16.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
+checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba"
 
 [[package]]
 name = "unicase"
@@ -2109,9 +2109,9 @@ dependencies = [
 
 [[package]]
 name = "webpki-roots"
-version = "0.22.5"
+version = "0.22.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "368bfe657969fb01238bb756d351dcade285e0f6fcbd36dcb23359a5169975be"
+checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87"
 dependencies = [
  "webpki",
 ]
@@ -2278,7 +2278,7 @@ dependencies = [
 
 [[package]]
 name = "xtask"
-version = "1.4.0"
+version = "1.5.0"
 dependencies = [
  "ansi_term",
  "anyhow",
diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml
index eb74353b2b..a1038384a5 100644
--- a/xtask/Cargo.toml
+++ b/xtask/Cargo.toml
@@ -2,7 +2,7 @@
 
 [package]
 name = "xtask"
-version = "1.4.0"
+version = "1.5.0"
 authors = ["Apollo Graph, Inc. "]
 edition = "2021"
 license = "LicenseRef-ELv2"
@@ -14,7 +14,7 @@ anyhow = "1"
 base64 = "0.13"
 camino = "1"
 cargo_metadata = "0.15"
-chrono = "0.4.19"
+chrono = "0.4.23"
 flate2 = "1"
 itertools = "0.10.5"
 libc = "0.2"
@@ -31,7 +31,7 @@ structopt = { version = "0.3", default-features = false }
 tar = "0.4"
 tempfile = "3"
 tap = "1.0.1"
-tokio = "1.17.0"
+tokio = "1.23.0"
 which = "4"
 zip = { version = "0.6", default-features = false }
 sha2 = "0.10"
diff --git a/xtask/src/commands/release.rs b/xtask/src/commands/release.rs
index 7e959c0117..15aee7f9cb 100644
--- a/xtask/src/commands/release.rs
+++ b/xtask/src/commands/release.rs
@@ -384,6 +384,17 @@ impl Prepare {
         for package in packages {
             cargo!(["set-version", &version, "--package", package])
         }
+        replace_in_file!(
+            "./apollo-router-scaffold/templates/base/Cargo.toml",
+            "^apollo-router\\s*=\\s*\"\\d+.\\d+.\\d+\"",
+            format!("apollo-router = \"{}\"", version)
+        );
+        replace_in_file!(
+            "./apollo-router-scaffold/templates/base/xtask/Cargo.toml",
+            "^apollo-router-scaffold = \\{ git=\"https://github.com/apollographql/router.git\", tag\\s*=\\s*\"v\\d+.\\d+.\\d+\"\\s*\\}",
+            format!("apollo-router-scaffold = {{ git=\"https://github.com/apollographql/router.git\", tag = \"v{}\" }}", version)
+        );
+
         Ok(version)
     }