From 5a6d35497cec9d2ba40147d47f1a3ee48fe0dc2d Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Mon, 29 Sep 2025 10:25:08 +1000 Subject: [PATCH 01/16] Add http method to http_client span --- .../config_new/http_client/attributes.rs | 144 ++++++++++++ .../telemetry/config_new/http_client/mod.rs | 3 + .../config_new/http_client/selectors.rs | 211 ++++++++++++++++++ .../telemetry/config_new/http_client/spans.rs | 135 +++++++++++ .../src/plugins/telemetry/config_new/mod.rs | 16 ++ .../src/plugins/telemetry/config_new/spans.rs | 14 ++ apollo-router/src/plugins/telemetry/mod.rs | 36 +++ apollo-router/src/services/http/service.rs | 12 +- .../src/services/subgraph_service.rs | 3 + 9 files changed, 570 insertions(+), 4 deletions(-) create mode 100644 apollo-router/src/plugins/telemetry/config_new/http_client/attributes.rs create mode 100644 apollo-router/src/plugins/telemetry/config_new/http_client/mod.rs create mode 100644 apollo-router/src/plugins/telemetry/config_new/http_client/selectors.rs create mode 100644 apollo-router/src/plugins/telemetry/config_new/http_client/spans.rs diff --git a/apollo-router/src/plugins/telemetry/config_new/http_client/attributes.rs b/apollo-router/src/plugins/telemetry/config_new/http_client/attributes.rs new file mode 100644 index 0000000000..d9028ec3d2 --- /dev/null +++ b/apollo-router/src/plugins/telemetry/config_new/http_client/attributes.rs @@ -0,0 +1,144 @@ +use std::fmt::Debug; + +use opentelemetry::KeyValue; +use opentelemetry_semantic_conventions::trace::HTTP_REQUEST_METHOD; +use schemars::JsonSchema; +use serde::Deserialize; +use tower::BoxError; + +use crate::Context; +use crate::plugins::telemetry::config_new::DefaultForLevel; +use crate::plugins::telemetry::config_new::Selectors; +use crate::plugins::telemetry::config_new::attributes::DefaultAttributeRequirementLevel; +use crate::plugins::telemetry::config_new::attributes::StandardAttribute; +use crate::plugins::telemetry::otlp::TelemetryDataKind; +use crate::services::http; + +#[derive(Deserialize, JsonSchema, Clone, Default, Debug)] +#[cfg_attr(test, derive(PartialEq))] +#[serde(deny_unknown_fields, default)] +pub(crate) struct HttpClientAttributes { + /// HTTP request method. + /// Examples: + /// + /// * GET + /// * POST + /// * HEAD + /// + /// Requirement level: Required + #[serde(rename = "http.request.method")] + pub(crate) http_request_method: Option, +} + +impl DefaultForLevel for HttpClientAttributes { + fn defaults_for_level( + &mut self, + requirement_level: DefaultAttributeRequirementLevel, + _kind: TelemetryDataKind, + ) { + match requirement_level { + DefaultAttributeRequirementLevel::Required => { + if self.http_request_method.is_none() { + self.http_request_method = Some(StandardAttribute::Bool(true)); + } + } + DefaultAttributeRequirementLevel::Recommended => { + if self.http_request_method.is_none() { + self.http_request_method = Some(StandardAttribute::Bool(true)); + } + } + DefaultAttributeRequirementLevel::None => {} + } + } +} + +impl Selectors for HttpClientAttributes { + fn on_request(&self, request: &http::HttpRequest) -> Vec { + let mut attrs = Vec::new(); + if let Some(key) = self + .http_request_method + .as_ref() + .and_then(|a| a.key(HTTP_REQUEST_METHOD.into())) + { + attrs.push(KeyValue::new( + key, + request.http_request.method().as_str().to_string(), + )); + } + + attrs + } + + fn on_response(&self, _response: &http::HttpResponse) -> Vec { + Vec::new() + } + + fn on_error(&self, _error: &BoxError, _ctx: &Context) -> Vec { + Vec::new() + } +} + +#[cfg(test)] +mod test { + use opentelemetry_semantic_conventions::trace::HTTP_REQUEST_METHOD; + + use super::*; + use crate::plugins::telemetry::config_new::Selectors; + use crate::services::http::HttpRequest; + + #[test] + fn test_http_client_request_method() { + let attributes = HttpClientAttributes { + http_request_method: Some(StandardAttribute::Bool(true)), + }; + + let http_request = ::http::Request::builder() + .method(::http::Method::POST) + .uri("http://localhost/graphql") + .body(crate::services::router::body::empty()) + .unwrap(); + + let request = HttpRequest { + http_request, + context: crate::Context::new(), + }; + + let attributes = attributes.on_request(&request); + assert_eq!( + attributes + .iter() + .find(|key_val| key_val.key.as_str() == HTTP_REQUEST_METHOD) + .map(|key_val| &key_val.value), + Some(&"POST".into()) + ); + } + + #[test] + fn test_http_client_request_method_aliased() { + let attributes = HttpClientAttributes { + http_request_method: Some(StandardAttribute::Aliased { + alias: "custom.request.method".to_string(), + }), + }; + + let http_request = ::http::Request::builder() + .method(::http::Method::GET) + .uri("http://localhost/graphql") + .body(crate::services::router::body::empty()) + .unwrap(); + + let request = HttpRequest { + http_request, + context: crate::Context::new(), + }; + + let attributes = attributes.on_request(&request); + assert_eq!( + attributes + .iter() + .find(|key_val| key_val.key.as_str() == "custom.request.method") + .map(|key_val| &key_val.value), + Some(&"GET".into()) + ); + } +} \ No newline at end of file diff --git a/apollo-router/src/plugins/telemetry/config_new/http_client/mod.rs b/apollo-router/src/plugins/telemetry/config_new/http_client/mod.rs new file mode 100644 index 0000000000..63d317c66e --- /dev/null +++ b/apollo-router/src/plugins/telemetry/config_new/http_client/mod.rs @@ -0,0 +1,3 @@ +pub(crate) mod attributes; +pub(crate) mod selectors; +pub(crate) mod spans; \ No newline at end of file diff --git a/apollo-router/src/plugins/telemetry/config_new/http_client/selectors.rs b/apollo-router/src/plugins/telemetry/config_new/http_client/selectors.rs new file mode 100644 index 0000000000..131b50b14b --- /dev/null +++ b/apollo-router/src/plugins/telemetry/config_new/http_client/selectors.rs @@ -0,0 +1,211 @@ +use derivative::Derivative; +use schemars::JsonSchema; +use serde::Deserialize; +use tower::BoxError; + +use crate::Context; +use crate::plugins::telemetry::config::AttributeValue; +use crate::plugins::telemetry::config_new::Selector; +use crate::plugins::telemetry::config_new::Stage; +use crate::plugins::telemetry::config_new::ToOtelValue; +use crate::plugins::telemetry::config_new::instruments::InstrumentValue; +use crate::plugins::telemetry::config_new::instruments::Standard; +use crate::services::http; + +#[derive(Deserialize, JsonSchema, Clone, Debug)] +#[serde(deny_unknown_fields, rename_all = "snake_case", untagged)] +pub(crate) enum HttpClientValue { + Standard(Standard), + Custom(HttpClientSelector), +} + +impl From<&HttpClientValue> for InstrumentValue { + fn from(value: &HttpClientValue) -> Self { + match value { + HttpClientValue::Standard(standard) => InstrumentValue::Standard(standard.clone()), + HttpClientValue::Custom(selector) => InstrumentValue::Custom(selector.clone()), + } + } +} + +#[derive(Derivative, Deserialize, JsonSchema, Clone)] +#[serde(deny_unknown_fields, untagged)] +#[derivative(Debug, PartialEq)] +pub(crate) enum HttpClientSelector { + /// A static field value + StaticField { + /// The static value. + #[serde(rename = "static")] + r#static: AttributeValue, + }, + /// A header from the HTTP request + HttpClientRequestHeader { + /// The name of the HTTP client request header. + http_client_request_header: String, + #[serde(skip)] + #[allow(dead_code)] + /// Optional redaction pattern. + redact: Option, + /// Optional default value. + default: Option, + }, + /// A header from the HTTP response + HttpClientResponseHeader { + /// The name of the HTTP client response header. + http_client_response_header: String, + #[serde(skip)] + #[allow(dead_code)] + /// Optional redaction pattern. + redact: Option, + /// Optional default value. + default: Option, + }, +} + +impl Selector for HttpClientSelector { + type Request = http::HttpRequest; + type Response = http::HttpResponse; + type EventResponse = (); + + fn on_request(&self, request: &http::HttpRequest) -> Option { + match self { + HttpClientSelector::StaticField { r#static } => r#static.maybe_to_otel_value(), + HttpClientSelector::HttpClientRequestHeader { + http_client_request_header, + default, + .. + } => request + .http_request + .headers() + .get(http_client_request_header) + .and_then(|h| h.to_str().ok()) + .map(|h| h.to_string()) + .or_else(|| default.clone()) + .map(opentelemetry::Value::from), + HttpClientSelector::HttpClientResponseHeader { default, .. } => { + default.clone().map(opentelemetry::Value::from) + } + } + } + + fn on_response(&self, response: &http::HttpResponse) -> Option { + match self { + HttpClientSelector::StaticField { r#static } => r#static.maybe_to_otel_value(), + HttpClientSelector::HttpClientRequestHeader { default, .. } => { + default.clone().map(opentelemetry::Value::from) + } + HttpClientSelector::HttpClientResponseHeader { + http_client_response_header, + default, + .. + } => response + .http_response + .headers() + .get(http_client_response_header) + .and_then(|h| h.to_str().ok()) + .map(|h| h.to_string()) + .or_else(|| default.clone()) + .map(opentelemetry::Value::from), + } + } + + fn on_error(&self, _error: &BoxError, _ctx: &Context) -> Option { + match self { + HttpClientSelector::StaticField { r#static } => r#static.maybe_to_otel_value(), + HttpClientSelector::HttpClientRequestHeader { default, .. } => { + default.clone().map(opentelemetry::Value::from) + } + HttpClientSelector::HttpClientResponseHeader { default, .. } => { + default.clone().map(opentelemetry::Value::from) + } + } + } + + fn is_active(&self, stage: Stage) -> bool { + match self { + HttpClientSelector::StaticField { .. } => true, + HttpClientSelector::HttpClientRequestHeader { .. } => matches!(stage, Stage::Request), + HttpClientSelector::HttpClientResponseHeader { .. } => matches!(stage, Stage::Response), + } + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::Context; + + #[test] + fn test_http_client_static_field() { + let selector = HttpClientSelector::StaticField { + r#static: AttributeValue::String("test_value".to_string()), + }; + + let http_request = ::http::Request::builder() + .method(::http::Method::GET) + .uri("http://localhost/graphql") + .body(crate::services::router::body::empty()) + .unwrap(); + + let request = http::HttpRequest { + http_request, + context: Context::new(), + }; + + assert_eq!( + selector.on_request(&request), + Some(opentelemetry::Value::String("test_value".to_string().into())) + ); + } + + #[test] + fn test_http_client_request_header() { + let selector = HttpClientSelector::HttpClientRequestHeader { + http_client_request_header: "content-type".to_string(), + redact: None, + default: None, + }; + + let http_request = ::http::Request::builder() + .method(::http::Method::GET) + .uri("http://localhost/graphql") + .header("content-type", "application/json") + .body(crate::services::router::body::empty()) + .unwrap(); + + let request = http::HttpRequest { + http_request, + context: Context::new(), + }; + + assert_eq!( + selector.on_request(&request), + Some(opentelemetry::Value::String("application/json".to_string().into())) + ); + } + + #[test] + fn test_http_client_response_header() { + let selector = HttpClientSelector::HttpClientResponseHeader { + http_client_response_header: "content-length".to_string(), + redact: None, + default: None, + }; + + let http_response = ::http::Response::builder() + .status(200) + .header("content-length", "1024") + .body(crate::services::router::body::empty()) + .unwrap(); + + let response = http::HttpResponse { + http_response, + context: Context::new(), + }; + + assert_eq!( + selector.on_response(&response), + Some(opentelemetry::Value::String("1024".to_string().into())) + ); + } +} \ No newline at end of file diff --git a/apollo-router/src/plugins/telemetry/config_new/http_client/spans.rs b/apollo-router/src/plugins/telemetry/config_new/http_client/spans.rs new file mode 100644 index 0000000000..59ee145cc9 --- /dev/null +++ b/apollo-router/src/plugins/telemetry/config_new/http_client/spans.rs @@ -0,0 +1,135 @@ +use schemars::JsonSchema; +use serde::Deserialize; + +use crate::plugins::telemetry::config_new::DefaultForLevel; +use crate::plugins::telemetry::config_new::attributes::DefaultAttributeRequirementLevel; +use crate::plugins::telemetry::config_new::conditional::Conditional; +use crate::plugins::telemetry::config_new::extendable::Extendable; +use crate::plugins::telemetry::config_new::http_client::attributes::HttpClientAttributes; +use crate::plugins::telemetry::config_new::http_client::selectors::HttpClientSelector; +use crate::plugins::telemetry::otlp::TelemetryDataKind; + +#[derive(Deserialize, JsonSchema, Clone, Debug, Default)] +#[serde(deny_unknown_fields, default)] +pub(crate) struct HttpClientSpans { + /// Custom attributes that are attached to the HTTP client span. + pub(crate) attributes: Extendable>, +} + +impl DefaultForLevel for HttpClientSpans { + fn defaults_for_level( + &mut self, + requirement_level: DefaultAttributeRequirementLevel, + kind: TelemetryDataKind, + ) { + self.attributes.defaults_for_level(requirement_level, kind); + } +} + +#[cfg(test)] +mod test { + use opentelemetry_semantic_conventions::trace::HTTP_REQUEST_METHOD; + + use super::*; + use crate::plugins::telemetry::config_new::DefaultForLevel; + use crate::plugins::telemetry::config_new::Selectors; + use crate::plugins::telemetry::config_new::attributes::DefaultAttributeRequirementLevel; + use crate::plugins::telemetry::otlp::TelemetryDataKind; + use crate::services::http::HttpRequest; + use crate::Context; + + #[test] + fn test_http_client_spans_level_none() { + let mut spans = HttpClientSpans::default(); + spans.defaults_for_levels( + DefaultAttributeRequirementLevel::None, + TelemetryDataKind::Traces, + ); + + let http_request = ::http::Request::builder() + .method(::http::Method::POST) + .uri("http://localhost/graphql") + .body(crate::services::router::body::empty()) + .unwrap(); + + let request = HttpRequest { + http_request, + context: Context::new(), + }; + + let values = spans.attributes.on_request(&request); + assert!( + !values + .iter() + .any(|key_val| key_val.key.as_str() == HTTP_REQUEST_METHOD) + ); + } + + #[test] + fn test_http_client_spans_level_required() { + let mut spans = HttpClientSpans::default(); + spans.defaults_for_levels( + DefaultAttributeRequirementLevel::Required, + TelemetryDataKind::Traces, + ); + + let http_request = ::http::Request::builder() + .method(::http::Method::POST) + .uri("http://localhost/graphql") + .body(crate::services::router::body::empty()) + .unwrap(); + + let request = HttpRequest { + http_request, + context: Context::new(), + }; + + let values = spans.attributes.on_request(&request); + assert!( + values + .iter() + .any(|key_val| key_val.key.as_str() == HTTP_REQUEST_METHOD) + ); + assert_eq!( + values + .iter() + .find(|key_val| key_val.key.as_str() == HTTP_REQUEST_METHOD) + .map(|key_val| &key_val.value), + Some(&"POST".into()) + ); + } + + #[test] + fn test_http_client_spans_level_recommended() { + let mut spans = HttpClientSpans::default(); + spans.defaults_for_levels( + DefaultAttributeRequirementLevel::Recommended, + TelemetryDataKind::Traces, + ); + + let http_request = ::http::Request::builder() + .method(::http::Method::GET) + .uri("http://localhost/graphql") + .body(crate::services::router::body::empty()) + .unwrap(); + + let request = HttpRequest { + http_request, + context: Context::new(), + }; + + let values = spans.attributes.on_request(&request); + assert!( + values + .iter() + .any(|key_val| key_val.key.as_str() == HTTP_REQUEST_METHOD) + ); + assert_eq!( + values + .iter() + .find(|key_val| key_val.key.as_str() == HTTP_REQUEST_METHOD) + .map(|key_val| &key_val.value), + Some(&"GET".into()) + ); + } +} \ No newline at end of file diff --git a/apollo-router/src/plugins/telemetry/config_new/mod.rs b/apollo-router/src/plugins/telemetry/config_new/mod.rs index bb9fc5dbf0..b965def760 100644 --- a/apollo-router/src/plugins/telemetry/config_new/mod.rs +++ b/apollo-router/src/plugins/telemetry/config_new/mod.rs @@ -26,6 +26,7 @@ pub(crate) mod cost; pub(crate) mod events; pub(crate) mod extendable; pub(crate) mod graphql; +pub(crate) mod http_client; pub(crate) mod http_common; pub(crate) mod http_server; pub(crate) mod instruments; @@ -238,6 +239,21 @@ macro_rules! impl_to_otel_value { impl_to_otel_value!(serde_json_bytes::Value); impl_to_otel_value!(serde_json::Value); +impl ToOtelValue for AttributeValue { + fn maybe_to_otel_value(&self) -> Option { + match self { + AttributeValue::Bool(value) => Some((*value).into()), + AttributeValue::I64(value) => Some((*value).into()), + AttributeValue::F64(value) => Some((*value).into()), + AttributeValue::String(value) => Some(value.clone().into()), + AttributeValue::Array(value) => { + // Convert array to opentelemetry value + Some(opentelemetry::Value::Array(value.clone().into())) + } + } + } +} + impl From for AttributeValue { fn from(value: opentelemetry::Value) -> Self { match value { diff --git a/apollo-router/src/plugins/telemetry/config_new/spans.rs b/apollo-router/src/plugins/telemetry/config_new/spans.rs index 5dcfdfe9ad..371a4f091d 100644 --- a/apollo-router/src/plugins/telemetry/config_new/spans.rs +++ b/apollo-router/src/plugins/telemetry/config_new/spans.rs @@ -2,6 +2,7 @@ use schemars::JsonSchema; use serde::Deserialize; use super::connector::spans::ConnectorSpans; +use super::http_client::spans::HttpClientSpans; use super::router::spans::RouterSpans; use super::subgraph::spans::SubgraphSpans; use super::supergraph::spans::SupergraphSpans; @@ -35,6 +36,10 @@ pub(crate) struct Spans { /// Attributes to include on the connector span. /// Connector spans contain information about the connector request and response and therefore contain connector specific attributes. pub(crate) connector: ConnectorSpans, + + /// Attributes to include on the HTTP client span. + /// HTTP client spans contain information about HTTP requests made to subgraphs and therefore contain HTTP client specific attributes. + pub(crate) http_client: HttpClientSpans, } impl Spans { @@ -52,6 +57,10 @@ impl Spans { self.default_attribute_requirement_level, TelemetryDataKind::Traces, ); + self.http_client.defaults_for_levels( + self.default_attribute_requirement_level, + TelemetryDataKind::Traces, + ); } pub(crate) fn validate(&self) -> Result<(), String> { @@ -70,6 +79,11 @@ impl Spans { .validate() .map_err(|err| format!("error for subgraph span attribute {name:?}: {err}"))?; } + for (name, custom) in &self.http_client.attributes.custom { + custom + .validate() + .map_err(|err| format!("error for http_client span attribute {name:?}: {err}"))?; + } Ok(()) } diff --git a/apollo-router/src/plugins/telemetry/mod.rs b/apollo-router/src/plugins/telemetry/mod.rs index e488352f50..837cde786e 100644 --- a/apollo-router/src/plugins/telemetry/mod.rs +++ b/apollo-router/src/plugins/telemetry/mod.rs @@ -1177,6 +1177,42 @@ impl PluginPrivate for Telemetry { .boxed() } + fn http_client_service( + &self, + subgraph_name: &str, + service: crate::services::http::BoxService, + ) -> crate::services::http::BoxService { + let req_fn_config = self.config.clone(); + let res_fn_config = self.config.clone(); + let subgraph_name = subgraph_name.to_string(); + + ServiceBuilder::new() + .map_request(move |request: crate::services::http::HttpRequest| { + // Generate the attributes that should be applied to the http_request span + let attributes = req_fn_config.instrumentation.spans.http_client.attributes.on_request(&request); + + // Store the attributes in the request context so they can be applied later + // when the http_request span is active + request.context.extensions().with_lock(|lock| { + lock.insert(attributes); + }); + + request + }) + .map_response(move |response: crate::services::http::HttpResponse| { + // Get the current span + let span = ::tracing::Span::current(); + + // Apply http_client attributes from config to the current span + let attributes = res_fn_config.instrumentation.spans.http_client.attributes.on_response(&response); + span.set_span_dyn_attributes(attributes); + + response + }) + .service(service) + .boxed() + } + fn web_endpoints(&self) -> MultiMap { self.custom_endpoints.clone() } diff --git a/apollo-router/src/services/http/service.rs b/apollo-router/src/services/http/service.rs index 1e07c2d69b..498a887f91 100644 --- a/apollo-router/src/services/http/service.rs +++ b/apollo-router/src/services/http/service.rs @@ -18,6 +18,7 @@ use hyper_util::client::legacy::connect::HttpConnector; use hyperlocal::UnixConnector; use opentelemetry::global; use rustls::ClientConfig; +use crate::plugins::telemetry::dynamic_attribute::SpanDynAttribute; use rustls::RootCertStore; use schemars::JsonSchema; use tower::BoxError; @@ -316,6 +317,11 @@ impl tower::Service for HttpClientService { //"apollo.subgraph.name" = %service_name, //"graphql.operation.name" = %operation_name, ); + + // Apply any attributes that were stored by telemetry middleware + if let Some(attributes) = context.extensions().with_lock(|lock| lock.get::>().cloned()) { + http_req_span.set_span_dyn_attributes(attributes); + } get_text_map_propagator(|propagator| { propagator.inject_context( &prepare_context(http_req_span.context()), @@ -353,15 +359,13 @@ impl tower::Service for HttpClientService { http_request }; - let http_response = do_fetch(client, &service_name, http_request) - .instrument(http_req_span) - .await?; + let http_response = do_fetch(client, &service_name, http_request).await?; Ok(HttpResponse { http_response, context, }) - }) + }.instrument(http_req_span)) } } diff --git a/apollo-router/src/services/subgraph_service.rs b/apollo-router/src/services/subgraph_service.rs index bcb2844c83..0235e57454 100644 --- a/apollo-router/src/services/subgraph_service.rs +++ b/apollo-router/src/services/subgraph_service.rs @@ -1247,7 +1247,10 @@ async fn call_http( })? } else { tracing::debug!("we called http"); + tracing::info!("we called http"); let client = client_factory.create(service_name); + + tracing::info!("client: {:?}", client); call_single_http(request, body, context, client, service_name).await } } From 9fb698f00178d6927fb519dde4460107cf491df2 Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Mon, 29 Sep 2025 11:42:51 +1000 Subject: [PATCH 02/16] Remove unused code --- .../config_new/http_client/selectors.rs | 35 ------------------- .../src/plugins/telemetry/config_new/mod.rs | 15 -------- .../src/plugins/telemetry/config_new/spans.rs | 2 +- apollo-router/src/plugins/telemetry/mod.rs | 15 ++------ apollo-router/src/services/http/service.rs | 6 ++-- 5 files changed, 8 insertions(+), 65 deletions(-) diff --git a/apollo-router/src/plugins/telemetry/config_new/http_client/selectors.rs b/apollo-router/src/plugins/telemetry/config_new/http_client/selectors.rs index 131b50b14b..5b31207606 100644 --- a/apollo-router/src/plugins/telemetry/config_new/http_client/selectors.rs +++ b/apollo-router/src/plugins/telemetry/config_new/http_client/selectors.rs @@ -4,10 +4,8 @@ use serde::Deserialize; use tower::BoxError; use crate::Context; -use crate::plugins::telemetry::config::AttributeValue; use crate::plugins::telemetry::config_new::Selector; use crate::plugins::telemetry::config_new::Stage; -use crate::plugins::telemetry::config_new::ToOtelValue; use crate::plugins::telemetry::config_new::instruments::InstrumentValue; use crate::plugins::telemetry::config_new::instruments::Standard; use crate::services::http; @@ -32,12 +30,6 @@ impl From<&HttpClientValue> for InstrumentValue { #[serde(deny_unknown_fields, untagged)] #[derivative(Debug, PartialEq)] pub(crate) enum HttpClientSelector { - /// A static field value - StaticField { - /// The static value. - #[serde(rename = "static")] - r#static: AttributeValue, - }, /// A header from the HTTP request HttpClientRequestHeader { /// The name of the HTTP client request header. @@ -69,7 +61,6 @@ impl Selector for HttpClientSelector { fn on_request(&self, request: &http::HttpRequest) -> Option { match self { - HttpClientSelector::StaticField { r#static } => r#static.maybe_to_otel_value(), HttpClientSelector::HttpClientRequestHeader { http_client_request_header, default, @@ -90,7 +81,6 @@ impl Selector for HttpClientSelector { fn on_response(&self, response: &http::HttpResponse) -> Option { match self { - HttpClientSelector::StaticField { r#static } => r#static.maybe_to_otel_value(), HttpClientSelector::HttpClientRequestHeader { default, .. } => { default.clone().map(opentelemetry::Value::from) } @@ -111,7 +101,6 @@ impl Selector for HttpClientSelector { fn on_error(&self, _error: &BoxError, _ctx: &Context) -> Option { match self { - HttpClientSelector::StaticField { r#static } => r#static.maybe_to_otel_value(), HttpClientSelector::HttpClientRequestHeader { default, .. } => { default.clone().map(opentelemetry::Value::from) } @@ -123,7 +112,6 @@ impl Selector for HttpClientSelector { fn is_active(&self, stage: Stage) -> bool { match self { - HttpClientSelector::StaticField { .. } => true, HttpClientSelector::HttpClientRequestHeader { .. } => matches!(stage, Stage::Request), HttpClientSelector::HttpClientResponseHeader { .. } => matches!(stage, Stage::Response), } @@ -135,29 +123,6 @@ mod test { use super::*; use crate::Context; - #[test] - fn test_http_client_static_field() { - let selector = HttpClientSelector::StaticField { - r#static: AttributeValue::String("test_value".to_string()), - }; - - let http_request = ::http::Request::builder() - .method(::http::Method::GET) - .uri("http://localhost/graphql") - .body(crate::services::router::body::empty()) - .unwrap(); - - let request = http::HttpRequest { - http_request, - context: Context::new(), - }; - - assert_eq!( - selector.on_request(&request), - Some(opentelemetry::Value::String("test_value".to_string().into())) - ); - } - #[test] fn test_http_client_request_header() { let selector = HttpClientSelector::HttpClientRequestHeader { diff --git a/apollo-router/src/plugins/telemetry/config_new/mod.rs b/apollo-router/src/plugins/telemetry/config_new/mod.rs index b965def760..c50cb5e75f 100644 --- a/apollo-router/src/plugins/telemetry/config_new/mod.rs +++ b/apollo-router/src/plugins/telemetry/config_new/mod.rs @@ -239,21 +239,6 @@ macro_rules! impl_to_otel_value { impl_to_otel_value!(serde_json_bytes::Value); impl_to_otel_value!(serde_json::Value); -impl ToOtelValue for AttributeValue { - fn maybe_to_otel_value(&self) -> Option { - match self { - AttributeValue::Bool(value) => Some((*value).into()), - AttributeValue::I64(value) => Some((*value).into()), - AttributeValue::F64(value) => Some((*value).into()), - AttributeValue::String(value) => Some(value.clone().into()), - AttributeValue::Array(value) => { - // Convert array to opentelemetry value - Some(opentelemetry::Value::Array(value.clone().into())) - } - } - } -} - impl From for AttributeValue { fn from(value: opentelemetry::Value) -> Self { match value { diff --git a/apollo-router/src/plugins/telemetry/config_new/spans.rs b/apollo-router/src/plugins/telemetry/config_new/spans.rs index 371a4f091d..f315626f71 100644 --- a/apollo-router/src/plugins/telemetry/config_new/spans.rs +++ b/apollo-router/src/plugins/telemetry/config_new/spans.rs @@ -38,7 +38,7 @@ pub(crate) struct Spans { pub(crate) connector: ConnectorSpans, /// Attributes to include on the HTTP client span. - /// HTTP client spans contain information about HTTP requests made to subgraphs and therefore contain HTTP client specific attributes. + /// HTTP client spans contain information about HTTP requests made to subgraphs, including any changes made by Rhai scripts. pub(crate) http_client: HttpClientSpans, } diff --git a/apollo-router/src/plugins/telemetry/mod.rs b/apollo-router/src/plugins/telemetry/mod.rs index 7d470aea1f..0ab13d5020 100644 --- a/apollo-router/src/plugins/telemetry/mod.rs +++ b/apollo-router/src/plugins/telemetry/mod.rs @@ -1179,20 +1179,16 @@ impl PluginPrivate for Telemetry { fn http_client_service( &self, - subgraph_name: &str, + _subgraph_name: &str, // todo config per subgraph service: crate::services::http::BoxService, ) -> crate::services::http::BoxService { let req_fn_config = self.config.clone(); let res_fn_config = self.config.clone(); - let subgraph_name = subgraph_name.to_string(); ServiceBuilder::new() .map_request(move |request: crate::services::http::HttpRequest| { - // Generate the attributes that should be applied to the http_request span + // Get and store attributes so that they can be applied later after the span is created let attributes = req_fn_config.instrumentation.spans.http_client.attributes.on_request(&request); - - // Store the attributes in the request context so they can be applied later - // when the http_request span is active request.context.extensions().with_lock(|lock| { lock.insert(attributes); }); @@ -1200,13 +1196,8 @@ impl PluginPrivate for Telemetry { request }) .map_response(move |response: crate::services::http::HttpResponse| { - // Get the current span - let span = ::tracing::Span::current(); - - // Apply http_client attributes from config to the current span let attributes = res_fn_config.instrumentation.spans.http_client.attributes.on_response(&response); - span.set_span_dyn_attributes(attributes); - + ::tracing::Span::current().set_span_dyn_attributes(attributes); response }) .service(service) diff --git a/apollo-router/src/services/http/service.rs b/apollo-router/src/services/http/service.rs index 498a887f91..fe7ba94247 100644 --- a/apollo-router/src/services/http/service.rs +++ b/apollo-router/src/services/http/service.rs @@ -359,13 +359,15 @@ impl tower::Service for HttpClientService { http_request }; - let http_response = do_fetch(client, &service_name, http_request).await?; + let http_response = do_fetch(client, &service_name, http_request) + .instrument(http_req_span) + .await?; Ok(HttpResponse { http_response, context, }) - }.instrument(http_req_span)) + }) } } From 10e4fd83bc5afb72fb0127c296fd64b0cc3da1c8 Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Mon, 29 Sep 2025 11:52:08 +1000 Subject: [PATCH 03/16] Update context key --- apollo-router/src/plugins/telemetry/mod.rs | 12 ++++++++++-- apollo-router/src/services/http/service.rs | 5 +++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/apollo-router/src/plugins/telemetry/mod.rs b/apollo-router/src/plugins/telemetry/mod.rs index 0ab13d5020..0622b66d4c 100644 --- a/apollo-router/src/plugins/telemetry/mod.rs +++ b/apollo-router/src/plugins/telemetry/mod.rs @@ -363,6 +363,12 @@ impl EnabledFeatures { } } +// Struct to hold request attributes for the http client in context +#[derive(Clone, Debug)] +pub(crate) struct HttpClientAttributes { + pub(crate) attributes: Vec, +} + #[async_trait::async_trait] impl PluginPrivate for Telemetry { type Config = config::Conf; @@ -1188,9 +1194,11 @@ impl PluginPrivate for Telemetry { ServiceBuilder::new() .map_request(move |request: crate::services::http::HttpRequest| { // Get and store attributes so that they can be applied later after the span is created - let attributes = req_fn_config.instrumentation.spans.http_client.attributes.on_request(&request); + let client_attributes = HttpClientAttributes { + attributes: req_fn_config.instrumentation.spans.http_client.attributes.on_request(&request), + }; request.context.extensions().with_lock(|lock| { - lock.insert(attributes); + lock.insert(client_attributes); }); request diff --git a/apollo-router/src/services/http/service.rs b/apollo-router/src/services/http/service.rs index fe7ba94247..bcdc23cce7 100644 --- a/apollo-router/src/services/http/service.rs +++ b/apollo-router/src/services/http/service.rs @@ -19,6 +19,7 @@ use hyperlocal::UnixConnector; use opentelemetry::global; use rustls::ClientConfig; use crate::plugins::telemetry::dynamic_attribute::SpanDynAttribute; +use crate::plugins::telemetry::HttpClientAttributes; use rustls::RootCertStore; use schemars::JsonSchema; use tower::BoxError; @@ -319,8 +320,8 @@ impl tower::Service for HttpClientService { ); // Apply any attributes that were stored by telemetry middleware - if let Some(attributes) = context.extensions().with_lock(|lock| lock.get::>().cloned()) { - http_req_span.set_span_dyn_attributes(attributes); + if let Some(client_attributes) = context.extensions().with_lock(|lock| lock.get::().cloned()) { + http_req_span.set_span_dyn_attributes(client_attributes.attributes); } get_text_map_propagator(|propagator| { propagator.inject_context( From fd887ea26608d5738bf2c3ef82cc456ad1cebdad Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Mon, 29 Sep 2025 11:58:52 +1000 Subject: [PATCH 04/16] Lint fix --- .../telemetry/config_new/http_client/attributes.rs | 2 +- .../telemetry/config_new/http_client/mod.rs | 2 +- .../telemetry/config_new/http_client/selectors.rs | 6 ++++-- .../telemetry/config_new/http_client/spans.rs | 4 ++-- apollo-router/src/plugins/telemetry/mod.rs | 14 ++++++++++++-- apollo-router/src/services/http/service.rs | 11 +++++++---- 6 files changed, 27 insertions(+), 12 deletions(-) diff --git a/apollo-router/src/plugins/telemetry/config_new/http_client/attributes.rs b/apollo-router/src/plugins/telemetry/config_new/http_client/attributes.rs index d9028ec3d2..0f13090748 100644 --- a/apollo-router/src/plugins/telemetry/config_new/http_client/attributes.rs +++ b/apollo-router/src/plugins/telemetry/config_new/http_client/attributes.rs @@ -141,4 +141,4 @@ mod test { Some(&"GET".into()) ); } -} \ No newline at end of file +} diff --git a/apollo-router/src/plugins/telemetry/config_new/http_client/mod.rs b/apollo-router/src/plugins/telemetry/config_new/http_client/mod.rs index 63d317c66e..a8e021de87 100644 --- a/apollo-router/src/plugins/telemetry/config_new/http_client/mod.rs +++ b/apollo-router/src/plugins/telemetry/config_new/http_client/mod.rs @@ -1,3 +1,3 @@ pub(crate) mod attributes; pub(crate) mod selectors; -pub(crate) mod spans; \ No newline at end of file +pub(crate) mod spans; diff --git a/apollo-router/src/plugins/telemetry/config_new/http_client/selectors.rs b/apollo-router/src/plugins/telemetry/config_new/http_client/selectors.rs index 5b31207606..0181002223 100644 --- a/apollo-router/src/plugins/telemetry/config_new/http_client/selectors.rs +++ b/apollo-router/src/plugins/telemetry/config_new/http_client/selectors.rs @@ -145,7 +145,9 @@ mod test { assert_eq!( selector.on_request(&request), - Some(opentelemetry::Value::String("application/json".to_string().into())) + Some(opentelemetry::Value::String( + "application/json".to_string().into() + )) ); } @@ -173,4 +175,4 @@ mod test { Some(opentelemetry::Value::String("1024".to_string().into())) ); } -} \ No newline at end of file +} diff --git a/apollo-router/src/plugins/telemetry/config_new/http_client/spans.rs b/apollo-router/src/plugins/telemetry/config_new/http_client/spans.rs index 59ee145cc9..a03b448e23 100644 --- a/apollo-router/src/plugins/telemetry/config_new/http_client/spans.rs +++ b/apollo-router/src/plugins/telemetry/config_new/http_client/spans.rs @@ -31,12 +31,12 @@ mod test { use opentelemetry_semantic_conventions::trace::HTTP_REQUEST_METHOD; use super::*; + use crate::Context; use crate::plugins::telemetry::config_new::DefaultForLevel; use crate::plugins::telemetry::config_new::Selectors; use crate::plugins::telemetry::config_new::attributes::DefaultAttributeRequirementLevel; use crate::plugins::telemetry::otlp::TelemetryDataKind; use crate::services::http::HttpRequest; - use crate::Context; #[test] fn test_http_client_spans_level_none() { @@ -132,4 +132,4 @@ mod test { Some(&"GET".into()) ); } -} \ No newline at end of file +} diff --git a/apollo-router/src/plugins/telemetry/mod.rs b/apollo-router/src/plugins/telemetry/mod.rs index 0622b66d4c..154339e2cb 100644 --- a/apollo-router/src/plugins/telemetry/mod.rs +++ b/apollo-router/src/plugins/telemetry/mod.rs @@ -1195,7 +1195,12 @@ impl PluginPrivate for Telemetry { .map_request(move |request: crate::services::http::HttpRequest| { // Get and store attributes so that they can be applied later after the span is created let client_attributes = HttpClientAttributes { - attributes: req_fn_config.instrumentation.spans.http_client.attributes.on_request(&request), + attributes: req_fn_config + .instrumentation + .spans + .http_client + .attributes + .on_request(&request), }; request.context.extensions().with_lock(|lock| { lock.insert(client_attributes); @@ -1204,7 +1209,12 @@ impl PluginPrivate for Telemetry { request }) .map_response(move |response: crate::services::http::HttpResponse| { - let attributes = res_fn_config.instrumentation.spans.http_client.attributes.on_response(&response); + let attributes = res_fn_config + .instrumentation + .spans + .http_client + .attributes + .on_response(&response); ::tracing::Span::current().set_span_dyn_attributes(attributes); response }) diff --git a/apollo-router/src/services/http/service.rs b/apollo-router/src/services/http/service.rs index bcdc23cce7..8e8acd9751 100644 --- a/apollo-router/src/services/http/service.rs +++ b/apollo-router/src/services/http/service.rs @@ -18,8 +18,6 @@ use hyper_util::client::legacy::connect::HttpConnector; use hyperlocal::UnixConnector; use opentelemetry::global; use rustls::ClientConfig; -use crate::plugins::telemetry::dynamic_attribute::SpanDynAttribute; -use crate::plugins::telemetry::HttpClientAttributes; use rustls::RootCertStore; use schemars::JsonSchema; use tower::BoxError; @@ -38,7 +36,9 @@ use crate::axum_factory::compression::Compressor; use crate::configuration::TlsClientAuth; use crate::error::FetchError; use crate::plugins::authentication::subgraph::SigningParamsConfig; +use crate::plugins::telemetry::HttpClientAttributes; use crate::plugins::telemetry::consts::HTTP_REQUEST_SPAN_NAME; +use crate::plugins::telemetry::dynamic_attribute::SpanDynAttribute; use crate::plugins::telemetry::otel::OpenTelemetrySpanExt; use crate::plugins::telemetry::reload::prepare_context; use crate::plugins::traffic_shaping::Http2Config; @@ -318,9 +318,12 @@ impl tower::Service for HttpClientService { //"apollo.subgraph.name" = %service_name, //"graphql.operation.name" = %operation_name, ); - + // Apply any attributes that were stored by telemetry middleware - if let Some(client_attributes) = context.extensions().with_lock(|lock| lock.get::().cloned()) { + if let Some(client_attributes) = context + .extensions() + .with_lock(|lock| lock.get::().cloned()) + { http_req_span.set_span_dyn_attributes(client_attributes.attributes); } get_text_map_propagator(|propagator| { From 700bd2c9e4f0f22056beef0efc696a32fd01d5b6 Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Mon, 29 Sep 2025 13:50:47 +1000 Subject: [PATCH 05/16] Snapshot updates, changed required config, removed useless tests --- ...nfiguration__tests__schema_generation.snap | 249 ++++++++++++++++++ .../config_new/http_client/attributes.rs | 4 +- .../telemetry/config_new/http_client/spans.rs | 108 -------- apollo-router/src/plugins/telemetry/mod.rs | 2 +- apollo-router/src/services/http/service.rs | 2 - ...ollo_otel_traces__batch_send_header-2.snap | 1 - ...apollo_otel_traces__batch_send_header.snap | 1 - .../apollo_otel_traces__batch_trace_id-2.snap | 1 - .../apollo_otel_traces__batch_trace_id.snap | 1 - .../apollo_otel_traces__client_name-2.snap | 1 - .../apollo_otel_traces__client_name.snap | 1 - .../apollo_otel_traces__client_version-2.snap | 1 - .../apollo_otel_traces__client_version.snap | 1 - .../apollo_otel_traces__condition_else-2.snap | 1 - .../apollo_otel_traces__condition_else.snap | 1 - .../apollo_otel_traces__condition_if-2.snap | 1 - .../apollo_otel_traces__condition_if.snap | 1 - .../apollo_otel_traces__non_defer-2.snap | 1 - .../apollo_otel_traces__non_defer.snap | 1 - .../apollo_otel_traces__send_header-2.snap | 1 - .../apollo_otel_traces__send_header.snap | 1 - ...lo_otel_traces__send_variable_value-2.snap | 1 - ...ollo_otel_traces__send_variable_value.snap | 1 - .../apollo_otel_traces__trace_id-2.snap | 1 - .../apollo_otel_traces__trace_id.snap | 1 - 25 files changed, 251 insertions(+), 134 deletions(-) diff --git a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap index 41c211d225..0e88b7c7a7 100644 --- a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap +++ b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap @@ -1157,6 +1157,133 @@ expression: "&schema" } ] }, + "ConditionHttpClientSelector": { + "description": "Specify a condition for when an [instrument][] should be mutated or an [event][] should be triggered.\n\n[instrument]: https://www.apollographql.com/docs/graphos/routing/observability/telemetry/instrumentation/instruments\n[event]: https://www.apollographql.com/docs/graphos/routing/observability/telemetry/instrumentation/events", + "oneOf": [ + { + "additionalProperties": false, + "description": "A condition to check a selection against a value.", + "properties": { + "eq": { + "items": { + "$ref": "#/definitions/HttpClientSelectorOrValue" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + } + }, + "required": [ + "eq" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "The first selection must be greater than the second selection.", + "properties": { + "gt": { + "items": { + "$ref": "#/definitions/HttpClientSelectorOrValue" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + } + }, + "required": [ + "gt" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "The first selection must be less than the second selection.", + "properties": { + "lt": { + "items": { + "$ref": "#/definitions/HttpClientSelectorOrValue" + }, + "maxItems": 2, + "minItems": 2, + "type": "array" + } + }, + "required": [ + "lt" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "A condition to check a selection against a selector.", + "properties": { + "exists": { + "$ref": "#/definitions/HttpClientSelector" + } + }, + "required": [ + "exists" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "All sub-conditions must be true.", + "properties": { + "all": { + "items": { + "$ref": "#/definitions/ConditionHttpClientSelector" + }, + "type": "array" + } + }, + "required": [ + "all" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "At least one sub-conditions must be true.", + "properties": { + "any": { + "items": { + "$ref": "#/definitions/ConditionHttpClientSelector" + }, + "type": "array" + } + }, + "required": [ + "any" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "The sub-condition must not be true", + "properties": { + "not": { + "$ref": "#/definitions/ConditionHttpClientSelector" + } + }, + "required": [ + "not" + ], + "type": "object" + }, + { + "const": "true", + "description": "Static true condition", + "type": "string" + }, + { + "const": "false", + "description": "Static false condition", + "type": "string" + } + ] + }, "ConditionRouterSelector": { "description": "Specify a condition for when an [instrument][] should be mutated or an [event][] should be triggered.\n\n[instrument]: https://www.apollographql.com/docs/graphos/routing/observability/telemetry/instrumentation/instruments\n[event]: https://www.apollographql.com/docs/graphos/routing/observability/telemetry/instrumentation/events", "oneOf": [ @@ -1553,6 +1680,21 @@ expression: "&schema" } ] }, + "ConditionalHttpClientSelector": { + "anyOf": [ + { + "$ref": "#/definitions/HttpClientSelector" + }, + { + "properties": { + "condition": { + "$ref": "#/definitions/ConditionHttpClientSelector" + } + }, + "type": "object" + } + ] + }, "ConditionalRouterSelector": { "anyOf": [ { @@ -3717,6 +3859,25 @@ expression: "&schema" }, "type": "object" }, + "ExtendedHttpClientAttributesWithConditionalHttpClientSelector": { + "additionalProperties": { + "$ref": "#/definitions/ConditionalHttpClientSelector" + }, + "properties": { + "http.request.method": { + "anyOf": [ + { + "$ref": "#/definitions/StandardAttribute" + }, + { + "type": "null" + } + ], + "description": "HTTP request method.\nExamples:\n\n* GET\n* POST\n* HEAD\n\nRequirement level: Required" + } + }, + "type": "object" + }, "ExtendedRouterAttributesWithConditionalRouterSelector": { "additionalProperties": { "$ref": "#/definitions/ConditionalRouterSelector" @@ -5246,6 +5407,86 @@ expression: "&schema" } ] }, + "HttpClientSelector": { + "anyOf": [ + { + "additionalProperties": false, + "description": "A header from the HTTP request", + "properties": { + "default": { + "description": "Optional default value.", + "type": [ + "string", + "null" + ] + }, + "http_client_request_header": { + "description": "The name of the HTTP client request header.", + "type": "string" + } + }, + "required": [ + "http_client_request_header" + ], + "type": "object" + }, + { + "additionalProperties": false, + "description": "A header from the HTTP response", + "properties": { + "default": { + "description": "Optional default value.", + "type": [ + "string", + "null" + ] + }, + "http_client_response_header": { + "description": "The name of the HTTP client response header.", + "type": "string" + } + }, + "required": [ + "http_client_response_header" + ], + "type": "object" + } + ] + }, + "HttpClientSelectorOrValue": { + "anyOf": [ + { + "allOf": [ + { + "$ref": "#/definitions/AttributeValue" + } + ], + "description": "A constant value." + }, + { + "allOf": [ + { + "$ref": "#/definitions/HttpClientSelector" + } + ], + "description": "Selector to extract a value from the pipeline." + } + ] + }, + "HttpClientSpans": { + "additionalProperties": false, + "properties": { + "attributes": { + "allOf": [ + { + "$ref": "#/definitions/ExtendedHttpClientAttributesWithConditionalHttpClientSelector" + } + ], + "description": "Custom attributes that are attached to the HTTP client span." + } + }, + "type": "object" + }, "HttpExporter": { "additionalProperties": false, "properties": { @@ -8245,6 +8486,14 @@ expression: "&schema" ], "description": "The attributes to include by default in spans based on their level as specified in the otel semantic conventions and Apollo documentation." }, + "http_client": { + "allOf": [ + { + "$ref": "#/definitions/HttpClientSpans" + } + ], + "description": "Attributes to include on the HTTP client span.\nHTTP client spans contain information about HTTP requests made to subgraphs, including any changes made by Rhai scripts." + }, "mode": { "allOf": [ { diff --git a/apollo-router/src/plugins/telemetry/config_new/http_client/attributes.rs b/apollo-router/src/plugins/telemetry/config_new/http_client/attributes.rs index 0f13090748..1f4ff7eeec 100644 --- a/apollo-router/src/plugins/telemetry/config_new/http_client/attributes.rs +++ b/apollo-router/src/plugins/telemetry/config_new/http_client/attributes.rs @@ -38,9 +38,6 @@ impl DefaultForLevel for HttpClientAttributes { ) { match requirement_level { DefaultAttributeRequirementLevel::Required => { - if self.http_request_method.is_none() { - self.http_request_method = Some(StandardAttribute::Bool(true)); - } } DefaultAttributeRequirementLevel::Recommended => { if self.http_request_method.is_none() { @@ -55,6 +52,7 @@ impl DefaultForLevel for HttpClientAttributes { impl Selectors for HttpClientAttributes { fn on_request(&self, request: &http::HttpRequest) -> Vec { let mut attrs = Vec::new(); + if let Some(key) = self .http_request_method .as_ref() diff --git a/apollo-router/src/plugins/telemetry/config_new/http_client/spans.rs b/apollo-router/src/plugins/telemetry/config_new/http_client/spans.rs index a03b448e23..b6b37bb046 100644 --- a/apollo-router/src/plugins/telemetry/config_new/http_client/spans.rs +++ b/apollo-router/src/plugins/telemetry/config_new/http_client/spans.rs @@ -25,111 +25,3 @@ impl DefaultForLevel for HttpClientSpans { self.attributes.defaults_for_level(requirement_level, kind); } } - -#[cfg(test)] -mod test { - use opentelemetry_semantic_conventions::trace::HTTP_REQUEST_METHOD; - - use super::*; - use crate::Context; - use crate::plugins::telemetry::config_new::DefaultForLevel; - use crate::plugins::telemetry::config_new::Selectors; - use crate::plugins::telemetry::config_new::attributes::DefaultAttributeRequirementLevel; - use crate::plugins::telemetry::otlp::TelemetryDataKind; - use crate::services::http::HttpRequest; - - #[test] - fn test_http_client_spans_level_none() { - let mut spans = HttpClientSpans::default(); - spans.defaults_for_levels( - DefaultAttributeRequirementLevel::None, - TelemetryDataKind::Traces, - ); - - let http_request = ::http::Request::builder() - .method(::http::Method::POST) - .uri("http://localhost/graphql") - .body(crate::services::router::body::empty()) - .unwrap(); - - let request = HttpRequest { - http_request, - context: Context::new(), - }; - - let values = spans.attributes.on_request(&request); - assert!( - !values - .iter() - .any(|key_val| key_val.key.as_str() == HTTP_REQUEST_METHOD) - ); - } - - #[test] - fn test_http_client_spans_level_required() { - let mut spans = HttpClientSpans::default(); - spans.defaults_for_levels( - DefaultAttributeRequirementLevel::Required, - TelemetryDataKind::Traces, - ); - - let http_request = ::http::Request::builder() - .method(::http::Method::POST) - .uri("http://localhost/graphql") - .body(crate::services::router::body::empty()) - .unwrap(); - - let request = HttpRequest { - http_request, - context: Context::new(), - }; - - let values = spans.attributes.on_request(&request); - assert!( - values - .iter() - .any(|key_val| key_val.key.as_str() == HTTP_REQUEST_METHOD) - ); - assert_eq!( - values - .iter() - .find(|key_val| key_val.key.as_str() == HTTP_REQUEST_METHOD) - .map(|key_val| &key_val.value), - Some(&"POST".into()) - ); - } - - #[test] - fn test_http_client_spans_level_recommended() { - let mut spans = HttpClientSpans::default(); - spans.defaults_for_levels( - DefaultAttributeRequirementLevel::Recommended, - TelemetryDataKind::Traces, - ); - - let http_request = ::http::Request::builder() - .method(::http::Method::GET) - .uri("http://localhost/graphql") - .body(crate::services::router::body::empty()) - .unwrap(); - - let request = HttpRequest { - http_request, - context: Context::new(), - }; - - let values = spans.attributes.on_request(&request); - assert!( - values - .iter() - .any(|key_val| key_val.key.as_str() == HTTP_REQUEST_METHOD) - ); - assert_eq!( - values - .iter() - .find(|key_val| key_val.key.as_str() == HTTP_REQUEST_METHOD) - .map(|key_val| &key_val.value), - Some(&"GET".into()) - ); - } -} diff --git a/apollo-router/src/plugins/telemetry/mod.rs b/apollo-router/src/plugins/telemetry/mod.rs index 154339e2cb..50f2e77436 100644 --- a/apollo-router/src/plugins/telemetry/mod.rs +++ b/apollo-router/src/plugins/telemetry/mod.rs @@ -1185,7 +1185,7 @@ impl PluginPrivate for Telemetry { fn http_client_service( &self, - _subgraph_name: &str, // todo config per subgraph + _subgraph_name: &str, service: crate::services::http::BoxService, ) -> crate::services::http::BoxService { let req_fn_config = self.config.clone(); diff --git a/apollo-router/src/services/http/service.rs b/apollo-router/src/services/http/service.rs index 8e8acd9751..f5d0e48d75 100644 --- a/apollo-router/src/services/http/service.rs +++ b/apollo-router/src/services/http/service.rs @@ -315,8 +315,6 @@ impl tower::Service for HttpClientService { "http.route" = %path, "http.url" = %schema_uri, "net.transport" = "ip_tcp", - //"apollo.subgraph.name" = %service_name, - //"graphql.operation.name" = %operation_name, ); // Apply any attributes that were stored by telemetry middleware diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__batch_send_header-2.snap b/apollo-router/tests/snapshots/apollo_otel_traces__batch_send_header-2.snap index 7f5bcc7b3e..d069ee55b1 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__batch_send_header-2.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__batch_send_header-2.snap @@ -1,7 +1,6 @@ --- source: apollo-router/tests/apollo_otel_traces.rs expression: report -snapshot_kind: text --- resourceSpans: - resource: diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__batch_send_header.snap b/apollo-router/tests/snapshots/apollo_otel_traces__batch_send_header.snap index 7f5bcc7b3e..d069ee55b1 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__batch_send_header.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__batch_send_header.snap @@ -1,7 +1,6 @@ --- source: apollo-router/tests/apollo_otel_traces.rs expression: report -snapshot_kind: text --- resourceSpans: - resource: diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__batch_trace_id-2.snap b/apollo-router/tests/snapshots/apollo_otel_traces__batch_trace_id-2.snap index 16a24340b1..fec5016370 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__batch_trace_id-2.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__batch_trace_id-2.snap @@ -1,7 +1,6 @@ --- source: apollo-router/tests/apollo_otel_traces.rs expression: report -snapshot_kind: text --- resourceSpans: - resource: diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__batch_trace_id.snap b/apollo-router/tests/snapshots/apollo_otel_traces__batch_trace_id.snap index 16a24340b1..fec5016370 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__batch_trace_id.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__batch_trace_id.snap @@ -1,7 +1,6 @@ --- source: apollo-router/tests/apollo_otel_traces.rs expression: report -snapshot_kind: text --- resourceSpans: - resource: diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__client_name-2.snap b/apollo-router/tests/snapshots/apollo_otel_traces__client_name-2.snap index 002cf3d71c..0791fe568d 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__client_name-2.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__client_name-2.snap @@ -1,7 +1,6 @@ --- source: apollo-router/tests/apollo_otel_traces.rs expression: report -snapshot_kind: text --- resourceSpans: - resource: diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__client_name.snap b/apollo-router/tests/snapshots/apollo_otel_traces__client_name.snap index 002cf3d71c..0791fe568d 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__client_name.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__client_name.snap @@ -1,7 +1,6 @@ --- source: apollo-router/tests/apollo_otel_traces.rs expression: report -snapshot_kind: text --- resourceSpans: - resource: diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__client_version-2.snap b/apollo-router/tests/snapshots/apollo_otel_traces__client_version-2.snap index 1354627235..eaf3ec9fd2 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__client_version-2.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__client_version-2.snap @@ -1,7 +1,6 @@ --- source: apollo-router/tests/apollo_otel_traces.rs expression: report -snapshot_kind: text --- resourceSpans: - resource: diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__client_version.snap b/apollo-router/tests/snapshots/apollo_otel_traces__client_version.snap index 1354627235..eaf3ec9fd2 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__client_version.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__client_version.snap @@ -1,7 +1,6 @@ --- source: apollo-router/tests/apollo_otel_traces.rs expression: report -snapshot_kind: text --- resourceSpans: - resource: diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__condition_else-2.snap b/apollo-router/tests/snapshots/apollo_otel_traces__condition_else-2.snap index 0921dc9256..0c45204836 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__condition_else-2.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__condition_else-2.snap @@ -1,7 +1,6 @@ --- source: apollo-router/tests/apollo_otel_traces.rs expression: report -snapshot_kind: text --- resourceSpans: - resource: diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__condition_else.snap b/apollo-router/tests/snapshots/apollo_otel_traces__condition_else.snap index 0921dc9256..0c45204836 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__condition_else.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__condition_else.snap @@ -1,7 +1,6 @@ --- source: apollo-router/tests/apollo_otel_traces.rs expression: report -snapshot_kind: text --- resourceSpans: - resource: diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__condition_if-2.snap b/apollo-router/tests/snapshots/apollo_otel_traces__condition_if-2.snap index 7ab763e2fc..2275283e09 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__condition_if-2.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__condition_if-2.snap @@ -1,7 +1,6 @@ --- source: apollo-router/tests/apollo_otel_traces.rs expression: report -snapshot_kind: text --- resourceSpans: - resource: diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__condition_if.snap b/apollo-router/tests/snapshots/apollo_otel_traces__condition_if.snap index 7ab763e2fc..2275283e09 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__condition_if.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__condition_if.snap @@ -1,7 +1,6 @@ --- source: apollo-router/tests/apollo_otel_traces.rs expression: report -snapshot_kind: text --- resourceSpans: - resource: diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__non_defer-2.snap b/apollo-router/tests/snapshots/apollo_otel_traces__non_defer-2.snap index e11f185c53..96a3eed566 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__non_defer-2.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__non_defer-2.snap @@ -1,7 +1,6 @@ --- source: apollo-router/tests/apollo_otel_traces.rs expression: report -snapshot_kind: text --- resourceSpans: - resource: diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__non_defer.snap b/apollo-router/tests/snapshots/apollo_otel_traces__non_defer.snap index e11f185c53..96a3eed566 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__non_defer.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__non_defer.snap @@ -1,7 +1,6 @@ --- source: apollo-router/tests/apollo_otel_traces.rs expression: report -snapshot_kind: text --- resourceSpans: - resource: diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__send_header-2.snap b/apollo-router/tests/snapshots/apollo_otel_traces__send_header-2.snap index 293fe3eb7c..d090a76689 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__send_header-2.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__send_header-2.snap @@ -1,7 +1,6 @@ --- source: apollo-router/tests/apollo_otel_traces.rs expression: report -snapshot_kind: text --- resourceSpans: - resource: diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__send_header.snap b/apollo-router/tests/snapshots/apollo_otel_traces__send_header.snap index 293fe3eb7c..d090a76689 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__send_header.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__send_header.snap @@ -1,7 +1,6 @@ --- source: apollo-router/tests/apollo_otel_traces.rs expression: report -snapshot_kind: text --- resourceSpans: - resource: diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__send_variable_value-2.snap b/apollo-router/tests/snapshots/apollo_otel_traces__send_variable_value-2.snap index 85ad5c7909..a6daaabc5b 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__send_variable_value-2.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__send_variable_value-2.snap @@ -1,7 +1,6 @@ --- source: apollo-router/tests/apollo_otel_traces.rs expression: report -snapshot_kind: text --- resourceSpans: - resource: diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__send_variable_value.snap b/apollo-router/tests/snapshots/apollo_otel_traces__send_variable_value.snap index 85ad5c7909..a6daaabc5b 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__send_variable_value.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__send_variable_value.snap @@ -1,7 +1,6 @@ --- source: apollo-router/tests/apollo_otel_traces.rs expression: report -snapshot_kind: text --- resourceSpans: - resource: diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__trace_id-2.snap b/apollo-router/tests/snapshots/apollo_otel_traces__trace_id-2.snap index e11f185c53..96a3eed566 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__trace_id-2.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__trace_id-2.snap @@ -1,7 +1,6 @@ --- source: apollo-router/tests/apollo_otel_traces.rs expression: report -snapshot_kind: text --- resourceSpans: - resource: diff --git a/apollo-router/tests/snapshots/apollo_otel_traces__trace_id.snap b/apollo-router/tests/snapshots/apollo_otel_traces__trace_id.snap index e11f185c53..96a3eed566 100644 --- a/apollo-router/tests/snapshots/apollo_otel_traces__trace_id.snap +++ b/apollo-router/tests/snapshots/apollo_otel_traces__trace_id.snap @@ -1,7 +1,6 @@ --- source: apollo-router/tests/apollo_otel_traces.rs expression: report -snapshot_kind: text --- resourceSpans: - resource: From 78b1e1aee94ecb2f24cf96c2e9fa16a68eb836d4 Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Mon, 29 Sep 2025 14:27:52 +1000 Subject: [PATCH 06/16] Remove unnecessary client attribute --- ...nfiguration__tests__schema_generation.snap | 13 --- .../config_new/http_client/attributes.rs | 106 +----------------- 2 files changed, 3 insertions(+), 116 deletions(-) diff --git a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap index 0e88b7c7a7..05bab80f58 100644 --- a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap +++ b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap @@ -3863,19 +3863,6 @@ expression: "&schema" "additionalProperties": { "$ref": "#/definitions/ConditionalHttpClientSelector" }, - "properties": { - "http.request.method": { - "anyOf": [ - { - "$ref": "#/definitions/StandardAttribute" - }, - { - "type": "null" - } - ], - "description": "HTTP request method.\nExamples:\n\n* GET\n* POST\n* HEAD\n\nRequirement level: Required" - } - }, "type": "object" }, "ExtendedRouterAttributesWithConditionalRouterSelector": { diff --git a/apollo-router/src/plugins/telemetry/config_new/http_client/attributes.rs b/apollo-router/src/plugins/telemetry/config_new/http_client/attributes.rs index 1f4ff7eeec..6c18a5d6dd 100644 --- a/apollo-router/src/plugins/telemetry/config_new/http_client/attributes.rs +++ b/apollo-router/src/plugins/telemetry/config_new/http_client/attributes.rs @@ -1,7 +1,6 @@ use std::fmt::Debug; use opentelemetry::KeyValue; -use opentelemetry_semantic_conventions::trace::HTTP_REQUEST_METHOD; use schemars::JsonSchema; use serde::Deserialize; use tower::BoxError; @@ -10,7 +9,6 @@ use crate::Context; use crate::plugins::telemetry::config_new::DefaultForLevel; use crate::plugins::telemetry::config_new::Selectors; use crate::plugins::telemetry::config_new::attributes::DefaultAttributeRequirementLevel; -use crate::plugins::telemetry::config_new::attributes::StandardAttribute; use crate::plugins::telemetry::otlp::TelemetryDataKind; use crate::services::http; @@ -18,53 +16,20 @@ use crate::services::http; #[cfg_attr(test, derive(PartialEq))] #[serde(deny_unknown_fields, default)] pub(crate) struct HttpClientAttributes { - /// HTTP request method. - /// Examples: - /// - /// * GET - /// * POST - /// * HEAD - /// - /// Requirement level: Required - #[serde(rename = "http.request.method")] - pub(crate) http_request_method: Option, } impl DefaultForLevel for HttpClientAttributes { fn defaults_for_level( &mut self, - requirement_level: DefaultAttributeRequirementLevel, + _requirement_level: DefaultAttributeRequirementLevel, _kind: TelemetryDataKind, ) { - match requirement_level { - DefaultAttributeRequirementLevel::Required => { - } - DefaultAttributeRequirementLevel::Recommended => { - if self.http_request_method.is_none() { - self.http_request_method = Some(StandardAttribute::Bool(true)); - } - } - DefaultAttributeRequirementLevel::None => {} - } } } impl Selectors for HttpClientAttributes { - fn on_request(&self, request: &http::HttpRequest) -> Vec { - let mut attrs = Vec::new(); - - if let Some(key) = self - .http_request_method - .as_ref() - .and_then(|a| a.key(HTTP_REQUEST_METHOD.into())) - { - attrs.push(KeyValue::new( - key, - request.http_request.method().as_str().to_string(), - )); - } - - attrs + fn on_request(&self, _request: &http::HttpRequest) -> Vec { + Vec::new() } fn on_response(&self, _response: &http::HttpResponse) -> Vec { @@ -75,68 +40,3 @@ impl Selectors for HttpClientAttribut Vec::new() } } - -#[cfg(test)] -mod test { - use opentelemetry_semantic_conventions::trace::HTTP_REQUEST_METHOD; - - use super::*; - use crate::plugins::telemetry::config_new::Selectors; - use crate::services::http::HttpRequest; - - #[test] - fn test_http_client_request_method() { - let attributes = HttpClientAttributes { - http_request_method: Some(StandardAttribute::Bool(true)), - }; - - let http_request = ::http::Request::builder() - .method(::http::Method::POST) - .uri("http://localhost/graphql") - .body(crate::services::router::body::empty()) - .unwrap(); - - let request = HttpRequest { - http_request, - context: crate::Context::new(), - }; - - let attributes = attributes.on_request(&request); - assert_eq!( - attributes - .iter() - .find(|key_val| key_val.key.as_str() == HTTP_REQUEST_METHOD) - .map(|key_val| &key_val.value), - Some(&"POST".into()) - ); - } - - #[test] - fn test_http_client_request_method_aliased() { - let attributes = HttpClientAttributes { - http_request_method: Some(StandardAttribute::Aliased { - alias: "custom.request.method".to_string(), - }), - }; - - let http_request = ::http::Request::builder() - .method(::http::Method::GET) - .uri("http://localhost/graphql") - .body(crate::services::router::body::empty()) - .unwrap(); - - let request = HttpRequest { - http_request, - context: crate::Context::new(), - }; - - let attributes = attributes.on_request(&request); - assert_eq!( - attributes - .iter() - .find(|key_val| key_val.key.as_str() == "custom.request.method") - .map(|key_val| &key_val.value), - Some(&"GET".into()) - ); - } -} From df2550a840b51d4217e6fe64300177e36e72e503 Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Mon, 29 Sep 2025 14:45:56 +1000 Subject: [PATCH 07/16] Lint fix --- .../src/plugins/telemetry/config_new/http_client/attributes.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apollo-router/src/plugins/telemetry/config_new/http_client/attributes.rs b/apollo-router/src/plugins/telemetry/config_new/http_client/attributes.rs index 6c18a5d6dd..0fdfcf8d1d 100644 --- a/apollo-router/src/plugins/telemetry/config_new/http_client/attributes.rs +++ b/apollo-router/src/plugins/telemetry/config_new/http_client/attributes.rs @@ -15,8 +15,7 @@ use crate::services::http; #[derive(Deserialize, JsonSchema, Clone, Default, Debug)] #[cfg_attr(test, derive(PartialEq))] #[serde(deny_unknown_fields, default)] -pub(crate) struct HttpClientAttributes { -} +pub(crate) struct HttpClientAttributes {} impl DefaultForLevel for HttpClientAttributes { fn defaults_for_level( From 571aa887984aa0f87354da558c6e81e1e8e18954 Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Mon, 29 Sep 2025 14:54:52 +1000 Subject: [PATCH 08/16] Add changeset --- .../config_http_client_header_telemetry.md | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .changesets/config_http_client_header_telemetry.md diff --git a/.changesets/config_http_client_header_telemetry.md b/.changesets/config_http_client_header_telemetry.md new file mode 100644 index 0000000000..987b0a4d9e --- /dev/null +++ b/.changesets/config_http_client_header_telemetry.md @@ -0,0 +1,20 @@ +### Telemetry instrumentation config for http_client headers ([PR #8349](https://github.com/apollographql/router/pull/8349)) + +Adds a new telemetry instrumentation configuration for the http_client spans. This setting allows request headers added by Rhai scripts to be attached to the http_client span. The `some_rhai_response_header` value is available on the subgraph span as before. + +```yaml +telemetry: + instrumentation: + spans: + mode: spec_compliant + subgraph: + attributes: + http.response.header.some_rhai_response_header: + subgraph_response_header: "some_rhai_response_header" + http_client: + attributes: + http.request.header.some_rhai_request_header: + http_client_request_header: "some_rhai_request_header" +``` + +By [@bonnici](https://github.com/bonnici) in https://github.com/apollographql/router/pull/8349 From 9dfe78482926058f459ee619ac590f00889dfbba Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Mon, 29 Sep 2025 14:56:37 +1000 Subject: [PATCH 09/16] Update docs --- .../router-telemetry-otel/enabling-telemetry/spans.mdx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/spans.mdx b/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/spans.mdx index 29a086917c..25fdcca845 100644 --- a/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/spans.mdx +++ b/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/spans.mdx @@ -18,7 +18,7 @@ A **span** captures contextual information about requests and responses as they' -The `router`, `supergraph`, `subgraph` and `connector` sections are used to define custom span configuration for each service: +The `router`, `supergraph`, `subgraph`, `connector` and `http_client` sections are used to define custom span configuration for each service: ```yaml title="router.yaml" telemetry: @@ -36,6 +36,9 @@ telemetry: connector: # highlight-line attributes: {} # ... + http_client: # highlight-line + attributes: {} + # ... ``` ### `attributes` @@ -246,6 +249,9 @@ telemetry: connector: attributes: {} # ... + http_client: + attributes: {} + # ... ``` ## Spans configuration reference From ffbb10922a4ccb2ed9cc95fdadba561fe781879b Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Mon, 29 Sep 2025 15:04:33 +1000 Subject: [PATCH 10/16] Librarian recommendation --- .../router-telemetry-otel/enabling-telemetry/spans.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/spans.mdx b/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/spans.mdx index 25fdcca845..de6e1d7835 100644 --- a/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/spans.mdx +++ b/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/spans.mdx @@ -18,7 +18,7 @@ A **span** captures contextual information about requests and responses as they' -The `router`, `supergraph`, `subgraph`, `connector` and `http_client` sections are used to define custom span configuration for each service: +The `router`, `supergraph`, `subgraph`, `connector`, and `http_client` sections are used to define custom span configuration for each service: ```yaml title="router.yaml" telemetry: From 08dca188e3c39be5bf9757f9047b49f036d80076 Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Tue, 30 Sep 2025 07:00:50 +1000 Subject: [PATCH 11/16] Rename config --- .../config_http_client_header_telemetry.md | 2 +- ...nfiguration__tests__schema_generation.snap | 12 +++++------ .../config_new/http_client/selectors.rs | 20 +++++++++---------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.changesets/config_http_client_header_telemetry.md b/.changesets/config_http_client_header_telemetry.md index 987b0a4d9e..3616e33394 100644 --- a/.changesets/config_http_client_header_telemetry.md +++ b/.changesets/config_http_client_header_telemetry.md @@ -14,7 +14,7 @@ telemetry: http_client: attributes: http.request.header.some_rhai_request_header: - http_client_request_header: "some_rhai_request_header" + request_header: "some_rhai_request_header" ``` By [@bonnici](https://github.com/bonnici) in https://github.com/apollographql/router/pull/8349 diff --git a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap index 05bab80f58..72c7f9a145 100644 --- a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap +++ b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap @@ -5407,13 +5407,13 @@ expression: "&schema" "null" ] }, - "http_client_request_header": { - "description": "The name of the HTTP client request header.", + "request_header": { + "description": "The name of the request header.", "type": "string" } }, "required": [ - "http_client_request_header" + "request_header" ], "type": "object" }, @@ -5428,13 +5428,13 @@ expression: "&schema" "null" ] }, - "http_client_response_header": { - "description": "The name of the HTTP client response header.", + "response_header": { + "description": "The name of the response header.", "type": "string" } }, "required": [ - "http_client_response_header" + "response_header" ], "type": "object" } diff --git a/apollo-router/src/plugins/telemetry/config_new/http_client/selectors.rs b/apollo-router/src/plugins/telemetry/config_new/http_client/selectors.rs index 0181002223..c0196a5d7e 100644 --- a/apollo-router/src/plugins/telemetry/config_new/http_client/selectors.rs +++ b/apollo-router/src/plugins/telemetry/config_new/http_client/selectors.rs @@ -32,8 +32,8 @@ impl From<&HttpClientValue> for InstrumentValue { pub(crate) enum HttpClientSelector { /// A header from the HTTP request HttpClientRequestHeader { - /// The name of the HTTP client request header. - http_client_request_header: String, + /// The name of the request header. + request_header: String, #[serde(skip)] #[allow(dead_code)] /// Optional redaction pattern. @@ -43,8 +43,8 @@ pub(crate) enum HttpClientSelector { }, /// A header from the HTTP response HttpClientResponseHeader { - /// The name of the HTTP client response header. - http_client_response_header: String, + /// The name of the response header. + response_header: String, #[serde(skip)] #[allow(dead_code)] /// Optional redaction pattern. @@ -62,13 +62,13 @@ impl Selector for HttpClientSelector { fn on_request(&self, request: &http::HttpRequest) -> Option { match self { HttpClientSelector::HttpClientRequestHeader { - http_client_request_header, + request_header, default, .. } => request .http_request .headers() - .get(http_client_request_header) + .get(request_header) .and_then(|h| h.to_str().ok()) .map(|h| h.to_string()) .or_else(|| default.clone()) @@ -85,13 +85,13 @@ impl Selector for HttpClientSelector { default.clone().map(opentelemetry::Value::from) } HttpClientSelector::HttpClientResponseHeader { - http_client_response_header, + response_header, default, .. } => response .http_response .headers() - .get(http_client_response_header) + .get(response_header) .and_then(|h| h.to_str().ok()) .map(|h| h.to_string()) .or_else(|| default.clone()) @@ -126,7 +126,7 @@ mod test { #[test] fn test_http_client_request_header() { let selector = HttpClientSelector::HttpClientRequestHeader { - http_client_request_header: "content-type".to_string(), + request_header: "content-type".to_string(), redact: None, default: None, }; @@ -154,7 +154,7 @@ mod test { #[test] fn test_http_client_response_header() { let selector = HttpClientSelector::HttpClientResponseHeader { - http_client_response_header: "content-length".to_string(), + response_header: "content-length".to_string(), redact: None, default: None, }; From 3161154f0d0bf7f9300e0dea9426a9eb64b52d17 Mon Sep 17 00:00:00 2001 From: Aaron Arinder Date: Tue, 30 Sep 2025 12:03:45 -0400 Subject: [PATCH 12/16] empty commit to trigger docs check From 9f2fc8183b3e60853e4731cb261d31219cf5fb02 Mon Sep 17 00:00:00 2001 From: Daniel Abdelsamed Date: Tue, 30 Sep 2025 14:02:44 -0400 Subject: [PATCH 13/16] bump for docs From 90873f989b944f923b873947910952ed857780a7 Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Wed, 1 Oct 2025 08:43:58 +1000 Subject: [PATCH 14/16] Merge from dev --- .config/mise/config.toml | 2 +- .config/nextest.toml | 25 + .gitleaks.toml | 2 + Cargo.lock | 427 +----- apollo-federation/Cargo.toml | 2 +- .../src/connectors/json_selection/apply_to.rs | 533 ++++++- .../json_selection/methods/public/get.rs | 29 +- .../src/connectors/json_selection/parser.rs | 4 + .../src/connectors/validation/expression.rs | 57 +- ...lidation_tests@body_selection.graphql.snap | 2 +- ...on_tests@url_properties__path.graphql.snap | 4 +- ...@url_properties__query_params.graphql.snap | 4 +- ...cd04a19fd394c234940976dd32bc507984fca.json | 16 - ...a71988b1ef61a8a4dd38e4ac71bcb968d489e.json | 20 - ...e261b56416439675a38b316017478797a56ab.json | 25 - ...cf4fdc174de48a1d7fd64c088a15d02c2c690.json | 22 - ...b27163e1b83fe86adf6e6a906a11547a1d05f.json | 22 - ...3813f86f0b8072bbd241ed1090b5c37b932c8.json | 25 - ...c848ed7767b3aa0e1b3eaeb612f4840f4765e.json | 46 - ...22ec531e1dc6f695c56268bb3546b1f14beab.json | 22 - ...53deffa2e7a2c8570ef7089701ebab9f02665.json | 46 - ...49b47ba93ab750c54cecec6ccf636f1145178.json | 16 - ...9cd980c07f640be2fea4bc9d76dfc451d437d.json | 22 - ...f57f456f3a15c9dfd1388e0b32a0978a08ae0.json | 14 - ...a0c610380ed3411e0e5d3cae03e94347410a3.json | 29 - ...cf21671f600ee645ed77e5a300b6bbb9590c3.json | 20 - apollo-router/Cargo.toml | 6 - .../20250516144204_creation.down.sql | 10 - .../migrations/20250516144204_creation.up.sql | 66 - apollo-router/src/cache/redis.rs | 155 +- apollo-router/src/configuration/metrics.rs | 6 +- apollo-router/src/configuration/mod.rs | 4 +- ...t__metrics@response_cache.router.yaml.snap | 3 +- ...nfiguration__tests__schema_generation.snap | 174 +-- .../metrics/response_cache.router.yaml | 6 +- .../src/plugins/cache/invalidation.rs | 2 +- .../plugins/response_cache/invalidation.rs | 4 +- .../response_cache/invalidation_endpoint.rs | 57 +- .../src/plugins/response_cache/metrics.rs | 114 +- .../src/plugins/response_cache/plugin.rs | 298 ++-- ...ache__tests__failure_mode_reconnect-4.snap | 2 +- ...gins__response_cache__tests__insert-3.snap | 2 +- ...tests__insert_with_nested_field_set-3.snap | 2 +- ..._cache__tests__insert_with_requires-3.snap | 2 +- ...che__tests__invalidate_by_cache_tag-3.snap | 2 +- ...che__tests__invalidate_by_cache_tag-5.snap | 2 +- ...se_cache__tests__invalidate_by_type-3.snap | 2 +- ...se_cache__tests__invalidate_by_type-5.snap | 2 +- ...ins__response_cache__tests__no_data-3.snap | 4 +- ...ts__polymorphic_private_and_public-11.snap | 4 +- ...sts__polymorphic_private_and_public-3.snap | 2 +- ...sts__polymorphic_private_and_public-5.snap | 4 +- ...sts__polymorphic_private_and_public-7.snap | 4 +- ...sts__polymorphic_private_and_public-9.snap | 4 +- ...se_cache__tests__private_and_public-3.snap | 4 +- ...se_cache__tests__private_and_public-5.snap | 2 +- ...response_cache__tests__private_only-3.snap | 2 +- .../plugins/response_cache/storage/config.rs | 153 ++ .../plugins/response_cache/storage/error.rs | 19 +- .../src/plugins/response_cache/storage/mod.rs | 7 +- .../response_cache/storage/postgres.rs | 720 --------- .../plugins/response_cache/storage/redis.rs | 1354 +++++++++++++++++ ...on_key_permutations@input____None____.snap | 7 + ...tations@input____None____invalidation.snap | 8 + ...dation1__invalidation2__invalidation3.snap | 10 + ...on_key_permutations@input____test____.snap | 7 + ...tations@input____test____invalidation.snap | 8 + ...dation1__invalidation2__invalidation3.snap | 10 + .../src/plugins/response_cache/tests.rs | 316 ++-- .../uplink/testdata/restricted.router.yaml | 4 +- apollo-router/tests/integration/README.md | 36 +- apollo-router/tests/integration/mod.rs | 2 +- .../tests/integration/response_cache.rs | 202 +-- .../graphos/graphos-reporting.mdx | 8 +- 74 files changed, 2805 insertions(+), 2453 deletions(-) delete mode 100644 apollo-router/.sqlx/query-119ea89f7b98079bd3d2ec81596cd04a19fd394c234940976dd32bc507984fca.json delete mode 100644 apollo-router/.sqlx/query-5070a632fa2c6fb2766a16d305ca71988b1ef61a8a4dd38e4ac71bcb968d489e.json delete mode 100644 apollo-router/.sqlx/query-602e9f11a9cf461010a1523d8e8e261b56416439675a38b316017478797a56ab.json delete mode 100644 apollo-router/.sqlx/query-6245ac300b03e217aa5ed15489acf4fdc174de48a1d7fd64c088a15d02c2c690.json delete mode 100644 apollo-router/.sqlx/query-6cb3643cda13c340a17084366d2b27163e1b83fe86adf6e6a906a11547a1d05f.json delete mode 100644 apollo-router/.sqlx/query-6e77a91b9716475f81f81e188b13813f86f0b8072bbd241ed1090b5c37b932c8.json delete mode 100644 apollo-router/.sqlx/query-92256ab4f44ce1b6a034cbb1bfac848ed7767b3aa0e1b3eaeb612f4840f4765e.json delete mode 100644 apollo-router/.sqlx/query-9cab35054c927272ec5fb820c6122ec531e1dc6f695c56268bb3546b1f14beab.json delete mode 100644 apollo-router/.sqlx/query-a14112a24ef08c184e9062837d553deffa2e7a2c8570ef7089701ebab9f02665.json delete mode 100644 apollo-router/.sqlx/query-a8ff75a83927cd264969cecc40449b47ba93ab750c54cecec6ccf636f1145178.json delete mode 100644 apollo-router/.sqlx/query-b1b1778f3d3f1ca73b5037d6d6e9cd980c07f640be2fea4bc9d76dfc451d437d.json delete mode 100644 apollo-router/.sqlx/query-b7f6dac1056d65e173a6f4f3d81f57f456f3a15c9dfd1388e0b32a0978a08ae0.json delete mode 100644 apollo-router/.sqlx/query-c2b5393c5c63e03e12e9d480e82a0c610380ed3411e0e5d3cae03e94347410a3.json delete mode 100644 apollo-router/.sqlx/query-cb60ba408aa989631c6257955d9cf21671f600ee645ed77e5a300b6bbb9590c3.json delete mode 100644 apollo-router/migrations/20250516144204_creation.down.sql delete mode 100644 apollo-router/migrations/20250516144204_creation.up.sql create mode 100644 apollo-router/src/plugins/response_cache/storage/config.rs delete mode 100644 apollo-router/src/plugins/response_cache/storage/postgres.rs create mode 100644 apollo-router/src/plugins/response_cache/storage/redis.rs create mode 100644 apollo-router/src/plugins/response_cache/storage/snapshots/apollo_router__plugins__response_cache__storage__redis__tests__invalidation_key_permutations@input____None____.snap create mode 100644 apollo-router/src/plugins/response_cache/storage/snapshots/apollo_router__plugins__response_cache__storage__redis__tests__invalidation_key_permutations@input____None____invalidation.snap create mode 100644 apollo-router/src/plugins/response_cache/storage/snapshots/apollo_router__plugins__response_cache__storage__redis__tests__invalidation_key_permutations@input____None____invalidation1__invalidation2__invalidation3.snap create mode 100644 apollo-router/src/plugins/response_cache/storage/snapshots/apollo_router__plugins__response_cache__storage__redis__tests__invalidation_key_permutations@input____test____.snap create mode 100644 apollo-router/src/plugins/response_cache/storage/snapshots/apollo_router__plugins__response_cache__storage__redis__tests__invalidation_key_permutations@input____test____invalidation.snap create mode 100644 apollo-router/src/plugins/response_cache/storage/snapshots/apollo_router__plugins__response_cache__storage__redis__tests__invalidation_key_permutations@input____test____invalidation1__invalidation2__invalidation3.snap diff --git a/.config/mise/config.toml b/.config/mise/config.toml index f484757f3c..61594351ee 100644 --- a/.config/mise/config.toml +++ b/.config/mise/config.toml @@ -16,4 +16,4 @@ gh = "2.72.0" helm = "3.19.0" helm-docs = "1.14.2" yq = "4.47.2" -jq = "1.7.1" +jq = "1.8.1" diff --git a/.config/nextest.toml b/.config/nextest.toml index 500a087cdf..93c3b595d4 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -35,6 +35,8 @@ or ( binary_id(=apollo-router::apollo_reports) & test(=test_condition_if) ) or ( binary_id(=apollo-router::apollo_reports) & test(=test_demand_control_stats) ) or ( binary_id(=apollo-router::apollo_reports) & test(=test_demand_control_trace_batched) ) or ( binary_id(=apollo-router::apollo_reports) & test(=test_demand_control_trace) ) +or ( binary_id(=apollo-router::apollo_reports) & test(=test_features_disabled) ) +or ( binary_id(=apollo-router::apollo_reports) & test(=test_features_enabled) ) or ( binary_id(=apollo-router::apollo_reports) & test(=test_new_field_stats) ) or ( binary_id(=apollo-router::apollo_reports) & test(=test_send_header) ) or ( binary_id(=apollo-router::apollo_reports) & test(=test_send_variable_value) ) @@ -61,6 +63,7 @@ or ( binary_id(=apollo-router::integration_tests) & test(=integration::batching: or ( binary_id(=apollo-router::integration_tests) & test(=integration::batching::it_supports_multi_subgraph_batching) ) or ( binary_id(=apollo-router::integration_tests) & test(=integration::batching::it_supports_single_subgraph_batching) ) or ( binary_id(=apollo-router::integration_tests) & test(=integration::connectors::authentication::incompatible_warnings_with_overrides) ) +or ( binary_id(=apollo-router::integration_tests) & test(=integration::connectors::authentication::test_aws_sig_v4_signing) ) or ( binary_id(=apollo-router::integration_tests) & test(=integration::coprocessor::test_coprocessor_response_handling) ) or ( binary_id(=apollo-router::integration_tests) & test(=integration::coprocessor::test_error_not_propagated_to_client) ) or ( binary_id(=apollo-router::integration_tests) & test(=integration::file_upload::it_fails_incompatible_query_order) ) @@ -83,9 +86,13 @@ or ( binary_id(=apollo-router::integration_tests) & test(=integration::lifecycle or ( binary_id(=apollo-router::integration_tests) & test(=integration::lifecycle::test_reload_config_with_broken_plugin_recovery) ) or ( binary_id(=apollo-router::integration_tests) & test(=integration::lifecycle::test_reload_config_with_broken_plugin) ) or ( binary_id(=apollo-router::integration_tests) & test(=integration::lifecycle::test_shutdown_with_idle_connection) ) +or ( binary_id(=apollo-router::integration_tests) & test(=integration::metrics::test_jemalloc_metrics_are_emitted) ) or ( binary_id(=apollo-router::integration_tests) & test(=integration::operation_limits::test_request_bytes_limit_with_coprocessor) ) or ( binary_id(=apollo-router::integration_tests) & test(=integration::operation_limits::test_request_bytes_limit) ) or ( binary_id(=apollo-router::integration_tests) & test(=integration::query_planner::context_with_new_qp) ) +or ( binary_id(=apollo-router::integration_tests) & test(=integration::query_planner::error_paths::test_multi_level_response_failure) ) +or ( binary_id(=apollo-router::integration_tests) & test(=integration::query_planner::error_paths::test_nested_response_failure) ) +or ( binary_id(=apollo-router::integration_tests) & test(=integration::query_planner::fed1_schema_with_new_qp) ) or ( binary_id(=apollo-router::integration_tests) & test(=integration::query_planner::max_evaluated_plans::reports_evaluated_plans) ) or ( binary_id(=apollo-router::integration_tests) & test(=integration::query_planner::overloaded_compute_job_pool) ) or ( binary_id(=apollo-router::integration_tests) & test(=integration::query_planner::progressive_override_with_legacy_qp_reload_to_both_best_effort_keep_previous_config) ) @@ -94,6 +101,7 @@ or ( binary_id(=apollo-router::integration_tests) & test(=integration::redis::ap or ( binary_id(=apollo-router::integration_tests) & test(=integration::redis::connection_failure_blocks_startup) ) or ( binary_id(=apollo-router::integration_tests) & test(=integration::redis::entity_cache_authorization) ) or ( binary_id(=apollo-router::integration_tests) & test(=integration::redis::entity_cache_basic) ) +or ( binary_id(=apollo-router::integration_tests) & test(=integration::redis::entity_cache_with_nested_field_set) ) or ( binary_id(=apollo-router::integration_tests) & test(=integration::redis::entity_cache) ) or ( binary_id(=apollo-router::integration_tests) & test(=integration::redis::query_planner_redis_update_defer) ) or ( binary_id(=apollo-router::integration_tests) & test(=integration::redis::query_planner_redis_update_introspection) ) @@ -105,6 +113,19 @@ or ( binary_id(=apollo-router::integration_tests) & test(=integration::rhai::tes or ( binary_id(=apollo-router::integration_tests) & test(=integration::subgraph_response::test_invalid_error_locations_contains_negative_one_location) ) or ( binary_id(=apollo-router::integration_tests) & test(=integration::subgraph_response::test_valid_extensions_service_for_subgraph_error) ) or ( binary_id(=apollo-router::integration_tests) & test(=integration::subgraph_response::test_valid_extensions_service_is_preserved_for_subgraph_error) ) +or ( binary_id(=apollo-router::integration_tests) & test(=integration::subscriptions::callback::test_subscription_callback_pure_error_payload) ) +or ( binary_id(=apollo-router::integration_tests) & test(=integration::subscriptions::callback::test_subscription_callback) ) +or ( binary_id(=apollo-router::integration_tests) & test(=integration::subscriptions::ws_passthrough::test_subscription_ws_passthrough_dedup_close_early) ) +or ( binary_id(=apollo-router::integration_tests) & test(=integration::subscriptions::ws_passthrough::test_subscription_ws_passthrough_dedup) ) +or ( binary_id(=apollo-router::integration_tests) & test(=integration::subscriptions::ws_passthrough::test_subscription_ws_passthrough) ) +or ( binary_id(=apollo-router::integration_tests) & test(=integration::supergraph::test_supergraph_errors_on_http1_header_that_does_not_fit_inside_buffer) ) +or ( binary_id(=apollo-router::integration_tests) & test(=integration::telemetry::apollo_otel_metrics::test_connector_request_emits_histogram) ) +or ( binary_id(=apollo-router::integration_tests) & test(=integration::telemetry::apollo_otel_metrics::test_execution_layer_error_emits_metric) ) +or ( binary_id(=apollo-router::integration_tests) & test(=integration::telemetry::apollo_otel_metrics::test_failed_connector_request_emits_histogram) ) +or ( binary_id(=apollo-router::integration_tests) & test(=integration::telemetry::apollo_otel_metrics::test_failed_subgraph_request_emits_histogram) ) +or ( binary_id(=apollo-router::integration_tests) & test(=integration::telemetry::apollo_otel_metrics::test_include_subgraph_error_disabled_does_not_redact_error_metrics) ) +or ( binary_id(=apollo-router::integration_tests) & test(=integration::telemetry::apollo_otel_metrics::test_router_layer_error_emits_metric) ) +or ( binary_id(=apollo-router::integration_tests) & test(=integration::telemetry::apollo_otel_metrics::test_subgraph_http_error_emits_metric) ) or ( binary_id(=apollo-router::integration_tests) & test(=integration::telemetry::datadog::test_basic) ) or ( binary_id(=apollo-router::integration_tests) & test(=integration::telemetry::datadog::test_priority_sampling_no_parent_propagated) ) or ( binary_id(=apollo-router::integration_tests) & test(=integration::telemetry::datadog::test_resource_mapping_default) ) @@ -146,6 +167,7 @@ or ( binary_id(=apollo-router::integration_tests) & test(=integration::traffic_s or ( binary_id(=apollo-router::integration_tests) & test(=integration::traffic_shaping::test_router_timeout) ) or ( binary_id(=apollo-router::integration_tests) & test(=integration::traffic_shaping::test_subgraph_rate_limit) ) or ( binary_id(=apollo-router::integration_tests) & test(=integration::traffic_shaping::test_subgraph_timeout) ) +or ( binary_id(=apollo-router::integration_tests) & test(=mutation_should_work_over_post) ) or ( binary_id(=apollo-router::integration_tests) & test(=normal_query_with_defer_accept_header) ) or ( binary_id(=apollo-router::integration_tests) & test(=persisted_queries) ) or ( binary_id(=apollo-router::integration_tests) & test(=queries_should_work_over_get) ) @@ -196,10 +218,13 @@ or ( binary_id(=apollo-router) & test(=axum_factory::tests::response_with_custom or ( binary_id(=apollo-router) & test(=axum_factory::tests::response_with_custom_prefix_endpoint) ) or ( binary_id(=apollo-router) & test(=axum_factory::tests::response_with_root_wildcard) ) or ( binary_id(=apollo-router) & test(=axum_factory::tests::response) ) +or ( binary_id(=apollo-router) & test(=cache::metrics::tests::test_redis_storage_with_mocks) ) or ( binary_id(=apollo-router) & test(=layers::map_first_graphql_response::tests::test_map_first_graphql_response) ) or ( binary_id(=apollo-router) & test(=notification::tests::it_test_ttl) ) or ( binary_id(=apollo-router) & test(=plugins::authentication::subgraph::test::test_credentials_provider_refresh_on_stale) ) +or ( binary_id(=apollo-router) & test(=plugins::connectors::tests::connect_on_type::batch_with_max_size_over_batch_size) ) or ( binary_id(=apollo-router) & test(=plugins::connectors::tests::quickstart::query_4) ) +or ( binary_id(=apollo-router) & test(=plugins::connectors::tests::test_entity_references) ) or ( binary_id(=apollo-router) & test(=plugins::connectors::tests::test_interface_object) ) or ( binary_id(=apollo-router) & test(=plugins::expose_query_plan::tests::it_expose_query_plan) ) or ( binary_id(=apollo-router) & test(=plugins::include_subgraph_errors::test::it_does_not_redact_all_explicit_allow_account_explict_redact_for_product_query) ) diff --git a/.gitleaks.toml b/.gitleaks.toml index 9a2489ffc7..b6b9b0c7b4 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -14,6 +14,8 @@ "d826844c8cf433f78938059f02feecc108468e49", # Not a key: https://github.com/apollographql/router/pull/8193#issuecomment-3249460075 "77700b93798ce98eaf75c3b02b70198e7b730ddb", + # not a key: https://github.com/apollographql/router/pull/8326#issuecomment-3325655427 + "58bca0271d5a2046dfa7705a5418781bc456c787", ] paths = [ diff --git a/Cargo.lock b/Cargo.lock index 5e864146f8..80fdc28338 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -393,7 +393,6 @@ dependencies = [ "sha2", "shellexpand", "similar", - "sqlx", "static_assertions", "strum 0.27.2", "strum_macros 0.27.2", @@ -824,15 +823,6 @@ dependencies = [ "syn 2.0.106", ] -[[package]] -name = "atoi" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" -dependencies = [ - "num-traits 0.2.19", -] - [[package]] name = "atomic-waker" version = "1.1.2" @@ -1916,21 +1906,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc" -version = "3.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" -dependencies = [ - "crc-catalog", -] - -[[package]] -name = "crc-catalog" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" - [[package]] name = "crc16" version = "0.4.0" @@ -2015,15 +1990,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "crossbeam-queue" -version = "0.3.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -2342,12 +2308,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "dotenvy" -version = "0.15.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" - [[package]] name = "downcast" version = "0.11.0" @@ -2421,9 +2381,6 @@ name = "either" version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" -dependencies = [ - "serde", -] [[package]] name = "elliptic-curve" @@ -2561,17 +2518,6 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5692dd7b5a1978a5aeb0ce83b7655c58ca8efdcb79d21036ea249da95afec2c6" -[[package]] -name = "etcetera" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" -dependencies = [ - "cfg-if", - "home", - "windows-sys 0.48.0", -] - [[package]] name = "event-listener" version = "2.5.3" @@ -2715,17 +2661,6 @@ dependencies = [ "serde", ] -[[package]] -name = "flume" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" -dependencies = [ - "futures-core", - "futures-sink", - "spin 0.9.8", -] - [[package]] name = "fnv" version = "1.0.7" @@ -2925,17 +2860,6 @@ dependencies = [ "num_cpus", ] -[[package]] -name = "futures-intrusive" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" -dependencies = [ - "futures-core", - "lock_api", - "parking_lot", -] - [[package]] name = "futures-io" version = "0.3.31" @@ -3293,15 +3217,6 @@ dependencies = [ "foldhash 0.2.0", ] -[[package]] -name = "hashlink" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" -dependencies = [ - "hashbrown 0.15.5", -] - [[package]] name = "hdrhistogram" version = "7.5.4" @@ -3424,15 +3339,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "hkdf" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" -dependencies = [ - "hmac", -] - [[package]] name = "hmac" version = "0.12.1" @@ -3442,15 +3348,6 @@ dependencies = [ "digest", ] -[[package]] -name = "home" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" -dependencies = [ - "windows-sys 0.59.0", -] - [[package]] name = "http" version = "0.2.12" @@ -4126,9 +4023,6 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" -dependencies = [ - "spin 0.9.8", -] [[package]] name = "levenshtein" @@ -4152,12 +4046,6 @@ dependencies = [ "cc", ] -[[package]] -name = "libm" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" - [[package]] name = "libredox" version = "0.1.9" @@ -4169,16 +4057,6 @@ dependencies = [ "redox_syscall", ] -[[package]] -name = "libsqlite3-sys" -version = "0.30.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" -dependencies = [ - "pkg-config", - "vcpkg", -] - [[package]] name = "libtest-mimic" version = "0.8.1" @@ -4310,16 +4188,6 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" -[[package]] -name = "md-5" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" -dependencies = [ - "cfg-if", - "digest", -] - [[package]] name = "mediatype" version = "0.20.0" @@ -4583,23 +4451,6 @@ dependencies = [ "num-traits 0.2.19", ] -[[package]] -name = "num-bigint-dig" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" -dependencies = [ - "byteorder", - "lazy_static", - "libm", - "num-integer", - "num-iter", - "num-traits 0.2.19", - "rand 0.8.5", - "smallvec", - "zeroize", -] - [[package]] name = "num-cmp" version = "0.1.0" @@ -4668,7 +4519,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", - "libm", ] [[package]] @@ -5208,17 +5058,6 @@ dependencies = [ "futures-io", ] -[[package]] -name = "pkcs1" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" -dependencies = [ - "der", - "pkcs8", - "spki", -] - [[package]] name = "pkcs8" version = "0.10.2" @@ -6037,26 +5876,6 @@ dependencies = [ "text-size", ] -[[package]] -name = "rsa" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78928ac1ed176a5ca1d17e578a1825f3d81ca54cf41053a592584b020cfd691b" -dependencies = [ - "const-oid", - "digest", - "num-bigint-dig", - "num-integer", - "num-traits 0.2.19", - "pkcs1", - "pkcs8", - "rand_core 0.6.4", - "signature", - "spki", - "subtle", - "zeroize", -] - [[package]] name = "rstack" version = "0.3.3" @@ -6526,9 +6345,9 @@ dependencies = [ [[package]] name = "shape" -version = "0.5.3" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "914e2afe9130bf8acf52c5e20b4222f7d2e5eb8327e05fb668fe70aad4b3a896" +checksum = "48f06e8e6e2486e2ca1fc86254acb38bca0cd7da30af443e8d63958c66738f88" dependencies = [ "apollo-compiler", "indexmap 2.11.4", @@ -6667,9 +6486,6 @@ name = "spin" version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" -dependencies = [ - "lock_api", -] [[package]] name = "spki" @@ -6681,200 +6497,6 @@ dependencies = [ "der", ] -[[package]] -name = "sqlx" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" -dependencies = [ - "sqlx-core", - "sqlx-macros", - "sqlx-mysql", - "sqlx-postgres", - "sqlx-sqlite", -] - -[[package]] -name = "sqlx-core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" -dependencies = [ - "base64 0.22.1", - "bytes", - "chrono", - "crc", - "crossbeam-queue", - "either", - "event-listener 5.4.1", - "futures-core", - "futures-intrusive", - "futures-io", - "futures-util", - "hashbrown 0.15.5", - "hashlink", - "indexmap 2.11.4", - "log", - "memchr", - "once_cell", - "percent-encoding", - "rustls", - "rustls-native-certs", - "serde", - "serde_json", - "sha2", - "smallvec", - "thiserror 2.0.16", - "tokio", - "tokio-stream", - "tracing", - "url", -] - -[[package]] -name = "sqlx-macros" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" -dependencies = [ - "proc-macro2", - "quote", - "sqlx-core", - "sqlx-macros-core", - "syn 2.0.106", -] - -[[package]] -name = "sqlx-macros-core" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" -dependencies = [ - "dotenvy", - "either", - "heck 0.5.0", - "hex", - "once_cell", - "proc-macro2", - "quote", - "serde", - "serde_json", - "sha2", - "sqlx-core", - "sqlx-mysql", - "sqlx-postgres", - "sqlx-sqlite", - "syn 2.0.106", - "tokio", - "url", -] - -[[package]] -name = "sqlx-mysql" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" -dependencies = [ - "atoi", - "base64 0.22.1", - "bitflags 2.9.3", - "byteorder", - "bytes", - "chrono", - "crc", - "digest", - "dotenvy", - "either", - "futures-channel", - "futures-core", - "futures-io", - "futures-util", - "generic-array", - "hex", - "hkdf", - "hmac", - "itoa", - "log", - "md-5", - "memchr", - "once_cell", - "percent-encoding", - "rand 0.8.5", - "rsa", - "serde", - "sha1", - "sha2", - "smallvec", - "sqlx-core", - "stringprep", - "thiserror 2.0.16", - "tracing", - "whoami", -] - -[[package]] -name = "sqlx-postgres" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" -dependencies = [ - "atoi", - "base64 0.22.1", - "bitflags 2.9.3", - "byteorder", - "chrono", - "crc", - "dotenvy", - "etcetera", - "futures-channel", - "futures-core", - "futures-util", - "hex", - "hkdf", - "hmac", - "home", - "itoa", - "log", - "md-5", - "memchr", - "once_cell", - "rand 0.8.5", - "serde", - "serde_json", - "sha2", - "smallvec", - "sqlx-core", - "stringprep", - "thiserror 2.0.16", - "tracing", - "whoami", -] - -[[package]] -name = "sqlx-sqlite" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" -dependencies = [ - "atoi", - "chrono", - "flume", - "futures-channel", - "futures-core", - "futures-executor", - "futures-intrusive", - "futures-util", - "libsqlite3-sys", - "log", - "percent-encoding", - "serde", - "serde_urlencoded", - "sqlx-core", - "thiserror 2.0.16", - "tracing", - "url", -] - [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -6903,17 +6525,6 @@ dependencies = [ "regex", ] -[[package]] -name = "stringprep" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" -dependencies = [ - "unicode-bidi", - "unicode-normalization", - "unicode-properties", -] - [[package]] name = "strsim" version = "0.11.1" @@ -7878,12 +7489,6 @@ version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" -[[package]] -name = "unicode-bidi" -version = "0.3.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" - [[package]] name = "unicode-ident" version = "1.0.18" @@ -7899,12 +7504,6 @@ dependencies = [ "tinyvec", ] -[[package]] -name = "unicode-properties" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" - [[package]] name = "unicode-segmentation" version = "1.12.0" @@ -8006,12 +7605,6 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "943ce29a8a743eb10d6082545d861b24f9d1b160b7d741e0f2cdf726bec909c5" -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - [[package]] name = "version_check" version = "0.9.5" @@ -8064,12 +7657,6 @@ dependencies = [ "wit-bindgen-rt", ] -[[package]] -name = "wasite" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" - [[package]] name = "wasm-bindgen" version = "0.2.100" @@ -8194,16 +7781,6 @@ dependencies = [ "winsafe", ] -[[package]] -name = "whoami" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d" -dependencies = [ - "libredox", - "wasite", -] - [[package]] name = "widestring" version = "1.2.0" diff --git a/apollo-federation/Cargo.toml b/apollo-federation/Cargo.toml index d574339339..52c56a623a 100644 --- a/apollo-federation/Cargo.toml +++ b/apollo-federation/Cargo.toml @@ -49,7 +49,7 @@ url = "2" either = "1.13.0" tracing = "0.1.40" ron = { version = "0.11.0", optional = true } -shape = "0.5.2" +shape = "0.6.0" form_urlencoded = "1.2.1" parking_lot = "0.12.4" mime = "0.3.17" diff --git a/apollo-federation/src/connectors/json_selection/apply_to.rs b/apollo-federation/src/connectors/json_selection/apply_to.rs index 1add221330..87346aa3e8 100644 --- a/apollo-federation/src/connectors/json_selection/apply_to.rs +++ b/apollo-federation/src/connectors/json_selection/apply_to.rs @@ -896,16 +896,23 @@ impl ApplyToInternal for WithRange { } PathList::Question(tail) => { - // Explicitly represent the possibility of None (or null mapped - // to None) in the output shape. + let q_shape = input_shape.question(self.shape_location(context.source_id())); ( - Shape::one( - [ - input_shape, - Shape::none().with_locations(self.shape_location(context.source_id())), - ], - self.shape_location(context.source_id()), - ), + if tail.is_empty() { + // If there is no tail, we do not need to account for + // the possibility that the whole path might evaluate to + // None, as that possibility will be encoded in the + // computed shape for this terminal/leaf ::Question. + q_shape + } else { + // Using the ? operator with a non-empty tail suggests + // the input shape could evaluate to None, so we always + // include None as a possible shape here. If we don't + // entertain this possibility, we might compute a + // non-optional object shape with missing fields instead + // of correctly computing One<{...}, None>. + Shape::one([q_shape, Shape::none()], []) + }, Some(tail), ) } @@ -1179,42 +1186,19 @@ impl ApplyToInternal for WithRange { }) .collect(); - if shapes.iter().any(|shape| match shape.case() { - ShapeCase::Name(..) => true, - ShapeCase::One(one) - if one.iter().any(|s| matches!(s.case(), ShapeCase::Name(..))) => - { - true - } - _ => false, - }) { - return Shape::unknown(locations); - } - match op.as_ref() { LitOp::NullishCoalescing => { if let Some(last_shape) = shapes.pop() { - if let Some(prefix) = match Shape::one(shapes.clone(), []).case() { - ShapeCase::None => None, - ShapeCase::Null => None, - ShapeCase::One(shapes) => { - let filtered = shapes - .iter() - .filter(|shape| !shape.is_none() && !shape.is_null()) - .cloned() - .collect::>(); - if filtered.is_empty() { - None - } else { - Some(Shape::one(filtered, locations.clone())) - } - } - _ => Some(Shape::one(shapes, locations.clone())), - } { - Shape::one([prefix, last_shape], locations) - } else { - last_shape - } + let mut new_shapes = shapes + .iter() + .map(|shape| { + shape + .question(locations.clone()) + .not_none(locations.clone()) + }) + .collect::>(); + new_shapes.push(last_shape); + Shape::one(new_shapes, locations) } else { Shape::one(shapes, locations) } @@ -1223,26 +1207,12 @@ impl ApplyToInternal for WithRange { // Just like NullishCoalescing except null is not excluded. LitOp::NoneCoalescing => { if let Some(last_shape) = shapes.pop() { - if let Some(prefix) = match Shape::one(shapes.clone(), []).case() { - ShapeCase::None => None, - ShapeCase::One(shapes) => { - let filtered = shapes - .iter() - .filter(|shape| !shape.is_none()) - .cloned() - .collect::>(); - if filtered.is_empty() { - None - } else { - Some(Shape::one(filtered, locations.clone())) - } - } - _ => Some(Shape::one(shapes, locations.clone())), - } { - Shape::one([prefix, last_shape], locations) - } else { - last_shape - } + let mut new_shapes = shapes + .iter() + .map(|shape| shape.not_none(locations.clone())) + .collect::>(); + new_shapes.push(last_shape); + Shape::one(new_shapes, locations) } else { Shape::one(shapes, locations) } @@ -3952,7 +3922,7 @@ mod tests { ); assert_eq!( author_selection.shape().pretty_print(), - "{ author: One<{ age: $root.*.author.*.age, middleName: One<$root.*.author.*.middleName, None> }, None> }", + "{ author: One<{ age: $root.*.author?.*.age, middleName: $root.*.author?.*.middleName? }, None> }", ); } @@ -5071,7 +5041,22 @@ mod tests { ); assert_eq!( complex_chain_no_fallback.shape().pretty_print(), - "One<{ __typename: \"Good\", message: $root.*.message }, { __typename: \"Bad\", error: $root.*.error }, None>", + // None should not be an option here, even though both message and + // error might not exist, because the ... spread operator spreads + // nothing in that case. + // "One<{ __typename: \"Good\", message: $root.*.message }, { __typename: \"Bad\", error: $root.*.error }, None>", + "One<{ __typename: \"Good\", message: $root.*.message }, { __typename: \"Bad\", error: $root.*.error }, {}>", + ); + assert_eq!( + complex_chain_no_fallback.apply_to(&json!({})), + (Some(json!({})), vec![ + ApplyToError::new( + "Inlined path produced no value".to_string(), + vec![], + Some(17..165), + spec, + ), + ]), ); } **/ @@ -5135,4 +5120,424 @@ mod tests { "One", ); } + + #[test] + fn question_operator_should_map_null_to_none() { + let spec = ConnectSpec::V0_3; + + let nullish_string_selection = selection!("$(stringOrNull?)", spec); + assert_eq!( + nullish_string_selection.apply_to(&json!({"stringOrNull": "a string"})), + (Some(json!("a string")), vec![]), + ); + assert_eq!( + nullish_string_selection.apply_to(&json!({"stringOrNull": null})), + (None, vec![]), + ); + assert_eq!( + nullish_string_selection.apply_to(&json!({})), + (None, vec![]), + ); + + let shape_context = { + let mut named_shapes = IndexMap::default(); + + named_shapes.insert( + "$root".to_string(), + Shape::record( + { + let mut map = Shape::empty_map(); + map.insert( + "stringOrNull".to_string(), + Shape::one([Shape::string([]), Shape::null([])], []), + ); + map + }, + [], + ), + ); + + ShapeContext::new(SourceId::Other("JSONSelection".into())) + .with_spec(spec) + .with_named_shapes(named_shapes) + }; + + let root_shape = shape_context.named_shapes().get("$root").unwrap().clone(); + + assert_eq!( + root_shape.pretty_print(), + "{ stringOrNull: One }", + ); + + assert_eq!( + nullish_string_selection + .compute_output_shape( + &shape_context, + shape_context.named_shapes().get("$root").unwrap().clone(), + ) + .pretty_print(), + // Note that null has been replaced with None. + "One", + ); + } + + #[test] + fn question_operator_should_add_none_to_named_shapes() { + let spec = ConnectSpec::V0_3; + + let string_or_null_expr = selection!("$(stringOrNull?)", spec); + + assert_eq!( + string_or_null_expr.shape().pretty_print(), + "$root.stringOrNull?", + ); + } + + #[test] + fn question_operator_with_nested_objects() { + let spec = ConnectSpec::V0_3; + + let nested_selection = selection!("$(user?.profile?.name)", spec); + assert_eq!( + nested_selection.apply_to(&json!({"user": {"profile": {"name": "Alice"}}})), + (Some(json!("Alice")), vec![]), + ); + assert_eq!( + nested_selection.apply_to(&json!({"user": null})), + (None, vec![]), + ); + assert_eq!( + nested_selection.apply_to(&json!({"user": {"profile": null}})), + (None, vec![]), + ); + assert_eq!(nested_selection.apply_to(&json!({})), (None, vec![])); + } + + #[test] + fn question_operator_with_array_access() { + let spec = ConnectSpec::V0_3; + + let array_selection = selection!("$(items?->first?.name)", spec); + assert_eq!( + array_selection.apply_to(&json!({"items": [{"name": "first"}]})), + (Some(json!("first")), vec![]), + ); + assert_eq!( + array_selection.apply_to(&json!({"items": []})), + (None, vec![]), + ); + assert_eq!( + array_selection.apply_to(&json!({"items": null})), + (None, vec![]), + ); + assert_eq!(array_selection.apply_to(&json!({})), (None, vec![])); + } + + #[test] + fn question_operator_with_union_shapes() { + let spec = ConnectSpec::V0_3; + + let shape_context = { + let mut named_shapes = IndexMap::default(); + + named_shapes.insert( + "$root".to_string(), + Shape::record( + { + let mut map = Shape::empty_map(); + map.insert( + "unionField".to_string(), + Shape::one([Shape::string([]), Shape::int([]), Shape::null([])], []), + ); + map + }, + [], + ), + ); + + ShapeContext::new(SourceId::Other("JSONSelection".into())) + .with_spec(spec) + .with_named_shapes(named_shapes) + }; + + let union_selection = selection!("$(unionField?)", spec); + + assert_eq!( + union_selection + .compute_output_shape( + &shape_context, + shape_context.named_shapes().get("$root").unwrap().clone(), + ) + .pretty_print(), + "One", + ); + } + + #[test] + fn question_operator_with_error_shapes() { + let spec = ConnectSpec::V0_3; + + let shape_context = { + let mut named_shapes = IndexMap::default(); + + named_shapes.insert( + "$root".to_string(), + Shape::record( + { + let mut map = Shape::empty_map(); + map.insert( + "errorField".to_string(), + Shape::error_with_partial( + "Test error".to_string(), + Shape::one([Shape::string([]), Shape::null([])], []), + [], + ), + ); + map + }, + [], + ), + ); + + ShapeContext::new(SourceId::Other("JSONSelection".into())) + .with_spec(spec) + .with_named_shapes(named_shapes) + }; + + let error_selection = selection!("$(errorField?)", spec); + + let result_shape = error_selection.compute_output_shape( + &shape_context, + shape_context.named_shapes().get("$root").unwrap().clone(), + ); + + // The question mark should be applied recursively to the partial shape within the error + assert!(result_shape.pretty_print().contains("Error")); + assert!(result_shape.pretty_print().contains("None")); + } + + #[test] + fn question_operator_with_all_shapes() { + let spec = ConnectSpec::V0_3; + + let shape_context = { + let mut named_shapes = IndexMap::default(); + + named_shapes.insert( + "$root".to_string(), + Shape::record( + { + let mut map = Shape::empty_map(); + map.insert( + "allField".to_string(), + Shape::all( + [ + Shape::string([]), + Shape::one([Shape::string([]), Shape::null([])], []), + ], + [], + ), + ); + map + }, + [], + ), + ); + + ShapeContext::new(SourceId::Other("JSONSelection".into())) + .with_spec(spec) + .with_named_shapes(named_shapes) + }; + + let all_selection = selection!("$(allField?)", spec); + + assert_eq!( + all_selection + .compute_output_shape( + &shape_context, + shape_context.named_shapes().get("$root").unwrap().clone(), + ) + .pretty_print(), + "One", + ); + } + + #[test] + fn question_operator_preserves_non_null_shapes() { + let spec = ConnectSpec::V0_3; + + let shape_context = { + let mut named_shapes = IndexMap::default(); + + named_shapes.insert( + "$root".to_string(), + Shape::record( + { + let mut map = Shape::empty_map(); + map.insert("nonNullString".to_string(), Shape::string([])); + map + }, + [], + ), + ); + + ShapeContext::new(SourceId::Other("JSONSelection".into())) + .with_spec(spec) + .with_named_shapes(named_shapes) + }; + + let non_null_selection = selection!("$(nonNullString?)", spec); + + assert_eq!( + non_null_selection + .compute_output_shape( + &shape_context, + shape_context.named_shapes().get("$root").unwrap().clone(), + ) + .pretty_print(), + "String", + ); + } + + #[test] + fn question_operator_with_multiple_operators_in_chain() { + let spec = ConnectSpec::V0_3; + + // Test combining ? with other operators + let mixed_chain_selection = selection!("$(field? ?? 'default')", spec); + assert_eq!( + mixed_chain_selection.apply_to(&json!({"field": "value"})), + (Some(json!("value")), vec![]), + ); + assert_eq!( + mixed_chain_selection.apply_to(&json!({"field": null})), + (Some(json!("default")), vec![]), + ); + assert_eq!( + mixed_chain_selection.apply_to(&json!({})), + (Some(json!("default")), vec![]), + ); + } + + #[test] + fn question_operator_direct_null_input_shape() { + let spec = ConnectSpec::V0_3; + + let shape_context = { + let mut named_shapes = IndexMap::default(); + + named_shapes.insert("$root".to_string(), Shape::null([])); + + ShapeContext::new(SourceId::Other("JSONSelection".into())) + .with_spec(spec) + .with_named_shapes(named_shapes) + }; + + let null_selection = selection!("$root?", spec); + + assert_eq!( + null_selection + .compute_output_shape( + &shape_context, + shape_context.named_shapes().get("$root").unwrap().clone(), + ) + .pretty_print(), + "None", + ); + } + + #[test] + fn test_unknown_name() { + let spec = ConnectSpec::V0_3; + let sel = selection!("book.author? { name age? }", spec); + assert_eq!( + sel.shape().pretty_print(), + "One<{ age: $root.book.author?.*.age?, name: $root.book.author?.*.name }, None>", + ); + } + + #[test] + fn test_nullish_coalescing_shape() { + let spec = ConnectSpec::V0_3; + let sel = selection!("$(a ?? b ?? c)", spec); + assert_eq!( + sel.shape().pretty_print(), + "One<$root.a?!, $root.b?!, $root.c>", + ); + + let mut named_shapes = IndexMap::default(); + named_shapes.insert( + "$root".to_string(), + Shape::record( + { + let mut map = Shape::empty_map(); + map.insert( + "a".to_string(), + Shape::one([Shape::string([]), Shape::null([])], []), + ); + map.insert("b".to_string(), Shape::string([])); + map.insert("c".to_string(), Shape::int([])); + map + }, + [], + ), + ); + + let shape_context = ShapeContext::new(SourceId::Other("JSONSelection".into())) + .with_spec(spec) + .with_named_shapes(named_shapes); + + assert_eq!( + sel.compute_output_shape( + &shape_context, + shape_context.named_shapes().get("$root").unwrap().clone(), + ) + .pretty_print(), + "One", + ); + } + + #[test] + fn test_none_coalescing_shape() { + let spec = ConnectSpec::V0_3; + let sel = selection!("$(a ?! b ?! c)", spec); + assert_eq!( + sel.shape().pretty_print(), + "One<$root.a!, $root.b!, $root.c>", + ); + + let mut named_shapes = IndexMap::default(); + named_shapes.insert( + "$root".to_string(), + Shape::record( + { + let mut map = Shape::empty_map(); + map.insert( + "a".to_string(), + Shape::one([Shape::string([]), Shape::null([])], []), + ); + map.insert( + "b".to_string(), + Shape::one([Shape::string([]), Shape::none()], []), + ); + map.insert("c".to_string(), Shape::null([])); + map + }, + [], + ), + ); + + let shape_context = ShapeContext::new(SourceId::Other("JSONSelection".into())) + .with_spec(spec) + .with_named_shapes(named_shapes); + + assert_eq!( + sel.compute_output_shape( + &shape_context, + shape_context.named_shapes().get("$root").unwrap().clone(), + ) + .pretty_print(), + "One", + ); + } } diff --git a/apollo-federation/src/connectors/json_selection/methods/public/get.rs b/apollo-federation/src/connectors/json_selection/methods/public/get.rs index c4be67a702..cf01d92f58 100644 --- a/apollo-federation/src/connectors/json_selection/methods/public/get.rs +++ b/apollo-federation/src/connectors/json_selection/methods/public/get.rs @@ -1182,22 +1182,23 @@ mod shape_tests { fn get_shape_should_return_unknown_for_object_with_unknown_key() { let fields = IndexMap::default(); let input_shape = Shape::object(fields, Shape::none(), []); + let test_shape = get_test_shape( + vec![WithRange::new( + LitExpr::Path(PathSelection { + path: PathList::Key( + Key::field("a").into_with_range(), + PathList::Empty.into_with_range(), + ) + .into_with_range(), + }), + None, + )], + input_shape.clone(), + ); assert_eq!( - get_test_shape( - vec![WithRange::new( - LitExpr::Path(PathSelection { - path: PathList::Key( - Key::field("a").into_with_range(), - PathList::Empty.into_with_range(), - ) - .into_with_range(), - }), - None - )], - input_shape.clone() - ), - input_shape.any_field([]) + test_shape, + input_shape.any_field(test_shape.locations.iter().cloned()), ); } diff --git a/apollo-federation/src/connectors/json_selection/parser.rs b/apollo-federation/src/connectors/json_selection/parser.rs index 4b61f50fdd..e244072742 100644 --- a/apollo-federation/src/connectors/json_selection/parser.rs +++ b/apollo-federation/src/connectors/json_selection/parser.rs @@ -924,6 +924,10 @@ pub(crate) enum PathList { } impl PathList { + pub(crate) fn is_empty(&self) -> bool { + matches!(self, PathList::Empty) + } + pub(super) fn parse(input: Span) -> ParseResult> { match Self::parse_with_depth(input.clone(), 0) { Ok((_, parsed)) if matches!(*parsed, Self::Empty) => Err(nom_error_message( diff --git a/apollo-federation/src/connectors/validation/expression.rs b/apollo-federation/src/connectors/validation/expression.rs index 1ec745c1dc..c6a13e26cb 100644 --- a/apollo-federation/src/connectors/validation/expression.rs +++ b/apollo-federation/src/connectors/validation/expression.rs @@ -373,7 +373,7 @@ fn resolve_shape( } Ok(Shape::all(inners, [])) } - ShapeCase::Name(name, key) => { + ShapeCase::Name(name, subpath) => { let mut resolved = if name.value == "$root" { // For response mapping, $root (aka the response body) is allowed so we will exit out early here // However, $root is not allowed for requests so we will error below @@ -381,10 +381,41 @@ fn resolve_shape( return Ok(Shape::unknown([])); } - let mut key_str = key.iter().map(|key| key.to_string()).join("."); - if !key_str.is_empty() { - key_str = format!("`{key_str}` "); + // This path implicitly starts with the $root variable, and may + // have a prefix like $root.*.foo or even $root?.**.foo. Since + // we want the error message to focus on the foo identifier, we + // skip not only the $root base name but any non-key/index path + // elements, like ?, .*, and .**. + let mut key_str = String::new(); + let mut skipping = true; + for key in subpath.iter() { + if skipping + && matches!( + key.value, + NamedShapePathKey::AnyIndex + | NamedShapePathKey::AnyField + | NamedShapePathKey::Question + | NamedShapePathKey::NotNone + ) + { + continue; + } else { + key_str.push_str(key.to_string().as_str()); + // We only skip until we stop skipping, and then all key + // variants become fair game. + skipping = false; + } + } + if key_str.is_empty() { + // If we ended up converting a path like $ to $root and then + // losing $root due to the logic above, fall back to + // printing the whole path for clarity/debuggability. + key_str = shape.to_string(); + } else if key_str.starts_with('.') { + // Remove initial . from field keys + key_str.remove(0); } + key_str = format!("`{key_str}` "); let locals_suffix = { let local_vars = expression.expression.local_var_names(); @@ -405,7 +436,8 @@ fn resolve_shape( namespaces = context.var_lookup.keys().map(|ns| ns.as_str()).join(", "), ), locations: transform_locations( - key.first() + subpath + .first() .map(|key| &key.locations) .unwrap_or(&shape.locations), context, @@ -447,16 +479,23 @@ fn resolve_shape( }; resolved.locations.extend(shape.locations.iter().cloned()); let mut path = name.value.clone(); - for key in key { + for key in subpath { let child = resolved.child(key.clone()); - if child.is_none() { + if child.is_none() || child.is_never() { + let key_string = key.to_string(); + let key_without_dot = key_string.trim_start_matches('.'); + let message = match key.value { NamedShapePathKey::AnyIndex | NamedShapePathKey::Index(_) => { format!("`{path}` is not an array or string") } NamedShapePathKey::AnyField | NamedShapePathKey::Field(_) => { - format!("`{path}` doesn't have a field named `{key}`") + format!("`{path}` doesn't have a field named `{key_without_dot}`") + } + + NamedShapePathKey::Question | NamedShapePathKey::NotNone => { + format!("`{path}{key}` evaluated to nothing") } }; return Err(Message { @@ -466,7 +505,7 @@ fn resolve_shape( }); } resolved = child; - path = format!("{path}.{key}"); + path = format!("{path}{key}"); } resolve_shape(&resolved, context, expression) } diff --git a/apollo-federation/src/connectors/validation/snapshots/validation_tests@body_selection.graphql.snap b/apollo-federation/src/connectors/validation/snapshots/validation_tests@body_selection.graphql.snap index 74f56bc37e..78e8f13e63 100644 --- a/apollo-federation/src/connectors/validation/snapshots/validation_tests@body_selection.graphql.snap +++ b/apollo-federation/src/connectors/validation/snapshots/validation_tests@body_selection.graphql.snap @@ -6,7 +6,7 @@ input_file: apollo-federation/src/connectors/validation/test_data/body_selection [ Message { code: InvalidBody, - message: "In `@connect(http: {body:})` on `Query.dollar`: must start with one of $args, $config, $context, $request, $env", + message: "In `@connect(http: {body:})` on `Query.dollar`: `$root` must start with one of $args, $config, $context, $request, $env", locations: [ 12:20..12:21, ], diff --git a/apollo-federation/src/connectors/validation/snapshots/validation_tests@url_properties__path.graphql.snap b/apollo-federation/src/connectors/validation/snapshots/validation_tests@url_properties__path.graphql.snap index 854a1e74ef..5d6afff0c4 100644 --- a/apollo-federation/src/connectors/validation/snapshots/validation_tests@url_properties__path.graphql.snap +++ b/apollo-federation/src/connectors/validation/snapshots/validation_tests@url_properties__path.graphql.snap @@ -6,7 +6,7 @@ input_file: apollo-federation/src/connectors/validation/test_data/url_properties [ Message { code: InvalidUrlProperty, - message: "In `@source(name: \"v2\")`, the `path` argument is invalid: `*.bad` must start with one of $config, $context, $request, $env", + message: "In `@source(name: \"v2\")`, the `path` argument is invalid: `bad` must start with one of $config, $context, $request, $env", locations: [ 13:70..13:73, ], @@ -27,7 +27,7 @@ input_file: apollo-federation/src/connectors/validation/test_data/url_properties }, Message { code: InvalidUrlProperty, - message: "In `@connect` on `Query.resources`, the `path` argument is invalid: `*.bad` must start with one of $args, $config, $context, $request, $env", + message: "In `@connect` on `Query.resources`, the `path` argument is invalid: `bad` must start with one of $args, $config, $context, $request, $env", locations: [ 35:53..35:56, ], diff --git a/apollo-federation/src/connectors/validation/snapshots/validation_tests@url_properties__query_params.graphql.snap b/apollo-federation/src/connectors/validation/snapshots/validation_tests@url_properties__query_params.graphql.snap index 8cc9d00cb6..fbad8a7ca7 100644 --- a/apollo-federation/src/connectors/validation/snapshots/validation_tests@url_properties__query_params.graphql.snap +++ b/apollo-federation/src/connectors/validation/snapshots/validation_tests@url_properties__query_params.graphql.snap @@ -6,7 +6,7 @@ input_file: apollo-federation/src/connectors/validation/test_data/url_properties [ Message { code: InvalidUrlProperty, - message: "In `@source(name: \"v3\")`, the `queryParams` argument is invalid: `*.bad` must start with one of $config, $context, $request, $env", + message: "In `@source(name: \"v3\")`, the `queryParams` argument is invalid: `bad` must start with one of $config, $context, $request, $env", locations: [ 19:59..19:62, ], @@ -27,7 +27,7 @@ input_file: apollo-federation/src/connectors/validation/test_data/url_properties }, Message { code: InvalidUrlProperty, - message: "In `@connect` on `Query.resources`, the `queryParams` argument is invalid: `*.bad` must start with one of $args, $config, $context, $request, $env", + message: "In `@connect` on `Query.resources`, the `queryParams` argument is invalid: `bad` must start with one of $args, $config, $context, $request, $env", locations: [ 47:39..47:42, ], diff --git a/apollo-router/.sqlx/query-119ea89f7b98079bd3d2ec81596cd04a19fd394c234940976dd32bc507984fca.json b/apollo-router/.sqlx/query-119ea89f7b98079bd3d2ec81596cd04a19fd394c234940976dd32bc507984fca.json deleted file mode 100644 index 09edf782c6..0000000000 --- a/apollo-router/.sqlx/query-119ea89f7b98079bd3d2ec81596cd04a19fd394c234940976dd32bc507984fca.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO invalidation_key (cache_key_id, invalidation_key, subgraph_name)\n SELECT * FROM UNNEST(\n $1::BIGINT[],\n $2::VARCHAR(255)[],\n $3::VARCHAR(255)[]\n ) ON CONFLICT (cache_key_id, invalidation_key, subgraph_name) DO NOTHING\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8Array", - "VarcharArray", - "VarcharArray" - ] - }, - "nullable": [] - }, - "hash": "119ea89f7b98079bd3d2ec81596cd04a19fd394c234940976dd32bc507984fca" -} diff --git a/apollo-router/.sqlx/query-5070a632fa2c6fb2766a16d305ca71988b1ef61a8a4dd38e4ac71bcb968d489e.json b/apollo-router/.sqlx/query-5070a632fa2c6fb2766a16d305ca71988b1ef61a8a4dd38e4ac71bcb968d489e.json deleted file mode 100644 index 431653ab5f..0000000000 --- a/apollo-router/.sqlx/query-5070a632fa2c6fb2766a16d305ca71988b1ef61a8a4dd38e4ac71bcb968d489e.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT COUNT(id) AS count FROM cache WHERE expires_at <= NOW()", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - null - ] - }, - "hash": "5070a632fa2c6fb2766a16d305ca71988b1ef61a8a4dd38e4ac71bcb968d489e" -} diff --git a/apollo-router/.sqlx/query-602e9f11a9cf461010a1523d8e8e261b56416439675a38b316017478797a56ab.json b/apollo-router/.sqlx/query-602e9f11a9cf461010a1523d8e8e261b56416439675a38b316017478797a56ab.json deleted file mode 100644 index 2fbd8f6806..0000000000 --- a/apollo-router/.sqlx/query-602e9f11a9cf461010a1523d8e8e261b56416439675a38b316017478797a56ab.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO cache\n ( cache_key, data, expires_at, control ) SELECT * FROM UNNEST(\n $1::VARCHAR(1024)[],\n $2::TEXT[],\n $3::TIMESTAMP WITH TIME ZONE[],\n $4::TEXT[]\n ) ON CONFLICT (cache_key) DO UPDATE SET data = excluded.data, control = excluded.control, expires_at = excluded.expires_at\n RETURNING id\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "VarcharArray", - "TextArray", - "TimestamptzArray", - "TextArray" - ] - }, - "nullable": [ - false - ] - }, - "hash": "602e9f11a9cf461010a1523d8e8e261b56416439675a38b316017478797a56ab" -} diff --git a/apollo-router/.sqlx/query-6245ac300b03e217aa5ed15489acf4fdc174de48a1d7fd64c088a15d02c2c690.json b/apollo-router/.sqlx/query-6245ac300b03e217aa5ed15489acf4fdc174de48a1d7fd64c088a15d02c2c690.json deleted file mode 100644 index 5147f2336a..0000000000 --- a/apollo-router/.sqlx/query-6245ac300b03e217aa5ed15489acf4fdc174de48a1d7fd64c088a15d02c2c690.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT COUNT(id) AS count FROM cache WHERE starts_with(cache_key, $1) AND expires_at <= NOW()", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "6245ac300b03e217aa5ed15489acf4fdc174de48a1d7fd64c088a15d02c2c690" -} diff --git a/apollo-router/.sqlx/query-6cb3643cda13c340a17084366d2b27163e1b83fe86adf6e6a906a11547a1d05f.json b/apollo-router/.sqlx/query-6cb3643cda13c340a17084366d2b27163e1b83fe86adf6e6a906a11547a1d05f.json deleted file mode 100644 index 8d0119ef9c..0000000000 --- a/apollo-router/.sqlx/query-6cb3643cda13c340a17084366d2b27163e1b83fe86adf6e6a906a11547a1d05f.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT cron.alter_job((SELECT jobid FROM cron.job WHERE jobname = 'delete-old-cache-entries'), $1)", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "alter_job", - "type_info": "Void" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - null - ] - }, - "hash": "6cb3643cda13c340a17084366d2b27163e1b83fe86adf6e6a906a11547a1d05f" -} diff --git a/apollo-router/.sqlx/query-6e77a91b9716475f81f81e188b13813f86f0b8072bbd241ed1090b5c37b932c8.json b/apollo-router/.sqlx/query-6e77a91b9716475f81f81e188b13813f86f0b8072bbd241ed1090b5c37b932c8.json deleted file mode 100644 index 7b03c87a7b..0000000000 --- a/apollo-router/.sqlx/query-6e77a91b9716475f81f81e188b13813f86f0b8072bbd241ed1090b5c37b932c8.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO cache ( cache_key, data, control, expires_at )\n VALUES ( $1, $2, $3, $4 )\n ON CONFLICT (cache_key) DO UPDATE SET data = $2, control = $3, expires_at = $4\n RETURNING id\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Varchar", - "Text", - "Text", - "Timestamptz" - ] - }, - "nullable": [ - false - ] - }, - "hash": "6e77a91b9716475f81f81e188b13813f86f0b8072bbd241ed1090b5c37b932c8" -} diff --git a/apollo-router/.sqlx/query-92256ab4f44ce1b6a034cbb1bfac848ed7767b3aa0e1b3eaeb612f4840f4765e.json b/apollo-router/.sqlx/query-92256ab4f44ce1b6a034cbb1bfac848ed7767b3aa0e1b3eaeb612f4840f4765e.json deleted file mode 100644 index ca1a34f7cb..0000000000 --- a/apollo-router/.sqlx/query-92256ab4f44ce1b6a034cbb1bfac848ed7767b3aa0e1b3eaeb612f4840f4765e.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT * FROM cache WHERE cache.cache_key = $1 AND expires_at >= NOW()", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "cache_key", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "data", - "type_info": "Text" - }, - { - "ordinal": 3, - "name": "control", - "type_info": "Text" - }, - { - "ordinal": 4, - "name": "expires_at", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - false, - false, - false, - false - ] - }, - "hash": "92256ab4f44ce1b6a034cbb1bfac848ed7767b3aa0e1b3eaeb612f4840f4765e" -} diff --git a/apollo-router/.sqlx/query-9cab35054c927272ec5fb820c6122ec531e1dc6f695c56268bb3546b1f14beab.json b/apollo-router/.sqlx/query-9cab35054c927272ec5fb820c6122ec531e1dc6f695c56268bb3546b1f14beab.json deleted file mode 100644 index 3737f26782..0000000000 --- a/apollo-router/.sqlx/query-9cab35054c927272ec5fb820c6122ec531e1dc6f695c56268bb3546b1f14beab.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT data FROM cache WHERE cache_key = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "data", - "type_info": "Text" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false - ] - }, - "hash": "9cab35054c927272ec5fb820c6122ec531e1dc6f695c56268bb3546b1f14beab" -} diff --git a/apollo-router/.sqlx/query-a14112a24ef08c184e9062837d553deffa2e7a2c8570ef7089701ebab9f02665.json b/apollo-router/.sqlx/query-a14112a24ef08c184e9062837d553deffa2e7a2c8570ef7089701ebab9f02665.json deleted file mode 100644 index 0483e1b425..0000000000 --- a/apollo-router/.sqlx/query-a14112a24ef08c184e9062837d553deffa2e7a2c8570ef7089701ebab9f02665.json +++ /dev/null @@ -1,46 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT * FROM cache WHERE cache.cache_key = ANY($1::VARCHAR(1024)[]) AND expires_at >= NOW()", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "cache_key", - "type_info": "Varchar" - }, - { - "ordinal": 2, - "name": "data", - "type_info": "Text" - }, - { - "ordinal": 3, - "name": "control", - "type_info": "Text" - }, - { - "ordinal": 4, - "name": "expires_at", - "type_info": "Timestamptz" - } - ], - "parameters": { - "Left": [ - "VarcharArray" - ] - }, - "nullable": [ - false, - false, - false, - false, - false - ] - }, - "hash": "a14112a24ef08c184e9062837d553deffa2e7a2c8570ef7089701ebab9f02665" -} diff --git a/apollo-router/.sqlx/query-a8ff75a83927cd264969cecc40449b47ba93ab750c54cecec6ccf636f1145178.json b/apollo-router/.sqlx/query-a8ff75a83927cd264969cecc40449b47ba93ab750c54cecec6ccf636f1145178.json deleted file mode 100644 index fdd920a80a..0000000000 --- a/apollo-router/.sqlx/query-a8ff75a83927cd264969cecc40449b47ba93ab750c54cecec6ccf636f1145178.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT into invalidation_key (cache_key_id, invalidation_key, subgraph_name) VALUES ($1, $2, $3) ON CONFLICT (cache_key_id, invalidation_key, subgraph_name) DO NOTHING", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Varchar", - "Varchar" - ] - }, - "nullable": [] - }, - "hash": "a8ff75a83927cd264969cecc40449b47ba93ab750c54cecec6ccf636f1145178" -} diff --git a/apollo-router/.sqlx/query-b1b1778f3d3f1ca73b5037d6d6e9cd980c07f640be2fea4bc9d76dfc451d437d.json b/apollo-router/.sqlx/query-b1b1778f3d3f1ca73b5037d6d6e9cd980c07f640be2fea4bc9d76dfc451d437d.json deleted file mode 100644 index 24194a9aec..0000000000 --- a/apollo-router/.sqlx/query-b1b1778f3d3f1ca73b5037d6d6e9cd980c07f640be2fea4bc9d76dfc451d437d.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH deleted AS\n (DELETE\n FROM cache\n USING invalidation_key\n WHERE invalidation_key.cache_key_id = cache.id AND invalidation_key.subgraph_name = ANY($1::text[]) RETURNING cache.cache_key, cache.expires_at\n )\n SELECT COUNT(*) AS count FROM deleted WHERE deleted.expires_at >= NOW()", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "TextArray" - ] - }, - "nullable": [ - null - ] - }, - "hash": "b1b1778f3d3f1ca73b5037d6d6e9cd980c07f640be2fea4bc9d76dfc451d437d" -} diff --git a/apollo-router/.sqlx/query-b7f6dac1056d65e173a6f4f3d81f57f456f3a15c9dfd1388e0b32a0978a08ae0.json b/apollo-router/.sqlx/query-b7f6dac1056d65e173a6f4f3d81f57f456f3a15c9dfd1388e0b32a0978a08ae0.json deleted file mode 100644 index ab6ce0062d..0000000000 --- a/apollo-router/.sqlx/query-b7f6dac1056d65e173a6f4f3d81f57f456f3a15c9dfd1388e0b32a0978a08ae0.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "DELETE FROM cache WHERE starts_with(cache_key, $1)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [] - }, - "hash": "b7f6dac1056d65e173a6f4f3d81f57f456f3a15c9dfd1388e0b32a0978a08ae0" -} diff --git a/apollo-router/.sqlx/query-c2b5393c5c63e03e12e9d480e82a0c610380ed3411e0e5d3cae03e94347410a3.json b/apollo-router/.sqlx/query-c2b5393c5c63e03e12e9d480e82a0c610380ed3411e0e5d3cae03e94347410a3.json deleted file mode 100644 index 939f23161e..0000000000 --- a/apollo-router/.sqlx/query-c2b5393c5c63e03e12e9d480e82a0c610380ed3411e0e5d3cae03e94347410a3.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH deleted AS\n (DELETE\n FROM cache\n USING invalidation_key\n WHERE invalidation_key.invalidation_key = ANY($1::text[])\n AND invalidation_key.cache_key_id = cache.id AND invalidation_key.subgraph_name = ANY($2::text[]) RETURNING cache.cache_key, cache.expires_at, invalidation_key.subgraph_name\n )\n SELECT subgraph_name, COUNT(deleted.cache_key) AS count FROM deleted WHERE deleted.expires_at >= NOW() GROUP BY deleted.subgraph_name", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "subgraph_name", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "count", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "TextArray", - "TextArray" - ] - }, - "nullable": [ - false, - null - ] - }, - "hash": "c2b5393c5c63e03e12e9d480e82a0c610380ed3411e0e5d3cae03e94347410a3" -} diff --git a/apollo-router/.sqlx/query-cb60ba408aa989631c6257955d9cf21671f600ee645ed77e5a300b6bbb9590c3.json b/apollo-router/.sqlx/query-cb60ba408aa989631c6257955d9cf21671f600ee645ed77e5a300b6bbb9590c3.json deleted file mode 100644 index abca13915f..0000000000 --- a/apollo-router/.sqlx/query-cb60ba408aa989631c6257955d9cf21671f600ee645ed77e5a300b6bbb9590c3.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT schedule FROM cron.job WHERE jobname = 'delete-old-cache-entries'", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "schedule", - "type_info": "Text" - } - ], - "parameters": { - "Left": [] - }, - "nullable": [ - false - ] - }, - "hash": "cb60ba408aa989631c6257955d9cf21671f600ee645ed77e5a300b6bbb9590c3" -} diff --git a/apollo-router/Cargo.toml b/apollo-router/Cargo.toml index 2e06a0533c..e2866cd0ba 100644 --- a/apollo-router/Cargo.toml +++ b/apollo-router/Cargo.toml @@ -224,12 +224,6 @@ serde_yaml = "0.8.26" static_assertions = "1.1.0" strum = "0.27.0" strum_macros = "0.27.0" -sqlx = { version = "0.8", features = [ - "postgres", - "runtime-tokio", - "tls-rustls-ring-native-roots", - "chrono", -] } sys-info = "0.9.1" sysinfo = { version = "0.37.0", features = [ "system", diff --git a/apollo-router/migrations/20250516144204_creation.down.sql b/apollo-router/migrations/20250516144204_creation.down.sql deleted file mode 100644 index baf9f6cff9..0000000000 --- a/apollo-router/migrations/20250516144204_creation.down.sql +++ /dev/null @@ -1,10 +0,0 @@ --- Add down migration script here --- -SELECT - cron.unschedule ('delete-old-cache-entries'); - -DROP EXTENSION IF EXISTS pg_cron; - -DROP TABLE "invalidation_key"; - -DROP TABLE "cache"; diff --git a/apollo-router/migrations/20250516144204_creation.up.sql b/apollo-router/migrations/20250516144204_creation.up.sql deleted file mode 100644 index 328eb297fb..0000000000 --- a/apollo-router/migrations/20250516144204_creation.up.sql +++ /dev/null @@ -1,66 +0,0 @@ --- Add up migration script here --- Add migration script here -CREATE EXTENSION IF NOT EXISTS pg_cron; -CREATE OR REPLACE FUNCTION create_index(table_name text, index_name text, column_name text) RETURNS void AS $$ -declare - l_count integer; -begin - select count(*) - into l_count - from pg_indexes - where schemaname = 'public' - and tablename = lower(table_name) - and indexname = lower(index_name); - - if l_count = 0 then - execute 'create index ' || index_name || ' on "' || table_name || '"(' || column_name || ')'; - end if; -end; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION create_unique_index(table_name text, index_name text, column_names text) RETURNS void AS $$ -declare - l_count integer; -begin - select count(*) - into l_count - from pg_indexes - where schemaname = 'public' - and tablename = lower(table_name) - and indexname = lower(index_name); - - if l_count = 0 then - execute 'create unique index ' || index_name || ' on "' || table_name || '"(' || array_to_string(string_to_array(column_names, ',') , ',') || ')'; - end if; -end; -$$ LANGUAGE plpgsql; - -CREATE OR REPLACE FUNCTION create_foreign_key(fk_name text, table_name_child text, table_name_parent text, column_name_child text, column_name_parent text) RETURNS void AS $$ -declare - l_count integer; -begin - select count(*) - into l_count - from information_schema.table_constraints as tc - where constraint_type = 'FOREIGN KEY' - and tc.table_name = lower(table_name_child) - and tc.constraint_name = lower(fk_name); - - if l_count = 0 then - execute 'alter table "' || table_name_child || '" ADD CONSTRAINT ' || fk_name || ' FOREIGN KEY(' || column_name_child || ') REFERENCES "' || table_name_parent || '"(' || column_name_parent || ')'; - end if; -end; -$$ LANGUAGE plpgsql; - - -CREATE UNLOGGED TABLE IF NOT EXISTS "invalidation_key" (cache_key_id BIGSERIAL NOT NULL, invalidation_key VARCHAR(255) NOT NULL, subgraph_name VARCHAR(255) NOT NULL, PRIMARY KEY(cache_key_id, invalidation_key, subgraph_name)); -CREATE UNLOGGED TABLE IF NOT EXISTS "cache" (id BIGSERIAL PRIMARY KEY, cache_key VARCHAR(1024) NOT NULL, data TEXT NOT NULL, control TEXT NOT NULL, expires_at TIMESTAMP WITH TIME ZONE NOT NULL); - -ALTER TABLE invalidation_key ADD CONSTRAINT FK_INVALIDATION_KEY_CACHE FOREIGN KEY (cache_key_id) references cache (id) ON delete cascade; -SELECT create_unique_index('cache', 'cache_key_idx', 'cache_key'); - --- Remove expired data every hour -SELECT cron.schedule('delete-old-cache-entries', '0 * * * *', $$ - DELETE FROM cache - WHERE expires_at < NOW() -$$); diff --git a/apollo-router/src/cache/redis.rs b/apollo-router/src/cache/redis.rs index af71044e51..6985151507 100644 --- a/apollo-router/src/cache/redis.rs +++ b/apollo-router/src/cache/redis.rs @@ -17,6 +17,7 @@ use fred::prelude::Error as RedisError; use fred::prelude::ErrorKind as RedisErrorKind; use fred::prelude::HeartbeatInterface; use fred::prelude::KeysInterface; +use fred::prelude::Options; use fred::prelude::Pool as RedisPool; use fred::prelude::TcpConfig; use fred::types::Builder; @@ -281,12 +282,37 @@ impl RedisCacheStorage { is_cluster, caller, config.metrics_interval, + config.required_to_start, ) .await } #[cfg(test)] pub(crate) async fn from_mocks(mocks: Arc) -> Result { + let config = RedisCache { + urls: vec![], + username: None, + password: None, + timeout: Duration::from_millis(2), + ttl: None, + namespace: None, + tls: None, + required_to_start: false, + reset_ttl: false, + pool_size: 1, + metrics_interval: Duration::from_millis(100), + }; + + Self::from_mocks_and_config(mocks, config, "test", false).await + } + + #[cfg(test)] + pub(crate) async fn from_mocks_and_config( + mocks: Arc, + config: RedisCache, + caller: &'static str, + is_cluster: bool, + ) -> Result { let client_config = RedisConfig { mocks: Some(mocks), ..Default::default() @@ -294,14 +320,15 @@ impl RedisCacheStorage { Self::create_client( client_config, - Duration::from_millis(2), - 1, - None, - None, - false, - false, - "test", - Duration::from_millis(100), + config.timeout, + config.pool_size as usize, + config.namespace, + config.ttl, + config.reset_ttl, + is_cluster, + caller, + config.metrics_interval, + true, ) .await } @@ -317,6 +344,7 @@ impl RedisCacheStorage { is_cluster: bool, caller: &'static str, metrics_interval: Duration, + required_to_start: bool, ) -> Result { let pooled_client = Builder::from_config(client_config) .with_connection_config(|config| { @@ -407,7 +435,10 @@ impl RedisCacheStorage { // NB: error is not recorded here as it will be observed by the task following `client.error_rx()` let client_handles = pooled_client.connect_pool(); - pooled_client.wait_for_connect().await?; + if required_to_start { + pooled_client.wait_for_connect().await?; + tracing::trace!("redis connections established"); + } tokio::spawn(async move { // the handles will resolve when the clients finish terminating. per the `fred` docs: @@ -428,7 +459,6 @@ impl RedisCacheStorage { let metrics_collector = RedisMetricsCollector::new(pooled_client_arc.clone(), caller, metrics_interval); - tracing::trace!("redis connection established"); Ok(Self { inner: Arc::new(DropSafeRedisPool { pool: pooled_client_arc, @@ -544,11 +574,15 @@ impl RedisCacheStorage { self.ttl = ttl; } - fn pipeline(&self) -> Pipeline { + pub(crate) fn client(&self) -> Client { + self.inner.next().clone() + } + + pub(crate) fn pipeline(&self) -> Pipeline { self.inner.next().pipeline() } - fn make_key(&self, key: RedisKey) -> String { + pub(crate) fn make_key(&self, key: RedisKey) -> String { match &self.namespace { Some(namespace) => format!("{namespace}:{key}"), None => key.to_string(), @@ -558,11 +592,19 @@ impl RedisCacheStorage { pub(crate) async fn get( &self, key: RedisKey, + ) -> Result, RedisError> { + self.get_with_options(key, Options::default()).await + } + + pub(crate) async fn get_with_options( + &self, + key: RedisKey, + options: Options, ) -> Result, RedisError> { let key = self.make_key(key); match self.ttl { Some(ttl) if self.reset_ttl => { - let pipeline = self.pipeline(); + let pipeline = self.pipeline().with_options(&options); let _: () = pipeline .get(&key) .await @@ -576,17 +618,25 @@ impl RedisCacheStorage { pipeline.all().await.inspect_err(|e| self.record_error(e))?; Ok(value) } - _ => self - .inner - .get(key) - .await - .inspect_err(|e| self.record_error(e)), + _ => { + let client = self.inner.next().with_options(&options); + client.get(key).await.inspect_err(|e| self.record_error(e)) + } } } pub(crate) async fn get_multiple( + &self, + keys: Vec>, + ) -> Vec>> { + self.get_multiple_with_options(keys, Options::default()) + .await + } + + pub(crate) async fn get_multiple_with_options( &self, mut keys: Vec>, + options: Options, ) -> Vec>> { // NB: MGET is different from GET in that it returns `Option`s rather than `Result`s. // > For every key that does not hold a string value or does not exist, the special value @@ -597,8 +647,8 @@ impl RedisCacheStorage { if keys.len() == 1 { let key = self.make_key(keys.remove(0)); - let res = self - .inner + let client = self.inner.next().with_options(&options); + let res = client .get(key) .await .inspect_err(|e| self.record_error(e)) @@ -621,7 +671,7 @@ impl RedisCacheStorage { // then we query all the key groups at the same time let mut tasks = Vec::new(); for (_shard, (indexes, keys)) in h { - let client = self.inner.next(); + let client = self.inner.next().with_options(&options); tasks.push(async move { let result: Result>>, _> = client.mget(keys).await; (indexes, result) @@ -651,7 +701,8 @@ impl RedisCacheStorage { .into_iter() .map(|k| self.make_key(k)) .collect::>(); - self.inner + let client = self.inner.next().with_options(&options); + client .mget(keys) .await .inspect_err(|e| self.record_error(e)) @@ -714,10 +765,24 @@ impl RedisCacheStorage { /// Delete keys *without* adding the `namespace` prefix because `keys` is from /// `scan_with_namespaced_results` and already includes it. - pub(crate) async fn delete_from_scan_result( + pub(crate) async fn delete_from_scan_result(&self, keys: I) -> Result + where + I: Iterator, + { + self.delete_from_scan_result_with_options(keys, Options::default()) + .await + } + + /// Delete keys *without* adding the `namespace` prefix because `keys` is from + /// `scan_with_namespaced_results` and already includes it. + pub(crate) async fn delete_from_scan_result_with_options( &self, - keys: Vec, - ) -> Result { + keys: I, + options: Options, + ) -> Result + where + I: Iterator, + { let mut h: HashMap> = HashMap::new(); for key in keys.into_iter() { let hash = ClusterRouting::hash_key(key.as_bytes()); @@ -726,8 +791,11 @@ impl RedisCacheStorage { } // then we query all the key groups at the same time - let results: Vec> = - join_all(h.into_values().map(|keys| self.inner.del(keys))).await; + let results: Vec> = join_all(h.into_values().map(|keys| async { + let client = self.inner.next().with_options(&options); + client.del(keys).await + })) + .await; let mut total = 0; for result in results { @@ -753,6 +821,39 @@ impl RedisCacheStorage { } } +#[cfg(all( + test, + any(not(feature = "ci"), all(target_arch = "x86_64", target_os = "linux")) +))] +impl RedisCacheStorage { + pub(crate) async fn truncate_namespace(&self) -> Result<(), RedisError> { + use fred::prelude::Key; + use futures::StreamExt; + + if self.namespace.is_none() { + return Ok(()); + } + + // find all members of this namespace via `SCAN` + let pattern = self.make_key(RedisKey("*")); + let client = self.client(); + let mut stream: Pin>>> = if self.is_cluster { + Box::pin(client.scan_cluster_buffered(pattern, None, None)) + } else { + Box::pin(client.scan_buffered(pattern, None, None)) + }; + + let mut keys = Vec::new(); + while let Some(key) = stream.next().await { + keys.push(key?); + } + + // remove all members of this namespace + self.delete_from_scan_result(keys.into_iter()).await?; + Ok(()) + } +} + #[cfg(test)] mod test { use std::collections::HashMap; diff --git a/apollo-router/src/configuration/metrics.rs b/apollo-router/src/configuration/metrics.rs index 9dd10eb405..8979af04d7 100644 --- a/apollo-router/src/configuration/metrics.rs +++ b/apollo-router/src/configuration/metrics.rs @@ -330,10 +330,8 @@ impl InstrumentData { "$[?(@.debug)]", opt.subgraph.enabled, "$[?(@.subgraph.all.enabled)]", - opt.subgraph.postgres.required_to_start, - "$[?(@.subgraph.all.postgres.required_to_start || @.subgraph.subgraphs..postgres.required_to_start)]", - opt.subgraph.postgres.cleanup_interval, - "$[?(@.subgraph.all.postgres.cleanup_interval || @.subgraph.subgraphs..postgres.cleanup_interval)]", + opt.subgraph.redis.required_to_start, + "$[?(@.subgraph.all.redis.required_to_start || @.subgraph.subgraphs..redis.required_to_start)]", opt.subgraph.enabled, "$[?(@.subgraph.subgraphs..enabled)]", opt.subgraph.ttl, diff --git a/apollo-router/src/configuration/mod.rs b/apollo-router/src/configuration/mod.rs index f6a076298e..ef959c6073 100644 --- a/apollo-router/src/configuration/mod.rs +++ b/apollo-router/src/configuration/mod.rs @@ -1101,11 +1101,11 @@ fn default_timeout() -> Duration { Duration::from_millis(500) } -fn default_required_to_start() -> bool { +pub(crate) fn default_required_to_start() -> bool { false } -fn default_pool_size() -> u32 { +pub(crate) fn default_pool_size() -> u32 { 1 } diff --git a/apollo-router/src/configuration/snapshots/apollo_router__configuration__metrics__test__metrics@response_cache.router.yaml.snap b/apollo-router/src/configuration/snapshots/apollo_router__configuration__metrics__test__metrics@response_cache.router.yaml.snap index 4a4deb211c..96c587d998 100644 --- a/apollo-router/src/configuration/snapshots/apollo_router__configuration__metrics__test__metrics@response_cache.router.yaml.snap +++ b/apollo-router/src/configuration/snapshots/apollo_router__configuration__metrics__test__metrics@response_cache.router.yaml.snap @@ -11,6 +11,5 @@ expression: "& metrics.non_zero()" opt.enabled: true opt.subgraph.enabled: true opt.subgraph.invalidation.enabled: true - opt.subgraph.postgres.cleanup_interval: true - opt.subgraph.postgres.required_to_start: true + opt.subgraph.redis.required_to_start: true opt.subgraph.ttl: true diff --git a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap index 72c7f9a145..3998910448 100644 --- a/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap +++ b/apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap @@ -1917,49 +1917,72 @@ expression: "&schema" }, "Config7": { "additionalProperties": false, - "description": "Postgres cache configuration", + "description": "Redis cache configuration", "properties": { - "acquire_timeout": { + "fetch_timeout": { "default": { - "nanos": 50000000, + "nanos": 0, "secs": 0 }, - "description": "PostgreSQL the maximum amount of time to spend waiting for a connection (default: 50ms)", - "type": "string" + "description": "Timeout for Redis fetch commands (default: 150ms)", + "type": [ + "string", + "null" + ] }, - "batch_size": { - "default": 100, - "description": "The size of batch when inserting cache entries in PG (default: 100)", - "format": "uint", - "minimum": 0, - "type": "integer" + "insert_timeout": { + "default": { + "nanos": 0, + "secs": 0 + }, + "description": "Timeout for Redis insert commands (default: 500ms)\n\nInserts are processed asynchronously, so this will not affect response duration.", + "type": [ + "string", + "null" + ] }, - "cleanup_interval": { + "invalidate_timeout": { "default": { "nanos": 0, - "secs": 3600 + "secs": 0 }, - "description": "Specifies the interval between cache cleanup operations (e.g., \"2 hours\", \"30min\"). Default: 1 hour", - "type": "string" + "description": "Timeout for Redis invalidation commands (default: 1s)", + "type": [ + "string", + "null" + ] }, - "idle_timeout": { + "maintenance_timeout": { "default": { "nanos": 0, - "secs": 60 + "secs": 0 }, - "description": "PostgreSQL maximum idle duration for individual connection (default: 1min)", - "type": "string" + "description": "Timeout for Redis maintenance commands (default: 500ms)\n\nMaintenance tasks are processed asynchronously, so this will not affect response duration.", + "type": [ + "string", + "null" + ] + }, + "metrics_interval": { + "default": { + "nanos": 0, + "secs": 0 + }, + "description": "Interval for collecting Redis metrics (default: 1s)", + "type": [ + "string", + "null" + ] }, "namespace": { - "default": null, - "description": "Useful when running tests in parallel to avoid conflicts", + "description": "namespace used to prefix Redis keys", "type": [ "string", "null" ] }, "password": { - "description": "PostgreSQL password if not provided in the URLs. This field takes precedence over the password in the URL", + "description": "Redis password if not provided in the URLs. This field takes precedence over the password in the URL", "type": [ "string", "null" @@ -1967,35 +1990,46 @@ expression: "&schema" }, "pool_size": { "default": 5, - "description": "The size of the PostgreSQL connection pool", + "description": "The size of the Redis connection pool (default: 5)", "format": "uint32", "minimum": 0, "type": "integer" }, "required_to_start": { "default": false, - "description": "Prevents the router from starting if it cannot connect to PostgreSQL", + "description": "Prevents the router from starting if it cannot connect to Redis", "type": "boolean" }, "tls": { - "allOf": [ + "anyOf": [ + { + "$ref": "#/definitions/TlsClient" + }, { - "$ref": "#/definitions/TlsConfig" + "type": "null" } ], - "default": { - "certificate_authorities": null, - "client_authentication": null - }, - "description": "Postgres TLS client configuration" + "default": null, + "description": "TLS client configuration" }, - "url": { - "description": "List of URL to Postgres", - "format": "uri", - "type": "string" + "ttl": { + "default": null, + "description": "TTL for entries", + "type": [ + "string", + "null" + ] + }, + "urls": { + "description": "List of URLs to the Redis cluster", + "items": { + "format": "uri", + "type": "string" + }, + "type": "array" }, "username": { - "description": "PostgreSQL username if not provided in the URLs. This field takes precedence over the username in the URL", + "description": "Redis username if not provided in the URLs. This field takes precedence over the username in the URL", "type": [ "string", "null" @@ -2003,7 +2037,7 @@ expression: "&schema" } }, "required": [ - "url" + "urls" ], "type": "object" }, @@ -8972,7 +9006,15 @@ expression: "&schema" "default": null, "description": "Invalidation configuration" }, - "postgres": { + "private_id": { + "default": null, + "description": "Context key used to separate cache sections per user", + "type": [ + "string", + "null" + ] + }, + "redis": { "anyOf": [ { "$ref": "#/definitions/Config7" @@ -8982,15 +9024,7 @@ expression: "&schema" } ], "default": null, - "description": "PostgreSQL configuration" - }, - "private_id": { - "default": null, - "description": "Context key used to separate cache sections per user", - "type": [ - "string", - "null" - ] + "description": "Redis configuration" }, "ttl": { "anyOf": [ @@ -10139,8 +10173,8 @@ expression: "&schema" "default": { "enabled": true, "invalidation": null, - "postgres": null, "private_id": null, + "redis": null, "ttl": null }, "description": "options applying to all subgraphs" @@ -11036,50 +11070,6 @@ expression: "&schema" ], "type": "object" }, - "TlsClientAuth2": { - "additionalProperties": false, - "description": "TLS client authentication", - "properties": { - "certificate": { - "description": "Sets the SSL client certificate as a PEM", - "type": "string" - }, - "key": { - "description": "key in PEM format", - "type": "string" - } - }, - "required": [ - "certificate", - "key" - ], - "type": "object" - }, - "TlsConfig": { - "additionalProperties": false, - "description": "Postgres TLS client configuration", - "properties": { - "certificate_authorities": { - "description": "list of certificate authorities in PEM format", - "type": "string" - }, - "client_authentication": { - "anyOf": [ - { - "$ref": "#/definitions/TlsClientAuth2" - }, - { - "type": "null" - } - ], - "description": "client certificate authentication" - } - }, - "required": [ - "certificate_authorities" - ], - "type": "object" - }, "TlsSupergraph": { "additionalProperties": false, "description": "Configuration options pertaining to the supergraph server component.", diff --git a/apollo-router/src/configuration/testdata/metrics/response_cache.router.yaml b/apollo-router/src/configuration/testdata/metrics/response_cache.router.yaml index 7902bd5fb3..f102c98521 100644 --- a/apollo-router/src/configuration/testdata/metrics/response_cache.router.yaml +++ b/apollo-router/src/configuration/testdata/metrics/response_cache.router.yaml @@ -6,11 +6,9 @@ preview_response_cache: path: /invalidation subgraph: all: - postgres: - url: "postgres://test" - acquire_timeout: 5ms + redis: + urls: [ "redis://test" ] required_to_start: true - cleanup_interval: 10mins enabled: true invalidation: enabled: true diff --git a/apollo-router/src/plugins/cache/invalidation.rs b/apollo-router/src/plugins/cache/invalidation.rs index 1c7c014234..b0beef4e2b 100644 --- a/apollo-router/src/plugins/cache/invalidation.rs +++ b/apollo-router/src/plugins/cache/invalidation.rs @@ -128,7 +128,7 @@ impl Invalidation { && !keys.is_empty() { let deleted = redis_storage - .delete_from_scan_result(keys) + .delete_from_scan_result(keys.into_iter()) .await .unwrap_or(0) as u64; count += deleted; diff --git a/apollo-router/src/plugins/response_cache/invalidation.rs b/apollo-router/src/plugins/response_cache/invalidation.rs index a7e63bb9f7..9b434686fd 100644 --- a/apollo-router/src/plugins/response_cache/invalidation.rs +++ b/apollo-router/src/plugins/response_cache/invalidation.rs @@ -18,7 +18,7 @@ use crate::plugins::response_cache::plugin::INTERNAL_CACHE_TAG_PREFIX; use crate::plugins::response_cache::plugin::RESPONSE_CACHE_VERSION; use crate::plugins::response_cache::storage; use crate::plugins::response_cache::storage::CacheStorage; -use crate::plugins::response_cache::storage::postgres::Storage; +use crate::plugins::response_cache::storage::redis::Storage; #[derive(Clone)] pub(crate) struct Invalidation { @@ -95,7 +95,7 @@ impl Invalidation { let (count, subgraphs) = match request { InvalidationRequest::Subgraph { subgraph } => { let count = storage - .invalidate_by_subgraph(subgraph.clone(), request.kind()) + .invalidate_by_subgraph(subgraph, request.kind()) .await .inspect_err(|err| { u64_counter_with_unit!( diff --git a/apollo-router/src/plugins/response_cache/invalidation_endpoint.rs b/apollo-router/src/plugins/response_cache/invalidation_endpoint.rs index 1239605114..9d64e73989 100644 --- a/apollo-router/src/plugins/response_cache/invalidation_endpoint.rs +++ b/apollo-router/src/plugins/response_cache/invalidation_endpoint.rs @@ -260,18 +260,23 @@ fn validate_shared_key( mod tests { use std::collections::HashMap; + use tokio::sync::broadcast; use tower::ServiceExt; use super::*; use crate::plugins::response_cache::plugin::StorageInterface; - use crate::plugins::response_cache::storage::postgres::Config; - use crate::plugins::response_cache::storage::postgres::Storage; + use crate::plugins::response_cache::storage::redis::Config; + use crate::plugins::response_cache::storage::redis::Storage; #[tokio::test] async fn test_invalidation_service_bad_shared_key() { - let storage = Storage::new(&Config::test("test_invalidation_service_bad_shared_key")) - .await - .unwrap(); + let (_drop_tx, drop_rx) = broadcast::channel(2); + let storage = Storage::new( + &Config::test(false, "test_invalidation_service_bad_shared_key"), + drop_rx, + ) + .await + .unwrap(); let storage = Arc::new(StorageInterface::from(storage)); let invalidation = Invalidation::new(storage.clone()).await.unwrap(); @@ -279,7 +284,7 @@ mod tests { all: Subgraph { ttl: None, enabled: Some(true), - postgres: None, + redis: None, private_id: None, invalidation: Some(SubgraphInvalidationConfig { enabled: true, @@ -316,9 +321,11 @@ mod tests { #[tokio::test] async fn test_invalidation_service_bad_shared_key_subgraph() { - let storage = Storage::new(&Config::test( - "test_invalidation_service_bad_shared_key_subgraph", - )) + let (_drop_tx, drop_rx) = broadcast::channel(2); + let storage = Storage::new( + &Config::test(false, "test_invalidation_service_bad_shared_key_subgraph"), + drop_rx, + ) .await .unwrap(); let storage = Arc::new(StorageInterface::from(storage)); @@ -328,7 +335,7 @@ mod tests { all: Subgraph { ttl: None, enabled: Some(true), - postgres: None, + redis: None, private_id: None, invalidation: Some(SubgraphInvalidationConfig { enabled: true, @@ -340,7 +347,7 @@ mod tests { Subgraph { ttl: None, enabled: Some(true), - postgres: None, + redis: None, private_id: None, invalidation: Some(SubgraphInvalidationConfig { enabled: true, @@ -374,9 +381,11 @@ mod tests { #[tokio::test] async fn test_invalidation_service_bad_shared_key_subgraphs() { - let storage = Storage::new(&Config::test( - "test_invalidation_service_bad_shared_key_subgraphs", - )) + let (_drop_tx, drop_rx) = broadcast::channel(2); + let storage = Storage::new( + &Config::test(false, "test_invalidation_service_bad_shared_key_subgraphs"), + drop_rx, + ) .await .unwrap(); let storage = Arc::new(StorageInterface::from(storage)); @@ -386,7 +395,7 @@ mod tests { all: Subgraph { ttl: None, enabled: Some(true), - postgres: None, + redis: None, private_id: None, invalidation: Some(SubgraphInvalidationConfig { enabled: true, @@ -399,7 +408,7 @@ mod tests { Subgraph { ttl: None, enabled: Some(true), - postgres: None, + redis: None, private_id: None, invalidation: Some(SubgraphInvalidationConfig { enabled: true, @@ -412,7 +421,7 @@ mod tests { Subgraph { ttl: None, enabled: Some(true), - postgres: None, + redis: None, private_id: None, invalidation: Some(SubgraphInvalidationConfig { enabled: true, @@ -452,9 +461,11 @@ mod tests { #[tokio::test] async fn test_invalidation_service_good_shared_key_subgraphs() { - let storage = Storage::new(&Config::test( - "test_invalidation_service_good_shared_key_subgraphs", - )) + let (_drop_tx, drop_rx) = broadcast::channel(2); + let storage = Storage::new( + &Config::test(false, "test_invalidation_service_good_shared_key_subgraphs"), + drop_rx, + ) .await .unwrap(); let storage = Arc::new(StorageInterface::from(storage)); @@ -464,7 +475,7 @@ mod tests { all: Subgraph { ttl: None, enabled: Some(true), - postgres: None, + redis: None, private_id: None, invalidation: Some(SubgraphInvalidationConfig { enabled: true, @@ -477,7 +488,7 @@ mod tests { Subgraph { ttl: None, enabled: Some(true), - postgres: None, + redis: None, private_id: None, invalidation: Some(SubgraphInvalidationConfig { enabled: true, @@ -490,7 +501,7 @@ mod tests { Subgraph { ttl: None, enabled: Some(true), - postgres: None, + redis: None, private_id: None, invalidation: Some(SubgraphInvalidationConfig { enabled: true, diff --git a/apollo-router/src/plugins/response_cache/metrics.rs b/apollo-router/src/plugins/response_cache/metrics.rs index 27e8e979c0..ccf8ba7a4e 100644 --- a/apollo-router/src/plugins/response_cache/metrics.rs +++ b/apollo-router/src/plugins/response_cache/metrics.rs @@ -1,19 +1,10 @@ -use std::sync::Arc; -use std::sync::atomic::AtomicU64; -use std::sync::atomic::Ordering; use std::time::Duration; -use opentelemetry::KeyValue; -use opentelemetry::metrics::MeterProvider; -use tokio::sync::broadcast; -use tokio_stream::StreamExt; -use tokio_stream::wrappers::IntervalStream; +use tokio::sync::mpsc::error::TrySendError; -use crate::metrics::meter_provider; use crate::plugins::response_cache::ErrorCode; use crate::plugins::response_cache::invalidation::InvalidationKind; use crate::plugins::response_cache::storage; -use crate::plugins::response_cache::storage::postgres::Storage; pub(crate) const CACHE_INFO_SUBGRAPH_CONTEXT_KEY: &str = "apollo::router::response_cache::cache_info_subgraph"; @@ -32,60 +23,6 @@ impl From for String { } } -/// This task counts all rows in the given Postgres DB that is expired and will be removed when pg_cron will be triggered -/// parameter subgraph_name is optional and is None when the database is the global one, and Some(...) when it's a database configured for a specific subgraph -pub(super) async fn expired_data_task( - storage: Storage, - mut abort_signal: broadcast::Receiver<()>, - subgraph_name: Option, -) { - let mut interval = IntervalStream::new(tokio::time::interval(std::time::Duration::from_secs( - (storage.cleanup_interval.num_seconds().max(60) / 2) as u64, - ))); - let expired_data_count = Arc::new(AtomicU64::new(0)); - let expired_data_count_clone = expired_data_count.clone(); - let meter = meter_provider().meter("apollo/router"); - let _gauge = meter - .u64_observable_gauge("apollo.router.response_cache.data.expired") - .with_description("Count of expired data entries still in database") - .with_unit("{entry}") - .with_callback(move |gauge| { - let attributes = match subgraph_name.clone() { - Some(subgraph_name) => { - vec![KeyValue::new( - "subgraph.name", - opentelemetry::Value::String(subgraph_name.into()), - )] - } - None => Vec::new(), - }; - gauge.observe( - expired_data_count_clone.load(Ordering::Relaxed), - &attributes, - ); - }) - .init(); - - loop { - tokio::select! { - biased; - _ = abort_signal.recv() => { - break; - } - _ = interval.next() => { - let exp_data = match storage.expired_data_count().await { - Ok(exp_data) => exp_data, - Err(err) => { - ::tracing::error!(error = ?err, "cannot get expired data count"); - continue; - } - }; - expired_data_count.store(exp_data, Ordering::Relaxed); - } - } - } -} - pub(super) fn record_fetch_error(error: &storage::Error, subgraph_name: &str) { u64_counter_with_unit!( "apollo.router.operations.response_cache.fetch.error", @@ -132,6 +69,49 @@ pub(super) fn record_insert_duration(duration: Duration, subgraph_name: &str, ba ); } +pub(super) fn record_maintenance_success(entries: u64) { + u64_counter_with_unit!( + "apollo.router.operations.response_cache.maintenance.removed_cache_tag_entries", + "Counter for removed items", + "{entry}", + entries + ); +} + +pub(super) fn record_maintenance_error(error: &storage::Error) { + u64_counter_with_unit!( + "apollo.router.operations.response_cache.maintenance.error", + "Errors while removing expired entries from cache tag set", + "{error}", + 1, + "code" = error.code() + ); + tracing::debug!(error = %error, "unable to perform maintenance on cache tag set in response cache"); +} + +pub(super) fn record_maintenance_duration(duration: Duration) { + f64_histogram_with_unit!( + "apollo.router.operations.response_cache.maintenance", + "Time to remove expired entries from cache tag set", + "s", + duration.as_secs_f64() + ); +} + +pub(super) fn record_maintenance_queue_error(error: &TrySendError) { + let kind = match error { + TrySendError::Closed(_) => "channel closed", + TrySendError::Full(_) => "channel full", + }; + u64_counter_with_unit!( + "apollo.router.operations.response_cache.maintenance.queue.error", + "Error while sending cache tag to maintenance queue", + "{error}", + 1, + "error" = kind + ); +} + pub(super) fn record_invalidation_duration( duration: Duration, invalidation_kind: InvalidationKind, @@ -147,13 +127,15 @@ pub(super) fn record_invalidation_duration( /// Restrict `batch_size` cardinality so that it can be used as a metric attribute. fn batch_size_str(batch_size: usize) -> &'static str { - if batch_size <= 10 { + if batch_size == 0 { + "0" + } else if batch_size <= 10 { "1-10" } else if batch_size <= 20 { "11-20" } else if batch_size <= 50 { "21-50" } else { - "50+" + "51+" } } diff --git a/apollo-router/src/plugins/response_cache/plugin.rs b/apollo-router/src/plugins/response_cache/plugin.rs index 6ca8b364ad..dad51d55b1 100644 --- a/apollo-router/src/plugins/response_cache/plugin.rs +++ b/apollo-router/src/plugins/response_cache/plugin.rs @@ -26,8 +26,6 @@ use serde_json_bytes::ByteString; use serde_json_bytes::Value; use tokio::sync::RwLock; use tokio::sync::broadcast; -use tokio::sync::broadcast::Receiver; -use tokio::sync::broadcast::Sender; use tokio_stream::StreamExt; use tokio_stream::wrappers::IntervalStream; use tower::BoxError; @@ -65,12 +63,11 @@ use crate::plugins::response_cache::cache_key::PrimaryCacheKeyEntity; use crate::plugins::response_cache::cache_key::PrimaryCacheKeyRoot; use crate::plugins::response_cache::cache_key::hash_additional_data; use crate::plugins::response_cache::cache_key::hash_query; -use crate::plugins::response_cache::metrics; use crate::plugins::response_cache::storage; use crate::plugins::response_cache::storage::CacheEntry; use crate::plugins::response_cache::storage::CacheStorage; use crate::plugins::response_cache::storage::Document; -use crate::plugins::response_cache::storage::postgres::Storage; +use crate::plugins::response_cache::storage::redis::Storage; use crate::plugins::telemetry::LruSizeInstrument; use crate::plugins::telemetry::dynamic_attribute::SpanDynAttribute; use crate::plugins::telemetry::span_ext::SpanMarkError; @@ -114,9 +111,9 @@ pub(crate) struct ResponseCache { supergraph_schema: Arc>, /// map containing the enum GRAPH subgraph_enums: Arc>, - /// To close all related tasks - drop_tx: Sender<()>, lru_size_instrument: LruSizeInstrument, + /// Sender to tell spawned tasks to abort when this struct is dropped + drop_tx: broadcast::Sender<()>, } #[derive(Debug, Clone, Hash, PartialEq, Eq)] @@ -125,13 +122,7 @@ struct PrivateQueryKey { has_private_id: bool, } -impl Drop for ResponseCache { - fn drop(&mut self) { - let _ = self.drop_tx.send(()); - } -} - -#[derive(Clone)] +#[derive(Clone, Default)] pub(crate) struct StorageInterface { all: Option>>, subgraphs: HashMap>>, @@ -142,54 +133,6 @@ impl StorageInterface { let storage = self.subgraphs.get(subgraph).or(self.all.as_ref())?; storage.get() } - - async fn migrate(&self) -> anyhow::Result<()> { - if let Some(all) = self.all.as_ref().and_then(|all| all.get()) { - all.migrate().await?; - } - futures::future::try_join_all( - self.subgraphs - .values() - .filter_map(|s| Some(s.get()?.migrate())), - ) - .await?; - - Ok(()) - } - - /// Spawn tokio task to refresh metrics about expired data count - fn expired_data_count_tasks(&self, drop_signal: Receiver<()>) { - if let Some(all) = self.all.as_ref().and_then(|all| all.get()) { - tokio::task::spawn(metrics::expired_data_task( - all.clone(), - drop_signal.resubscribe(), - None, - )); - } - for (subgraph_name, subgraph_cache_storage) in &self.subgraphs { - if let Some(subgraph_cache_storage) = subgraph_cache_storage.get() { - tokio::task::spawn(metrics::expired_data_task( - subgraph_cache_storage.clone(), - drop_signal.resubscribe(), - subgraph_name.clone().into(), - )); - } - } - } - - async fn update_cron(&self) -> anyhow::Result<()> { - if let Some(all) = self.all.as_ref().and_then(|all| all.get()) { - all.update_cron().await?; - } - futures::future::try_join_all( - self.subgraphs - .values() - .filter_map(|s| Some(s.get()?.update_cron())), - ) - .await?; - - Ok(()) - } } #[cfg(all( @@ -250,8 +193,8 @@ const fn default_lru_private_queries_size() -> NonZeroUsize { #[derive(Clone, Debug, JsonSchema, Deserialize, Serialize)] #[serde(rename_all = "snake_case", deny_unknown_fields, default)] pub(crate) struct Subgraph { - /// PostgreSQL configuration - pub(crate) postgres: Option, + /// Redis configuration + pub(crate) redis: Option, /// expiration for all keys for this subgraph, unless overridden by the `Cache-Control` header in subgraph responses pub(crate) ttl: Option, @@ -269,7 +212,7 @@ pub(crate) struct Subgraph { impl Default for Subgraph { fn default() -> Self { Self { - postgres: None, + redis: None, enabled: Some(true), ttl: Default::default(), private_id: Default::default(), @@ -314,71 +257,6 @@ impl PluginPrivate for ResponseCache { .as_ref() .map(|q| q.name.to_string()); - let mut all = None; - let (drop_tx, drop_rx) = broadcast::channel(2); - let mut task_aborts = Vec::new(); - if let Some(postgres) = &init.config.subgraph.all.postgres { - let postgres_config = postgres.clone(); - let required_to_start = postgres_config.required_to_start; - all = match Storage::new(&postgres_config).await { - Ok(storage) => Some(Arc::new(OnceLock::from(storage))), - Err(e) => { - tracing::error!( - cache = "response", - error = %e, - "could not open connection to Postgres for caching", - ); - if required_to_start { - return Err(e.into()); - } else { - let storage = Arc::new(OnceLock::new()); - task_aborts.push( - tokio::spawn(check_connection( - postgres_config, - storage.clone(), - drop_rx, - None, - )) - .abort_handle(), - ); - Some(storage) - } - } - }; - } - let mut subgraph_storages = HashMap::new(); - for (subgraph, config) in &init.config.subgraph.subgraphs { - if let Some(postgres) = &config.postgres { - let required_to_start = postgres.required_to_start; - let storage = match Storage::new(postgres).await { - Ok(storage) => Arc::new(OnceLock::from(storage)), - Err(e) => { - tracing::error!( - cache = "response", - error = %e, - "could not open connection to Postgres for caching", - ); - if required_to_start { - return Err(e.into()); - } else { - let storage = Arc::new(OnceLock::new()); - task_aborts.push( - tokio::spawn(check_connection( - postgres.clone(), - storage.clone(), - drop_tx.subscribe(), - subgraph.clone().into(), - )) - .abort_handle(), - ); - storage - } - } - }; - subgraph_storages.insert(subgraph.clone(), storage); - } - } - if init.config.subgraph.all.ttl.is_none() && init .config @@ -408,17 +286,30 @@ impl PluginPrivate for ResponseCache { ); } - let storage = Arc::new(StorageInterface { - all, - subgraphs: subgraph_storages, - }); - storage.migrate().await?; - storage.update_cron().await?; + let mut storage_interface = StorageInterface::default(); - let invalidation = Invalidation::new(storage.clone()).await?; + let (drop_tx, drop_rx) = tokio::sync::broadcast::channel(2); + if let Some(config) = init.config.subgraph.all.redis.clone() { + let storage = Arc::new(OnceLock::new()); + storage_interface.all = Some(storage.clone()); + connect_or_spawn_reconnection_task(config, storage, drop_rx).await?; + } + + for (subgraph, subgraph_config) in &init.config.subgraph.subgraphs { + if let Some(config) = subgraph_config.redis.clone() { + let storage = Arc::new(OnceLock::new()); + storage_interface + .subgraphs + .insert(subgraph.clone(), storage.clone()); + connect_or_spawn_reconnection_task(config, storage, drop_tx.subscribe()).await?; + } + } + + let storage_interface = Arc::new(storage_interface); + let invalidation = Invalidation::new(storage_interface.clone()).await?; Ok(Self { - storage, + storage: storage_interface, entity_type, enabled: init.config.enabled, debug: init.config.debug, @@ -430,15 +321,12 @@ impl PluginPrivate for ResponseCache { invalidation, subgraph_enums: Arc::new(get_subgraph_enums(&init.supergraph_schema)), supergraph_schema: init.supergraph_schema, - drop_tx, lru_size_instrument: LruSizeInstrument::new(LRU_PRIVATE_QUERIES_INSTRUMENT_NAME), + drop_tx, }) } - fn activate(&self) { - self.storage - .expired_data_count_tasks(self.drop_tx.subscribe()); - } + fn activate(&self) {} fn supergraph_service(&self, service: supergraph::BoxService) -> supergraph::BoxService { let debug = self.debug; @@ -582,7 +470,7 @@ impl ResponseCache { subgraphs: HashMap, supergraph_schema: Arc>, truncate_namespace: bool, - update_cron: bool, + drop_tx: broadcast::Sender<()>, ) -> Result where Self: Sized, @@ -590,10 +478,6 @@ impl ResponseCache { use std::net::IpAddr; use std::net::Ipv4Addr; use std::net::SocketAddr; - storage.migrate().await?; - if update_cron { - storage.update_cron().await?; - } if truncate_namespace { storage.truncate_namespace().await?; } @@ -603,7 +487,6 @@ impl ResponseCache { subgraphs: HashMap::new(), }); let invalidation = Invalidation::new(storage.clone()).await?; - let (drop_tx, _drop_rx) = broadcast::channel(2); Ok(Self { storage, entity_type: None, @@ -630,8 +513,8 @@ impl ResponseCache { invalidation, subgraph_enums: Arc::new(get_subgraph_enums(&supergraph_schema)), supergraph_schema, - drop_tx, lru_size_instrument: LruSizeInstrument::new(LRU_PRIVATE_QUERIES_INSTRUMENT_NAME), + drop_tx, }) } #[cfg(all( @@ -683,8 +566,8 @@ impl ResponseCache { invalidation, subgraph_enums: Arc::new(get_subgraph_enums(&supergraph_schema)), supergraph_schema, - drop_tx, lru_size_instrument: LruSizeInstrument::new(LRU_PRIVATE_QUERIES_INSTRUMENT_NAME), + drop_tx, }) } @@ -714,6 +597,12 @@ impl ResponseCache { } } +impl Drop for ResponseCache { + fn drop(&mut self) { + let _ = self.drop_tx.send(()); + } +} + /// Get the map of subgraph enum variant mapped with subgraph name fn get_subgraph_enums(supergraph_schema: &Valid) -> HashMap { let mut subgraph_enums = HashMap::new(); @@ -2439,45 +2328,6 @@ fn assemble_response_from_errors( (new_entities, new_errors) } -async fn check_connection( - postgres_config: storage::postgres::Config, - cache_storage: Arc>, - mut abort_signal: Receiver<()>, - subgraph_name: Option, -) { - let mut interval = - IntervalStream::new(tokio::time::interval(std::time::Duration::from_secs(30))); - let abort_signal_cloned = abort_signal.resubscribe(); - loop { - tokio::select! { - biased; - _ = abort_signal.recv() => { - break; - } - _ = interval.next() => { - u64_counter_with_unit!( - "apollo.router.response_cache.reconnection", - "Number of reconnections to the cache storage", - "{retry}", - 1, - "subgraph.name" = subgraph_name.clone().unwrap_or_default() - ); - if let Ok(storage) = Storage::new(&postgres_config).await { - if let Err(err) = storage.migrate().await { - tracing::error!(error = %err, "cannot migrate storage"); - } - if let Err(err) = storage.update_cron().await { - tracing::error!(error = %err, "cannot update cron storage"); - } - let _ = cache_storage.set(storage.clone()); - tokio::task::spawn(metrics::expired_data_task(storage, abort_signal_cloned, None)); - break; - } - } - } - } -} - pub(crate) type CacheKeysContext = Vec; #[derive(Clone, Debug, Deserialize, Serialize)] @@ -2539,6 +2389,61 @@ impl Ord for CacheKeySource { } } +async fn connect_or_spawn_reconnection_task( + config: storage::redis::Config, + storage: Arc>, + abort_signal: broadcast::Receiver<()>, +) -> Result<(), BoxError> { + match attempt_connection(&config, storage.clone(), abort_signal.resubscribe()).await { + Ok(()) => Ok(()), + Err(err) if config.required_to_start => Err(err), + Err(_) => { + tokio::spawn(reattempt_connection(config.clone(), storage, abort_signal)); + Ok(()) + } + } +} + +async fn attempt_connection( + config: &storage::redis::Config, + cache_storage: Arc>, + abort_signal: broadcast::Receiver<()>, +) -> Result<(), BoxError> { + let storage = Storage::new(config, abort_signal) + .await + .inspect_err(|err| { + tracing::error!( + cache = "response", + error = %err, + "could not open connection to Redis for response caching", + ) + })?; + let _ = cache_storage.set(storage); + + Ok(()) +} + +async fn reattempt_connection( + config: storage::redis::Config, + cache_storage: Arc>, + mut abort_signal: broadcast::Receiver<()>, +) { + let mut interval = IntervalStream::new(tokio::time::interval(Duration::from_secs(30))); + loop { + tokio::select! { + biased; + _ = abort_signal.recv() => { + break; + } + _ = interval.next() => { + if attempt_connection(&config, cache_storage.clone(), abort_signal.resubscribe()).await.is_ok() { + break; + } + } + } + } +} + #[cfg(all( test, any(not(feature = "ci"), all(target_arch = "x86_64", target_os = "linux")) @@ -2548,20 +2453,22 @@ mod tests { use std::time::Duration; use apollo_compiler::Schema; + use tokio::sync::broadcast; use super::Subgraph; use super::Ttl; use crate::configuration::subgraph::SubgraphConfiguration; use crate::plugins::response_cache::plugin::ResponseCache; - use crate::plugins::response_cache::storage::postgres::Config; - use crate::plugins::response_cache::storage::postgres::Storage; + use crate::plugins::response_cache::storage::redis::Config; + use crate::plugins::response_cache::storage::redis::Storage; const SCHEMA: &str = include_str!("../../testdata/orga_supergraph_cache_key.graphql"); #[tokio::test] async fn test_subgraph_enabled() { let valid_schema = Arc::new(Schema::parse_and_validate(SCHEMA, "test.graphql").unwrap()); - let storage = Storage::new(&Config::test("test_subgraph_enabled")) + let (drop_tx, drop_rx) = broadcast::channel(2); + let storage = Storage::new(&Config::test(false, "test_subgraph_enabled"), drop_rx) .await .unwrap(); let map = serde_json::json!({ @@ -2583,7 +2490,7 @@ mod tests { serde_json::from_value(map).unwrap(), valid_schema.clone(), true, - false, + drop_tx, ) .await .unwrap(); @@ -2605,7 +2512,8 @@ mod tests { #[tokio::test] async fn test_subgraph_ttl() { let valid_schema = Arc::new(Schema::parse_and_validate(SCHEMA, "test.graphql").unwrap()); - let storage = Storage::new(&Config::test("test_subgraph_ttl")) + let (drop_tx, drop_rx) = broadcast::channel(2); + let storage = Storage::new(&Config::test(false, "test_subgraph_ttl"), drop_rx) .await .unwrap(); let map = serde_json::json!({ @@ -2629,7 +2537,7 @@ mod tests { serde_json::from_value(map).unwrap(), valid_schema.clone(), true, - false, + drop_tx, ) .await .unwrap(); diff --git a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__failure_mode_reconnect-4.snap b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__failure_mode_reconnect-4.snap index 5787ab9581..075978b642 100644 --- a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__failure_mode_reconnect-4.snap +++ b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__failure_mode_reconnect-4.snap @@ -5,7 +5,7 @@ expression: cache_keys --- [ { - "key": "failure_mode_reconnect-version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", + "key": "version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", "invalidationKeys": [ "currentUser" ], diff --git a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__insert-3.snap b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__insert-3.snap index e3841c2984..075978b642 100644 --- a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__insert-3.snap +++ b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__insert-3.snap @@ -5,7 +5,7 @@ expression: cache_keys --- [ { - "key": "test_insert_simple-version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", + "key": "version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", "invalidationKeys": [ "currentUser" ], diff --git a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__insert_with_nested_field_set-3.snap b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__insert_with_nested_field_set-3.snap index 132434c0a6..80526b0d42 100644 --- a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__insert_with_nested_field_set-3.snap +++ b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__insert_with_nested_field_set-3.snap @@ -5,7 +5,7 @@ expression: cache_keys --- [ { - "key": "test_insert_with_nested_field_set-version:1.0:subgraph:products:type:Query:hash:b59f4823f96ad499bceb011d1bbe7b26214cfaadc8e0e553fd3f11f96b11e3a9:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", + "key": "version:1.0:subgraph:products:type:Query:hash:b59f4823f96ad499bceb011d1bbe7b26214cfaadc8e0e553fd3f11f96b11e3a9:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", "invalidationKeys": [ "allProducts" ], diff --git a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__insert_with_requires-3.snap b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__insert_with_requires-3.snap index c4052e6499..265b1d12c0 100644 --- a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__insert_with_requires-3.snap +++ b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__insert_with_requires-3.snap @@ -36,7 +36,7 @@ expression: cache_keys } }, { - "key": "test_insert_with_requires-version:1.0:subgraph:products:type:Query:hash:950d73004047e5e88793cb22c0c1d0dedc48244c0a1e0c0b41e2078584b4cb65:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", + "key": "version:1.0:subgraph:products:type:Query:hash:950d73004047e5e88793cb22c0c1d0dedc48244c0a1e0c0b41e2078584b4cb65:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", "invalidationKeys": [ "topProducts", "topProducts-5" diff --git a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__invalidate_by_cache_tag-3.snap b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__invalidate_by_cache_tag-3.snap index a387d89ef5..075c0e6484 100644 --- a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__invalidate_by_cache_tag-3.snap +++ b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__invalidate_by_cache_tag-3.snap @@ -4,7 +4,7 @@ expression: cache_keys --- [ { - "key": "test_invalidate_by_cache_tag-version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", + "key": "version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", "invalidationKeys": [ "currentUser" ], diff --git a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__invalidate_by_cache_tag-5.snap b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__invalidate_by_cache_tag-5.snap index 179c9068d2..95d28ac21a 100644 --- a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__invalidate_by_cache_tag-5.snap +++ b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__invalidate_by_cache_tag-5.snap @@ -4,7 +4,7 @@ expression: cache_keys --- [ { - "key": "test_invalidate_by_cache_tag-version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", + "key": "version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", "invalidationKeys": [ "currentUser" ], diff --git a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__invalidate_by_type-3.snap b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__invalidate_by_type-3.snap index 8c9b18cc0e..075c0e6484 100644 --- a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__invalidate_by_type-3.snap +++ b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__invalidate_by_type-3.snap @@ -4,7 +4,7 @@ expression: cache_keys --- [ { - "key": "test_invalidate_by_subgraph-version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", + "key": "version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", "invalidationKeys": [ "currentUser" ], diff --git a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__invalidate_by_type-5.snap b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__invalidate_by_type-5.snap index 0501078789..95d28ac21a 100644 --- a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__invalidate_by_type-5.snap +++ b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__invalidate_by_type-5.snap @@ -4,7 +4,7 @@ expression: cache_keys --- [ { - "key": "test_invalidate_by_subgraph-version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", + "key": "version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", "invalidationKeys": [ "currentUser" ], diff --git a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__no_data-3.snap b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__no_data-3.snap index b3c0689707..4bdfc94920 100644 --- a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__no_data-3.snap +++ b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__no_data-3.snap @@ -45,7 +45,7 @@ expression: cache_keys } }, { - "key": "no_data-version:1.0:subgraph:orga:type:Organization:entity:07f0ad9351c409fd3acfae0be59e64f218dc486d6dbe0081e68794673b96df73:representation::hash:b477abf3b559d50444a6790a53a232701071574572433de2f9424d902a402de6:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", + "key": "version:1.0:subgraph:orga:type:Organization:entity:07f0ad9351c409fd3acfae0be59e64f218dc486d6dbe0081e68794673b96df73:representation::hash:b477abf3b559d50444a6790a53a232701071574572433de2f9424d902a402de6:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", "invalidationKeys": [ "organization", "organization-1" @@ -81,7 +81,7 @@ expression: cache_keys } }, { - "key": "no_data-version:1.0:subgraph:orga:type:Organization:entity:374f6019af287de3375bef883e121cd7951908eb8c1d544f17b56980aeafc376:representation::hash:b477abf3b559d50444a6790a53a232701071574572433de2f9424d902a402de6:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", + "key": "version:1.0:subgraph:orga:type:Organization:entity:374f6019af287de3375bef883e121cd7951908eb8c1d544f17b56980aeafc376:representation::hash:b477abf3b559d50444a6790a53a232701071574572433de2f9424d902a402de6:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", "invalidationKeys": [ "organization", "organization-3" diff --git a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__polymorphic_private_and_public-11.snap b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__polymorphic_private_and_public-11.snap index 47583a9d26..92c84ecda9 100644 --- a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__polymorphic_private_and_public-11.snap +++ b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__polymorphic_private_and_public-11.snap @@ -4,7 +4,7 @@ expression: cache_keys --- [ { - "key": "polymorphic_private_and_public-version:1.0:subgraph:orga:type:Query:hash:d2121420bf697ae407e8b32d950b8778890172bf1f013f687f9c52cf8e54c10a:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", + "key": "version:1.0:subgraph:orga:type:Query:hash:d2121420bf697ae407e8b32d950b8778890172bf1f013f687f9c52cf8e54c10a:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", "invalidationKeys": [], "kind": { "rootFields": [ @@ -30,7 +30,7 @@ expression: cache_keys } }, { - "key": "polymorphic_private_and_public-version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", + "key": "version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", "invalidationKeys": [ "currentUser" ], diff --git a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__polymorphic_private_and_public-3.snap b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__polymorphic_private_and_public-3.snap index d9268dd198..df4537e310 100644 --- a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__polymorphic_private_and_public-3.snap +++ b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__polymorphic_private_and_public-3.snap @@ -30,7 +30,7 @@ expression: cache_keys } }, { - "key": "polymorphic_private_and_public-version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", + "key": "version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", "invalidationKeys": [ "currentUser" ], diff --git a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__polymorphic_private_and_public-5.snap b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__polymorphic_private_and_public-5.snap index 438eca507f..fc0a694247 100644 --- a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__polymorphic_private_and_public-5.snap +++ b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__polymorphic_private_and_public-5.snap @@ -4,7 +4,7 @@ expression: cache_keys --- [ { - "key": "polymorphic_private_and_public-version:1.0:subgraph:orga:type:Query:hash:d2121420bf697ae407e8b32d950b8778890172bf1f013f687f9c52cf8e54c10a:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6:cde13a55f41e387480391c47238acfe9c0136dd56bf365b01416aec03eec7dc4", + "key": "version:1.0:subgraph:orga:type:Query:hash:d2121420bf697ae407e8b32d950b8778890172bf1f013f687f9c52cf8e54c10a:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6:cde13a55f41e387480391c47238acfe9c0136dd56bf365b01416aec03eec7dc4", "invalidationKeys": [], "kind": { "rootFields": [ @@ -31,7 +31,7 @@ expression: cache_keys } }, { - "key": "polymorphic_private_and_public-version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", + "key": "version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", "invalidationKeys": [ "currentUser" ], diff --git a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__polymorphic_private_and_public-7.snap b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__polymorphic_private_and_public-7.snap index 47583a9d26..92c84ecda9 100644 --- a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__polymorphic_private_and_public-7.snap +++ b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__polymorphic_private_and_public-7.snap @@ -4,7 +4,7 @@ expression: cache_keys --- [ { - "key": "polymorphic_private_and_public-version:1.0:subgraph:orga:type:Query:hash:d2121420bf697ae407e8b32d950b8778890172bf1f013f687f9c52cf8e54c10a:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", + "key": "version:1.0:subgraph:orga:type:Query:hash:d2121420bf697ae407e8b32d950b8778890172bf1f013f687f9c52cf8e54c10a:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", "invalidationKeys": [], "kind": { "rootFields": [ @@ -30,7 +30,7 @@ expression: cache_keys } }, { - "key": "polymorphic_private_and_public-version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", + "key": "version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", "invalidationKeys": [ "currentUser" ], diff --git a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__polymorphic_private_and_public-9.snap b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__polymorphic_private_and_public-9.snap index 438eca507f..fc0a694247 100644 --- a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__polymorphic_private_and_public-9.snap +++ b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__polymorphic_private_and_public-9.snap @@ -4,7 +4,7 @@ expression: cache_keys --- [ { - "key": "polymorphic_private_and_public-version:1.0:subgraph:orga:type:Query:hash:d2121420bf697ae407e8b32d950b8778890172bf1f013f687f9c52cf8e54c10a:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6:cde13a55f41e387480391c47238acfe9c0136dd56bf365b01416aec03eec7dc4", + "key": "version:1.0:subgraph:orga:type:Query:hash:d2121420bf697ae407e8b32d950b8778890172bf1f013f687f9c52cf8e54c10a:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6:cde13a55f41e387480391c47238acfe9c0136dd56bf365b01416aec03eec7dc4", "invalidationKeys": [], "kind": { "rootFields": [ @@ -31,7 +31,7 @@ expression: cache_keys } }, { - "key": "polymorphic_private_and_public-version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", + "key": "version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", "invalidationKeys": [ "currentUser" ], diff --git a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__private_and_public-3.snap b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__private_and_public-3.snap index 196d0cde3f..fc0a694247 100644 --- a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__private_and_public-3.snap +++ b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__private_and_public-3.snap @@ -4,7 +4,7 @@ expression: cache_keys --- [ { - "key": "private_and_public-version:1.0:subgraph:orga:type:Query:hash:d2121420bf697ae407e8b32d950b8778890172bf1f013f687f9c52cf8e54c10a:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6:cde13a55f41e387480391c47238acfe9c0136dd56bf365b01416aec03eec7dc4", + "key": "version:1.0:subgraph:orga:type:Query:hash:d2121420bf697ae407e8b32d950b8778890172bf1f013f687f9c52cf8e54c10a:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6:cde13a55f41e387480391c47238acfe9c0136dd56bf365b01416aec03eec7dc4", "invalidationKeys": [], "kind": { "rootFields": [ @@ -31,7 +31,7 @@ expression: cache_keys } }, { - "key": "private_and_public-version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", + "key": "version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", "invalidationKeys": [ "currentUser" ], diff --git a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__private_and_public-5.snap b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__private_and_public-5.snap index b69a77b06b..731b390555 100644 --- a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__private_and_public-5.snap +++ b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__private_and_public-5.snap @@ -31,7 +31,7 @@ expression: cache_keys } }, { - "key": "private_and_public-version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", + "key": "version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6", "invalidationKeys": [ "currentUser" ], diff --git a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__private_only-3.snap b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__private_only-3.snap index 46c0de1a4e..c515745b9a 100644 --- a/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__private_only-3.snap +++ b/apollo-router/src/plugins/response_cache/snapshots/apollo_router__plugins__response_cache__tests__private_only-3.snap @@ -4,7 +4,7 @@ expression: cache_keys --- [ { - "key": "private_only-version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6:cde13a55f41e387480391c47238acfe9c0136dd56bf365b01416aec03eec7dc4", + "key": "version:1.0:subgraph:user:type:Query:hash:fa8ff6a034f5ccb3f49e51bc30ff031104ba420310e67764f9e9f82702437592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6:cde13a55f41e387480391c47238acfe9c0136dd56bf365b01416aec03eec7dc4", "invalidationKeys": [ "currentUser" ], diff --git a/apollo-router/src/plugins/response_cache/storage/config.rs b/apollo-router/src/plugins/response_cache/storage/config.rs new file mode 100644 index 0000000000..8b267ebd00 --- /dev/null +++ b/apollo-router/src/plugins/response_cache/storage/config.rs @@ -0,0 +1,153 @@ +use std::time::Duration; + +use schemars::JsonSchema; +use serde::Deserialize; +use serde::Serialize; + +use crate::configuration::RedisCache; +use crate::configuration::TlsClient; +use crate::configuration::default_metrics_interval; +use crate::configuration::default_required_to_start; + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] +#[serde(deny_unknown_fields)] +/// Redis cache configuration +pub(crate) struct Config { + /// List of URLs to the Redis cluster + pub(crate) urls: Vec, + + /// Redis username if not provided in the URLs. This field takes precedence over the username in the URL + pub(crate) username: Option, + /// Redis password if not provided in the URLs. This field takes precedence over the password in the URL + pub(crate) password: Option, + + #[serde( + deserialize_with = "humantime_serde::deserialize", + default = "default_fetch_timeout" + )] + #[schemars(with = "Option", default)] + /// Timeout for Redis fetch commands (default: 150ms) + pub(crate) fetch_timeout: Duration, + + #[serde( + deserialize_with = "humantime_serde::deserialize", + default = "default_insert_timeout" + )] + #[schemars(with = "Option", default)] + /// Timeout for Redis insert commands (default: 500ms) + /// + /// Inserts are processed asynchronously, so this will not affect response duration. + pub(crate) insert_timeout: Duration, + + #[serde( + deserialize_with = "humantime_serde::deserialize", + default = "default_invalidate_timeout" + )] + #[schemars(with = "Option", default)] + /// Timeout for Redis invalidation commands (default: 1s) + pub(crate) invalidate_timeout: Duration, + + #[serde( + deserialize_with = "humantime_serde::deserialize", + default = "default_maintenance_timeout" + )] + #[schemars(with = "Option", default)] + /// Timeout for Redis maintenance commands (default: 500ms) + /// + /// Maintenance tasks are processed asynchronously, so this will not affect response duration. + pub(crate) maintenance_timeout: Duration, + + #[serde(deserialize_with = "humantime_serde::deserialize", default)] + #[schemars(with = "Option", default)] + /// TTL for entries + pub(crate) ttl: Option, + + /// namespace used to prefix Redis keys + pub(crate) namespace: Option, + + #[serde(default)] + /// TLS client configuration + pub(crate) tls: Option, + + #[serde(default = "default_required_to_start")] + /// Prevents the router from starting if it cannot connect to Redis + pub(crate) required_to_start: bool, + + #[serde(default = "default_pool_size")] + /// The size of the Redis connection pool (default: 5) + pub(crate) pool_size: u32, + + #[serde( + deserialize_with = "humantime_serde::deserialize", + default = "default_metrics_interval" + )] + #[schemars(with = "Option", default)] + /// Interval for collecting Redis metrics (default: 1s) + pub(crate) metrics_interval: Duration, +} + +fn default_fetch_timeout() -> Duration { + Duration::from_millis(150) +} + +fn default_insert_timeout() -> Duration { + Duration::from_millis(500) +} + +fn default_invalidate_timeout() -> Duration { + Duration::from_secs(1) +} + +fn default_maintenance_timeout() -> Duration { + Duration::from_millis(500) +} + +fn default_pool_size() -> u32 { + 5 +} + +impl From<&Config> for RedisCache { + fn from(value: &Config) -> Self { + let timeout = value + .fetch_timeout + .max(value.insert_timeout) + .max(value.invalidate_timeout) + .max(value.maintenance_timeout); + Self { + urls: value.urls.clone(), + username: value.username.clone(), + password: value.password.clone(), + timeout, + ttl: value.ttl, + namespace: value.namespace.clone(), + tls: value.tls.clone(), + required_to_start: value.required_to_start, + reset_ttl: false, + pool_size: value.pool_size, + metrics_interval: value.metrics_interval, + } + } +} + +#[cfg(all( + test, + any(not(feature = "ci"), all(target_arch = "x86_64", target_os = "linux")) +))] +impl Config { + pub(crate) fn test(clustered: bool, namespace: &str) -> Self { + let url = if clustered { + "redis-cluster://127.0.0.1:7000" + } else { + "redis://127.0.0.1:6379" + }; + + serde_json::from_value(serde_json::json!({ + "urls": [url], + "namespace": namespace, + "pool_size": 1, + "required_to_start": true, + "ttl": "5m" + })) + .unwrap() + } +} diff --git a/apollo-router/src/plugins/response_cache/storage/error.rs b/apollo-router/src/plugins/response_cache/storage/error.rs index 3b949425c1..a5d6f0d992 100644 --- a/apollo-router/src/plugins/response_cache/storage/error.rs +++ b/apollo-router/src/plugins/response_cache/storage/error.rs @@ -5,7 +5,10 @@ use crate::plugins::response_cache::ErrorCode; #[derive(Debug, thiserror::Error)] pub(crate) enum Error { #[error("{0}")] - Database(#[from] sqlx::Error), + Database(#[from] fred::error::Error), + + #[error("{0}")] + Join(#[from] tokio::task::JoinError), #[error("NO_STORAGE")] NoStorage, @@ -19,17 +22,21 @@ pub(crate) enum Error { impl Error { pub(crate) fn is_row_not_found(&self) -> bool { - match self { - Error::Database(err) => matches!(err, &sqlx::Error::RowNotFound), - Error::NoStorage | Error::Serialize(_) | Error::Timeout(_) => false, - } + matches!(self, Error::Database(err) if err.is_not_found()) } } impl ErrorCode for Error { fn code(&self) -> &'static str { match self { - Error::Database(err) => err.code(), + Error::Database(err) => err.kind().to_str(), + Error::Join(err) => { + if err.is_cancelled() { + "CANCELLED" + } else { + "PANICKED" + } + } Error::NoStorage => "NO_STORAGE", Error::Serialize(err) => match err.classify() { Category::Io => "Serialize::IO", diff --git a/apollo-router/src/plugins/response_cache/storage/mod.rs b/apollo-router/src/plugins/response_cache/storage/mod.rs index 16f73013e9..f034391fa1 100644 --- a/apollo-router/src/plugins/response_cache/storage/mod.rs +++ b/apollo-router/src/plugins/response_cache/storage/mod.rs @@ -1,5 +1,6 @@ +mod config; mod error; -pub(super) mod postgres; +pub(super) mod redis; use std::collections::HashMap; use std::time::Duration; @@ -136,13 +137,13 @@ pub(super) trait CacheStorage { } #[doc(hidden)] - async fn internal_invalidate_by_subgraph(&self, subgraph_name: String) -> StorageResult; + async fn internal_invalidate_by_subgraph(&self, subgraph_name: &str) -> StorageResult; /// Invalidate all data associated with `subgraph_names`. Command will be timed out after /// `self.invalidate_timeout()`. async fn invalidate_by_subgraph( &self, - subgraph_name: String, + subgraph_name: &str, invalidation_kind: InvalidationKind, ) -> StorageResult { let now = Instant::now(); diff --git a/apollo-router/src/plugins/response_cache/storage/postgres.rs b/apollo-router/src/plugins/response_cache/storage/postgres.rs deleted file mode 100644 index 3256b777e0..0000000000 --- a/apollo-router/src/plugins/response_cache/storage/postgres.rs +++ /dev/null @@ -1,720 +0,0 @@ -use std::collections::HashMap; -use std::sync::Arc; -use std::time::Duration; - -use chrono::TimeDelta; -use log::LevelFilter; -use schemars::JsonSchema; -use serde::Deserialize; -use serde::Serialize; -use sqlx::Acquire; -use sqlx::PgPool; -use sqlx::postgres::PgConnectOptions; -use sqlx::postgres::PgPoolOptions; -use sqlx::types::chrono::DateTime; -use sqlx::types::chrono::Utc; - -use super::CacheEntry; -use crate::plugins::response_cache::ErrorCode; -use crate::plugins::response_cache::storage::CacheStorage; -use crate::plugins::response_cache::storage::Document; -use crate::plugins::response_cache::storage::StorageResult; - -#[derive(sqlx::FromRow, Debug, Clone)] -pub(crate) struct CacheEntryRow { - #[allow(unused)] - pub(crate) id: i64, - pub(crate) cache_key: String, - pub(crate) data: String, - #[allow(unused)] - pub(crate) expires_at: DateTime, - pub(crate) control: String, -} - -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] -#[serde(deny_unknown_fields)] -/// Postgres cache configuration -pub(crate) struct Config { - /// List of URL to Postgres - pub(crate) url: url::Url, - - /// PostgreSQL username if not provided in the URLs. This field takes precedence over the username in the URL - pub(crate) username: Option, - /// PostgreSQL password if not provided in the URLs. This field takes precedence over the password in the URL - pub(crate) password: Option, - - #[serde( - deserialize_with = "humantime_serde::deserialize", - default = "default_idle_timeout" - )] - #[schemars(with = "String")] - /// PostgreSQL maximum idle duration for individual connection (default: 1min) - pub(crate) idle_timeout: Duration, - - #[serde( - deserialize_with = "humantime_serde::deserialize", - default = "default_acquire_timeout" - )] - #[schemars(with = "String")] - /// PostgreSQL the maximum amount of time to spend waiting for a connection (default: 50ms) - pub(crate) acquire_timeout: Duration, - - #[serde(default = "default_required_to_start")] - /// Prevents the router from starting if it cannot connect to PostgreSQL - pub(crate) required_to_start: bool, - - #[serde(default = "default_pool_size")] - /// The size of the PostgreSQL connection pool - pub(crate) pool_size: u32, - #[serde(default = "default_batch_size")] - /// The size of batch when inserting cache entries in PG (default: 100) - pub(crate) batch_size: usize, - /// Useful when running tests in parallel to avoid conflicts - #[serde(default)] - pub(crate) namespace: Option, - - #[serde( - deserialize_with = "humantime_serde::deserialize", - default = "default_cleanup_interval" - )] - #[schemars(with = "String")] - /// Specifies the interval between cache cleanup operations (e.g., "2 hours", "30min"). Default: 1 hour - pub(crate) cleanup_interval: Duration, - - /// Postgres TLS client configuration - #[serde(default)] - pub(crate) tls: TlsConfig, -} - -#[cfg(all( - test, - any(not(feature = "ci"), all(target_arch = "x86_64", target_os = "linux")) -))] -impl Config { - pub(crate) fn test(namespace: &str) -> Self { - Self { - cleanup_interval: default_cleanup_interval(), - tls: Default::default(), - url: "postgres://127.0.0.1".parse().unwrap(), - username: None, - password: None, - idle_timeout: Duration::from_secs(5), - acquire_timeout: Duration::from_millis(50), - required_to_start: true, - pool_size: default_pool_size(), - batch_size: default_batch_size(), - namespace: Some(String::from(namespace)), - } - } -} - -const fn default_required_to_start() -> bool { - false -} - -const fn default_pool_size() -> u32 { - 5 -} - -const fn default_cleanup_interval() -> Duration { - Duration::from_secs(60 * 60) -} - -const fn default_idle_timeout() -> Duration { - Duration::from_secs(60) -} - -const fn default_acquire_timeout() -> Duration { - Duration::from_millis(50) -} - -const fn default_batch_size() -> usize { - 100 -} - -#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema, Default)] -#[serde(deny_unknown_fields)] -/// Postgres TLS client configuration -pub(crate) struct TlsConfig { - /// list of certificate authorities in PEM format - #[schemars(with = "String")] - pub(crate) certificate_authorities: Option>, - /// client certificate authentication - pub(crate) client_authentication: Option>, -} - -/// TLS client authentication -#[derive(Debug, Deserialize, Serialize, JsonSchema, Clone)] -#[serde(deny_unknown_fields)] -pub(crate) struct TlsClientAuth { - /// Sets the SSL client certificate as a PEM - #[schemars(with = "String")] - pub(crate) certificate: Vec, - /// key in PEM format - #[schemars(with = "String")] - pub(crate) key: Vec, -} - -impl TryFrom for CacheEntry { - type Error = serde_json::Error; - - fn try_from(value: CacheEntryRow) -> Result { - let data = serde_json::from_str(&value.data)?; - let control = serde_json::from_str(&value.control)?; - Ok(Self { - key: value.cache_key, - data, - control, - }) - } -} - -#[derive(Clone)] -pub(crate) struct Storage { - batch_size: usize, - pg_pool: PgPool, - namespace: Option, - pub(in crate::plugins::response_cache) cleanup_interval: TimeDelta, -} - -#[derive(thiserror::Error, Debug)] -pub(crate) enum PostgresCacheStorageError { - #[error("postgres error: {0}")] - PgError(#[from] sqlx::Error), - #[error("cleanup_interval configuration is out of range: {0}")] - OutOfRangeError(#[from] chrono::OutOfRangeError), - #[error("cleanup_interval configuration is invalid: {0}")] - InvalidCleanupInterval(String), -} - -impl Storage { - pub(crate) async fn new(conf: &Config) -> Result { - // After 500ms trying to get a connection from PG pool it will return a warning in logs - const ACQUIRE_SLOW_THRESHOLD: std::time::Duration = std::time::Duration::from_millis(500); - let mut pg_connection: PgConnectOptions = conf.url.as_ref().parse()?; - if let Some(user) = &conf.username { - pg_connection = pg_connection.username(user); - } - if let Some(password) = &conf.password { - pg_connection = pg_connection.password(password); - } - if let Some(ca) = &conf.tls.certificate_authorities { - pg_connection = pg_connection.ssl_root_cert_from_pem(ca.clone()); - } - if let Some(tls_client_auth) = &conf.tls.client_authentication { - pg_connection = pg_connection - .ssl_client_cert_from_pem(&tls_client_auth.certificate) - .ssl_client_key_from_pem(&tls_client_auth.key); - } - let pg_pool = PgPoolOptions::new() - .max_connections(conf.pool_size) - .idle_timeout(conf.idle_timeout) - .acquire_timeout(conf.acquire_timeout) - .acquire_slow_threshold(ACQUIRE_SLOW_THRESHOLD) - .acquire_slow_level(LevelFilter::Warn) - .connect_with(pg_connection) - .await?; - - Ok(Self { - pg_pool, - batch_size: conf.batch_size, - namespace: conf.namespace.clone(), - cleanup_interval: TimeDelta::from_std(conf.cleanup_interval)?, - }) - } - - pub(crate) async fn migrate(&self) -> sqlx::Result<()> { - sqlx::migrate!().run(&self.pg_pool).await?; - Ok(()) - } - - fn namespaced(&self, key: &str) -> String { - if let Some(ns) = &self.namespace { - format!("{ns}-{key}") - } else { - key.into() - } - } -} - -impl CacheStorage for Storage { - fn insert_timeout(&self) -> Duration { - // NB: this will be replaced - Duration::from_secs(1) - } - - fn fetch_timeout(&self) -> Duration { - // NB: this will be replaced - Duration::from_secs(1) - } - - fn invalidate_timeout(&self) -> Duration { - // NB: this will be replaced - Duration::from_secs(1) - } - - async fn internal_insert(&self, document: Document, subgraph_name: &str) -> StorageResult<()> { - let mut conn = self.pg_pool.acquire().await?; - let mut transaction = conn.begin().await?; - let tx = &mut transaction; - - let expired_at = Utc::now() + document.expire; - let value_str = serde_json::to_string(&document.data) - .map_err(|err| sqlx::Error::Encode(Box::new(err)))?; - let control_str = serde_json::to_string(&document.control) - .map_err(|err| sqlx::Error::Encode(Box::new(err)))?; - let cache_key = self.namespaced(&document.key); - let rec = sqlx::query!( - r#" - INSERT INTO cache ( cache_key, data, control, expires_at ) - VALUES ( $1, $2, $3, $4 ) - ON CONFLICT (cache_key) DO UPDATE SET data = $2, control = $3, expires_at = $4 - RETURNING id - "#, - &cache_key, - value_str, - control_str, - expired_at - ) - .fetch_one(&mut **tx) - .await?; - - for invalidation_key in &document.invalidation_keys { - let invalidation_key = self.namespaced(invalidation_key); - sqlx::query!( - r#"INSERT into invalidation_key (cache_key_id, invalidation_key, subgraph_name) VALUES ($1, $2, $3) ON CONFLICT (cache_key_id, invalidation_key, subgraph_name) DO NOTHING"#, - rec.id, - &invalidation_key, - subgraph_name - ) - .execute(&mut **tx) - .await?; - } - - transaction.commit().await?; - - Ok(()) - } - - async fn internal_insert_in_batch( - &self, - documents: Vec, - subgraph_name: &str, - ) -> StorageResult<()> { - // order batch_docs to prevent deadlocks! don't need namespaced as we just need to make sure - // that transaction 1 can't lock A and wait for B, and transaction 2 can't lock B and wait for A - let mut batch_docs = documents.clone(); - batch_docs.sort_by(|a, b| a.key.cmp(&b.key)); - - let mut conn = self.pg_pool.acquire().await?; - let batch_docs = batch_docs.chunks(self.batch_size); - for batch_docs in batch_docs { - let mut transaction = conn.begin().await?; - let tx = &mut transaction; - let cache_keys = batch_docs - .iter() - .map(|b| self.namespaced(&b.key)) - .collect::>(); - - let data = batch_docs - .iter() - .map(|b| b.data.clone()) - .flat_map(|d| serde_json::to_string(&d)) - .collect::>(); - let controls = batch_docs - .iter() - .map(|b| b.control.clone()) - .flat_map(|d| serde_json::to_string(&d)) - .collect::>(); - let expires = batch_docs - .iter() - .map(|b| Utc::now() + b.expire) - .collect::>>(); - - let resp = sqlx::query!( - r#" - INSERT INTO cache - ( cache_key, data, expires_at, control ) SELECT * FROM UNNEST( - $1::VARCHAR(1024)[], - $2::TEXT[], - $3::TIMESTAMP WITH TIME ZONE[], - $4::TEXT[] - ) ON CONFLICT (cache_key) DO UPDATE SET data = excluded.data, control = excluded.control, expires_at = excluded.expires_at - RETURNING id - "#, - &cache_keys, - &data, - &expires, - &controls - ) - .fetch_all(&mut **tx) - .await?; - - let invalidation_keys: Vec<(i64, String)> = resp - .iter() - .enumerate() - .flat_map(|(idx, resp)| { - let cache_key_id = resp.id; - batch_docs - .get(idx) - .unwrap() - .invalidation_keys - .iter() - .map(move |k| (cache_key_id, k.clone())) - }) - .collect(); - - let cache_key_ids: Vec = invalidation_keys.iter().map(|(idx, _)| *idx).collect(); - - let subgraph_names: Vec = (0..invalidation_keys.len()) - .map(|_| subgraph_name.to_string()) - .collect(); - let invalidation_keys: Vec = invalidation_keys - .iter() - .map(|(_, invalidation_key)| self.namespaced(invalidation_key)) - .collect(); - sqlx::query!( - r#" - INSERT INTO invalidation_key (cache_key_id, invalidation_key, subgraph_name) - SELECT * FROM UNNEST( - $1::BIGINT[], - $2::VARCHAR(255)[], - $3::VARCHAR(255)[] - ) ON CONFLICT (cache_key_id, invalidation_key, subgraph_name) DO NOTHING - "#, - &cache_key_ids, - &invalidation_keys, - &subgraph_names, - ) - .execute(&mut **tx) - .await?; - - transaction.commit().await?; - } - - Ok(()) - } - - async fn internal_fetch(&self, cache_key: &str) -> StorageResult { - let cache_key = self.namespaced(cache_key); - let resp = sqlx::query_as!( - CacheEntryRow, - "SELECT * FROM cache WHERE cache.cache_key = $1 AND expires_at >= NOW()", - &cache_key - ) - .fetch_one(&self.pg_pool) - .await?; - - let cache_entry_json = resp - .try_into() - .map_err(|err| sqlx::Error::Decode(Box::new(err)))?; - - Ok(cache_entry_json) - } - - async fn internal_fetch_multiple( - &self, - cache_keys: &[&str], - ) -> StorageResult>> { - let cache_keys: Vec<_> = cache_keys.iter().map(|ck| self.namespaced(ck)).collect(); - let resp = sqlx::query_as!( - CacheEntryRow, - "SELECT * FROM cache WHERE cache.cache_key = ANY($1::VARCHAR(1024)[]) AND expires_at >= NOW()", - &cache_keys - ) - .fetch_all(&self.pg_pool) - .await?; - - let cache_key_entries: Result, serde_json::Error> = resp - .into_iter() - .map(|e| { - let entry: CacheEntry = e.try_into()?; - Ok((entry.key.clone(), entry)) - }) - .collect(); - let mut cache_key_entries = - cache_key_entries.map_err(|err| sqlx::Error::Encode(Box::new(err)))?; - - Ok(cache_keys - .iter() - .map(|ck| cache_key_entries.remove(ck)) - .collect()) - } - - async fn internal_invalidate_by_subgraph(&self, subgraph_name: String) -> StorageResult { - let rec = sqlx::query!( - r#"WITH deleted AS - (DELETE - FROM cache - USING invalidation_key - WHERE invalidation_key.cache_key_id = cache.id AND invalidation_key.subgraph_name = ANY($1::text[]) RETURNING cache.cache_key, cache.expires_at - ) - SELECT COUNT(*) AS count FROM deleted WHERE deleted.expires_at >= NOW()"#, - &vec![subgraph_name] - ) - .fetch_one(&self.pg_pool) - .await?; - - Ok(rec.count.unwrap_or_default() as u64) - } - - async fn internal_invalidate( - &self, - invalidation_keys: Vec, - subgraph_names: Vec, - ) -> StorageResult> { - let invalidation_keys: Vec = invalidation_keys - .iter() - .map(|ck| self.namespaced(ck)) - .collect(); - // In this query the 'deleted' view contains the number of data we deleted from 'cache' - // The SELECT on 'deleted' happening at the end is to filter the data to only count for deleted fresh data and get it by subgraph to be able to use it in a metric - let rec = sqlx::query!( - r#"WITH deleted AS - (DELETE - FROM cache - USING invalidation_key - WHERE invalidation_key.invalidation_key = ANY($1::text[]) - AND invalidation_key.cache_key_id = cache.id AND invalidation_key.subgraph_name = ANY($2::text[]) RETURNING cache.cache_key, cache.expires_at, invalidation_key.subgraph_name - ) - SELECT subgraph_name, COUNT(deleted.cache_key) AS count FROM deleted WHERE deleted.expires_at >= NOW() GROUP BY deleted.subgraph_name"#, - &invalidation_keys, - &subgraph_names - ) - .fetch_all(&self.pg_pool) - .await?; - - Ok(rec - .into_iter() - .map(|rec| (rec.subgraph_name, rec.count.unwrap_or_default() as u64)) - .collect()) - } - - #[cfg(all( - test, - any(not(feature = "ci"), all(target_arch = "x86_64", target_os = "linux")) - ))] - async fn truncate_namespace(&self) -> StorageResult<()> { - if let Some(ns) = &self.namespace { - sqlx::query!("DELETE FROM cache WHERE starts_with(cache_key, $1)", ns) - .execute(&self.pg_pool) - .await?; - } - - Ok(()) - } -} - -impl Storage { - pub(crate) async fn expired_data_count(&self) -> anyhow::Result { - match &self.namespace { - Some(ns) => { - let resp = sqlx::query!("SELECT COUNT(id) AS count FROM cache WHERE starts_with(cache_key, $1) AND expires_at <= NOW()", ns) - .fetch_one(&self.pg_pool) - .await?; - - Ok(resp.count.unwrap_or_default() as u64) - } - None => { - let resp = - sqlx::query!("SELECT COUNT(id) AS count FROM cache WHERE expires_at <= NOW()") - .fetch_one(&self.pg_pool) - .await?; - - Ok(resp.count.unwrap_or_default() as u64) - } - } - } - - pub(crate) async fn update_cron(&self) -> sqlx::Result<()> { - let cron = Cron::try_from(&self.cleanup_interval).map_err(|err| { - sqlx::Error::Configuration(Box::new(PostgresCacheStorageError::InvalidCleanupInterval( - err, - ))) - })?; - sqlx::query!("SELECT cron.alter_job((SELECT jobid FROM cron.job WHERE jobname = 'delete-old-cache-entries'), $1)", &cron.0) - .execute(&self.pg_pool) - .await?; - log::trace!( - "Configured `delete-old-cache-entries` cron to have interval = `{}`", - &cron.0 - ); - - Ok(()) - } - - #[cfg(all( - test, - any(not(feature = "ci"), all(target_arch = "x86_64", target_os = "linux")) - ))] - pub(crate) async fn get_cron(&self) -> sqlx::Result { - let rec = sqlx::query!( - "SELECT schedule FROM cron.job WHERE jobname = 'delete-old-cache-entries'" - ) - .fetch_one(&self.pg_pool) - .await?; - - Ok(Cron(rec.schedule)) - } -} - -#[derive(Debug, sqlx::Type)] -#[sqlx(transparent)] -pub(crate) struct Cron(pub(crate) String); - -impl TryFrom<&TimeDelta> for Cron { - type Error = String; - fn try_from(value: &TimeDelta) -> Result { - let num_days = value.num_days(); - let num_hours = value.num_hours(); - let num_mins = value.num_minutes(); - if num_days > 366 { - Err(String::from("interval cannot exceed 1 year")) - } else if num_days > 31 { - // multiple months - let months = (num_days / 30).min(12); - Ok(Cron(format!("0 0 1 */{months} *"))) - } else if num_days > 28 { - // treat as one month - Ok(Cron(String::from("0 0 1 * *"))) - } else if num_days > 0 { - Ok(Cron(format!("0 0 */{num_days} * *"))) - } else if num_hours > 0 { - Ok(Cron(format!("0 */{num_hours} * * *"))) - } else if num_mins > 0 { - Ok(Cron(format!("*/{num_mins} * * * *"))) - } else { - Err(String::from( - "interval lower than 1 minute is not supported", - )) - } - } -} - -impl ErrorCode for sqlx::Error { - fn code(&self) -> &'static str { - match &self { - sqlx::Error::Configuration(_) => "CONFIGURATION", - sqlx::Error::InvalidArgument(_) => "INVALID_ARGUMENT", - sqlx::Error::Database(_) => "DATABASE", - sqlx::Error::Io(_) => "IO", - sqlx::Error::Tls(_) => "TLS", - sqlx::Error::Protocol(_) => "PROTOCOL", - sqlx::Error::RowNotFound => "ROW_NOT_FOUND", - sqlx::Error::TypeNotFound { .. } => "TYPE_NOT_FOUND", - sqlx::Error::ColumnIndexOutOfBounds { .. } => "COLUMN_INDEX_OUT_OF_BOUNDS", - sqlx::Error::ColumnNotFound(_) => "COLUMN_NOT_FOUND", - sqlx::Error::ColumnDecode { .. } => "COLUMN_DECODE", - sqlx::Error::Encode(..) => "ENCODE", - sqlx::Error::Decode(..) => "DECODE", - sqlx::Error::AnyDriverError(..) => "DRIVER_ERROR", - sqlx::Error::PoolTimedOut => "POOL_TIMED_OUT", - sqlx::Error::PoolClosed => "POOL_CLOSED", - sqlx::Error::WorkerCrashed => "WORKER_CRASHED", - sqlx::Error::Migrate(_) => "MIGRATE", - sqlx::Error::InvalidSavePointStatement => "INVALID_SAVE_POINT_STATEMENT", - sqlx::Error::BeginFailed => "BEGIN_FAILED", - _ => "UNKNOWN", - } - } -} - -#[cfg(test)] -mod tests { - use std::time::Duration; - - use chrono::TimeDelta; - - use super::Cron; - - #[rstest::rstest] - #[case(TimeDelta::minutes(1), "*/1 * * * *")] - #[case(TimeDelta::minutes(5), "*/5 * * * *")] - #[case(TimeDelta::minutes(30), "*/30 * * * *")] - #[case(TimeDelta::minutes(59), "*/59 * * * *")] - #[case(TimeDelta::minutes(60), "0 */1 * * *")] - #[case(TimeDelta::hours(1), "0 */1 * * *")] - #[case(TimeDelta::hours(3), "0 */3 * * *")] - #[case(TimeDelta::hours(12), "0 */12 * * *")] - #[case(TimeDelta::hours(23), "0 */23 * * *")] - #[case(TimeDelta::hours(24), "0 0 */1 * *")] - #[case(TimeDelta::days(1), "0 0 */1 * *")] - #[case(TimeDelta::days(7), "0 0 */7 * *")] - #[case(TimeDelta::days(15), "0 0 */15 * *")] - #[case(TimeDelta::days(27), "0 0 */27 * *")] - #[case(TimeDelta::days(28), "0 0 */28 * *")] - #[case::monthly(TimeDelta::days(29), "0 0 1 * *")] - #[case::monthly(TimeDelta::days(30), "0 0 1 * *")] - #[case::monthly(TimeDelta::days(31), "0 0 1 * *")] - #[case::two_months(TimeDelta::days(60), "0 0 1 */2 *")] - #[case::three_months(TimeDelta::days(90), "0 0 1 */3 *")] - #[case::six_months(TimeDelta::days(180), "0 0 1 */6 *")] - #[case::year(TimeDelta::days(360), "0 0 1 */12 *")] - #[case::year(TimeDelta::days(365), "0 0 1 */12 *")] - #[case::year(TimeDelta::days(366), "0 0 1 */12 *")] - #[case::six_weeks_rounds_down(TimeDelta::days(42), "0 0 1 */1 *")] - #[case::complex(TimeDelta::minutes(90), "0 */1 * * *")] - #[case::complex(TimeDelta::hours(36), "0 0 */1 * *")] - fn check_passing_conversion(#[case] interval: TimeDelta, #[case] expected: &str) { - let cron = Cron::try_from(&interval); - assert!(cron.is_ok()); - - let cron_str = cron.unwrap().0; - assert_eq!(cron_str, expected); - } - - #[rstest::rstest] - #[case("1m", "*/1 * * * *")] - #[case("5m", "*/5 * * * *")] - #[case("30m", "*/30 * * * *")] - #[case("59m", "*/59 * * * *")] - #[case("60m", "0 */1 * * *")] - #[case("1h", "0 */1 * * *")] - #[case("3h", "0 */3 * * *")] - #[case("12h", "0 */12 * * *")] - #[case("23h", "0 */23 * * *")] - #[case("24h", "0 0 */1 * *")] - #[case("1d", "0 0 */1 * *")] - #[case("7d", "0 0 */7 * *")] - #[case("1w", "0 0 */7 * *")] - #[case("15d", "0 0 */15 * *")] - #[case("27d", "0 0 */27 * *")] - #[case("28d", "0 0 */28 * *")] - #[case::monthly("29d", "0 0 1 * *")] - #[case::monthly("30d", "0 0 1 * *")] - #[case::monthly("31d", "0 0 1 * *")] - #[case::monthly("1month", "0 0 1 * *")] - #[case::two_months("2months", "0 0 1 */2 *")] - #[case::three_months("3months", "0 0 1 */3 *")] - #[case::six_months("6months", "0 0 1 */6 *")] - #[case::year("365d", "0 0 1 */12 *")] - #[case::year("366d", "0 0 1 */12 *")] - #[case::year("12months", "0 0 1 */12 *")] - #[case::year("1y", "0 0 1 */12 *")] - #[case::six_weeks_rounds_down("6w", "0 0 1 */1 *")] - #[case::complex("90m", "0 */1 * * *")] - #[case::complex("36h", "0 0 */1 * *")] - fn check_passing_conversion_from_humantime(#[case] interval: &str, #[case] expected: &str) { - let interval_dur: Duration = humantime::parse_duration(interval).unwrap(); - let interval = TimeDelta::from_std(interval_dur).unwrap(); - - let cron = Cron::try_from(&interval); - assert!(cron.is_ok()); - - let cron_str = cron.unwrap().0; - assert_eq!(cron_str, expected); - } - - #[rstest::rstest] - #[case::zero(TimeDelta::minutes(0), "interval lower than 1 minute is not supported")] - #[case::negative(TimeDelta::minutes(-1), "interval lower than 1 minute is not supported")] - #[case::too_small(TimeDelta::seconds(1), "interval lower than 1 minute is not supported")] - #[case::too_large(TimeDelta::days(367), "interval cannot exceed 1 year")] - fn check_error_conversion(#[case] interval: TimeDelta, #[case] expected_err: &str) { - let cron = Cron::try_from(&interval); - assert!(cron.is_err()); - - let err_str = cron.unwrap_err(); - assert_eq!(err_str, expected_err); - } -} diff --git a/apollo-router/src/plugins/response_cache/storage/redis.rs b/apollo-router/src/plugins/response_cache/storage/redis.rs new file mode 100644 index 0000000000..105c6b7d62 --- /dev/null +++ b/apollo-router/src/plugins/response_cache/storage/redis.rs @@ -0,0 +1,1354 @@ +use std::collections::HashMap; +use std::collections::HashSet; +use std::time::Duration; +use std::time::Instant; +use std::time::SystemTime; +use std::time::UNIX_EPOCH; + +use fred::interfaces::ClientLike; +use fred::interfaces::KeysInterface; +use fred::interfaces::SortedSetsInterface; +use fred::prelude::Options; +use fred::types::Expiration; +use fred::types::ExpireOptions; +use fred::types::Value; +use fred::types::sorted_sets::Ordering; +use serde::Deserialize; +use serde::Serialize; +use tokio::sync::broadcast; +use tokio::sync::mpsc; +use tokio::task::JoinSet; +use tokio_util::future::FutureExt; +use tower::BoxError; + +use super::CacheEntry; +use super::CacheStorage; +use super::Document; +use super::StorageResult; +use crate::cache::redis::RedisCacheStorage; +use crate::cache::redis::RedisKey; +use crate::cache::redis::RedisValue; +use crate::cache::storage::KeyType; +use crate::cache::storage::ValueType; +use crate::plugins::response_cache::cache_control::CacheControl; +use crate::plugins::response_cache::metrics::record_maintenance_duration; +use crate::plugins::response_cache::metrics::record_maintenance_error; +use crate::plugins::response_cache::metrics::record_maintenance_queue_error; +use crate::plugins::response_cache::metrics::record_maintenance_success; + +pub(crate) type Config = super::config::Config; + +#[derive(Deserialize, Debug, Clone, Serialize)] +struct CacheValue { + data: serde_json_bytes::Value, + cache_control: CacheControl, +} + +impl ValueType for CacheValue {} + +impl From<(&str, CacheValue)> for CacheEntry { + fn from((cache_key, cache_value): (&str, CacheValue)) -> Self { + CacheEntry { + key: cache_key.to_string(), + data: cache_value.data, + control: cache_value.cache_control, + } + } +} + +#[derive(Clone)] +pub(crate) struct Storage { + storage: RedisCacheStorage, + cache_tag_tx: mpsc::Sender, + fetch_timeout: Duration, + insert_timeout: Duration, + invalidate_timeout: Duration, + maintenance_timeout: Duration, +} + +impl Storage { + pub(crate) async fn new( + config: &Config, + drop_rx: broadcast::Receiver<()>, + ) -> Result { + // NB: sorted set cleanup happens via an async task, reading from `cache_tag_rx`. + // Items are added to it via `try_send` to avoid blocking, but this does mean that some items + // won't be added to the channel. This is probably acceptable given the limited number of options + // for the cache tag: + // * frequently used - another insert will eventually add the cache tag to the queue + // * not frequently used - small memory footprint, so probably doesn't need much cleanup + // * never used again - will be removed via TTL + // There are opportunities for improvement here to make sure that we don't try to do maintenance + // on the same cache tag multiple times a second, and perhaps a world where we actually want multiple + // consumers running at the same time. + + let storage = RedisCacheStorage::new(config.into(), "response-cache").await?; + let (cache_tag_tx, cache_tag_rx) = mpsc::channel(1000); + let s = Self { + storage, + cache_tag_tx, + fetch_timeout: config.fetch_timeout, + insert_timeout: config.insert_timeout, + invalidate_timeout: config.invalidate_timeout, + maintenance_timeout: config.maintenance_timeout, + }; + s.perform_periodic_maintenance(cache_tag_rx, drop_rx).await; + Ok(s) + } + + fn make_key(&self, key: K) -> String { + self.storage.make_key(RedisKey(key)) + } + + async fn invalidate_keys(&self, invalidation_keys: Vec) -> StorageResult { + let options = Options { + timeout: Some(self.invalidate_timeout()), + ..Options::default() + }; + let pipeline = self.storage.pipeline().with_options(&options); + for invalidation_key in &invalidation_keys { + let invalidation_key = self.make_key(format!("cache-tag:{invalidation_key}")); + self.send_to_maintenance_queue(invalidation_key.clone()); + let _: () = pipeline + .zrange(invalidation_key.clone(), 0, -1, None, false, None, false) + .await?; + } + + let results: Vec> = pipeline.all().await?; + let all_keys: HashSet = results.into_iter().flatten().collect(); + if all_keys.is_empty() { + return Ok(0); + } + + let keys = all_keys.into_iter().map(fred::types::Key::from); + let deleted = self + .storage + .delete_from_scan_result_with_options(keys, options) + .await?; + + // NOTE: we don't delete elements from the cache tag sorted sets. if we did, we would likely + // encounter a race condition - if another router inserted a value associated with this cache + // tag between when we run the `zrange` and the `delete`. + // it's safer to just rely on the TTL-based cleanup. + Ok(deleted as u64) + } + + fn send_to_maintenance_queue(&self, cache_tag_key: String) { + if let Err(err) = self.cache_tag_tx.try_send(cache_tag_key) { + record_maintenance_queue_error(&err); + } + } + + pub(crate) async fn perform_periodic_maintenance( + &self, + mut cache_tag_rx: mpsc::Receiver, + mut drop_rx: broadcast::Receiver<()>, + ) { + let storage = self.clone(); + + // spawn a task that reads from cache_tag_rx and uses `zremrangebyscore` on each cache tag + tokio::spawn(async move { + loop { + tokio::select! { + biased; + _ = drop_rx.recv() => break, + Some(cache_tag) = cache_tag_rx.recv() => storage.perform_maintenance_on_cache_tag(cache_tag).await + } + } + }); + } + + async fn perform_maintenance_on_cache_tag(&self, cache_tag: String) { + // NB: `cache_tag` already includes namespace + let cutoff = now() - 1; + + let now = Instant::now(); + let removed_items_result = super::flatten_storage_error( + self.remove_keys_from_cache_tag_by_cutoff(cache_tag, cutoff as f64) + .timeout(self.maintenance_timeout()) + .await, + ); + record_maintenance_duration(now.elapsed()); + + match removed_items_result { + Ok(removed_items) => record_maintenance_success(removed_items), + Err(err) => record_maintenance_error(&err), + } + } + + async fn remove_keys_from_cache_tag_by_cutoff( + &self, + cache_tag_key: String, + cutoff_time: f64, + ) -> StorageResult { + // Returns number of items removed + let options = Options { + timeout: Some(self.maintenance_timeout()), + ..Options::default() + }; + Ok(self + .storage + .client() + .with_options(&options) + .zremrangebyscore(&cache_tag_key, f64::NEG_INFINITY, cutoff_time) + .await?) + } + + /// Create a list of the cache tags that describe this document, with associated namespaces. + /// + /// For a given subgraph `s` and invalidation keys `i1`, `i2`, ..., we need to store the + /// following subgraph-invalidation-key permutations: + /// * `subgraph-{s}` (whole subgraph) + /// * `subgraph-{s}:key-{i1}`, `subgraph-{s}:key-{i2}`, ... (invalidation key per subgraph) + /// + /// These are then turned into redis keys by adding the namespace and a `cache-tag:` prefix, ie: + /// * `{namespace}:cache-tag:subgraph-{s}` + /// * `{namespace}:cache-tag:subgraph-{s}:key-{i1}`, ... + fn namespaced_cache_tags( + &self, + document_invalidation_keys: &[String], + subgraph_name: &str, + ) -> Vec { + let mut cache_tags = Vec::with_capacity(1 + document_invalidation_keys.len()); + cache_tags.push(format!("subgraph-{subgraph_name}")); + for invalidation_key in document_invalidation_keys { + cache_tags.push(format!("subgraph-{subgraph_name}:key-{invalidation_key}")); + } + + for cache_tag in cache_tags.iter_mut() { + *cache_tag = self.make_key(format!("cache-tag:{cache_tag}")); + } + + cache_tags + } + + fn maintenance_timeout(&self) -> Duration { + self.maintenance_timeout + } +} + +impl CacheStorage for Storage { + fn insert_timeout(&self) -> Duration { + self.insert_timeout + } + + fn fetch_timeout(&self) -> Duration { + self.fetch_timeout + } + + fn invalidate_timeout(&self) -> Duration { + self.invalidate_timeout + } + + async fn internal_insert(&self, document: Document, subgraph_name: &str) -> StorageResult<()> { + self.internal_insert_in_batch(vec![document], subgraph_name) + .await + } + + async fn internal_insert_in_batch( + &self, + mut batch_docs: Vec, + subgraph_name: &str, + ) -> StorageResult<()> { + // three phases: + // 1 - update keys, cache tags to include namespace so that we don't have to do it in each phase + // 2 - update each cache tag with new keys + // 3 - update each key + // a failure in any phase will cause the function to return, which prevents invalid states + + let now = now(); + + // phase 1 + for document in &mut batch_docs { + document.key = self.make_key(&document.key); + document.invalidation_keys = + self.namespaced_cache_tags(&document.invalidation_keys, subgraph_name); + } + + // phase 2 + let num_cache_tags_estimate = 2 * batch_docs.len(); + let mut cache_tags_to_pcks: HashMap> = + HashMap::with_capacity(num_cache_tags_estimate); + for document in &mut batch_docs { + for cache_tag_key in document.invalidation_keys.drain(..) { + let cache_tag_value = ( + (now + document.expire.as_secs()) as f64, + document.key.clone(), + ); + // NB: performance concerns with `entry` API + if let Some(entry) = cache_tags_to_pcks.get_mut(&cache_tag_key) { + entry.push(cache_tag_value); + } else { + cache_tags_to_pcks.insert(cache_tag_key, vec![cache_tag_value]); + } + } + } + + let options = Options { + timeout: Some(self.insert_timeout()), + ..Options::default() + }; + let pipeline = self.storage.client().pipeline().with_options(&options); + for (cache_tag_key, elements) in cache_tags_to_pcks.into_iter() { + self.send_to_maintenance_queue(cache_tag_key.clone()); + + // NB: expiry time being max + 1 is important! if you use a volatile TTL eviction policy, + // Redis will evict the keys with the shortest TTLs - we have to make sure that the cache + // tag will outlive any of the keys it refers to. + let max_expiry_time = elements + .iter() + .map(|(exp_time, _)| *exp_time) + .fold(now as f64, f64::max); + let cache_tag_expiry_time = max_expiry_time as i64 + 1; + + let _: Result<(), _> = pipeline + .zadd( + cache_tag_key.clone(), + None, + Some(Ordering::GreaterThan), + false, + false, + elements, + ) + .await; + + // > A non-volatile key is treated as an infinite TTL for the purpose of GT and LT. + // > The GT, LT and NX options are mutually exclusive. + // - https://redis.io/docs/latest/commands/expire/ + // + // what we want are NX (set when key has no expiry) AND GT (set when new expiry is greater + // than the current one). + // that means we have to call `expire_at` twice :( + for exp_opt in [ExpireOptions::NX, ExpireOptions::GT] { + let _: Result<(), _> = pipeline + .expire_at(cache_tag_key.clone(), cache_tag_expiry_time, Some(exp_opt)) + .await; + } + } + + let result_vec = pipeline.try_all::().await; + for result in result_vec { + if let Err(err) = result { + tracing::debug!("Caught error during cache tag update: {err:?}"); + return Err(err.into()); + } + } + + // phase 3 + let pipeline = self.storage.client().pipeline().with_options(&options); + for document in batch_docs.into_iter() { + let value = CacheValue { + data: document.data, + cache_control: document.control, + }; + let _: () = pipeline + .set::<(), _, _>( + document.key, + &serde_json::to_string(&value)?, + Some(Expiration::EXAT((now + document.expire.as_secs()) as i64)), + None, + false, + ) + .await?; + } + + let result_vec = pipeline.try_all::().await; + for result in result_vec { + if let Err(err) = result { + tracing::debug!("Caught error during document insert: {err:?}"); + return Err(err.into()); + } + } + + Ok(()) + } + + async fn internal_fetch(&self, cache_key: &str) -> StorageResult { + // NB: don't need `make_key` for `get` - the storage layer already runs it + let options = Options { + timeout: Some(self.fetch_timeout()), + ..Options::default() + }; + let value: RedisValue = self + .storage + .get_with_options(RedisKey(cache_key), options) + .await?; + Ok(CacheEntry::from((cache_key, value.0))) + } + + async fn internal_fetch_multiple( + &self, + cache_keys: &[&str], + ) -> StorageResult>> { + let keys: Vec> = cache_keys + .iter() + .map(|key| RedisKey(key.to_string())) + .collect(); + let options = Options { + timeout: Some(self.fetch_timeout()), + ..Options::default() + }; + let values: Vec>> = + self.storage.get_multiple_with_options(keys, options).await; + + let entries = values + .into_iter() + .zip(cache_keys) + .map(|(opt_value, cache_key)| { + opt_value.map(|value| CacheEntry::from((*cache_key, value.0))) + }) + .collect(); + + Ok(entries) + } + + async fn internal_invalidate_by_subgraph(&self, subgraph_name: &str) -> StorageResult { + self.invalidate_keys(vec![format!("subgraph-{subgraph_name}")]) + .await + } + + async fn internal_invalidate( + &self, + invalidation_keys: Vec, + subgraph_names: Vec, + ) -> StorageResult> { + let mut join_set = JoinSet::default(); + let num_subgraphs = subgraph_names.len(); + + for subgraph_name in subgraph_names { + let keys: Vec = invalidation_keys + .iter() + .map(|invalidation_key| format!("subgraph-{subgraph_name}:key-{invalidation_key}")) + .collect(); + let storage = self.clone(); + join_set.spawn(async move { (subgraph_name, storage.invalidate_keys(keys).await) }); + } + + let mut counts = HashMap::with_capacity(num_subgraphs); + while let Some(result) = join_set.join_next().await { + let (subgraph_name, count) = result?; + counts.insert(subgraph_name, count.unwrap_or(0)); + } + + Ok(counts) + } + + #[cfg(all( + test, + any(not(feature = "ci"), all(target_arch = "x86_64", target_os = "linux")) + ))] + async fn truncate_namespace(&self) -> StorageResult<()> { + self.storage.truncate_namespace().await?; + Ok(()) + } +} + +fn now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs() +} + +#[cfg(all( + test, + any(not(feature = "ci"), all(target_arch = "x86_64", target_os = "linux")) +))] +impl Storage { + async fn mocked( + config: &Config, + is_cluster: bool, + mock_storage: std::sync::Arc, + drop_rx: broadcast::Receiver<()>, + ) -> Result { + let storage = RedisCacheStorage::from_mocks_and_config( + mock_storage, + config.into(), + "response-cache", + is_cluster, + ) + .await?; + let (cache_tag_tx, cache_tag_rx) = mpsc::channel(100); + let s = Self { + storage, + cache_tag_tx, + fetch_timeout: config.fetch_timeout, + insert_timeout: config.insert_timeout, + invalidate_timeout: config.invalidate_timeout, + maintenance_timeout: config.maintenance_timeout, + }; + s.perform_periodic_maintenance(cache_tag_rx, drop_rx).await; + Ok(s) + } + + async fn all_keys_in_namespace(&self) -> Result, BoxError> { + use fred::types::scan::Scanner; + use tokio_stream::StreamExt; + + let mut scan_stream = self + .storage + .scan_with_namespaced_results(String::from("*"), None); + let mut keys = Vec::default(); + while let Some(result) = scan_stream.next().await { + if let Some(page_keys) = result?.take_results() { + let mut str_keys: Vec = page_keys + .into_iter() + .map(|k| k.into_string().unwrap()) + .collect(); + keys.append(&mut str_keys); + } + } + + Ok(keys) + } + + async fn ttl(&self, key: &str) -> StorageResult { + Ok(self.storage.client().ttl(key).await?) + } + + async fn expire_time(&self, key: &str) -> StorageResult { + Ok(self.storage.client().expire_time(key).await?) + } + + async fn zscore(&self, sorted_set_key: &str, member: &str) -> Result { + let score: String = self.storage.client().zscore(sorted_set_key, member).await?; + Ok(score.parse()?) + } + + async fn zcard(&self, sorted_set_key: &str) -> StorageResult { + let cardinality = self.storage.client().zcard(sorted_set_key).await?; + Ok(cardinality) + } + + async fn zexists(&self, sorted_set_key: &str, member: &str) -> StorageResult { + let score: Option = self.storage.client().zscore(sorted_set_key, member).await?; + Ok(score.is_some()) + } + + async fn exists(&self, key: &str) -> StorageResult { + Ok(self.storage.client().exists(key).await?) + } +} + +#[cfg(all( + test, + any(not(feature = "ci"), all(target_arch = "x86_64", target_os = "linux")) +))] +mod tests { + use std::sync::Arc; + use std::time::Duration; + + use insta::assert_debug_snapshot; + use itertools::Itertools; + use tokio::sync::broadcast; + use tower::BoxError; + use uuid::Uuid; + + use super::Config; + use super::Storage; + use super::now; + use crate::plugins::response_cache::storage::CacheStorage; + use crate::plugins::response_cache::storage::Document; + + const SUBGRAPH_NAME: &str = "test"; + + fn redis_config(clustered: bool) -> Config { + Config::test(clustered, &random_namespace()) + } + + fn random_namespace() -> String { + Uuid::new_v4().to_string() + } + + fn common_document() -> Document { + Document { + key: "key".to_string(), + data: Default::default(), + control: Default::default(), + invalidation_keys: vec!["invalidate".to_string()], + expire: Duration::from_secs(60), + } + } + + #[tokio::test] + #[rstest::rstest] + async fn test_invalidation_key_permutations( + #[values(None, Some("test"))] namespace: Option<&str>, + #[values(vec![], vec!["invalidation"], vec!["invalidation1", "invalidation2", "invalidation3"])] + invalidation_keys: Vec<&str>, + ) { + // Set up insta snapshot to support test parameterization + let mut settings = insta::Settings::clone_current(); + settings.set_snapshot_suffix(format!( + "input____{}____{}", + namespace.unwrap_or("None"), + invalidation_keys.iter().join("__") + )); + let _guard = settings.bind_to_scope(); + + let mock_storage = Arc::new(fred::mocks::Echo); + let config = Config { + namespace: namespace.map(ToString::to_string), + ..redis_config(false) + }; + let (_drop_tx, drop_rx) = broadcast::channel(2); + let storage = Storage::mocked(&config, false, mock_storage, drop_rx) + .await + .expect("could not build storage"); + + let invalidation_keys: Vec = invalidation_keys + .into_iter() + .map(ToString::to_string) + .collect(); + + let mut cache_tags = storage.namespaced_cache_tags(&invalidation_keys, "products"); + cache_tags.sort(); + assert_debug_snapshot!(cache_tags); + } + + /// Tests that validate the following TTL behaviors: + /// * a document's TTL must be shorter than the TTL of all its related cache tags + /// * a document's TTL will always be less than or equal to its score in all its related cache tags + /// * only expired keys will be removed via the cache maintenance + mod ttl_guarantees { + use std::collections::HashMap; + use std::time::Duration; + + use itertools::Itertools; + use tokio::sync::broadcast; + use tower::BoxError; + + use super::SUBGRAPH_NAME; + use super::common_document; + use super::redis_config; + use crate::plugins::response_cache::storage::CacheStorage; + use crate::plugins::response_cache::storage::Document; + use crate::plugins::response_cache::storage::redis::Storage; + + #[tokio::test] + #[rstest::rstest] + async fn single_document(#[values(true, false)] clustered: bool) -> Result<(), BoxError> { + let (_drop_tx, drop_rx) = broadcast::channel(2); + let storage = Storage::new(&redis_config(clustered), drop_rx).await?; + storage.truncate_namespace().await?; + + // every element of this namespace must have a TTL associated with it, and the TTL of the + // cache keys must be greater than the TTL of the document + let document = common_document(); + storage.insert(document.clone(), SUBGRAPH_NAME).await?; + + let document_key = storage.make_key(document.key.clone()); + let expected_cache_tag_keys = + storage.namespaced_cache_tags(&document.invalidation_keys, SUBGRAPH_NAME); + + // iterate over all the keys in the namespace and make sure we have everything we'd expect + let keys = storage.all_keys_in_namespace().await?; + assert!(keys.contains(&document_key)); + for key in &expected_cache_tag_keys { + assert!(keys.contains(key), "missing {key}"); + } + assert_eq!(keys.len(), 3); // 1 document + 2 cache tags + + // extract the TTL for each key. the TTL for the document must be less than the TTL for each + // of the invalidation keys. + let document_ttl = storage.ttl(&document_key).await?; + assert!(document_ttl > 0); + + for cache_tag_key in &expected_cache_tag_keys { + let cache_tag_ttl = storage.ttl(cache_tag_key).await?; + assert!(cache_tag_ttl > 0, "{cache_tag_key}"); + assert!(document_ttl < cache_tag_ttl, "{cache_tag_key}") + } + + // extract the expiry time for the document key. it should match the sorted set score in each + // of the cache tags. + let document_expire_time = storage.expire_time(&document_key).await?; + assert!(document_expire_time > 0); + + for cache_tag_key in &expected_cache_tag_keys { + let document_score = storage.zscore(cache_tag_key, &document_key).await?; + assert_eq!(document_expire_time, document_score); + } + + Ok(()) + } + + #[tokio::test] + #[rstest::rstest] + async fn multiple_documents( + #[values(true, false)] clustered: bool, + ) -> Result<(), BoxError> { + let (_drop_tx, drop_rx) = broadcast::channel(2); + let storage = Storage::new(&redis_config(clustered), drop_rx).await?; + storage.truncate_namespace().await?; + + // set up two documents with a shared key and different TTLs + let documents = vec![ + Document { + key: "key1".to_string(), + invalidation_keys: vec![ + "invalidation".to_string(), + "invalidation1".to_string(), + ], + expire: Duration::from_secs(30), + ..common_document() + }, + Document { + key: "key2".to_string(), + invalidation_keys: vec![ + "invalidation".to_string(), + "invalidation2".to_string(), + ], + expire: Duration::from_secs(60), + ..common_document() + }, + ]; + storage + .insert_in_batch(documents.clone(), SUBGRAPH_NAME) + .await?; + + // based on these documents, we expect: + // * subgraph cache-tag TTL ~60s + // * `invalidation` cache-tag TTL ~60s + // * `invalidation1` cache-tag TTL ~30s + // * `invalidation2` cache-tag TTL ~60s + // since those are the maximums observed + + let mut expected_document_keys = Vec::new(); + let mut expected_cache_tag_keys = Vec::new(); + for document in &documents { + expected_document_keys.push(storage.make_key(&document.key)); + expected_cache_tag_keys.push( + storage.namespaced_cache_tags(&document.invalidation_keys, SUBGRAPH_NAME), + ); + } + + let all_expected_cache_tag_keys: Vec = expected_cache_tag_keys + .iter() + .flatten() + .cloned() + .unique() + .collect(); + + // we should have a few shared keys + assert!( + all_expected_cache_tag_keys.len() + < expected_cache_tag_keys.iter().map(|keys| keys.len()).sum() + ); + + // iterate over all the keys in the namespace and make sure we have everything we'd expect + let keys = storage.all_keys_in_namespace().await?; + for expected_document_key in &expected_document_keys { + assert!(keys.contains(expected_document_key)); + } + for expected_cache_tag_key in &all_expected_cache_tag_keys { + assert!(keys.contains(expected_cache_tag_key)); + } + assert_eq!(keys.len(), 6); // 2 documents + 4 cache tags + + // extract all TTLs + let mut ttls: HashMap = HashMap::default(); + for key in &keys { + let ttl = storage.ttl(key).await?; + assert!(ttl > 0); + ttls.insert(key.clone(), ttl); + } + + // for each document, make sure that its cache tags have a TTL greater than its own + for (index, document) in documents.iter().enumerate() { + let document_key = &expected_document_keys[index]; + let cache_tag_keys = &expected_cache_tag_keys[index]; + + let document_ttl = ttls.get(document_key).unwrap(); + + // the document TTL should be close to the expiry time on the document (within some range + // of acceptable redis latency - 10s for now) + assert!(document.expire.as_secs() as i64 - *document_ttl < 10); + + for cache_tag_key in cache_tag_keys { + let cache_tag_ttl = ttls.get(cache_tag_key).unwrap(); + assert!(document_ttl < cache_tag_ttl); + } + } + + // for each document, make sure the expiry time matches its score in each cache tag set + for index in 0..documents.len() { + let document_key = &expected_document_keys[index]; + let cache_tag_keys = &expected_cache_tag_keys[index]; + + let document_expire_time = storage.expire_time(document_key).await?; + assert!(document_expire_time > 0); + + for cache_tag_key in cache_tag_keys { + let document_score = storage.zscore(cache_tag_key, document_key).await?; + assert_eq!(document_expire_time, document_score); + } + } + + Ok(()) + } + + #[tokio::test] + #[rstest::rstest] + async fn cache_tag_ttl_will_only_increase( + #[values(true, false)] clustered: bool, + ) -> Result<(), BoxError> { + let (_drop_tx, drop_rx) = broadcast::channel(2); + let storage = Storage::new(&redis_config(clustered), drop_rx).await?; + storage.truncate_namespace().await?; + + let document = Document { + key: "key1".to_string(), + expire: Duration::from_secs(60), + ..common_document() + }; + storage.insert(document.clone(), SUBGRAPH_NAME).await?; + + let keys = storage.all_keys_in_namespace().await?; + + // save current expiry times + let mut expire_times: HashMap = HashMap::default(); + for key in &keys { + let expire_time = storage.expire_time(key).await?; + assert!(expire_time > 0); + expire_times.insert(key.clone(), expire_time); + } + + // add another document with a very short expiry time but the same cache tags + let document = Document { + key: "key2".to_string(), + expire: Duration::from_secs(1), + ..common_document() + }; + storage.insert(document, SUBGRAPH_NAME).await?; + + // fetch new expiry times; they should be the same + for key in keys { + let new_expire_time = storage.expire_time(&key).await?; + assert!(new_expire_time > 0); + assert_eq!(*expire_times.get(&key).unwrap(), new_expire_time); + } + + Ok(()) + } + + /// When re-inserting the same key with a lower TTL, the score in the sorted set will not + /// decrease. + /// + /// This might seem strange, but it's a defensive mechanism in case the insert fails midway + /// through - we don't want to lower the cache tag score only to not change the TTL on the key. + #[tokio::test] + #[rstest::rstest] + async fn cache_tag_score_will_not_decrease( + #[values(true, false)] clustered: bool, + ) -> Result<(), BoxError> { + let (_drop_tx, drop_rx) = broadcast::channel(2); + let storage = Storage::new(&redis_config(clustered), drop_rx).await?; + storage.truncate_namespace().await?; + + let document = Document { + expire: Duration::from_secs(60), + data: serde_json_bytes::Value::Number(1.into()), + ..common_document() + }; + let document_key = storage.make_key(document.key.clone()); + storage.insert(document.clone(), SUBGRAPH_NAME).await?; + + // make sure the document was stored + let stored_data = storage.fetch(&common_document().key, SUBGRAPH_NAME).await?; + assert_eq!(stored_data.data, document.data); + + let keys = storage.namespaced_cache_tags(&document.invalidation_keys, SUBGRAPH_NAME); + + // save current scores + let mut scores: HashMap = HashMap::default(); + let mut expire_times: HashMap = HashMap::default(); + for key in &keys { + let score = storage.zscore(key, &document_key).await?; + assert!(score > 0); + scores.insert(key.clone(), score); + + let expire_time = storage.expire_time(key).await?; + assert!(expire_time > 0); + expire_times.insert(key.clone(), expire_time); + } + + // update the document with new data and a shorter TTL + let document = Document { + expire: Duration::from_secs(10), + data: serde_json_bytes::Value::Number(2.into()), + ..common_document() + }; + storage.insert(document.clone(), SUBGRAPH_NAME).await?; + + // make sure the document was updated + let stored_data = storage.fetch(&document.key, SUBGRAPH_NAME).await?; + assert_eq!(stored_data.data, document.data); + + // the TTL on the document should be aligned with the new document expiry time + let ttl = storage.ttl(&document_key).await?; + assert!(ttl <= document.expire.as_secs() as i64); + + // however, the TTL on the cache tags and the score in the cache tags will be the same + for key in keys { + let score = storage.zscore(&key, &document_key).await?; + assert!(score > 0); + assert_eq!(*scores.get(&key).unwrap(), score); + + let expire_time = storage.expire_time(&key).await?; + assert!(expire_time > 0); + assert_eq!(*expire_times.get(&key).unwrap(), expire_time); + } + + Ok(()) + } + + /// When re-inserting the same key with a later expiry time, the score in the sorted set will + /// increase. + #[tokio::test] + #[rstest::rstest] + async fn cache_tag_score_will_increase( + #[values(true, false)] clustered: bool, + ) -> Result<(), BoxError> { + let (_drop_tx, drop_rx) = broadcast::channel(2); + let storage = Storage::new(&redis_config(clustered), drop_rx).await?; + storage.truncate_namespace().await?; + + let document = Document { + expire: Duration::from_secs(60), + data: serde_json_bytes::Value::Number(1.into()), + ..common_document() + }; + let document_key = storage.make_key(document.key.clone()); + storage.insert(document.clone(), SUBGRAPH_NAME).await?; + + // make sure the document was stored + let stored_data = storage.fetch(&common_document().key, SUBGRAPH_NAME).await?; + assert_eq!(stored_data.data, document.data); + + let keys = storage.namespaced_cache_tags(&document.invalidation_keys, SUBGRAPH_NAME); + + // update the document with new data and a longer TTL + let old_ttl = document.expire; + let document = Document { + expire: old_ttl * 2, + data: serde_json_bytes::Value::Number(2.into()), + ..common_document() + }; + storage.insert(document.clone(), SUBGRAPH_NAME).await?; + + // make sure the document was updated + let stored_data = storage.fetch(&document.key, SUBGRAPH_NAME).await?; + assert_eq!(stored_data.data, document.data); + + // the TTL on the document should be aligned with the new document expiry time + let ttl = storage.ttl(&document_key).await?; + assert!(ttl <= document.expire.as_secs() as i64); + assert!(ttl > old_ttl.as_secs() as i64); + + let doc_expire_time = storage.expire_time(&document_key).await?; + + // the TTL on the cache tags and the score in the cache tags should have also increased + for key in keys { + let score = storage.zscore(&key, &document_key).await?; + assert!(doc_expire_time <= score); + + let expire_time = storage.expire_time(&key).await?; + assert!(doc_expire_time < expire_time); + } + + Ok(()) + } + } + + /// Tests that ensure that if a key's cache tag cannot be updated, the key will not be updated. + mod cache_tag_insert_failure_should_abort_key_insertion { + use std::sync::Arc; + + use fred::error::Error; + use fred::error::ErrorKind; + use fred::interfaces::KeysInterface; + use fred::mocks::MockCommand; + use fred::mocks::Mocks; + use fred::prelude::Expiration; + use fred::prelude::Value; + use parking_lot::RwLock; + use tokio::sync::broadcast; + use tower::BoxError; + + use super::SUBGRAPH_NAME; + use super::common_document; + use super::redis_config; + use crate::plugins::response_cache::storage::CacheStorage; + use crate::plugins::response_cache::storage::Document; + use crate::plugins::response_cache::storage::redis::Storage; + + /// Trigger failure by pre-setting the cache tag to an invalid type. + #[tokio::test] + #[rstest::rstest] + async fn type_failure(#[values(true, false)] clustered: bool) -> Result<(), BoxError> { + let (_drop_tx, drop_rx) = broadcast::channel(2); + let config = redis_config(clustered); + let storage = Storage::new(&config, drop_rx).await?; + storage.truncate_namespace().await?; + + let document = common_document(); + let document_key = storage.make_key(document.key.clone()); + let cache_tag_keys = + storage.namespaced_cache_tags(&document.invalidation_keys, SUBGRAPH_NAME); + + let insert_invalid_cache_tag = |key: String| async { + let expiration = config.ttl.map(|ttl| Expiration::EX(ttl.as_secs() as i64)); + let _: () = storage + .storage + .client() + .set(key, 1, expiration, None, false) + .await?; + Ok::<(), BoxError>(()) + }; + let inserted_data = |key: String| async { + let exists = storage.storage.client().exists(key).await?; + Ok::(exists) + }; + + // try performing the insert with one of the cache_tag_keys set to a string so that the ZADD + // is guaranteed to fail. + // NB: we do this for each key because fred might report a failure at the beginning of a pipeline + // differently than a failure at the end. + for key in cache_tag_keys { + storage.truncate_namespace().await?; + insert_invalid_cache_tag(key.clone()).await?; + + let result = storage.insert(document.clone(), SUBGRAPH_NAME).await; + result.expect_err(&format!( + "cache tag {key} should have caused insertion failure" + )); + + assert!(!inserted_data(document_key.clone()).await?); + } + + // this should also be true if inserting multiple documents, even if only one of the + // documents' cache tags couldn't be updated. + let documents = vec![ + Document { + key: "key1".to_string(), + invalidation_keys: vec![], + ..common_document() + }, + Document { + key: "key2".to_string(), + invalidation_keys: vec!["invalidate".to_string()], + ..common_document() + }, + ]; + + let cache_tag_keys = + storage.namespaced_cache_tags(&documents[1].invalidation_keys, SUBGRAPH_NAME); + for key in cache_tag_keys { + storage.truncate_namespace().await?; + insert_invalid_cache_tag(key.clone()).await?; + + storage + .insert_in_batch(documents.clone(), SUBGRAPH_NAME) + .await + .expect_err(&format!( + "cache tag {key} should have caused insertion failure" + )); + + for document in &documents { + assert!(!inserted_data(storage.make_key(document.key.clone())).await?); + } + } + + Ok(()) + } + + #[tokio::test] + #[rstest::rstest] + async fn timeout_failure(#[values(true, false)] clustered: bool) -> Result<(), BoxError> { + use crate::plugins::response_cache::storage::error::Error as StorageError; + + // Mock the Redis connection to be able to simulate a timeout error coming from within + // the `fred` client + #[derive(Default, Debug, Clone)] + struct MockStorage(Arc>>); + impl Mocks for MockStorage { + fn process_command(&self, command: MockCommand) -> Result { + self.0.write().push(command); + Err(Error::new(ErrorKind::Timeout, "timeout")) + } + } + + let (_drop_tx, drop_rx) = broadcast::channel(2); + let mock_storage = Arc::new(MockStorage::default()); + let storage = Storage::mocked( + &redis_config(clustered), + clustered, + mock_storage.clone(), + drop_rx, + ) + .await?; + + let document = common_document(); + let document_key = Value::from(storage.make_key(document.key.clone())); + + let result = storage.insert(document, SUBGRAPH_NAME).await; + let error = result.expect_err("should have timed out via redis"); + assert!(matches!(error, StorageError::Database(e) if e.details() == "timeout")); + + // make sure the insert function did not try to operate on the document key + for command in mock_storage.0.read().iter() { + if command.cmd.contains("SET") && command.args.contains(&document_key) { + panic!("Command {command:?} set the document key"); + } + } + + Ok(()) + } + } + + #[tokio::test] + #[rstest::rstest] + async fn maintenance_removes_expired_data( + #[values(true, false)] clustered: bool, + ) -> Result<(), BoxError> { + let (_drop_tx, drop_rx) = broadcast::channel(2); + let storage = Storage::new(&redis_config(clustered), drop_rx).await?; + storage.truncate_namespace().await?; + + // set up two documents with a shared key and different TTLs + let documents = vec![ + Document { + key: "key1".to_string(), + expire: Duration::from_secs(2), + ..common_document() + }, + Document { + key: "key2".to_string(), + expire: Duration::from_secs(60), + ..common_document() + }, + Document { + key: "key3".to_string(), + expire: Duration::from_secs(60), + ..common_document() + }, + ]; + storage + .insert_in_batch(documents.clone(), SUBGRAPH_NAME) + .await?; + + // ensure that we have three elements in the 'whole-subgraph' invalidation key + let invalidation_key = storage.namespaced_cache_tags(&[], SUBGRAPH_NAME).remove(0); + assert_eq!(storage.zcard(&invalidation_key).await?, 3); + + let doc_key1 = storage.make_key("key1"); + let doc_key2 = storage.make_key("key2"); + let doc_key3 = storage.make_key("key3"); + for key in [&doc_key1, &doc_key2, &doc_key3] { + assert!(storage.zexists(&invalidation_key, key).await?); + } + + // manually trigger maintenance with a time in the future, in between the expiry times of doc1 + // and docs 2 and 3. therefore, we should remove `key1` and leave `key2` and `key3` + let cutoff = now() + 10; + assert!(storage.zscore(&invalidation_key, &doc_key1).await? < cutoff as i64); + let removed_keys = storage + .remove_keys_from_cache_tag_by_cutoff(invalidation_key.clone(), cutoff as f64) + .await?; + assert_eq!(removed_keys, 1); + + // now we should have two elements in the 'whole-subgraph' invalidation key + assert_eq!(storage.zcard(&invalidation_key).await?, 2); + assert!(!storage.zexists(&invalidation_key, &doc_key1).await?); + assert!(storage.zexists(&invalidation_key, &doc_key2).await?); + assert!(storage.zexists(&invalidation_key, &doc_key3).await?); + + // manually trigger maintenance with the time set way in the future + let cutoff = now() + 1000; + let removed_keys = storage + .remove_keys_from_cache_tag_by_cutoff(invalidation_key.clone(), cutoff as f64) + .await?; + assert_eq!(removed_keys, 2); + + // now we should have zero elements in the 'whole-subgraph' invalidation key + assert_eq!(storage.zcard(&invalidation_key).await?, 0); + for key in [&doc_key1, &doc_key2, &doc_key3] { + assert!(!storage.zexists(&invalidation_key, key).await?); + } + + Ok(()) + } + + mod invalidation { + use tokio::sync::broadcast; + use tower::BoxError; + + use super::common_document; + use super::redis_config; + use crate::plugins::response_cache::storage::CacheStorage; + use crate::plugins::response_cache::storage::Document; + use crate::plugins::response_cache::storage::redis::Storage; + + #[tokio::test] + #[rstest::rstest] + async fn invalidation_by_subgraph_removes_everything_associated_with_that_subgraph( + #[values(true, false)] clustered: bool, + ) -> Result<(), BoxError> { + let (_drop_tx, drop_rx) = broadcast::channel(2); + let storage = Storage::new(&redis_config(clustered), drop_rx).await?; + storage.truncate_namespace().await?; + + let document1 = Document { + key: "key1".to_string(), + ..common_document() + }; + + let document2 = Document { + key: "key2".to_string(), + ..common_document() + }; + + let document3 = Document { + key: "key3".to_string(), + ..common_document() + }; + + storage.insert(document1.clone(), "S1").await?; + storage.insert(document2.clone(), "S2").await?; + storage.insert(document3.clone(), "S2").await?; + + // invalidate just subgraph1 + let num_invalidated = storage.invalidate_by_subgraph("S1", "subgraph").await?; + assert_eq!(num_invalidated, 1); + assert!(!storage.exists(&storage.make_key("key1")).await?); + assert!(storage.exists(&storage.make_key("key2")).await?); + + // invalidate subgraph2 + let num_invalidated = storage.invalidate_by_subgraph("S2", "subgraph").await?; + assert_eq!(num_invalidated, 2); + assert!(!storage.exists(&storage.make_key("key2")).await?); + assert!(!storage.exists(&storage.make_key("key3")).await?); + + Ok(()) + } + + #[tokio::test] + #[rstest::rstest] + async fn arguments_are_restrictive_rather_than_additive( + #[values(true, false)] clustered: bool, + ) -> Result<(), BoxError> { + // invalidate takes a list of invalidation keys and a list of subgraphs; the two are combined + // to form a list of cache tags to remove from + let (_drop_tx, drop_rx) = broadcast::channel(2); + let storage = Storage::new(&redis_config(clustered), drop_rx).await?; + storage.truncate_namespace().await?; + + let document1 = Document { + key: "key1".to_string(), + invalidation_keys: vec!["A".to_string()], + ..common_document() + }; + + let document2 = Document { + key: "key2".to_string(), + invalidation_keys: vec!["A".to_string()], + ..common_document() + }; + + let document3 = Document { + key: "key3".to_string(), + invalidation_keys: vec!["B".to_string()], + ..common_document() + }; + + storage.insert(document1.clone(), "S1").await?; + storage.insert(document2.clone(), "S2").await?; + storage.insert(document3.clone(), "S2").await?; + + // invalidate(A, S2) will invalidate key2, NOT key1 or key3 + let invalidated = storage + .invalidate(vec!["A".to_string()], vec!["S2".to_string()], "cache_tag") + .await?; + assert_eq!(invalidated.len(), 1); + assert_eq!(*invalidated.get("S2").unwrap(), 1); + assert!(storage.exists(&storage.make_key("key1")).await?); + assert!(!storage.exists(&storage.make_key("key2")).await?); + assert!(storage.exists(&storage.make_key("key3")).await?); + + Ok(()) + } + + #[tokio::test] + #[rstest::rstest] + async fn invalidating_missing_subgraph_will_not_error( + #[values(true, false)] clustered: bool, + ) -> Result<(), BoxError> { + let (_drop_tx, drop_rx) = broadcast::channel(2); + let storage = Storage::new(&redis_config(clustered), drop_rx).await?; + storage.truncate_namespace().await?; + + storage.insert(common_document(), "S1").await?; + + let invalidated = storage.invalidate_by_subgraph("S2", "subgraph").await?; + assert_eq!(invalidated, 0); + + let invalidated = storage + .invalidate(vec!["key".to_string()], vec!["S2".to_string()], "cache_tag") + .await?; + assert_eq!(invalidated.len(), 1); + assert_eq!(*invalidated.get("S2").unwrap(), 0); + + Ok(()) + } + + #[tokio::test] + #[rstest::rstest] + async fn invalidating_missing_invalidation_key_will_not_error( + #[values(true, false)] clustered: bool, + ) -> Result<(), BoxError> { + let (_drop_tx, drop_rx) = broadcast::channel(2); + let storage = Storage::new(&redis_config(clustered), drop_rx).await?; + storage.truncate_namespace().await?; + + storage.insert(common_document(), "S1").await?; + + let invalidated = storage + .invalidate(vec!["key".to_string()], vec!["S1".to_string()], "cache_tag") + .await?; + assert_eq!(invalidated.len(), 1); + assert_eq!(*invalidated.get("S1").unwrap(), 0); + + Ok(()) + } + + #[tokio::test] + #[rstest::rstest] + async fn invalidation_is_idempotent( + #[values(true, false)] clustered: bool, + ) -> Result<(), BoxError> { + let (_drop_tx, drop_rx) = broadcast::channel(2); + let storage = Storage::new(&redis_config(clustered), drop_rx).await?; + storage.truncate_namespace().await?; + + let document = common_document(); + let document_key = storage.make_key(&document.key); + + storage.insert(document, "S1").await?; + assert!(storage.exists(&document_key).await?); + + let invalidated = storage.invalidate_by_subgraph("S1", "subgraph").await?; + assert_eq!(invalidated, 1); + + assert!(!storage.exists(&document_key).await?); + + // re-invalidate - storage still shouldn't have the key in it, and it shouldn't + // encounter an error + let invalidated = storage.invalidate_by_subgraph("S1", "subgraph").await?; + assert_eq!(invalidated, 0); + assert!(!storage.exists(&document_key).await?); + + Ok(()) + } + } +} diff --git a/apollo-router/src/plugins/response_cache/storage/snapshots/apollo_router__plugins__response_cache__storage__redis__tests__invalidation_key_permutations@input____None____.snap b/apollo-router/src/plugins/response_cache/storage/snapshots/apollo_router__plugins__response_cache__storage__redis__tests__invalidation_key_permutations@input____None____.snap new file mode 100644 index 0000000000..2e442ba775 --- /dev/null +++ b/apollo-router/src/plugins/response_cache/storage/snapshots/apollo_router__plugins__response_cache__storage__redis__tests__invalidation_key_permutations@input____None____.snap @@ -0,0 +1,7 @@ +--- +source: apollo-router/src/plugins/response_cache/storage/redis.rs +expression: cache_tags +--- +[ + "cache-tag:subgraph-products", +] diff --git a/apollo-router/src/plugins/response_cache/storage/snapshots/apollo_router__plugins__response_cache__storage__redis__tests__invalidation_key_permutations@input____None____invalidation.snap b/apollo-router/src/plugins/response_cache/storage/snapshots/apollo_router__plugins__response_cache__storage__redis__tests__invalidation_key_permutations@input____None____invalidation.snap new file mode 100644 index 0000000000..798eedc930 --- /dev/null +++ b/apollo-router/src/plugins/response_cache/storage/snapshots/apollo_router__plugins__response_cache__storage__redis__tests__invalidation_key_permutations@input____None____invalidation.snap @@ -0,0 +1,8 @@ +--- +source: apollo-router/src/plugins/response_cache/storage/redis.rs +expression: cache_tags +--- +[ + "cache-tag:subgraph-products", + "cache-tag:subgraph-products:key-invalidation", +] diff --git a/apollo-router/src/plugins/response_cache/storage/snapshots/apollo_router__plugins__response_cache__storage__redis__tests__invalidation_key_permutations@input____None____invalidation1__invalidation2__invalidation3.snap b/apollo-router/src/plugins/response_cache/storage/snapshots/apollo_router__plugins__response_cache__storage__redis__tests__invalidation_key_permutations@input____None____invalidation1__invalidation2__invalidation3.snap new file mode 100644 index 0000000000..49f795d69b --- /dev/null +++ b/apollo-router/src/plugins/response_cache/storage/snapshots/apollo_router__plugins__response_cache__storage__redis__tests__invalidation_key_permutations@input____None____invalidation1__invalidation2__invalidation3.snap @@ -0,0 +1,10 @@ +--- +source: apollo-router/src/plugins/response_cache/storage/redis.rs +expression: cache_tags +--- +[ + "cache-tag:subgraph-products", + "cache-tag:subgraph-products:key-invalidation1", + "cache-tag:subgraph-products:key-invalidation2", + "cache-tag:subgraph-products:key-invalidation3", +] diff --git a/apollo-router/src/plugins/response_cache/storage/snapshots/apollo_router__plugins__response_cache__storage__redis__tests__invalidation_key_permutations@input____test____.snap b/apollo-router/src/plugins/response_cache/storage/snapshots/apollo_router__plugins__response_cache__storage__redis__tests__invalidation_key_permutations@input____test____.snap new file mode 100644 index 0000000000..7378d3fa64 --- /dev/null +++ b/apollo-router/src/plugins/response_cache/storage/snapshots/apollo_router__plugins__response_cache__storage__redis__tests__invalidation_key_permutations@input____test____.snap @@ -0,0 +1,7 @@ +--- +source: apollo-router/src/plugins/response_cache/storage/redis.rs +expression: cache_tags +--- +[ + "test:cache-tag:subgraph-products", +] diff --git a/apollo-router/src/plugins/response_cache/storage/snapshots/apollo_router__plugins__response_cache__storage__redis__tests__invalidation_key_permutations@input____test____invalidation.snap b/apollo-router/src/plugins/response_cache/storage/snapshots/apollo_router__plugins__response_cache__storage__redis__tests__invalidation_key_permutations@input____test____invalidation.snap new file mode 100644 index 0000000000..e03ea36c0a --- /dev/null +++ b/apollo-router/src/plugins/response_cache/storage/snapshots/apollo_router__plugins__response_cache__storage__redis__tests__invalidation_key_permutations@input____test____invalidation.snap @@ -0,0 +1,8 @@ +--- +source: apollo-router/src/plugins/response_cache/storage/redis.rs +expression: cache_tags +--- +[ + "test:cache-tag:subgraph-products", + "test:cache-tag:subgraph-products:key-invalidation", +] diff --git a/apollo-router/src/plugins/response_cache/storage/snapshots/apollo_router__plugins__response_cache__storage__redis__tests__invalidation_key_permutations@input____test____invalidation1__invalidation2__invalidation3.snap b/apollo-router/src/plugins/response_cache/storage/snapshots/apollo_router__plugins__response_cache__storage__redis__tests__invalidation_key_permutations@input____test____invalidation1__invalidation2__invalidation3.snap new file mode 100644 index 0000000000..a9acd78e2c --- /dev/null +++ b/apollo-router/src/plugins/response_cache/storage/snapshots/apollo_router__plugins__response_cache__storage__redis__tests__invalidation_key_permutations@input____test____invalidation1__invalidation2__invalidation3.snap @@ -0,0 +1,10 @@ +--- +source: apollo-router/src/plugins/response_cache/storage/redis.rs +expression: cache_tags +--- +[ + "test:cache-tag:subgraph-products", + "test:cache-tag:subgraph-products:key-invalidation1", + "test:cache-tag:subgraph-products:key-invalidation2", + "test:cache-tag:subgraph-products:key-invalidation3", +] diff --git a/apollo-router/src/plugins/response_cache/tests.rs b/apollo-router/src/plugins/response_cache/tests.rs index 77032ac45e..0dcb9442e8 100644 --- a/apollo-router/src/plugins/response_cache/tests.rs +++ b/apollo-router/src/plugins/response_cache/tests.rs @@ -7,7 +7,6 @@ use futures::StreamExt; use http::HeaderName; use http::HeaderValue; use http::header::CACHE_CONTROL; -use tokio::sync::broadcast; use tokio_stream::wrappers::IntervalStream; use tower::Service; use tower::ServiceExt; @@ -20,16 +19,13 @@ use crate::graphql; use crate::metrics::FutureMetricsExt; use crate::plugin::test::MockSubgraph; use crate::plugin::test::MockSubgraphService; -use crate::plugins::response_cache::cache_control::CacheControl; use crate::plugins::response_cache::invalidation::InvalidationRequest; -use crate::plugins::response_cache::metrics; use crate::plugins::response_cache::plugin::CACHE_DEBUG_HEADER_NAME; use crate::plugins::response_cache::plugin::CacheKeysContext; use crate::plugins::response_cache::plugin::Subgraph; use crate::plugins::response_cache::storage::CacheStorage; -use crate::plugins::response_cache::storage::Document; -use crate::plugins::response_cache::storage::postgres::Config; -use crate::plugins::response_cache::storage::postgres::Storage; +use crate::plugins::response_cache::storage::redis::Config; +use crate::plugins::response_cache::storage::redis::Storage; use crate::services::subgraph; use crate::services::supergraph; @@ -164,14 +160,15 @@ async fn insert() { }, }); - let storage = Storage::new(&Config::test("test_insert_simple")) + let (drop_tx, drop_rx) = tokio::sync::broadcast::channel(2); + let storage = Storage::new(&Config::test(false, "test_insert_simple"), drop_rx) .await .unwrap(); let map = [ ( "user".to_string(), Subgraph { - postgres: None, + redis: None, private_id: Some("sub".to_string()), enabled: true.into(), ttl: None, @@ -181,7 +178,7 @@ async fn insert() { ( "orga".to_string(), Subgraph { - postgres: None, + redis: None, private_id: Some("sub".to_string()), enabled: true.into(), ttl: None, @@ -192,7 +189,7 @@ async fn insert() { .into_iter() .collect(); let response_cache = - ResponseCache::for_test(storage.clone(), map, valid_schema.clone(), true, false) + ResponseCache::for_test(storage.clone(), map, valid_schema.clone(), true, drop_tx) .await .unwrap(); @@ -330,14 +327,15 @@ async fn insert_without_debug_header() { }, }); - let storage = Storage::new(&Config::test("insert_without_debug_header")) + let (drop_tx, drop_rx) = tokio::sync::broadcast::channel(2); + let storage = Storage::new(&Config::test(false, "insert_without_debug_header"), drop_rx) .await .unwrap(); let map = [ ( "user".to_string(), Subgraph { - postgres: None, + redis: None, private_id: Some("sub".to_string()), enabled: true.into(), ttl: None, @@ -347,7 +345,7 @@ async fn insert_without_debug_header() { ( "orga".to_string(), Subgraph { - postgres: None, + redis: None, private_id: Some("sub".to_string()), enabled: true.into(), ttl: None, @@ -358,7 +356,7 @@ async fn insert_without_debug_header() { .into_iter() .collect(); let response_cache = - ResponseCache::for_test(storage.clone(), map, valid_schema.clone(), true, false) + ResponseCache::for_test(storage.clone(), map, valid_schema.clone(), true, drop_tx) .await .unwrap(); @@ -483,14 +481,15 @@ async fn insert_with_requires() { ).with_header(CACHE_CONTROL, HeaderValue::from_static("public")).build()) ].into_iter().collect()); - let storage = Storage::new(&Config::test("test_insert_with_requires")) + let (drop_tx, drop_rx) = tokio::sync::broadcast::channel(2); + let storage = Storage::new(&Config::test(false, "test_insert_with_requires"), drop_rx) .await .unwrap(); let map: HashMap = [ ( "products".to_string(), Subgraph { - postgres: None, + redis: None, private_id: Some("sub".to_string()), enabled: true.into(), ttl: None, @@ -500,7 +499,7 @@ async fn insert_with_requires() { ( "inventory".to_string(), Subgraph { - postgres: None, + redis: None, private_id: Some("sub".to_string()), enabled: true.into(), ttl: None, @@ -515,7 +514,7 @@ async fn insert_with_requires() { map.clone(), valid_schema.clone(), true, - false, + drop_tx, ) .await .unwrap(); @@ -646,14 +645,18 @@ async fn insert_with_nested_field_set() { } }); - let storage = Storage::new(&Config::test("test_insert_with_nested_field_set")) - .await - .unwrap(); + let (drop_tx, drop_rx) = tokio::sync::broadcast::channel(2); + let storage = Storage::new( + &Config::test(false, "test_insert_with_nested_field_set"), + drop_rx, + ) + .await + .unwrap(); let map = [ ( "products".to_string(), Subgraph { - postgres: None, + redis: None, private_id: Some("sub".to_string()), enabled: true.into(), ttl: None, @@ -663,7 +666,7 @@ async fn insert_with_nested_field_set() { ( "users".to_string(), Subgraph { - postgres: None, + redis: None, private_id: Some("sub".to_string()), enabled: true.into(), ttl: None, @@ -674,7 +677,7 @@ async fn insert_with_nested_field_set() { .into_iter() .collect(); let response_cache = - ResponseCache::for_test(storage.clone(), map, valid_schema.clone(), true, false) + ResponseCache::for_test(storage.clone(), map, valid_schema.clone(), true, drop_tx) .await .unwrap(); @@ -813,7 +816,8 @@ async fn no_cache_control() { }, }); - let storage = Storage::new(&Config::test("test_no_cache_control")) + let (drop_tx, drop_rx) = tokio::sync::broadcast::channel(2); + let storage = Storage::new(&Config::test(false, "test_no_cache_control"), drop_rx) .await .unwrap(); let response_cache = ResponseCache::for_test( @@ -821,7 +825,7 @@ async fn no_cache_control() { HashMap::new(), valid_schema.clone(), false, - false, + drop_tx, ) .await .unwrap(); @@ -939,7 +943,8 @@ async fn no_store_from_request() { }, }); - let storage = Storage::new(&Config::test("test_no_store_from_client")) + let (drop_tx, drop_rx) = tokio::sync::broadcast::channel(2); + let storage = Storage::new(&Config::test(false, "test_no_store_from_client"), drop_rx) .await .unwrap(); let response_cache = ResponseCache::for_test( @@ -947,7 +952,7 @@ async fn no_store_from_request() { HashMap::new(), valid_schema.clone(), false, - false, + drop_tx, ) .await .unwrap(); @@ -1002,21 +1007,19 @@ async fn no_store_from_request() { "#); // Just to make sure it doesn't invalidate anything, which means nothing has been stored - assert!( - storage - .invalidate( - vec![ - "user".to_string(), - "organization".to_string(), - "currentUser".to_string() - ], - vec!["orga".to_string(), "user".to_string()], - "test_bulk_invalidation" - ) - .await - .unwrap() - .is_empty() - ); + let invalidations_by_subgraph = storage + .invalidate( + vec![ + "user".to_string(), + "organization".to_string(), + "currentUser".to_string(), + ], + vec!["orga".to_string(), "user".to_string()], + "test_bulk_invalidation", + ) + .await + .unwrap(); + assert_eq!(invalidations_by_subgraph.into_values().sum::(), 0); let service = TestHarness::builder() .configuration_json(serde_json::json!({"include_subgraph_errors": { "all": true }, "experimental_mock_subgraphs": subgraphs.clone(), "headers": { @@ -1069,21 +1072,19 @@ async fn no_store_from_request() { "#); // Just to make sure it doesn't invalidate anything, which means nothing has been stored - assert!( - storage - .invalidate( - vec![ - "user".to_string(), - "organization".to_string(), - "currentUser".to_string() - ], - vec!["orga".to_string(), "user".to_string()], - "test_bulk_invalidation" - ) - .await - .unwrap() - .is_empty() - ); + let invalidations_by_subgraph = storage + .invalidate( + vec![ + "user".to_string(), + "organization".to_string(), + "currentUser".to_string(), + ], + vec!["orga".to_string(), "user".to_string()], + "test_bulk_invalidate", + ) + .await + .unwrap(); + assert_eq!(invalidations_by_subgraph.into_values().sum::(), 0); } #[tokio::test] @@ -1119,14 +1120,15 @@ async fn private_only() { }, }); - let storage = Storage::new(&Config::test("private_only")) + let (drop_tx, drop_rx) = tokio::sync::broadcast::channel(2); + let storage = Storage::new(&Config::test(false,"private_only"), drop_rx) .await .unwrap(); let map = [ ( "user".to_string(), Subgraph { - postgres: None, + redis: None, private_id: Some("sub".to_string()), enabled: true.into(), ttl: None, @@ -1136,7 +1138,7 @@ async fn private_only() { ( "orga".to_string(), Subgraph { - postgres: None, + redis: None, private_id: Some("sub".to_string()), enabled: true.into(), ttl: None, @@ -1147,7 +1149,7 @@ async fn private_only() { .into_iter() .collect(); let response_cache = - ResponseCache::for_test(storage.clone(), map, valid_schema.clone(), true, false) + ResponseCache::for_test(storage.clone(), map, valid_schema.clone(), true, drop_tx) .await .unwrap(); @@ -1320,14 +1322,15 @@ async fn private_and_public() { }, }); - let storage = Storage::new(&Config::test("private_and_public")) + let (drop_tx, drop_rx) = tokio::sync::broadcast::channel(2); + let storage = Storage::new(&Config::test(false, "private_and_public"), drop_rx) .await .unwrap(); let map = [ ( "user".to_string(), Subgraph { - postgres: None, + redis: None, private_id: Some("sub".to_string()), enabled: true.into(), ttl: None, @@ -1337,7 +1340,7 @@ async fn private_and_public() { ( "orga".to_string(), Subgraph { - postgres: None, + redis: None, private_id: Some("sub".to_string()), enabled: true.into(), ttl: None, @@ -1348,7 +1351,7 @@ async fn private_and_public() { .into_iter() .collect(); let response_cache = - ResponseCache::for_test(storage.clone(), map, valid_schema.clone(), true, false) + ResponseCache::for_test(storage.clone(), map, valid_schema.clone(), true, drop_tx) .await .unwrap(); @@ -1528,14 +1531,15 @@ async fn polymorphic_private_and_public() { }, }); - let storage = Storage::new(&Config::test("polymorphic_private_and_public")) + let (drop_tx, drop_rx) = tokio::sync::broadcast::channel(2); + let storage = Storage::new(&Config::test(false,"polymorphic_private_and_public"), drop_rx) .await .unwrap(); let map = [ ( "user".to_string(), Subgraph { - postgres: None, + redis: None, private_id: Some("sub".to_string()), enabled: true.into(), ttl: None, @@ -1545,7 +1549,7 @@ async fn polymorphic_private_and_public() { ( "orga".to_string(), Subgraph { - postgres: None, + redis: None, private_id: Some("sub".to_string()), enabled: true.into(), ttl: None, @@ -1556,7 +1560,7 @@ async fn polymorphic_private_and_public() { .into_iter() .collect(); let response_cache = - ResponseCache::for_test(storage.clone(), map, valid_schema.clone(), true, false) + ResponseCache::for_test(storage.clone(), map, valid_schema.clone(), true, drop_tx) .await .unwrap(); @@ -1914,14 +1918,15 @@ async fn private_without_private_id() { }, }); - let storage = Storage::new(&Config::test("private_without_private_id")) + let (drop_tx, drop_rx) = tokio::sync::broadcast::channel(2); + let storage = Storage::new(&Config::test(false,"private_without_private_id"), drop_rx) .await .unwrap(); let map = [ ( "user".to_string(), Subgraph { - postgres: None, + redis: None, enabled: true.into(), ttl: None, ..Default::default() @@ -1930,7 +1935,7 @@ async fn private_without_private_id() { ( "orga".to_string(), Subgraph { - postgres: None, + redis: None, enabled: true.into(), ttl: None, ..Default::default() @@ -1940,7 +1945,7 @@ async fn private_without_private_id() { .into_iter() .collect(); let response_cache = - ResponseCache::for_test(storage.clone(), map, valid_schema.clone(), true, false) + ResponseCache::for_test(storage.clone(), map, valid_schema.clone(), true, drop_tx) .await .unwrap(); @@ -2084,12 +2089,15 @@ async fn no_data() { ).with_header(CACHE_CONTROL, HeaderValue::from_static("public, max-age=3600")).build()) ].into_iter().collect()); - let storage = Storage::new(&Config::test("no_data")).await.unwrap(); + let (drop_tx, drop_rx) = tokio::sync::broadcast::channel(2); + let storage = Storage::new(&Config::test(false, "no_data"), drop_rx) + .await + .unwrap(); let map = [ ( "user".to_string(), Subgraph { - postgres: None, + redis: None, private_id: Some("sub".to_string()), enabled: true.into(), ttl: None, @@ -2099,7 +2107,7 @@ async fn no_data() { ( "orga".to_string(), Subgraph { - postgres: None, + redis: None, private_id: Some("sub".to_string()), enabled: true.into(), ttl: None, @@ -2109,8 +2117,9 @@ async fn no_data() { ] .into_iter() .collect(); + let response_cache = - ResponseCache::for_test(storage.clone(), map, valid_schema.clone(), true, false) + ResponseCache::for_test(storage.clone(), map, valid_schema.clone(), true, drop_tx) .await .unwrap(); @@ -2320,14 +2329,11 @@ async fn missing_entities() { ).with_header(CACHE_CONTROL, HeaderValue::from_static("public, max-age=3600")).build()) ].into_iter().collect()); - let storage = Storage::new(&Config::test("missing_entities")) - .await - .unwrap(); let map = [ ( "user".to_string(), Subgraph { - postgres: None, + redis: None, private_id: Some("sub".to_string()), enabled: true.into(), ttl: None, @@ -2337,7 +2343,7 @@ async fn missing_entities() { ( "orga".to_string(), Subgraph { - postgres: None, + redis: None, private_id: Some("sub".to_string()), enabled: true.into(), ttl: None, @@ -2347,8 +2353,13 @@ async fn missing_entities() { ] .into_iter() .collect(); + + let (drop_tx, drop_rx) = tokio::sync::broadcast::channel(2); + let storage = Storage::new(&Config::test(false, "missing_entities"), drop_rx) + .await + .unwrap(); let response_cache = - ResponseCache::for_test(storage.clone(), map, valid_schema.clone(), true, false) + ResponseCache::for_test(storage.clone(), map, valid_schema.clone(), true, drop_tx) .await .unwrap(); @@ -2376,12 +2387,16 @@ async fn missing_entities() { assert!(remove_debug_extensions_key(&mut response)); insta::assert_json_snapshot!(response); + let (drop_tx, drop_rx) = tokio::sync::broadcast::channel(2); + let storage = Storage::new(&Config::test(false, "missing_entities"), drop_rx) + .await + .unwrap(); let response_cache = ResponseCache::for_test( storage.clone(), HashMap::new(), valid_schema.clone(), false, - false, + drop_tx, ) .await .unwrap(); @@ -2482,14 +2497,15 @@ async fn invalidate_by_cache_tag() { }, }); - let storage = Storage::new(&Config::test("test_invalidate_by_cache_tag")) + let (drop_tx, drop_rx) = tokio::sync::broadcast::channel(2); + let storage = Storage::new(&Config::test(false,"test_invalidate_by_cache_tag"), drop_rx) .await .unwrap(); let map = [ ( "user".to_string(), Subgraph { - postgres: None, + redis: None, private_id: Some("sub".to_string()), enabled: true.into(), ttl: None, @@ -2499,7 +2515,7 @@ async fn invalidate_by_cache_tag() { ( "orga".to_string(), Subgraph { - postgres: None, + redis: None, private_id: Some("sub".to_string()), enabled: true.into(), ttl: None, @@ -2510,7 +2526,7 @@ async fn invalidate_by_cache_tag() { .into_iter() .collect(); let response_cache = - ResponseCache::for_test(storage.clone(), map, valid_schema.clone(), true, false) + ResponseCache::for_test(storage.clone(), map, valid_schema.clone(), true, drop_tx) .await .unwrap(); @@ -2696,14 +2712,15 @@ async fn invalidate_by_type() { }, }); - let storage = Storage::new(&Config::test("test_invalidate_by_subgraph")) + let (drop_tx, drop_rx) = tokio::sync::broadcast::channel(2); + let storage = Storage::new(&Config::test(false,"test_invalidate_by_subgraph"), drop_rx) .await .unwrap(); let map = [ ( "user".to_string(), Subgraph { - postgres: None, + redis: None, private_id: Some("sub".to_string()), enabled: true.into(), ttl: None, @@ -2713,7 +2730,7 @@ async fn invalidate_by_type() { ( "orga".to_string(), Subgraph { - postgres: None, + redis: None, private_id: Some("sub".to_string()), enabled: true.into(), ttl: None, @@ -2724,7 +2741,7 @@ async fn invalidate_by_type() { .into_iter() .collect(); let response_cache = - ResponseCache::for_test(storage.clone(), map, valid_schema.clone(), true, false) + ResponseCache::for_test(storage.clone(), map, valid_schema.clone(), true, drop_tx) .await .unwrap(); @@ -2874,68 +2891,6 @@ async fn invalidate_by_type() { }.with_metrics().await; } -#[tokio::test(flavor = "multi_thread")] -async fn interval_cleanup_config() { - let valid_schema = Arc::new(Schema::parse_and_validate(SCHEMA, "test.graphql").unwrap()); - - let storage = Storage::new(&Config { - cleanup_interval: std::time::Duration::from_secs(60 * 7), // Every 7 minutes - ..Config::test("interval_cleanup_config_1") - }) - .await - .unwrap(); - let _response_cache = ResponseCache::for_test( - storage.clone(), - Default::default(), - valid_schema.clone(), - true, - true, - ) - .await - .unwrap(); - - let cron = storage.get_cron().await.unwrap(); - assert_eq!(cron.0, String::from("*/7 * * * *")); - - let storage = Storage::new(&Config { - cleanup_interval: std::time::Duration::from_secs(60 * 60 * 7), // Every 7 hours - ..Config::test("interval_cleanup_config_2") - }) - .await - .unwrap(); - let _response_cache = ResponseCache::for_test( - storage.clone(), - Default::default(), - valid_schema.clone(), - true, - true, - ) - .await - .unwrap(); - - let cron = storage.get_cron().await.unwrap(); - assert_eq!(cron.0, String::from("0 */7 * * *")); - - let storage = Storage::new(&Config { - cleanup_interval: std::time::Duration::from_secs(60 * 60 * 24 * 7), // Every 7 days - ..Config::test("interval_cleanup_config_2") - }) - .await - .unwrap(); - let _response_cache = ResponseCache::for_test( - storage.clone(), - Default::default(), - valid_schema.clone(), - true, - true, - ) - .await - .unwrap(); - - let cron = storage.get_cron().await.unwrap(); - assert_eq!(cron.0, String::from("0 0 */7 * *")); -} - #[tokio::test] async fn failure_mode() { async { @@ -2974,7 +2929,7 @@ async fn failure_mode() { ( "user".to_string(), Subgraph { - postgres: None, + redis: None, private_id: Some("sub".to_string()), enabled: true.into(), ttl: None, @@ -2984,7 +2939,7 @@ async fn failure_mode() { ( "orga".to_string(), Subgraph { - postgres: None, + redis: None, private_id: Some("sub".to_string()), enabled: true.into(), ttl: None, @@ -3109,45 +3064,6 @@ async fn failure_mode() { .await; } -#[tokio::test(flavor = "multi_thread")] -async fn expired_data_count() { - async { - let valid_schema = Arc::new(Schema::parse_and_validate(SCHEMA, "test.graphql").unwrap()); - - let storage = Storage::new(&Config::test("expired_data_count")) - .await - .unwrap(); - let _response_cache = ResponseCache::for_test( - storage.clone(), - Default::default(), - valid_schema.clone(), - true, - true, - ) - .await - .unwrap(); - let cache_key = uuid::Uuid::new_v4().to_string(); - let document = Document { - key: cache_key, - data: Default::default(), - control: CacheControl::default(), - invalidation_keys: vec![], - expire: Duration::from_millis(2), - }; - storage.insert(document, "test").await.unwrap(); - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - let (_drop_rx, drop_tx) = broadcast::channel(2); - tokio::spawn( - metrics::expired_data_task(storage.clone(), drop_tx, None) - .with_current_meter_provider(), - ); - tokio::time::sleep(std::time::Duration::from_millis(100)).await; - assert_gauge!("apollo.router.response_cache.data.expired", 1); - } - .with_metrics() - .await; -} - #[tokio::test] async fn failure_mode_reconnect() { async { @@ -3186,7 +3102,7 @@ async fn failure_mode_reconnect() { ( "user".to_string(), Subgraph { - postgres: None, + redis: None, private_id: Some("sub".to_string()), enabled: true.into(), ttl: None, @@ -3196,7 +3112,7 @@ async fn failure_mode_reconnect() { ( "orga".to_string(), Subgraph { - postgres: None, + redis: None, private_id: Some("sub".to_string()), enabled: true.into(), ttl: None, @@ -3206,10 +3122,10 @@ async fn failure_mode_reconnect() { ] .into_iter() .collect(); - let storage = Storage::new(&Config::test("failure_mode_reconnect")) + let (_drop_tx, drop_rx) = tokio::sync::broadcast::channel(2); + let storage = Storage::new(&Config::test(false,"failure_mode_reconnect"), drop_rx) .await .unwrap(); - storage.migrate().await.unwrap(); storage.truncate_namespace().await.unwrap(); let response_cache = diff --git a/apollo-router/src/uplink/testdata/restricted.router.yaml b/apollo-router/src/uplink/testdata/restricted.router.yaml index efc54c336f..15a8fc2ab5 100644 --- a/apollo-router/src/uplink/testdata/restricted.router.yaml +++ b/apollo-router/src/uplink/testdata/restricted.router.yaml @@ -60,8 +60,8 @@ preview_response_cache: subgraph: all: enabled: true - postgres: - url: "postgres://localhost:5432" + redis: + urls: [ "redis://localhost:6379" ] ttl: 360s # overrides the global TTL telemetry: diff --git a/apollo-router/tests/integration/README.md b/apollo-router/tests/integration/README.md index 246e77c530..7af85d1f75 100644 --- a/apollo-router/tests/integration/README.md +++ b/apollo-router/tests/integration/README.md @@ -1,6 +1,7 @@ # Apollo Router Integration Tests -This directory contains integration tests for the Apollo Router. These tests verify the router's behavior in realistic scenarios by starting actual router processes and testing their functionality. +This directory contains integration tests for the Apollo Router. These tests verify the router's behavior in realistic +scenarios by starting actual router processes and testing their functionality. ## Prerequisites @@ -12,28 +13,37 @@ docker-compose up -d ``` **For enterprise feature tests** (optional): + ```shell export TEST_APOLLO_KEY="your-apollo-api-key" export TEST_APOLLO_GRAPH_REF="your-graph-ref@variant" ``` -Without these environment variables, enterprise tests will be skipped. See notes in the [development README](../../DEVELOPMENT.md#testing) for nuances of how tests are run. +Without these environment variables, enterprise tests will be skipped. See notes in +the [development README](../../DEVELOPMENT.md#testing) for nuances of how tests are run. ## Test Categories **Standard tests:** Work with just Docker Compose services (most will fail if services aren't running) **Enterprise tests:** Require GraphOS credentials (entity caching, Apollo reporting, etc.) -**CI-only tests:** Some Redis/PostgreSQL tests only run on Linux x86_64 in CI +**CI-only tests:** Some Redis tests only run on Linux x86_64 in CI -**Note:** Most integration tests are configured with `required_to_start: true` for Redis/PostgreSQL, meaning they will fail at startup if these services aren't available. A few tests use `required_to_start: false` and will continue without the services. +**Note:** Most integration tests are configured with `required_to_start: true` for Redis, meaning they will +fail at startup if these services aren't available. A few tests use `required_to_start: false` and will continue without +the services. ## Using `xtask` -See the [xtask README](../../xtask/README.md) for the commands that are useful for checks, linting, and running the tests. Using `xtask` is an easy way to run the integration tests! +See the [xtask README](../../xtask/README.md) for the commands that are useful for checks, linting, and running the +tests. Using `xtask` is an easy way to run the integration tests! ## Using `cargo nextest` -We use [nextest](https://nexte.st/) as the recommended test runner. See the [development README](../../DEVELOPMENT.md#testing) for details, but nextest provides faster and more reliable test execution! There are more details there on how to install `nextest` and use it to target all integration tests or individual tests, along with using `RUST_LOG`. See the section below on log-levels to understand why you might want to use different levels while testing (especially if you encounter flaky tests locally). +We use [nextest](https://nexte.st/) as the recommended test runner. See +the [development README](../../DEVELOPMENT.md#testing) for details, but nextest provides faster and more reliable test +execution! There are more details there on how to install `nextest` and use it to target all integration tests or +individual tests, along with using `RUST_LOG`. See the section below on log-levels to understand why you might want to +use different levels while testing (especially if you encounter flaky tests locally). ## Using `cargo` to run the integration tests @@ -45,10 +55,10 @@ Running all of the integration tests is fairly straightforward: cargo test --test integration_tests ``` - ### Running Individual Integration Tests -Sometimes, though, you might want to work on a specific integration test. You can run specific test modules or individual tests like this: +Sometimes, though, you might want to work on a specific integration test. You can run specific test modules or +individual tests like this: ```shell # Run all lifecycle module tests @@ -60,7 +70,8 @@ cargo test --test integration_tests integration::lifecycle::test_happy ## Log-level configuration -Integration tests often require examining log output for debugging. The `RUST_LOG` environment variable controls logging verbosity. +Integration tests often require examining log output for debugging. The `RUST_LOG` environment variable controls logging +verbosity. ### Common Log Level Examples @@ -74,9 +85,12 @@ RUST_LOG=debug cargo test --test integration_tests integration::lifecycle::test_ ### Reducing Log Noise -Some third-party libraries produce excessive debug output that can obscure useful information or take a significant amount of processing time. This can be a problem! We match on a magic string to denote a healthy router coming online, and if a dependency is producing more logs than we can process before timing out that check, tests will fail. +Some third-party libraries produce excessive debug output that can obscure useful information or take a significant +amount of processing time. This can be a problem! We match on a magic string to denote a healthy router coming online, +and if a dependency is producing more logs than we can process before timing out that check, tests will fail. -So, if you're experiencing flaky tests while using debug-level logging, excessive logs might be the culprit. Find the noisiest dependency and tune its level: +So, if you're experiencing flaky tests while using debug-level logging, excessive logs might be the culprit. Find the +noisiest dependency and tune its level: ```shell # Reduce verbose jsonpath_lib logging while keeping debug for everything else diff --git a/apollo-router/tests/integration/mod.rs b/apollo-router/tests/integration/mod.rs index 0106efc700..7ee952595c 100644 --- a/apollo-router/tests/integration/mod.rs +++ b/apollo-router/tests/integration/mod.rs @@ -25,7 +25,7 @@ mod supergraph; mod traffic_shaping; mod typename; -// In the CI environment we only install PostgreSQL on x86_64 Linux +// In the CI environment we only install Redis on x86_64 Linux #[cfg(any(not(feature = "ci"), all(target_arch = "x86_64", target_os = "linux")))] mod response_cache; // In the CI environment we only install Redis on x86_64 Linux diff --git a/apollo-router/tests/integration/response_cache.rs b/apollo-router/tests/integration/response_cache.rs index 18e52a8a4e..89ef0a85bf 100644 --- a/apollo-router/tests/integration/response_cache.rs +++ b/apollo-router/tests/integration/response_cache.rs @@ -7,21 +7,25 @@ use apollo_router::graphql; use apollo_router::services::router; use apollo_router::services::supergraph; use apollo_router::test_harness::HttpService; +use fred::clients::Client; +use fred::interfaces::ClientLike; +use fred::interfaces::KeysInterface; +use fred::types::Builder; use http::HeaderMap; use http::HeaderValue; use http_body_util::BodyExt as _; use indexmap::IndexMap; use serde_json::Value; use serde_json::json; -use sqlx::Connection; -use sqlx::FromRow; -use sqlx::PgConnection; +use tokio::time::sleep; +use tokio_util::future::FutureExt; use tower::BoxError; use tower::Service as _; use tower::ServiceExt as _; use crate::integration::common::graph_os_enabled; +const REDIS_URL: &str = "redis://127.0.0.1:6379"; const INVALIDATION_PATH: &str = "/invalidation"; const INVALIDATION_SHARED_KEY: &str = "supersecret"; @@ -30,6 +34,13 @@ pub(crate) fn namespace() -> String { uuid::Uuid::new_v4().simple().to_string() } +async fn redis_client() -> Result { + let client = + Builder::from_config(fred::prelude::Config::from_url(REDIS_URL).unwrap()).build()?; + client.init().await?; + Ok(client) +} + fn base_config() -> Value { json!({ "include_subgraph_errors": { @@ -39,8 +50,8 @@ fn base_config() -> Value { "enabled": true, "subgraph": { "all": { - "postgres": { - "url": "postgres://127.0.0.1", + "redis": { + "urls": ["redis://127.0.0.1:6379"], "pool_size": 3, "namespace": namespace(), "required_to_start": true, @@ -69,8 +80,8 @@ fn failure_config() -> Value { "enabled": true, "subgraph": { "all": { - "postgres": { - "url": "postgres://test", + "redis": { + "urls": ["redis://invalid"], "pool_size": 3, "namespace": namespace(), "required_to_start": false, @@ -502,31 +513,34 @@ fn parse_max_age(cache_control: &str) -> u32 { .unwrap_or_else(|| panic!("expected 'max-age={{seconds}},public', got '{cache_control}'")) } -#[derive(FromRow)] -struct Record { - data: String, -} - macro_rules! check_cache_key { - ($cache_key: expr, $conn: expr) => { - let mut record = None; - // Because insert is async + ($namespace: expr, $cache_key: expr, $client: expr) => { + let mut record: Option = None; + let key = format!("{}:{}", $namespace, $cache_key); + // Retry a few times because insert is asynchronous for _ in 0..10 { - if let Ok(resp) = sqlx::query_as!( - Record, - "SELECT data FROM cache WHERE cache_key = $1", - $cache_key - ) - .fetch_one(&mut $conn) - .await + match $client + .get(key.clone()) + .timeout(Duration::from_secs(5)) + .await { - record = Some(resp); - break; + Ok(Ok(resp)) => { + record = Some(resp); + break; + } + Ok(Err(_)) => { + sleep(Duration::from_secs(1)).await; + } + Err(_) => { + panic!("long timeout connecting to redis - did you call client.init()?"); + } } } + match record { Some(s) => { - let v: Value = serde_json::from_str(&s.data).unwrap(); + let cache_value: Value = serde_json::from_str(&s).unwrap(); + let v: Value = cache_value.get("data").unwrap().clone(); insta::assert_json_snapshot!(v); } None => panic!("cannot get cache key {}", $cache_key), @@ -534,15 +548,24 @@ macro_rules! check_cache_key { }; } +async fn cache_key_exists( + namespace: &str, + cache_key: &str, + client: &Client, +) -> Result { + let key = format!("{namespace}:{cache_key}"); + let count: u32 = client.exists(key).await?; + Ok(count == 1) +} + #[tokio::test(flavor = "multi_thread")] async fn integration_test_basic() -> Result<(), BoxError> { if !graph_os_enabled() { return Ok(()); } let namespace = namespace(); + let client = redis_client().await?; - let mut conn = PgConnection::connect("postgres://127.0.0.1").await?; - sqlx::migrate!().run(&mut conn).await.unwrap(); let subgraphs = json!({ "products": { "query": {"topProducts": [{ @@ -611,8 +634,8 @@ async fn integration_test_basic() -> Result<(), BoxError> { "subgraph": { "all": { "enabled": true, - "postgres": { - "url": "postgres://127.0.0.1", + "redis": { + "urls": ["redis://127.0.0.1:6379"], "namespace": namespace, "pool_size": 3 }, @@ -656,16 +679,11 @@ async fn integration_test_basic() -> Result<(), BoxError> { ".extensions.apolloCacheDebugging.data[].cacheControl.created" => 0 }); - let cache_key = format!( - "{namespace}-version:1.0:subgraph:products:type:Query:hash:bf44683f0c222652b509d6efb8f324610c8671181de540a96a5016bd71daa7cc:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6" - ); + let cache_key = "version:1.0:subgraph:products:type:Query:hash:bf44683f0c222652b509d6efb8f324610c8671181de540a96a5016bd71daa7cc:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6"; + check_cache_key!(&namespace, cache_key, &client); - check_cache_key!(&cache_key, conn); - - let cache_key = format!( - "{namespace}-version:1.0:subgraph:reviews:type:Product:entity:cf4952a1e511b1bf2561a6193b4cdfc95f265a79e5cae4fd3e46fd9e75bc512f:representation::hash:06a24c8b3861c95f53d224071ee9627ee81b4826d23bc3de69bdc0031edde6ed:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6" - ); - check_cache_key!(&cache_key, conn); + let cache_key = "version:1.0:subgraph:reviews:type:Product:entity:cf4952a1e511b1bf2561a6193b4cdfc95f265a79e5cae4fd3e46fd9e75bc512f:representation::hash:06a24c8b3861c95f53d224071ee9627ee81b4826d23bc3de69bdc0031edde6ed:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6"; + check_cache_key!(&namespace, cache_key, &client); let supergraph = apollo_router::TestHarness::builder() .configuration_json(json!({ @@ -679,8 +697,8 @@ async fn integration_test_basic() -> Result<(), BoxError> { "subgraph": { "all": { "enabled": false, - "postgres": { - "url": "postgres://127.0.0.1", + "redis": { + "urls": ["redis://127.0.0.1:6379"], "namespace": namespace, }, }, @@ -722,10 +740,8 @@ async fn integration_test_basic() -> Result<(), BoxError> { ".extensions.apolloCacheDebugging.data[].cacheControl.created" => 0 }); - let cache_key = format!( - "{namespace}-version:1.0:subgraph:reviews:type:Product:entity:cf4952a1e511b1bf2561a6193b4cdfc95f265a79e5cae4fd3e46fd9e75bc512f:representation::hash:06a24c8b3861c95f53d224071ee9627ee81b4826d23bc3de69bdc0031edde6ed:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6" - ); - check_cache_key!(&cache_key, conn); + let cache_key = "version:1.0:subgraph:reviews:type:Product:entity:cf4952a1e511b1bf2561a6193b4cdfc95f265a79e5cae4fd3e46fd9e75bc512f:representation::hash:06a24c8b3861c95f53d224071ee9627ee81b4826d23bc3de69bdc0031edde6ed:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6"; + check_cache_key!(&namespace, cache_key, &client); const SECRET_SHARED_KEY: &str = "supersecret"; let http_service = apollo_router::TestHarness::builder() @@ -739,8 +755,8 @@ async fn integration_test_basic() -> Result<(), BoxError> { "subgraph": { "all": { "enabled": true, - "postgres": { - "url": "postgres://127.0.0.1", + "redis": { + "urls": ["redis://127.0.0.1:6379"], "namespace": namespace, }, "invalidation": { @@ -819,33 +835,12 @@ async fn integration_test_basic() -> Result<(), BoxError> { assert!(response_status.is_success()); // This should be in error because we invalidated this entity - let cache_key = format!( - "{namespace}-version:1.0:subgraph:reviews:type:Product:entity:cf4952a1e511b1bf2561a6193b4cdfc95f265a79e5cae4fd3e46fd9e75bc512f:representation::hash:06a24c8b3861c95f53d224071ee9627ee81b4826d23bc3de69bdc0031edde6ed:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6" - ); - assert!( - sqlx::query_as!( - Record, - "SELECT data FROM cache WHERE cache_key = $1", - cache_key - ) - .fetch_one(&mut conn) - .await - .is_err() - ); + let cache_key = "version:1.0:subgraph:reviews:type:Product:entity:cf4952a1e511b1bf2561a6193b4cdfc95f265a79e5cae4fd3e46fd9e75bc512f:representation::hash:06a24c8b3861c95f53d224071ee9627ee81b4826d23bc3de69bdc0031edde6ed:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6"; + assert!(!cache_key_exists(&namespace, cache_key, &client).await?); + // This entry should still be in redis because we didn't invalidate this entry - let cache_key = format!( - "{namespace}-version:1.0:subgraph:products:type:Query:hash:bf44683f0c222652b509d6efb8f324610c8671181de540a96a5016bd71daa7cc:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6" - ); - assert!( - sqlx::query_as!( - Record, - "SELECT data FROM cache WHERE cache_key = $1", - cache_key - ) - .fetch_one(&mut conn) - .await - .is_ok() - ); + let cache_key = "version:1.0:subgraph:products:type:Query:hash:bf44683f0c222652b509d6efb8f324610c8671181de540a96a5016bd71daa7cc:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6"; + assert!(cache_key_exists(&namespace, cache_key, &client).await?); Ok(()) } @@ -858,8 +853,7 @@ async fn integration_test_with_nested_field_set() -> Result<(), BoxError> { let namespace = namespace(); let schema = include_str!("../../src/testdata/supergraph_nested_fields.graphql"); - let mut conn = PgConnection::connect("postgres://127.0.0.1").await?; - sqlx::migrate!().run(&mut conn).await.unwrap(); + let client = redis_client().await?; let subgraphs = json!({ "products": { @@ -896,8 +890,8 @@ async fn integration_test_with_nested_field_set() -> Result<(), BoxError> { "subgraph": { "all": { "enabled": true, - "postgres": { - "url": "postgres://127.0.0.1", + "redis": { + "urls": ["redis://127.0.0.1:6379"], "namespace": namespace, "pool_size": 3 }, @@ -931,15 +925,11 @@ async fn integration_test_with_nested_field_set() -> Result<(), BoxError> { ".extensions.apolloCacheDebugging.data[].cacheControl.created" => 0 }); - let cache_key = format!( - "{namespace}-version:1.0:subgraph:products:type:Query:hash:f4f41cfa309494d41648c3a3c398c61cb00197696102199454a25a0dcdd2f592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6" - ); - check_cache_key!(&cache_key, conn); + let cache_key = "version:1.0:subgraph:products:type:Query:hash:f4f41cfa309494d41648c3a3c398c61cb00197696102199454a25a0dcdd2f592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6"; + check_cache_key!(&namespace, cache_key, &client); - let cache_key = format!( - "{namespace}-version:1.0:subgraph:users:type:User:entity:b41dfad85edaabac7bb681098e9b23e21b3b8b9b8b1849babbd5a1300af64b43:representation:68fd4df7c06fd234bd0feb24e3300abcc06136ea8a9dd7533b7378f5fce7cfc4:hash:460b70e698b8c9d8496b0567e0f0848b9f7fef36e841a8a0b0771891150c35e5:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6" - ); - check_cache_key!(&cache_key, conn); + let cache_key = "version:1.0:subgraph:users:type:User:entity:b41dfad85edaabac7bb681098e9b23e21b3b8b9b8b1849babbd5a1300af64b43:representation:68fd4df7c06fd234bd0feb24e3300abcc06136ea8a9dd7533b7378f5fce7cfc4:hash:460b70e698b8c9d8496b0567e0f0848b9f7fef36e841a8a0b0771891150c35e5:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6"; + check_cache_key!(&namespace, cache_key, &client); let supergraph = apollo_router::TestHarness::builder() .configuration_json(json!({ @@ -953,8 +943,8 @@ async fn integration_test_with_nested_field_set() -> Result<(), BoxError> { "subgraph": { "all": { "enabled": false, - "postgres": { - "url": "postgres://127.0.0.1", + "redis": { + "urls": ["redis://127.0.0.1:6379"], "namespace": namespace, }, }, @@ -995,10 +985,8 @@ async fn integration_test_with_nested_field_set() -> Result<(), BoxError> { ".extensions.apolloCacheDebugging.data[].cacheControl.created" => 0 }); - let cache_key = format!( - "{namespace}-version:1.0:subgraph:users:type:User:entity:b41dfad85edaabac7bb681098e9b23e21b3b8b9b8b1849babbd5a1300af64b43:representation:68fd4df7c06fd234bd0feb24e3300abcc06136ea8a9dd7533b7378f5fce7cfc4:hash:460b70e698b8c9d8496b0567e0f0848b9f7fef36e841a8a0b0771891150c35e5:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6" - ); - check_cache_key!(&cache_key, conn); + let cache_key = "version:1.0:subgraph:users:type:User:entity:b41dfad85edaabac7bb681098e9b23e21b3b8b9b8b1849babbd5a1300af64b43:representation:68fd4df7c06fd234bd0feb24e3300abcc06136ea8a9dd7533b7378f5fce7cfc4:hash:460b70e698b8c9d8496b0567e0f0848b9f7fef36e841a8a0b0771891150c35e5:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6"; + check_cache_key!(&namespace, cache_key, &client); const SECRET_SHARED_KEY: &str = "supersecret"; let http_service = apollo_router::TestHarness::builder() @@ -1013,8 +1001,8 @@ async fn integration_test_with_nested_field_set() -> Result<(), BoxError> { "subgraph": { "all": { "enabled": true, - "postgres": { - "url": "postgres://127.0.0.1", + "redis": { + "urls": ["redis://127.0.0.1:6379"], "namespace": namespace, }, "invalidation": { @@ -1093,34 +1081,12 @@ async fn integration_test_with_nested_field_set() -> Result<(), BoxError> { assert!(response_status.is_success()); // This should be in error because we invalidated this entity - let cache_key = format!( - "{namespace}-version:1.0:subgraph:users:type:User:entity:b41dfad85edaabac7bb681098e9b23e21b3b8b9b8b1849babbd5a1300af64b43:representation:68fd4df7c06fd234bd0feb24e3300abcc06136ea8a9dd7533b7378f5fce7cfc4:hash:460b70e698b8c9d8496b0567e0f0848b9f7fef36e841a8a0b0771891150c35e5:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6" - ); - assert!( - sqlx::query_as!( - Record, - "SELECT data FROM cache WHERE cache_key = $1", - cache_key - ) - .fetch_one(&mut conn) - .await - .is_err() - ); + let cache_key = "version:1.0:subgraph:users:type:User:entity:b41dfad85edaabac7bb681098e9b23e21b3b8b9b8b1849babbd5a1300af64b43:representation:68fd4df7c06fd234bd0feb24e3300abcc06136ea8a9dd7533b7378f5fce7cfc4:hash:460b70e698b8c9d8496b0567e0f0848b9f7fef36e841a8a0b0771891150c35e5:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6"; + assert!(!cache_key_exists(&namespace, cache_key, &client).await?); // This entry should still be in redis because we didn't invalidate this entry - let cache_key = format!( - "{namespace}-version:1.0:subgraph:products:type:Query:hash:f4f41cfa309494d41648c3a3c398c61cb00197696102199454a25a0dcdd2f592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6" - ); - assert!( - sqlx::query_as!( - Record, - "SELECT data FROM cache WHERE cache_key = $1", - cache_key - ) - .fetch_one(&mut conn) - .await - .is_ok() - ); + let cache_key = "version:1.0:subgraph:products:type:Query:hash:f4f41cfa309494d41648c3a3c398c61cb00197696102199454a25a0dcdd2f592:data:070af9367f9025bd796a1b7e0cd1335246f658aa4857c3a4d6284673b7d07fa6"; + assert!(cache_key_exists(&namespace, cache_key, &client).await?); Ok(()) } diff --git a/docs/source/routing/observability/graphos/graphos-reporting.mdx b/docs/source/routing/observability/graphos/graphos-reporting.mdx index 8a47eab746..87b6036b23 100644 --- a/docs/source/routing/observability/graphos/graphos-reporting.mdx +++ b/docs/source/routing/observability/graphos/graphos-reporting.mdx @@ -423,12 +423,10 @@ with additional attributes including the `service` and `code` found in [GraphQL Additional diagnostic capabilities available in Studio now support viewing errors by `service` or `code`. The `service` dimension refers to the subgraph or connector where the error originated from. The `code` refers to the specific type of error that was raised by the router, federated subgraphs, or connectors. -##### Cardinality limitations +#### Cardinality limitations -At scale, this feature is known to hit cardinality limitations in the OTel reporting agent. When this happens, some of the extended metrics attributes may no longer -be visible in Studio. To reduce cardinality warnings in your logs, adjust the Apollo metrics OTLP batch processor configuration to send reports more frequently. - -Additionally, this feature may increase the Router's memory usage profile. Similarly, lowering the `scheduled_delay` can help to alleviate that as well. See the example below. +At scale, some metrics are known to hit cardinality limitations in the OTel reporting agent. When this happens, some of the metrics' attributes may no longer +be visible in Studio. To reduce cardinality warnings in your logs, adjust the Apollo metrics OTLP batch processor configuration to send reports more frequently. You can lower the `scheduled_delay` to reduce the cardinality stored in the router before the metric is flushed, as shown in the example: ```yaml title="router.yaml" telemetry: From dca190c3e8dc7aaa6bcd567099a9f3b3d50a8d32 Mon Sep 17 00:00:00 2001 From: Nick Marsh Date: Fri, 3 Oct 2025 11:15:08 +1000 Subject: [PATCH 15/16] Update selectors.mdx --- .../enabling-telemetry/selectors.mdx | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/selectors.mdx b/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/selectors.mdx index 42c40daafd..f252aa344e 100644 --- a/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/selectors.mdx +++ b/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/selectors.mdx @@ -105,6 +105,15 @@ The subgraph service executes multiple times during query execution, with each e | `error` | No | `reason` | A string value containing error reason when it's a critical error | | `cache` | No | `hit` \| `miss` | Returns the number of cache hit or miss for this subgraph request | +### HTTP Client + +The HTTP client service also executes multiple times, with each execution representing a HTTP request to a single subgraph. Importantly, this service executes after any Rhai scripts that modify the subgraph requests, so these selectors can be used to observe any headers added by the Rhai script. + +| Selector | Defaultable | Values | Description | +|--------------------|-------------|--------|-------------------------------| +| `request_header` | Yes | | The name of a request header | +| `response_header` | Yes | | The name of a response header | + ### Connector #### HTTP From ed6f1446f1711ba224a8de81790aa170d7df4d52 Mon Sep 17 00:00:00 2001 From: Bryn Cooke Date: Fri, 3 Oct 2025 09:08:32 +0100 Subject: [PATCH 16/16] Update docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/selectors.mdx --- .../router-telemetry-otel/enabling-telemetry/selectors.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/selectors.mdx b/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/selectors.mdx index f252aa344e..1b51256976 100644 --- a/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/selectors.mdx +++ b/docs/source/routing/observability/router-telemetry-otel/enabling-telemetry/selectors.mdx @@ -107,7 +107,7 @@ The subgraph service executes multiple times during query execution, with each e ### HTTP Client -The HTTP client service also executes multiple times, with each execution representing a HTTP request to a single subgraph. Importantly, this service executes after any Rhai scripts that modify the subgraph requests, so these selectors can be used to observe any headers added by the Rhai script. +The HTTP client service also executes multiple times, with each execution representing a HTTP request to a single subgraph or REST service. Importantly, this service executes after any Rhai scripts that modify the subgraph requests, so these selectors can be used to observe any headers added by the Rhai script. | Selector | Defaultable | Values | Description | |--------------------|-------------|--------|-------------------------------|