[Subgraph Insights] Add Apollo Subgraph Fetch Histogram to Telemetry Plugin

.changesets/feat_rreg_experimental_subgraph_metrics.md

@@ -0,0 +1,18 @@
+### [Subgraph Insights] Experimental Apollo Subgraph Fetch Histogram ([PR #8013](https://github.com/apollographql/router/pull/8013))
+
+This change adds a new, experimental histogram to capture subgraph fetch duration for GraphOS. This will
+eventually be used to power subgraph-level insights in Apollo Studio.
+
+This can be toggled on using a new boolean config flag:
+
+```yaml
+telemetry:
+  apollo:
+    experimental_subgraph_metrics: true
+```
+
+The new instrument is only sent to GraphOS and is not available in 3rd-party OTel export targets. It is not currently 
+customizable. Users requiring a customizable alternative can use the existing `http.client.request.duration` 
+instrument, which measures the same value.
+
+By [@rregitsky](https://github.com/rregitsky) in https://github.com/apollographql/router/pull/8013

apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap

@@ -1,6 +1,7 @@
 ---
 source: apollo-router/src/configuration/tests.rs
 expression: "&schema"
+snapshot_kind: text
 ---
 {
   "$schema": "http://json-schema.org/draft-07/schema#",
@@ -1496,6 +1497,11 @@ expression: "&schema"
           "$ref": "#/definitions/Protocol",
           "description": "#/definitions/Protocol"
         },
+        "experimental_subgraph_metrics": {
+          "default": false,
+          "description": "Enable sending additional subgraph metrics to Apollo Studio via OTLP",
+          "type": "boolean"
+        },
         "field_level_instrumentation_sampler": {
           "$ref": "#/definitions/SamplerOption",
           "description": "#/definitions/SamplerOption"

apollo-router/src/metrics/filter.rs

@@ -89,7 +89,8 @@ impl FilterMeterProvider {
     }
 
     fn get_private_realtime_regex() -> Regex {
-        Regex::new(r"apollo\.router\.operations\.error").expect("regex should have been valid")
+        Regex::new(r"apollo\.router\.operations\.(?:error|fetch\.duration)")
+            .expect("regex should have been valid")
     }
 
     pub(crate) fn private_realtime<T: Into<MeterProvider>>(delegate: T) -> Self {

apollo-router/src/plugins/telemetry/apollo.rs

@@ -119,6 +119,9 @@ pub(crate) struct Config {
 
     /// Enable field metrics that are generated without FTV1 to be sent to Apollo Studio.
     pub(crate) experimental_local_field_metrics: bool,
+
+    /// Enable sending additional subgraph metrics to Apollo Studio via OTLP
+    pub(crate) experimental_subgraph_metrics: bool,
 }
 
 #[derive(Debug, Clone, Deserialize, JsonSchema, Default)]
@@ -253,6 +256,7 @@ impl Default for Config {
             signature_normalization_algorithm: ApolloSignatureNormalizationAlgorithm::default(),
             experimental_local_field_metrics: false,
             metrics_reference_mode: ApolloMetricsReferenceMode::default(),
+            experimental_subgraph_metrics: false,
         }
     }
 }

apollo-router/src/plugins/telemetry/config_new/apollo/instruments.rs

@@ -0,0 +1,176 @@
+use std::collections::HashMap;
+use std::sync::Arc;
+
+use opentelemetry::metrics::MeterProvider;
+use tokio::time::Instant;
+use tower::BoxError;
+
+use crate::Context;
+use crate::metrics;
+use crate::plugins::telemetry::APOLLO_CLIENT_NAME_ATTRIBUTE;
+use crate::plugins::telemetry::APOLLO_CLIENT_VERSION_ATTRIBUTE;
+use crate::plugins::telemetry::APOLLO_HAS_ERRORS_ATTRIBUTE;
+use crate::plugins::telemetry::APOLLO_OPERATION_ID_ATTRIBUTE;
+use crate::plugins::telemetry::CLIENT_NAME;
+use crate::plugins::telemetry::CLIENT_VERSION;
+use crate::plugins::telemetry::GRAPHQL_OPERATION_NAME_ATTRIBUTE;
+use crate::plugins::telemetry::GRAPHQL_OPERATION_TYPE_ATTRIBUTE;
+use crate::plugins::telemetry::apollo::Config;
+use crate::plugins::telemetry::config_new::attributes::StandardAttribute;
+use crate::plugins::telemetry::config_new::extendable::Extendable;
+use crate::plugins::telemetry::config_new::instruments::APOLLO_ROUTER_OPERATIONS_FETCH_DURATION;
+use crate::plugins::telemetry::config_new::instruments::CustomHistogram;
+use crate::plugins::telemetry::config_new::instruments::Increment;
+use crate::plugins::telemetry::config_new::instruments::Instrumented;
+use crate::plugins::telemetry::config_new::instruments::METER_NAME;
+use crate::plugins::telemetry::config_new::instruments::StaticInstrument;
+use crate::plugins::telemetry::config_new::selectors::OperationKind;
+use crate::plugins::telemetry::config_new::selectors::OperationName;
+use crate::plugins::telemetry::config_new::subgraph::attributes::SubgraphAttributes;
+use crate::plugins::telemetry::config_new::subgraph::selectors::SubgraphSelector;
+use crate::query_planner::APOLLO_OPERATION_ID;
+use crate::services::subgraph;
+
+pub(crate) struct ApolloSubgraphInstruments {
+    pub(crate) apollo_router_operations_fetch_duration: Option<
+        CustomHistogram<
+            subgraph::Request,
+            subgraph::Response,
+            (),
+            SubgraphAttributes,
+            SubgraphSelector,
+        >,
+    >,
+}
+
+impl ApolloSubgraphInstruments {
+    pub(crate) fn new(
+        static_instruments: Arc<HashMap<String, StaticInstrument>>,
+        apollo_config: Config,
+    ) -> Self {
+        let selectors = Extendable {
+            attributes: SubgraphAttributes::builder()
+                .subgraph_name(StandardAttribute::Bool(true))
+                .build(),
+            custom: HashMap::from([
+                (
+                    APOLLO_CLIENT_NAME_ATTRIBUTE.to_string(),
+                    SubgraphSelector::ResponseContext {
+                        response_context: CLIENT_NAME.to_string(),
+                        redact: None,
+                        default: None,
+                    },
+                ),
+                (
+                    APOLLO_CLIENT_VERSION_ATTRIBUTE.to_string(),
+                    SubgraphSelector::ResponseContext {
+                        response_context: CLIENT_VERSION.to_string(),
+                        redact: None,
+                        default: None,
+                    },
+                ),
+                (
+                    GRAPHQL_OPERATION_NAME_ATTRIBUTE.to_string(),
+                    SubgraphSelector::SupergraphOperationName {
+                        supergraph_operation_name: OperationName::String,
+                        redact: None,
+                        default: None,
+                    },
+                ),
+                (
+                    GRAPHQL_OPERATION_TYPE_ATTRIBUTE.to_string(),
+                    SubgraphSelector::SupergraphOperationKind {
+                        supergraph_operation_kind: OperationKind::String,
+                    },
+                ),
+                (
+                    APOLLO_OPERATION_ID_ATTRIBUTE.to_string(),
+                    SubgraphSelector::ResponseContext {
+                        response_context: APOLLO_OPERATION_ID.to_string(),
+                        redact: None,
+                        default: None,
+                    },
+                ),
+                (
+                    APOLLO_HAS_ERRORS_ATTRIBUTE.to_string(),
+                    SubgraphSelector::OnGraphQLError {
+                        subgraph_on_graphql_error: true,
+                    },
+                ),
+            ]),
+        };
+        let attribute_count = selectors.custom.len() + 1; // 1 for subgraph_name on attributes
+
+        let apollo_router_operations_fetch_duration =
+            apollo_config.experimental_subgraph_metrics.then(|| {
+                CustomHistogram::builder()
+                    .increment(Increment::Duration(Instant::now()))
+                    .attributes(Vec::with_capacity(attribute_count))
+                    .selectors(Arc::new(selectors))
+                    .histogram(static_instruments
+                        .get(APOLLO_ROUTER_OPERATIONS_FETCH_DURATION)
+                        .expect(
+                            "cannot get apollo static instrument for subgraph; this should not happen",
+                        )
+                        .as_histogram()
+                        .cloned()
+                        .expect(
+                            "cannot convert apollo instrument to histogram for subgraph; this should not happen",
+                        )
+                    )
+                    .build()
+            });
+
+        Self {
+            apollo_router_operations_fetch_duration,
+        }
+    }
+
+    pub(crate) fn new_builtin() -> HashMap<String, StaticInstrument> {
+        let meter = metrics::meter_provider().meter(METER_NAME);
+        let mut static_instruments = HashMap::with_capacity(1);
+
+        static_instruments.insert(
+            APOLLO_ROUTER_OPERATIONS_FETCH_DURATION.to_string(),
+            StaticInstrument::Histogram(
+                meter
+                    .f64_histogram(APOLLO_ROUTER_OPERATIONS_FETCH_DURATION)
+                    .with_unit("s")
+                    .with_description("Duration of a subgraph fetch.")
+                    .init(),
+            ),
+        );
+
+        static_instruments
+    }
+}
+
+impl Instrumented for ApolloSubgraphInstruments {
+    type Request = subgraph::Request;
+    type Response = subgraph::Response;
+    type EventResponse = ();
+
+    fn on_request(&self, request: &Self::Request) {
+        if let Some(apollo_router_operations_fetch_duration) =
+            &self.apollo_router_operations_fetch_duration
+        {
+            apollo_router_operations_fetch_duration.on_request(request);
+        }
+    }
+
+    fn on_response(&self, response: &Self::Response) {
+        if let Some(apollo_router_operations_fetch_duration) =
+            &self.apollo_router_operations_fetch_duration
+        {
+            apollo_router_operations_fetch_duration.on_response(response);
+        }
+    }
+
+    fn on_error(&self, error: &BoxError, ctx: &Context) {
+        if let Some(apollo_router_operations_fetch_duration) =
+            &self.apollo_router_operations_fetch_duration
+        {
+            apollo_router_operations_fetch_duration.on_error(error, ctx);
+        }
+    }
+}

apollo-router/src/plugins/telemetry/config_new/apollo/mod.rs

@@ -0,0 +1 @@
+pub(crate) mod instruments;

apollo-router/src/plugins/telemetry/config_new/fixtures/apollo/subgraph_fetch_duration/metrics.snap

@@ -0,0 +1,29 @@
+---
+source: apollo-router/src/plugins/telemetry/config_new/instruments.rs
+description: Apollo subgraph fetch duration histogram
+expression: "&metrics.all()"
+info:
+  telemetry:
+    apollo:
+      experimental_subgraph_metrics: true
+    instrumentation:
+      instruments:
+        subgraph:
+          http.client.request.duration: false
+snapshot_kind: text
+---
+- name: apollo.router.operations.fetch.duration
+  description: Duration of a subgraph fetch.
+  unit: s
+  data:
+    datapoints:
+      - sum: 0.1
+        count: 1
+        attributes:
+          apollo.client.name: myClient
+          apollo.client.version: v0.1.0
+          apollo.operation.id: myOperationID
+          graphql.operation.name: Test
+          graphql.operation.type: query
+          has_errors: false
+          subgraph.name: products

apollo-router/src/plugins/telemetry/config_new/fixtures/apollo/subgraph_fetch_duration/router.yaml

@@ -0,0 +1,7 @@
+telemetry:
+  apollo:
+    experimental_subgraph_metrics: true
+  instrumentation:
+    instruments:
+      subgraph:
+        http.client.request.duration: false

apollo-router/src/plugins/telemetry/config_new/fixtures/apollo/subgraph_fetch_duration/test.yaml

@@ -0,0 +1,18 @@
+description: Apollo subgraph fetch duration histogram
+events:
+  - - context:
+        map:
+          "apollo::supergraph::operation_name": "Test"
+          "apollo::supergraph::operation_id": "myOperationID"
+          "apollo::supergraph::operation_kind": "query"
+          "apollo::telemetry::client_name": "myClient"
+          "apollo::telemetry::client_version": "v0.1.0"
+    - subgraph_request:
+        query: "query { hello }"
+        operation_name: "Products"
+        operation_kind: query
+        subgraph_name: "products"
+    - subgraph_response:
+        status: 200
+        data:
+          hello: "world"