fix(telemetry): export properly resources configured on prometheus (backport #7394)

.changesets/fix_bnjjj_fix_telemetry_prom_resource.md

@@ -0,0 +1,19 @@
+### Telemetry: export properly resources on metrics configured on prometheus ([PR #7394](https://github.com/apollographql/router/pull/7394))
+
+When configuring `telemetry.exporters.metrics.common.resource` to globally add labels on metrics, these labels were not exported to every metrics on prometheus. This can now be done if you set `resource_selector` to `all` (default is `none`).
+
+```yaml
+telemetry:
+  exporters:
+    metrics:
+      common:
+        resource:
+          "test-resource": "test"
+      prometheus:
+        enabled: true
+        resource_selector: all # This will add resources on every metrics
+```
+
+This only occurred with Prometheus and not OTLP.
+
+By [@bnjjj](https://github.com/bnjjj) in https://github.com/apollographql/router/pull/7394

apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap

@@ -1813,6 +1813,10 @@ expression: "&schema"
           "description": "The optional output name",
           "nullable": true,
           "type": "string"
+        },
+        "resource_selector": {
+          "$ref": "#/definitions/ResourceSelectorConfig",
+          "description": "#/definitions/ResourceSelectorConfig"
         }
       },
       "required": [
@@ -4664,6 +4668,24 @@ expression: "&schema"
       ],
       "type": "object"
     },
+    "ResourceSelectorConfig": {
+      "oneOf": [
+        {
+          "description": "Export all resource attributes with every metrics.",
+          "enum": [
+            "all"
+          ],
+          "type": "string"
+        },
+        {
+          "description": "Do not export any resource attributes with every metrics.",
+          "enum": [
+            "none"
+          ],
+          "type": "string"
+        }
+      ]
+    },
     "ResponseStatus": {
       "oneOf": [
         {

apollo-router/src/plugins/telemetry/metrics/prometheus.rs

@@ -5,9 +5,17 @@ use std::task::Poll;
 use futures::future::BoxFuture;
 use http::StatusCode;
 use once_cell::sync::Lazy;
+<<<<<<< HEAD
 use opentelemetry::sdk::Resource;
 use opentelemetry::sdk::metrics::MeterProvider;
 use opentelemetry::sdk::metrics::View;
+=======
+use opentelemetry_prometheus::ResourceSelector;
+use opentelemetry_sdk::Resource;
+use opentelemetry_sdk::metrics::SdkMeterProvider;
+use opentelemetry_sdk::metrics::View;
+use parking_lot::Mutex;
+>>>>>>> 731fd23c (fix(telemetry): export properly resources configured on prometheus (#7394))
 use prometheus::Encoder;
 use prometheus::Registry;
 use prometheus::TextEncoder;
@@ -33,16 +41,38 @@ use crate::services::router::Body;
 pub(crate) struct Config {
     /// Set to true to enable
     pub(crate) enabled: bool,
+    /// resource_selector is used to select which resource to export with every metrics.
+    pub(crate) resource_selector: ResourceSelectorConfig,
     /// The listen address
     pub(crate) listen: ListenAddr,
     /// The path where prometheus will be exposed
     pub(crate) path: String,
 }
 
+#[derive(Debug, Clone, Copy, Deserialize, JsonSchema, Default)]
+#[serde(rename_all = "snake_case")]
+pub(crate) enum ResourceSelectorConfig {
+    /// Export all resource attributes with every metrics.
+    All,
+    #[default]
+    /// Do not export any resource attributes with every metrics.
+    None,
+}
+
+impl From<ResourceSelectorConfig> for ResourceSelector {
+    fn from(value: ResourceSelectorConfig) -> Self {
+        match value {
+            ResourceSelectorConfig::All => ResourceSelector::All,
+            ResourceSelectorConfig::None => ResourceSelector::None,
+        }
+    }
+}
+
 impl Default for Config {
     fn default() -> Self {
         Self {
             enabled: false,
+            resource_selector: ResourceSelectorConfig::default(),
             listen: ListenAddr::SocketAddr("127.0.0.1:9090".parse().expect("valid listenAddr")),
             path: "/metrics".to_string(),
         }
@@ -133,6 +163,7 @@ impl MetricsConfigurator for Config {
                     .record_min_max(true)
                     .build(),
             )
+            .with_resource_selector(self.resource_selector)
             .with_registry(registry.clone())
             .build()?;
 

apollo-router/src/plugins/telemetry/mod.rs

@@ -2109,6 +2109,7 @@ struct MetricsAttributes(HashMap<String, AttributeValue>);
 struct SubgraphMetricsAttributes(HashMap<String, AttributeValue>);
 
 struct EnableSubgraphFtv1;
+
 //
 // Please ensure that any tests added to the tests module use the tokio multi-threaded test executor.
 //
@@ -2185,10 +2186,31 @@ mod tests {
     use crate::services::SupergraphResponse;
     use crate::services::router::body::get_body_bytes;
 
+<<<<<<< HEAD
     async fn create_plugin_with_config(config: &str) -> Box<dyn DynPlugin> {
         let prometheus_support = config.contains("prometheus");
         let config: Value = serde_yaml::from_str(config).expect("yaml must be valid");
         let telemetry_config = config
+=======
+    macro_rules! assert_prometheus_metrics {
+        ($plugin:expr) => {{
+            let prometheus_metrics = get_prometheus_metrics($plugin.as_ref()).await;
+            let regexp = regex::Regex::new(
+                r#"process_executable_name="(?P<process>[^"]+)",?|service_name="(?P<service>[^"]+)",?"#,
+            )
+            .unwrap();
+            let prometheus_metrics = regexp.replace_all(&prometheus_metrics, "").to_owned();
+            assert_snapshot!(prometheus_metrics.replace(
+                &format!(r#"service_version="{}""#, std::env!("CARGO_PKG_VERSION")),
+                r#"service_version="X""#
+            ));
+        }};
+    }
+
+    async fn create_plugin_with_config(full_config: &str) -> Box<dyn DynPlugin> {
+        let full_config = serde_yaml::from_str::<Value>(full_config).expect("yaml must be valid");
+        let telemetry_config = full_config
+>>>>>>> 731fd23c (fix(telemetry): export properly resources configured on prometheus (#7394))
             .as_object()
             .expect("must be an object")
             .get("telemetry")
@@ -3099,8 +3121,7 @@ mod tests {
             let plugin =
                 create_plugin_with_config(include_str!("testdata/prometheus.router.yaml")).await;
             make_supergraph_request(plugin.as_ref()).await;
-            let prometheus_metrics = get_prometheus_metrics(plugin.as_ref()).await;
-            assert_snapshot!(prometheus_metrics);
+            assert_prometheus_metrics!(plugin);
         }
         .with_metrics()
         .await;
@@ -3114,9 +3135,7 @@ mod tests {
             ))
             .await;
             make_supergraph_request(plugin.as_ref()).await;
-            let prometheus_metrics = get_prometheus_metrics(plugin.as_ref()).await;
-
-            assert_snapshot!(prometheus_metrics);
+            assert_prometheus_metrics!(plugin);
         }
         .with_metrics()
         .await;
@@ -3130,9 +3149,14 @@ mod tests {
             ))
             .await;
             make_supergraph_request(plugin.as_ref()).await;
+<<<<<<< HEAD
             let prometheus_metrics = get_prometheus_metrics(plugin.as_ref()).await;
 
             assert_snapshot!(prometheus_metrics);
+=======
+            u64_histogram!("apollo.test.histo", "it's a test", 1u64);
+            assert_prometheus_metrics!(plugin);
+>>>>>>> 731fd23c (fix(telemetry): export properly resources configured on prometheus (#7394))
         }
         .with_metrics()
         .await;
@@ -3146,14 +3170,30 @@ mod tests {
             ))
             .await;
             make_supergraph_request(plugin.as_ref()).await;
-            let prometheus_metrics = get_prometheus_metrics(plugin.as_ref()).await;
+            assert_prometheus_metrics!(plugin);
+        }
+        .with_metrics()
+        .await;
+    }
 
-            assert!(prometheus_metrics.is_empty());
+<<<<<<< HEAD
+=======
+    #[tokio::test(flavor = "multi_thread")]
+    async fn it_test_prometheus_metrics_units_are_included() {
+        async {
+            let plugin =
+                create_plugin_with_config(include_str!("testdata/prometheus.router.yaml")).await;
+            u64_histogram_with_unit!("apollo.test.histo1", "no unit", "{request}", 1u64);
+            f64_histogram_with_unit!("apollo.test.histo2", "unit", "s", 1f64);
+
+            make_supergraph_request(plugin.as_ref()).await;
+            assert_prometheus_metrics!(plugin);
         }
         .with_metrics()
         .await;
     }
 
+>>>>>>> 731fd23c (fix(telemetry): export properly resources configured on prometheus (#7394))
     #[test]
     fn it_test_send_headers_to_studio() {
         let fw_headers = ForwardHeaders::Only(vec![

apollo-router/src/plugins/telemetry/snapshots/apollo_router__plugins__telemetry__tests__it_test_prometheus_metrics.snap

@@ -1,7 +1,8 @@
 ---
 source: apollo-router/src/plugins/telemetry/mod.rs
-expression: prometheus_metrics
+expression: "prometheus_metrics.replace(& format!\n(r#\"service_version=\"{}\"\"#, std :: env! (\"CARGO_PKG_VERSION\")),\nr#\"service_version=\"X\"\"#)"
 ---
+<<<<<<< HEAD
 apollo_router_http_request_duration_seconds_bucket{status="200",otel_scope_name="apollo/router",le="+Inf"} 1
 apollo_router_http_request_duration_seconds_bucket{status="200",otel_scope_name="apollo/router",le="0.001"} 1
 apollo_router_http_request_duration_seconds_bucket{status="200",otel_scope_name="apollo/router",le="0.005"} 1
@@ -15,3 +16,18 @@ apollo_router_http_request_duration_seconds_bucket{status="200",otel_scope_name=
 apollo_router_http_request_duration_seconds_bucket{status="200",otel_scope_name="apollo/router",le="1"} 1
 apollo_router_http_request_duration_seconds_bucket{status="200",otel_scope_name="apollo/router",le="10"} 1
 apollo_router_http_request_duration_seconds_bucket{status="200",otel_scope_name="apollo/router",le="5"} 1
+=======
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="+Inf"} 1
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.001"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.005"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.015"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.05"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.1"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.2"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.3"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.4"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.5"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="1"} 1
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="10"} 1
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="5"} 1
+>>>>>>> 731fd23c (fix(telemetry): export properly resources configured on prometheus (#7394))

apollo-router/src/plugins/telemetry/snapshots/apollo_router__plugins__telemetry__tests__it_test_prometheus_metrics_custom_buckets.snap

@@ -1,6 +1,6 @@
 ---
 source: apollo-router/src/plugins/telemetry/mod.rs
-expression: prometheus_metrics
+expression: "prometheus_metrics.replace(& format!\n(r#\"service_version=\"{}\"\"#, std :: env! (\"CARGO_PKG_VERSION\")),\nr#\"service_version=\"X\"\"#)"
 ---
 apollo_router_http_request_duration_seconds_bucket{status="200",otel_scope_name="apollo/router",le="+Inf"} 1
 apollo_router_http_request_duration_seconds_bucket{status="200",otel_scope_name="apollo/router",le="10"} 1

apollo-router/src/plugins/telemetry/snapshots/apollo_router__plugins__telemetry__tests__it_test_prometheus_metrics_custom_buckets_for_specific_metrics.snap

@@ -1,6 +1,6 @@
 ---
 source: apollo-router/src/plugins/telemetry/mod.rs
-expression: prometheus_metrics
+expression: "prometheus_metrics.replace(& format!\n(r#\"service_version=\"{}\"\"#, std :: env! (\"CARGO_PKG_VERSION\")),\nr#\"service_version=\"X\"\"#)"
 ---
 apollo_router_http_request_duration_seconds_bucket{otel_scope_name="apollo/router",le="+Inf"} 1
 apollo_router_http_request_duration_seconds_bucket{otel_scope_name="apollo/router",le="1"} 1

apollo-router/src/plugins/telemetry/snapshots/apollo_router__plugins__telemetry__tests__it_test_prometheus_metrics_custom_view_drop.snap

@@ -0,0 +1,5 @@
+---
+source: apollo-router/src/plugins/telemetry/mod.rs
+expression: "prometheus_metrics.replace(& format!\n(r#\"service_version=\"{}\"\"#, std :: env! (\"CARGO_PKG_VERSION\")),\nr#\"service_version=\"X\"\"#)"
+---
+

apollo-router/src/plugins/telemetry/snapshots/apollo_router__plugins__telemetry__tests__it_test_prometheus_metrics_units_are_included.snap

@@ -0,0 +1,30 @@
+---
+source: apollo-router/src/plugins/telemetry/mod.rs
+expression: "prometheus_metrics.replace(& format!\n(r#\"service_version=\"{}\"\"#, std :: env! (\"CARGO_PKG_VERSION\")),\nr#\"service_version=\"X\"\"#)"
+---
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="+Inf"} 1
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.001"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.005"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.015"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.05"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.1"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.2"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.3"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.4"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.5"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="1"} 1
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="10"} 1
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="5"} 1
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="+Inf"} 1
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.001"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.005"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.015"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.05"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.1"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.2"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.3"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.4"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.5"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="1"} 1
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="10"} 1
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="5"} 1

apollo-router/src/plugins/telemetry/testdata/prometheus.router.yaml

@@ -4,5 +4,9 @@ telemetry:
     client_version_header: version_header
   exporters:
     metrics:
+      common:
+        resource:
+          "test-resource": "test"
       prometheus:
         enabled: true
+        resource_selector: all