fix(telemetry): export properly resources configured on prometheus

.changesets/fix_bnjjj_fix_telemetry_prom_resource.md

@@ -0,0 +1,19 @@
+### Telemetry: export properly resources on metrics configured on prometheus ([PR #7394](https://github.com/apollographql/router/pull/7394))
+
+When configuring `telemetry.exporters.metrics.common.resource` to globally add labels on metrics, these labels were not exported to every metrics on prometheus. This can now be done if you set `resource_selector` to `all` (default is `none`).
+
+```yaml
+telemetry:
+  exporters:
+    metrics:
+      common:
+        resource:
+          "test-resource": "test"
+      prometheus:
+        enabled: true
+        resource_selector: all # This will add resources on every metrics
+```
+
+This only occurred with Prometheus and not OTLP.
+
+By [@bnjjj](https://github.com/bnjjj) in https://github.com/apollographql/router/pull/7394

apollo-router/src/configuration/snapshots/apollo_router__configuration__tests__schema_generation.snap

@@ -1568,6 +1568,10 @@ expression: "&schema"
           "default": "/metrics",
           "description": "The path where prometheus will be exposed",
           "type": "string"
+        },
+        "resource_selector": {
+          "$ref": "#/definitions/ResourceSelectorConfig",
+          "description": "#/definitions/ResourceSelectorConfig"
         }
       },
       "type": "object"
@@ -5234,6 +5238,24 @@ expression: "&schema"
       ],
       "type": "object"
     },
+    "ResourceSelectorConfig": {
+      "oneOf": [
+        {
+          "description": "Export all resource attributes with every metrics.",
+          "enum": [
+            "all"
+          ],
+          "type": "string"
+        },
+        {
+          "description": "Do not export any resource attributes with every metrics.",
+          "enum": [
+            "none"
+          ],
+          "type": "string"
+        }
+      ]
+    },
     "ResponseStatus": {
       "oneOf": [
         {

apollo-router/src/plugins/telemetry/metrics/prometheus.rs

@@ -4,6 +4,7 @@ use std::task::Poll;
 use futures::future::BoxFuture;
 use http::StatusCode;
 use once_cell::sync::Lazy;
+use opentelemetry_prometheus::ResourceSelector;
 use opentelemetry_sdk::Resource;
 use opentelemetry_sdk::metrics::SdkMeterProvider;
 use opentelemetry_sdk::metrics::View;
@@ -32,16 +33,38 @@ use crate::services::router;
 pub(crate) struct Config {
     /// Set to true to enable
     pub(crate) enabled: bool,
+    /// resource_selector is used to select which resource to export with every metrics.
+    pub(crate) resource_selector: ResourceSelectorConfig,
     /// The listen address
     pub(crate) listen: ListenAddr,
     /// The path where prometheus will be exposed
     pub(crate) path: String,
 }
 
+#[derive(Debug, Clone, Copy, Deserialize, JsonSchema, Default)]
+#[serde(rename_all = "snake_case")]
+pub(crate) enum ResourceSelectorConfig {
+    /// Export all resource attributes with every metrics.
+    All,
+    #[default]
+    /// Do not export any resource attributes with every metrics.
+    None,
+}
+
+impl From<ResourceSelectorConfig> for ResourceSelector {
+    fn from(value: ResourceSelectorConfig) -> Self {
+        match value {
+            ResourceSelectorConfig::All => ResourceSelector::All,
+            ResourceSelectorConfig::None => ResourceSelector::None,
+        }
+    }
+}
+
 impl Default for Config {
     fn default() -> Self {
         Self {
             enabled: false,
+            resource_selector: ResourceSelectorConfig::default(),
             listen: ListenAddr::SocketAddr("127.0.0.1:9090".parse().expect("valid listenAddr")),
             path: "/metrics".to_string(),
         }
@@ -127,6 +150,7 @@ impl MetricsConfigurator for Config {
                     .record_min_max(true)
                     .build(),
             )
+            .with_resource_selector(self.resource_selector)
             .with_registry(registry.clone())
             .build()?;
 

apollo-router/src/plugins/telemetry/mod.rs

@@ -1993,6 +1993,7 @@ pub(crate) fn add_query_attributes(context: &Context, custom_attributes: &mut Ve
 }
 
 struct EnableSubgraphFtv1;
+
 //
 // Please ensure that any tests added to the tests module use the tokio multi-threaded test executor.
 //
@@ -2057,6 +2058,21 @@ mod tests {
     use crate::services::SupergraphResponse;
     use crate::services::router;
 
+    macro_rules! assert_prometheus_metrics {
+        ($plugin:expr) => {{
+            let prometheus_metrics = get_prometheus_metrics($plugin.as_ref()).await;
+            let regexp = regex::Regex::new(
+                r#"process_executable_name="(?P<process>[^"]+)",?|service_name="(?P<service>[^"]+)",?"#,
+            )
+            .unwrap();
+            let prometheus_metrics = regexp.replace_all(&prometheus_metrics, "").to_owned();
+            assert_snapshot!(prometheus_metrics.replace(
+                &format!(r#"service_version="{}""#, std::env!("CARGO_PKG_VERSION")),
+                r#"service_version="X""#
+            ));
+        }};
+    }
+
     async fn create_plugin_with_config(full_config: &str) -> Box<dyn DynPlugin> {
         let full_config = serde_yaml::from_str::<Value>(full_config).expect("yaml must be valid");
         let telemetry_config = full_config
@@ -3027,8 +3043,7 @@ mod tests {
             u64_histogram!("apollo.test.histo", "it's a test", 1u64);
 
             make_supergraph_request(plugin.as_ref()).await;
-            let prometheus_metrics = get_prometheus_metrics(plugin.as_ref()).await;
-            assert_snapshot!(prometheus_metrics);
+            assert_prometheus_metrics!(plugin);
         }
         .with_metrics()
         .await;
@@ -3044,9 +3059,7 @@ mod tests {
             u64_histogram!("apollo.test.histo", "it's a test", 1u64);
 
             make_supergraph_request(plugin.as_ref()).await;
-            let prometheus_metrics = get_prometheus_metrics(plugin.as_ref()).await;
-
-            assert_snapshot!(prometheus_metrics);
+            assert_prometheus_metrics!(plugin);
         }
         .with_metrics()
         .await;
@@ -3061,9 +3074,7 @@ mod tests {
             .await;
             make_supergraph_request(plugin.as_ref()).await;
             u64_histogram!("apollo.test.histo", "it's a test", 1u64);
-            let prometheus_metrics = get_prometheus_metrics(plugin.as_ref()).await;
-
-            assert_snapshot!(prometheus_metrics);
+            assert_prometheus_metrics!(plugin);
         }
         .with_metrics()
         .await;
@@ -3077,9 +3088,7 @@ mod tests {
             ))
             .await;
             make_supergraph_request(plugin.as_ref()).await;
-            let prometheus_metrics = get_prometheus_metrics(plugin.as_ref()).await;
-
-            assert!(prometheus_metrics.is_empty());
+            assert_prometheus_metrics!(plugin);
         }
         .with_metrics()
         .await;
@@ -3094,8 +3103,7 @@ mod tests {
             f64_histogram_with_unit!("apollo.test.histo2", "unit", "s", 1f64);
 
             make_supergraph_request(plugin.as_ref()).await;
-            let prometheus_metrics = get_prometheus_metrics(plugin.as_ref()).await;
-            assert_snapshot!(prometheus_metrics);
+            assert_prometheus_metrics!(plugin);
         }
         .with_metrics()
         .await;

apollo-router/src/plugins/telemetry/snapshots/apollo_router__plugins__telemetry__tests__it_test_prometheus_metrics.snap

@@ -1,17 +1,17 @@
 ---
 source: apollo-router/src/plugins/telemetry/mod.rs
-expression: prometheus_metrics
+expression: "prometheus_metrics.replace(& format!\n(r#\"service_version=\"{}\"\"#, std :: env! (\"CARGO_PKG_VERSION\")),\nr#\"service_version=\"X\"\"#)"
 ---
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="+Inf"} 1
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="0.001"} 0
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="0.005"} 0
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="0.015"} 0
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="0.05"} 0
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="0.1"} 0
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="0.2"} 0
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="0.3"} 0
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="0.4"} 0
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="0.5"} 0
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="1"} 1
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="10"} 1
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="5"} 1
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="+Inf"} 1
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.001"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.005"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.015"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.05"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.1"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.2"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.3"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.4"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.5"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="1"} 1
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="10"} 1
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="5"} 1

apollo-router/src/plugins/telemetry/snapshots/apollo_router__plugins__telemetry__tests__it_test_prometheus_metrics_custom_buckets.snap

@@ -1,6 +1,6 @@
 ---
 source: apollo-router/src/plugins/telemetry/mod.rs
-expression: prometheus_metrics
+expression: "prometheus_metrics.replace(& format!\n(r#\"service_version=\"{}\"\"#, std :: env! (\"CARGO_PKG_VERSION\")),\nr#\"service_version=\"X\"\"#)"
 ---
 apollo_test_histo_bucket{otel_scope_name="apollo/router",le="+Inf"} 1
 apollo_test_histo_bucket{otel_scope_name="apollo/router",le="10"} 1

apollo-router/src/plugins/telemetry/snapshots/apollo_router__plugins__telemetry__tests__it_test_prometheus_metrics_custom_buckets_for_specific_metrics.snap

@@ -1,6 +1,6 @@
 ---
 source: apollo-router/src/plugins/telemetry/mod.rs
-expression: prometheus_metrics
+expression: "prometheus_metrics.replace(& format!\n(r#\"service_version=\"{}\"\"#, std :: env! (\"CARGO_PKG_VERSION\")),\nr#\"service_version=\"X\"\"#)"
 ---
 apollo_test_histo_bucket{otel_scope_name="apollo/router",le="+Inf"} 1
 apollo_test_histo_bucket{otel_scope_name="apollo/router",le="1"} 1

apollo-router/src/plugins/telemetry/snapshots/apollo_router__plugins__telemetry__tests__it_test_prometheus_metrics_custom_view_drop.snap

@@ -0,0 +1,5 @@
+---
+source: apollo-router/src/plugins/telemetry/mod.rs
+expression: "prometheus_metrics.replace(& format!\n(r#\"service_version=\"{}\"\"#, std :: env! (\"CARGO_PKG_VERSION\")),\nr#\"service_version=\"X\"\"#)"
+---
+

apollo-router/src/plugins/telemetry/snapshots/apollo_router__plugins__telemetry__tests__it_test_prometheus_metrics_units_are_included.snap

@@ -1,30 +1,30 @@
 ---
 source: apollo-router/src/plugins/telemetry/mod.rs
-expression: prometheus_metrics
+expression: "prometheus_metrics.replace(& format!\n(r#\"service_version=\"{}\"\"#, std :: env! (\"CARGO_PKG_VERSION\")),\nr#\"service_version=\"X\"\"#)"
 ---
-apollo_test_histo1_bucket{otel_scope_name="apollo/router",le="+Inf"} 1
-apollo_test_histo1_bucket{otel_scope_name="apollo/router",le="0.001"} 0
-apollo_test_histo1_bucket{otel_scope_name="apollo/router",le="0.005"} 0
-apollo_test_histo1_bucket{otel_scope_name="apollo/router",le="0.015"} 0
-apollo_test_histo1_bucket{otel_scope_name="apollo/router",le="0.05"} 0
-apollo_test_histo1_bucket{otel_scope_name="apollo/router",le="0.1"} 0
-apollo_test_histo1_bucket{otel_scope_name="apollo/router",le="0.2"} 0
-apollo_test_histo1_bucket{otel_scope_name="apollo/router",le="0.3"} 0
-apollo_test_histo1_bucket{otel_scope_name="apollo/router",le="0.4"} 0
-apollo_test_histo1_bucket{otel_scope_name="apollo/router",le="0.5"} 0
-apollo_test_histo1_bucket{otel_scope_name="apollo/router",le="1"} 1
-apollo_test_histo1_bucket{otel_scope_name="apollo/router",le="10"} 1
-apollo_test_histo1_bucket{otel_scope_name="apollo/router",le="5"} 1
-apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",le="+Inf"} 1
-apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",le="0.001"} 0
-apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",le="0.005"} 0
-apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",le="0.015"} 0
-apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",le="0.05"} 0
-apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",le="0.1"} 0
-apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",le="0.2"} 0
-apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",le="0.3"} 0
-apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",le="0.4"} 0
-apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",le="0.5"} 0
-apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",le="1"} 1
-apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",le="10"} 1
-apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",le="5"} 1
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="+Inf"} 1
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.001"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.005"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.015"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.05"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.1"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.2"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.3"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.4"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.5"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="1"} 1
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="10"} 1
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="5"} 1
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="+Inf"} 1
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.001"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.005"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.015"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.05"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.1"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.2"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.3"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.4"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.5"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="1"} 1
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="10"} 1
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="5"} 1

apollo-router/src/plugins/telemetry/testdata/prometheus.router.yaml

@@ -4,5 +4,9 @@ telemetry:
     client_version_header: version_header
   exporters:
     metrics:
+      common:
+        resource:
+          "test-resource": "test"
       prometheus:
         enabled: true
+        resource_selector: all