fix(telemetry): export properly resources configured on prometheus

.changesets/fix_bnjjj_fix_telemetry_prom_resource.md

@@ -0,0 +1,22 @@
+### Telemetry: export properly resources on metrics configured on prometheus ([PR #7394](https://github.com/apollographql/router/pull/7394))
+
+When configuring `resource` to globally add labels on metrics like this:
+
+```yaml
+telemetry:
+  apollo:
+    client_name_header: name_header
+    client_version_header: version_header
+  exporters:
+    metrics:
+      common:
+        resource:
+          "test-resource": "test"
+      prometheus:
+        enabled: true
+```
+
+`test-resource` label was never exported to prometheus, this bug only occurs with prometheus and not otlp. 
+This PR fixes this behavior and will no longer filter `resource`s.
+
+By [@bnjjj](https://github.com/bnjjj) in https://github.com/apollographql/router/pull/7394

apollo-router/src/plugins/telemetry/metrics/prometheus.rs

@@ -4,6 +4,7 @@ use std::task::Poll;
 use futures::future::BoxFuture;
 use http::StatusCode;
 use once_cell::sync::Lazy;
+use opentelemetry_prometheus::ResourceSelector;
 use opentelemetry_sdk::Resource;
 use opentelemetry_sdk::metrics::SdkMeterProvider;
 use opentelemetry_sdk::metrics::View;
@@ -127,6 +128,7 @@ impl MetricsConfigurator for Config {
                     .record_min_max(true)
                     .build(),
             )
+            .with_resource_selector(ResourceSelector::All)
             .with_registry(registry.clone())
             .build()?;
 

apollo-router/src/plugins/telemetry/mod.rs

@@ -1924,6 +1924,7 @@ pub(crate) fn add_query_attributes(context: &Context, custom_attributes: &mut Ve
 }
 
 struct EnableSubgraphFtv1;
+
 //
 // Please ensure that any tests added to the tests module use the tokio multi-threaded test executor.
 //
@@ -1986,6 +1987,21 @@ mod tests {
     use crate::services::SupergraphResponse;
     use crate::services::router;
 
+    macro_rules! assert_prometheus_metrics {
+        ($plugin:expr) => {{
+            let prometheus_metrics = get_prometheus_metrics($plugin.as_ref()).await;
+            let regexp = regex::Regex::new(
+                r#"process_executable_name="(?P<process>[^"]+)",?|service_name="(?P<service>[^"]+)",?"#,
+            )
+            .unwrap();
+            let prometheus_metrics = regexp.replace_all(&prometheus_metrics, "").to_owned();
+            assert_snapshot!(prometheus_metrics.replace(
+                &format!(r#"service_version="{}""#, std::env!("CARGO_PKG_VERSION")),
+                r#"service_version="X""#
+            ));
+        }};
+    }
+
     async fn create_plugin_with_config(config: &str) -> Box<dyn DynPlugin> {
         let prometheus_support = config.contains("prometheus");
         let config: Value = serde_yaml::from_str(config).expect("yaml must be valid");
@@ -2859,8 +2875,7 @@ mod tests {
             u64_histogram!("apollo.test.histo", "it's a test", 1u64);
 
             make_supergraph_request(plugin.as_ref()).await;
-            let prometheus_metrics = get_prometheus_metrics(plugin.as_ref()).await;
-            assert_snapshot!(prometheus_metrics);
+            assert_prometheus_metrics!(plugin);
         }
         .with_metrics()
         .await;
@@ -2876,9 +2891,7 @@ mod tests {
             u64_histogram!("apollo.test.histo", "it's a test", 1u64);
 
             make_supergraph_request(plugin.as_ref()).await;
-            let prometheus_metrics = get_prometheus_metrics(plugin.as_ref()).await;
-
-            assert_snapshot!(prometheus_metrics);
+            assert_prometheus_metrics!(plugin);
         }
         .with_metrics()
         .await;
@@ -2893,9 +2906,7 @@ mod tests {
             .await;
             make_supergraph_request(plugin.as_ref()).await;
             u64_histogram!("apollo.test.histo", "it's a test", 1u64);
-            let prometheus_metrics = get_prometheus_metrics(plugin.as_ref()).await;
-
-            assert_snapshot!(prometheus_metrics);
+            assert_prometheus_metrics!(plugin);
         }
         .with_metrics()
         .await;
@@ -2909,9 +2920,7 @@ mod tests {
             ))
             .await;
             make_supergraph_request(plugin.as_ref()).await;
-            let prometheus_metrics = get_prometheus_metrics(plugin.as_ref()).await;
-
-            assert!(prometheus_metrics.is_empty());
+            assert_prometheus_metrics!(plugin);
         }
         .with_metrics()
         .await;
@@ -2926,8 +2935,7 @@ mod tests {
             f64_histogram_with_unit!("apollo.test.histo2", "unit", "s", 1f64);
 
             make_supergraph_request(plugin.as_ref()).await;
-            let prometheus_metrics = get_prometheus_metrics(plugin.as_ref()).await;
-            assert_snapshot!(prometheus_metrics);
+            assert_prometheus_metrics!(plugin);
         }
         .with_metrics()
         .await;

apollo-router/src/plugins/telemetry/snapshots/apollo_router__plugins__telemetry__tests__it_test_prometheus_metrics.snap

@@ -1,17 +1,17 @@
 ---
 source: apollo-router/src/plugins/telemetry/mod.rs
-expression: prometheus_metrics
+expression: "prometheus_metrics.replace(& format!\n(r#\"service_version=\"{}\"\"#, std :: env! (\"CARGO_PKG_VERSION\")),\nr#\"service_version=\"X\"\"#)"
 ---
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="+Inf"} 1
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="0.001"} 0
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="0.005"} 0
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="0.015"} 0
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="0.05"} 0
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="0.1"} 0
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="0.2"} 0
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="0.3"} 0
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="0.4"} 0
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="0.5"} 0
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="1"} 1
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="10"} 1
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="5"} 1
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="+Inf"} 1
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.001"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.005"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.015"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.05"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.1"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.2"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.3"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.4"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.5"} 0
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="1"} 1
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="10"} 1
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="5"} 1

apollo-router/src/plugins/telemetry/snapshots/apollo_router__plugins__telemetry__tests__it_test_prometheus_metrics_custom_buckets.snap

@@ -1,8 +1,8 @@
 ---
 source: apollo-router/src/plugins/telemetry/mod.rs
-expression: prometheus_metrics
+expression: "prometheus_metrics.replace(& format!\n(r#\"service_version=\"{}\"\"#, std :: env! (\"CARGO_PKG_VERSION\")),\nr#\"service_version=\"X\"\"#)"
 ---
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="+Inf"} 1
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="10"} 1
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="20"} 1
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="5"} 1
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",le="+Inf"} 1
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",le="10"} 1
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",le="20"} 1
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",le="5"} 1

apollo-router/src/plugins/telemetry/snapshots/apollo_router__plugins__telemetry__tests__it_test_prometheus_metrics_custom_buckets_for_specific_metrics.snap

@@ -1,10 +1,10 @@
 ---
 source: apollo-router/src/plugins/telemetry/mod.rs
-expression: prometheus_metrics
+expression: "prometheus_metrics.replace(& format!\n(r#\"service_version=\"{}\"\"#, std :: env! (\"CARGO_PKG_VERSION\")),\nr#\"service_version=\"X\"\"#)"
 ---
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="+Inf"} 1
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="1"} 1
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="2"} 1
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="3"} 1
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="4"} 1
-apollo_test_histo_bucket{otel_scope_name="apollo/router",le="5"} 1
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",le="+Inf"} 1
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",le="1"} 1
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",le="2"} 1
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",le="3"} 1
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",le="4"} 1
+apollo_test_histo_bucket{otel_scope_name="apollo/router",service_version="X",le="5"} 1

apollo-router/src/plugins/telemetry/snapshots/apollo_router__plugins__telemetry__tests__it_test_prometheus_metrics_custom_view_drop.snap

@@ -0,0 +1,5 @@
+---
+source: apollo-router/src/plugins/telemetry/mod.rs
+expression: "prometheus_metrics.replace(& format!\n(r#\"service_version=\"{}\"\"#, std :: env! (\"CARGO_PKG_VERSION\")),\nr#\"service_version=\"X\"\"#)"
+---
+

apollo-router/src/plugins/telemetry/snapshots/apollo_router__plugins__telemetry__tests__it_test_prometheus_metrics_units_are_included.snap

@@ -1,30 +1,30 @@
 ---
 source: apollo-router/src/plugins/telemetry/mod.rs
-expression: prometheus_metrics
+expression: "prometheus_metrics.replace(& format!\n(r#\"service_version=\"{}\"\"#, std :: env! (\"CARGO_PKG_VERSION\")),\nr#\"service_version=\"X\"\"#)"
 ---
-apollo_test_histo1_bucket{otel_scope_name="apollo/router",le="+Inf"} 1
-apollo_test_histo1_bucket{otel_scope_name="apollo/router",le="0.001"} 0
-apollo_test_histo1_bucket{otel_scope_name="apollo/router",le="0.005"} 0
-apollo_test_histo1_bucket{otel_scope_name="apollo/router",le="0.015"} 0
-apollo_test_histo1_bucket{otel_scope_name="apollo/router",le="0.05"} 0
-apollo_test_histo1_bucket{otel_scope_name="apollo/router",le="0.1"} 0
-apollo_test_histo1_bucket{otel_scope_name="apollo/router",le="0.2"} 0
-apollo_test_histo1_bucket{otel_scope_name="apollo/router",le="0.3"} 0
-apollo_test_histo1_bucket{otel_scope_name="apollo/router",le="0.4"} 0
-apollo_test_histo1_bucket{otel_scope_name="apollo/router",le="0.5"} 0
-apollo_test_histo1_bucket{otel_scope_name="apollo/router",le="1"} 1
-apollo_test_histo1_bucket{otel_scope_name="apollo/router",le="10"} 1
-apollo_test_histo1_bucket{otel_scope_name="apollo/router",le="5"} 1
-apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",le="+Inf"} 1
-apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",le="0.001"} 0
-apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",le="0.005"} 0
-apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",le="0.015"} 0
-apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",le="0.05"} 0
-apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",le="0.1"} 0
-apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",le="0.2"} 0
-apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",le="0.3"} 0
-apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",le="0.4"} 0
-apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",le="0.5"} 0
-apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",le="1"} 1
-apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",le="10"} 1
-apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",le="5"} 1
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="+Inf"} 1
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.001"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.005"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.015"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.05"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.1"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.2"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.3"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.4"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.5"} 0
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="1"} 1
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="10"} 1
+apollo_test_histo1_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="5"} 1
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="+Inf"} 1
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.001"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.005"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.015"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.05"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.1"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.2"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.3"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.4"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="0.5"} 0
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="1"} 1
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="10"} 1
+apollo_test_histo2_seconds_bucket{otel_scope_name="apollo/router",service_version="X",test_resource="test",le="5"} 1

apollo-router/src/plugins/telemetry/testdata/prometheus.router.yaml

@@ -4,5 +4,8 @@ telemetry:
     client_version_header: version_header
   exporters:
     metrics:
+      common:
+        resource:
+          "test-resource": "test"
       prometheus:
         enabled: true

apollo-router/tests/integration/traffic_shaping.rs

@@ -283,7 +283,7 @@ async fn test_subgraph_rate_limit() -> Result<(), BoxError> {
     assert!(response.contains("REQUEST_RATE_LIMITED"));
     assert_yaml_snapshot!(response);
 
-    router.assert_metrics_contains(r#"apollo_router_graphql_error_total{code="REQUEST_RATE_LIMITED",otel_scope_name="apollo/router"} 1"#, None).await;
+    router.assert_metrics_contains(&format!(r#"apollo_router_graphql_error_total{{code="REQUEST_RATE_LIMITED",otel_scope_name="apollo/router",process_executable_name="router",service_name="unknown_service:router",service_version="{}"}} 1"#, std::env!("CARGO_PKG_VERSION")), None).await;
 
     router.graceful_shutdown().await;
     Ok(())