diff --git a/.changesets/feat_marcelo_router_1609.md b/.changesets/feat_marcelo_router_1609.md new file mode 100644 index 0000000000..206cc006ae --- /dev/null +++ b/.changesets/feat_marcelo_router_1609.md @@ -0,0 +1,12 @@ +### Block Router startup when certain OTEL environment variables are set (PR #8915) + +The Apollo Router will now *fail to start* if any of the following OpenTelemetry (OTEL) keyword environment variables are set: + • OTEL_EXPORTER_OTLP_ENDPOINT + • OTEL_EXPORTER_OTLP_TRACES_ENDPOINT + • OTEL_EXPORTER_OTLP_METRICS_ENDPOINT + +Using these variables *is not supported by the Router* since it can override or interfere with its built-in telemetry configuration, leading to unexpected behavior. + +Previously, the Router emitted a warning when OTEL_EXPORTER_OTLP_ENDPOINT was present. With this change, *startup is now blocked* to prevent unintended telemetry configuration conflicts. + +If your deployment defines any of these environment variables (for example, through base container images, platform defaults, or infrastructure tooling), they must be removed before starting the Router. diff --git a/apollo-router/src/executable.rs b/apollo-router/src/executable.rs index 513a1f9671..d6ba909f54 100644 --- a/apollo-router/src/executable.rs +++ b/apollo-router/src/executable.rs @@ -55,6 +55,12 @@ pub(crate) static APOLLO_ROUTER_HOT_RELOAD_CLI: AtomicBool = AtomicBool::new(fal const INITIAL_UPLINK_POLL_INTERVAL: Duration = Duration::from_secs(10); const INITIAL_OCI_POLL_INTERVAL: Duration = Duration::from_secs(30); +const FORBIDDEN_OTEL_VARS: [&str; 3] = [ + "OTEL_EXPORTER_OTLP_ENDPOINT", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", +]; + /// Subcommands #[derive(Subcommand, Debug)] enum Commands { @@ -286,6 +292,10 @@ impl Opt { fn err_require_opt(env_var: &str) -> anyhow::Error { anyhow!("Use of Apollo Graph OS requires setting the {env_var} environment variable") } + + fn prohibit_env_vars(env_vars: &[&'static str]) -> Result<(), anyhow::Error> { + reject_environment_variables(&env_variables_set(env_vars)) + } } /// This is the main router entrypoint. @@ -475,6 +485,9 @@ impl Executable { // Enable hot reload when dev mode is enabled opt.hot_reload = opt.hot_reload || opt.dev; + // ROUTER-1609: prevent router from starting if OTEL environment variables are set. + Opt::prohibit_env_vars(&FORBIDDEN_OTEL_VARS)?; + let configuration = match (config, opt.config_path.as_ref()) { (Some(_), Some(_)) => { return Err(anyhow!( @@ -738,14 +751,6 @@ impl Executable { ); } - // Warn users that OTEL_EXPORTER_OTLP_ENDPOINT takes precedence over default configurations - // and may override trace export to Apollo Studio - if std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT").is_ok() { - tracing::warn!( - "The OTEL_EXPORTER_OTLP_ENDPOINT environment variable is set. This takes precedence over default configurations and may override trace export to Apollo Studio." - ); - } - let router = RouterHttpServer::builder() .is_telemetry_disabled(opt.anonymous_telemetry_disabled) .configuration(configuration) @@ -768,6 +773,27 @@ fn graph_os() -> bool { && crate::services::APOLLO_GRAPH_REF.lock().is_some() } +/// Of the environment variable names provided, return a list of those which are set in the environment. +fn env_variables_set(variables: &[&'static str]) -> Vec<&'static str> { + variables + .iter() + .filter(|v| !matches!(std::env::var(v), Err(std::env::VarError::NotPresent))) + .cloned() + .collect() +} + +/// Return an error if the list of environment variables provided is not empty +fn reject_environment_variables(variables: &[&str]) -> Result<(), anyhow::Error> { + if variables.is_empty() { + Ok(()) + } else { + Err(anyhow!( + "the following environment variables must not be set: {}", + variables.join(", ") + )) + } +} + fn setup_panic_handler() { // Redirect panics to the logs. let backtrace_env = std::env::var("RUST_BACKTRACE"); @@ -796,6 +822,8 @@ fn setup_panic_handler() { #[cfg(test)] mod tests { use crate::executable::add_log_filter; + use crate::executable::env_variables_set; + use crate::executable::reject_environment_variables; #[test] fn simplest_logging_modifications() { @@ -1033,4 +1061,33 @@ mod tests { ); } } + + #[test] + fn it_observes_environment_variables() { + const VALID_ENV_VAR: &str = "CARGO_HOME"; + + // if we're running tests, we should have a CARGO_HOME env variable present + assert!(std::env::var(VALID_ENV_VAR).is_ok()); + + // make sure the env_variables_set function can see that, both alone and in a list + assert!(env_variables_set(&[VALID_ENV_VAR]).contains(&VALID_ENV_VAR)); + assert!( + env_variables_set(&[VALID_ENV_VAR, "ANOTHER_ENV_VARIABLE"]).contains(&VALID_ENV_VAR) + ); + + // make sure the env_variables_set variable doesn't find not-present environment variables + assert!(env_variables_set(&["AN_EXTREMELY_UNLIKELY_TO_BE_SET_VARIABLE"]).is_empty()); + } + + #[test] + fn it_returns_an_error_when_env_variable_provided() { + assert!(reject_environment_variables(&[]).is_ok()); + + let err = reject_environment_variables(&["env1"]).unwrap_err(); + assert!(err.to_string().contains("env1")); + + let err = reject_environment_variables(&["env1", "env2"]).unwrap_err(); + assert!(err.to_string().contains("env1")); + assert!(err.to_string().contains("env2")); + } } diff --git a/docs/shared/otel-envvars-caution.mdx b/docs/shared/otel-envvars-caution.mdx new file mode 100644 index 0000000000..52043ce91c --- /dev/null +++ b/docs/shared/otel-envvars-caution.mdx @@ -0,0 +1,15 @@ + + +The following OpenTelemetry (OTEL) environment variables will override router's built-in telemetry configuration: + +- `OTEL_EXPORTER_OTLP_ENDPOINT` +- `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` +- `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` + +In Apollo Router v2.12.0 and earlier, these variables will override router's telemetry settings and may cause traces or metrics to be sent to an unintended destination. + +

+ +In Apollo Router v2.13.0 and later, your router will not start if any of these variables are set. Remove them from your environment before launching. + + \ No newline at end of file diff --git a/docs/source/routing/observability/graphos/graphos-reporting.mdx b/docs/source/routing/observability/graphos/graphos-reporting.mdx index 563ad13efd..a3ece4eccf 100644 --- a/docs/source/routing/observability/graphos/graphos-reporting.mdx +++ b/docs/source/routing/observability/graphos/graphos-reporting.mdx @@ -6,6 +6,8 @@ redirectFrom: - /graphos/routing/graphos-reporting --- +import OTelEnvVarsCaution from '../../../../shared/otel-envvars-caution.mdx'; + The GraphOS Router and Apollo Router Core can report operation usage metrics to [GraphOS](/graphos/) that you can then visualize in GraphOS Studio. These metrics also enable powerful GraphOS features like [schema checks](/graphos/delivery/schema-checks/). ## Enabling usage reporting @@ -67,11 +69,7 @@ telemetry: sampler: 0.01 ``` - - -If your environment defines `OTEL_EXPORTER_OTLP_ENDPOINT` or `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, they take precedence over your Router settings and may send traces to a different destination. Verify whether these variables are set and unset them if needed. For details, see the [OTLP exporter documentation](/router/configuration/telemetry/exporters/tracing/otlp). - - + ## Reporting field-level traces diff --git a/docs/source/routing/observability/router-telemetry-otel/telemetry-pipelines/metrics-exporters/otlp.mdx b/docs/source/routing/observability/router-telemetry-otel/telemetry-pipelines/metrics-exporters/otlp.mdx index 460d7122ab..dd1411dff8 100644 --- a/docs/source/routing/observability/router-telemetry-otel/telemetry-pipelines/metrics-exporters/otlp.mdx +++ b/docs/source/routing/observability/router-telemetry-otel/telemetry-pipelines/metrics-exporters/otlp.mdx @@ -9,6 +9,7 @@ redirectFrom: --- import BatchProcessorPreamble from '../../../../../../shared/batch-processor-preamble.mdx'; import BatchProcessorRef from '../../../../../../shared/batch-processor-ref.mdx'; +import OTelEnvVarsCaution from '../../../../../../shared/otel-envvars-caution.mdx'; Enable and configure the [OpenTelemetry Protocol (OTLP)](https://www.opentelemetry.io/) exporter for metrics in the GraphOS Router or Apollo Router Core. @@ -69,11 +70,7 @@ Defaults to: * http://127.0.0.1:4317 for gRPC * http://127.0.0.1:4318 for HTTP - - -If your environment sets `OTEL_EXPORTER_OTLP_ENDPOINT` or `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT`, these variables take precedence over the `endpoint` configured in your `router.yaml`. If metrics aren't appearing at your expected endpoint, check if these environment variables are set and unset them if necessary. - - + ### `grpc` diff --git a/docs/source/routing/observability/router-telemetry-otel/telemetry-pipelines/trace-exporters/otlp.mdx b/docs/source/routing/observability/router-telemetry-otel/telemetry-pipelines/trace-exporters/otlp.mdx index 0f1143eb97..baf99543c0 100644 --- a/docs/source/routing/observability/router-telemetry-otel/telemetry-pipelines/trace-exporters/otlp.mdx +++ b/docs/source/routing/observability/router-telemetry-otel/telemetry-pipelines/trace-exporters/otlp.mdx @@ -9,6 +9,7 @@ redirectFrom: --- import BatchProcessorPreamble from '../../../../../../shared/batch-processor-preamble.mdx'; import BatchProcessorRef from '../../../../../../shared/batch-processor-ref.mdx'; +import OTelEnvVarsCaution from '../../../../../../shared/otel-envvars-caution.mdx'; Enable and configure the [OpenTelemetry Protocol (OTLP)](https://github.com/open-telemetry/opentelemetry-proto/blob/main/docs/specification.md) exporter for tracing in the GraphOS Router or Apollo Router Core. @@ -65,11 +66,7 @@ Specify only the base URL in the endpoint parameter. The router automatically ad - - -If your environment sets `OTEL_EXPORTER_OTLP_ENDPOINT` or `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, these variables take precedence over the `endpoint` configured in your `router.yaml`. If traces aren't appearing at your expected endpoint, check if these environment variables are set and unset them if necessary. - - + ### `grpc` Settings specific to the gRPC protocol for setting a custom SSL certificate, domain name, and metadata.