From edc7d44250d055e28f19cfa2b84701c9e50c409a Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Mon, 17 Aug 2026 01:54:45 +0200 Subject: [PATCH 01/10] =?UTF-8?q?feat(receiver):=20composite=20tenant=20ru?= =?UTF-8?q?le=20+=20receiver.tenant=20config=20(RFC=200045=20slices=201?= =?UTF-8?q?=E2=80=932)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit TenantRule becomes an ordered, non-empty key list: single-key rules derive verbatim (RFC0045.6 — byte-identical default), composite rules percent-escape % and / and join with / (RFC0045.2/.4, injectivity as a proptest), every rule key required (RFC0045.3). receiver.tenant.{rule, watch,watch_capacity} resolve through FileConfig → ReceiverParams → ReceiverConfig (RFC0045.1); the server no longer hard-codes service_name(). Watch keys are carried; the detector lands next. Signed-off-by: Jens Holdgaard Pedersen --- crates/ourios-ingester/src/receiver.rs | 2 +- crates/ourios-ingester/src/receiver/tenant.rs | 299 ++++++++++++++++-- crates/ourios-server/src/config/file.rs | 78 ++++- crates/ourios-server/src/main.rs | 99 +++++- crates/ourios-server/src/receiver.rs | 12 +- docs/guides/configuration.md | 11 + 6 files changed, 465 insertions(+), 36 deletions(-) diff --git a/crates/ourios-ingester/src/receiver.rs b/crates/ourios-ingester/src/receiver.rs index 0ded64eb4..004a3903b 100644 --- a/crates/ourios-ingester/src/receiver.rs +++ b/crates/ourios-ingester/src/receiver.rs @@ -51,4 +51,4 @@ pub use pipeline::{IngestPipeline, Journal, ReceiveError, SharedPipeline}; pub use propagation::{ HeaderExtractor, MetadataExtractor, extract_context, extract_context_from_metadata, }; -pub use tenant::{TenantResolutionError, TenantRule, fan_out}; +pub use tenant::{TenantDerivation, TenantResolutionError, TenantRule, TenantRuleError, fan_out}; diff --git a/crates/ourios-ingester/src/receiver/tenant.rs b/crates/ourios-ingester/src/receiver/tenant.rs index 80fdee2b1..bedf18cd5 100644 --- a/crates/ourios-ingester/src/receiver/tenant.rs +++ b/crates/ourios-ingester/src/receiver/tenant.rs @@ -5,7 +5,8 @@ //! `Resource.attributes`, so one OTLP export can route records to //! several tenants. The default rule reads `service.name` — the //! OTel-canonical "what application emitted this", which maps onto -//! Ourios's per-tenant template-tree partitioning (`[§3.7]`). If any +//! Ourios's per-tenant template-tree partitioning (`[§3.7]`); the +//! operator may configure a composite of several keys (RFC 0045). If any //! group's Resource resolves to no tenant, the **entire** export is //! rejected (RFC0003.4) — no silent default tenant, no per-Resource //! partial acceptance. @@ -19,15 +20,16 @@ use ourios_core::tenant::TenantId; use crate::receiver::materialize::materialize_resource_logs; /// The operator-configured rule that derives a `tenant_id` from a -/// `ResourceLogs`' `Resource.attributes`. +/// `ResourceLogs`' `Resource.attributes` (RFC 0045 §3.1): an ordered, +/// non-empty list of resource-attribute keys, every one required. /// -/// Today the rule reads a single string-valued resource attribute (the -/// default key is `service.name`). RFC 0001 §6.1 reserves richer -/// operator models (per-namespace, composite of several attributes) for -/// when they're actually configured; this stays a single key until then. -#[derive(Debug, Clone)] +/// A single-key rule (the default `[service.name]`) derives the +/// attribute's string value verbatim. A composite rule (two or more keys) +/// percent-escapes `%` and `/` in each value and joins the values with +/// `/`, so distinct value tuples never collide (RFC 0045 §3.2). +#[derive(Debug, Clone, PartialEq, Eq)] pub struct TenantRule { - attribute_key: String, + keys: Vec, } impl TenantRule { @@ -38,48 +40,168 @@ impl TenantRule { Self::by_attribute("service.name") } - /// A rule reading an operator-chosen resource attribute key. + /// A single-key rule reading an operator-chosen resource attribute. pub fn by_attribute(key: impl Into) -> Self { Self { - attribute_key: key.into(), + keys: vec![key.into()], } } - /// The resource attribute key this rule reads. + /// An ordered rule over `keys` (RFC0045.1). + /// + /// # Errors + /// + /// [`TenantRuleError::Empty`] for no keys; [`TenantRuleError::Duplicate`] + /// naming the first repeated key. + pub fn from_keys(keys: I) -> Result + where + I: IntoIterator, + K: Into, + { + let keys: Vec = keys.into_iter().map(Into::into).collect(); + if keys.is_empty() { + return Err(TenantRuleError::Empty); + } + let mut seen = std::collections::HashSet::new(); + if let Some(duplicate) = keys.iter().find(|key| !seen.insert(key.as_str())) { + return Err(TenantRuleError::Duplicate { + key: duplicate.clone(), + }); + } + Ok(Self { keys }) + } + + /// The resource attribute keys this rule reads, in join order. + #[must_use] + pub fn keys(&self) -> &[String] { + &self.keys + } + + /// Whether `key` is one of the rule's keys. #[must_use] - pub fn attribute_key(&self) -> &str { - &self.attribute_key + pub fn contains(&self, key: &str) -> bool { + self.keys.iter().any(|k| k == key) } /// Derive the tenant for one Resource from its `attributes`. /// - /// Resolves to the rule's attribute when it is present with a - /// non-empty string value. + /// Resolves when every key is present with a non-empty string value. /// /// # Errors /// - /// [`TenantResolutionError`] (naming the attribute) when the - /// attribute is absent, not a string, or an empty string — the - /// receiver never invents a tenant the operator hasn't declared. + /// [`TenantResolutionError`] naming the first key that is absent, not a + /// string, or an empty string — the receiver never invents a tenant the + /// operator hasn't declared, and never joins a partial tuple. pub fn derive( &self, resource_attributes: &[KeyValue], ) -> Result { - resource_attributes - .iter() - .find(|kv| kv.key == self.attribute_key) - .and_then(|kv| kv.value.as_ref()) - .and_then(|value| match value.value.as_ref() { - Some(Value::StringValue(s)) if !s.is_empty() => Some(TenantId::new(s.clone())), - _ => None, - }) - .ok_or_else(|| TenantResolutionError { - attribute: self.attribute_key.clone(), - resource_index: None, - }) + let mut values = Vec::with_capacity(self.keys.len()); + for key in &self.keys { + let value = string_attribute(resource_attributes, key).ok_or_else(|| { + TenantResolutionError { + attribute: key.clone(), + resource_index: None, + } + })?; + values.push(value); + } + Ok(TenantId::new(join_components(&values))) + } +} + +/// The non-empty string value of `key` in `attributes`, if any. +fn string_attribute<'a>(attributes: &'a [KeyValue], key: &str) -> Option<&'a str> { + attributes + .iter() + .find(|kv| kv.key == key) + .and_then(|kv| kv.value.as_ref()) + .and_then(|value| match value.value.as_ref() { + Some(Value::StringValue(s)) if !s.is_empty() => Some(s.as_str()), + _ => None, + }) +} + +/// RFC 0045 §3.2: one component is the value verbatim; two or more are +/// `%`/`/`-escaped and `/`-joined. +fn join_components(values: &[&str]) -> String { + match values { + [single] => (*single).to_owned(), + many => { + let mut out = String::new(); + for (i, value) in many.iter().enumerate() { + if i > 0 { + out.push('/'); + } + for c in value.chars() { + match c { + '%' => out.push_str("%25"), + '/' => out.push_str("%2F"), + other => out.push(other), + } + } + } + out + } + } +} + +/// The resolved `receiver.tenant` section (RFC 0045 §3.1): the derivation +/// rule plus the divergence-watch keys and state bound (§3.4). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TenantDerivation { + pub rule: TenantRule, + /// Keys watched for divergence; a key also in `rule` is skipped. + pub watch: Vec, + /// Upper bound on remembered (tenant, key) pairs. + pub watch_capacity: usize, +} + +impl TenantDerivation { + /// The RFC 0045 §3.4 default watch key. + pub const DEFAULT_WATCH: &'static str = "k8s.cluster.name"; + /// The RFC 0045 §3.4 default state bound. + pub const DEFAULT_WATCH_CAPACITY: usize = 10_000; +} + +impl Default for TenantDerivation { + fn default() -> Self { + Self { + rule: TenantRule::service_name(), + watch: vec![Self::DEFAULT_WATCH.to_owned()], + watch_capacity: Self::DEFAULT_WATCH_CAPACITY, + } } } +/// A `receiver.tenant.rule` that cannot be a rule (RFC0045.1). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TenantRuleError { + /// No keys at all. + Empty, + /// The same key listed twice. + Duplicate { key: String }, +} + +impl std::fmt::Display for TenantRuleError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Empty => write!( + f, + "tenant rule must list at least one resource attribute key" + ), + Self::Duplicate { key } => { + write!( + f, + "tenant rule lists resource attribute key `{key}` more than once" + ) + } + } + } +} + +impl std::error::Error for TenantRuleError {} + impl Default for TenantRule { fn default() -> Self { Self::service_name() @@ -199,8 +321,9 @@ pub(crate) fn derive_for_group( #[cfg(test)] mod tests { - use super::{TenantRule, Value}; + use super::{TenantRule, TenantRuleError, Value}; use opentelemetry_proto::tonic::common::v1::{AnyValue, KeyValue}; + use proptest::strategy::Strategy; fn string_attr(key: &str, value: &str) -> KeyValue { KeyValue { @@ -266,4 +389,118 @@ mod tests { // Assert assert_eq!(tenant.as_str(), "acme"); } + + // RFC0045.6 — the single-key path never escapes. + #[test] + fn single_key_rule_is_verbatim_even_with_slash_and_percent() { + for raw in ["a/b", "100%", "a%2Fb"] { + let attrs = [string_attr("service.name", raw)]; + let tenant = TenantRule::service_name().derive(&attrs).expect("resolves"); + assert_eq!(tenant.as_str(), raw); + } + } + + // RFC0045.2 — composite join. + #[test] + fn composite_rule_joins_in_key_order() { + let rule = TenantRule::from_keys(["k8s.cluster.name", "service.name"]).expect("valid"); + let attrs = [ + string_attr("service.name", "fluxcd"), + string_attr("k8s.cluster.name", "cluster1"), + ]; + let tenant = rule.derive(&attrs).expect("resolves"); + assert_eq!(tenant.as_str(), "cluster1/fluxcd"); + } + + // RFC0045.4 — the two canonical colliding tuples stay apart. + #[test] + fn composite_join_escapes_separator_and_escape_char() { + let rule = TenantRule::from_keys(["a", "b"]).expect("valid"); + let left = rule + .derive(&[string_attr("a", "a"), string_attr("b", "b/c")]) + .expect("resolves"); + let right = rule + .derive(&[string_attr("a", "a/b"), string_attr("b", "c")]) + .expect("resolves"); + assert_eq!(left.as_str(), "a/b%2Fc"); + assert_eq!(right.as_str(), "a%2Fb/c"); + let pct = rule + .derive(&[string_attr("a", "50%"), string_attr("b", "x")]) + .expect("resolves"); + assert_eq!(pct.as_str(), "50%25/x"); + } + + // RFC0045.3 — every rule key is required; the error names the missing one. + #[test] + fn composite_rule_rejects_missing_empty_or_non_string_component() { + let rule = TenantRule::from_keys(["k8s.cluster.name", "service.name"]).expect("valid"); + let missing = [string_attr("service.name", "fluxcd")]; + let err = rule.derive(&missing).unwrap_err(); + assert_eq!(err.attribute(), "k8s.cluster.name"); + + let empty = [ + string_attr("k8s.cluster.name", ""), + string_attr("service.name", "fluxcd"), + ]; + assert_eq!( + rule.derive(&empty).unwrap_err().attribute(), + "k8s.cluster.name" + ); + + let non_string = [ + KeyValue { + key: "k8s.cluster.name".to_owned(), + value: Some(AnyValue { + value: Some(Value::IntValue(1)), + }), + ..Default::default() + }, + string_attr("service.name", "fluxcd"), + ]; + assert_eq!( + rule.derive(&non_string).unwrap_err().attribute(), + "k8s.cluster.name" + ); + } + + // RFC0045.1 — rule validation. + #[test] + fn from_keys_rejects_empty_and_duplicate() { + assert_eq!( + TenantRule::from_keys(Vec::::new()).unwrap_err(), + TenantRuleError::Empty + ); + assert_eq!( + TenantRule::from_keys(["service.name", "service.name"]).unwrap_err(), + TenantRuleError::Duplicate { + key: "service.name".to_owned() + } + ); + assert_eq!( + TenantRule::from_keys(["service.name"]).expect("valid"), + TenantRule::service_name() + ); + } + + // RFC0045.4 in property form: for a fixed composite rule, distinct + // value tuples derive distinct tenant ids. + proptest::proptest! { + #![proptest_config(proptest::prelude::ProptestConfig::with_cases(512))] + #[test] + fn composite_join_is_injective( + (left, right) in (2usize..=3).prop_flat_map(|arity| ( + proptest::collection::vec("[a-c/%]{1,4}", arity), + proptest::collection::vec("[a-c/%]{1,4}", arity), + )), + ) { + let keys: Vec = (0..left.len()).map(|i| format!("k{i}")).collect(); + let rule = TenantRule::from_keys(keys.clone()).expect("valid"); + let attrs = |values: &[String]| -> Vec { + keys.iter().zip(values).map(|(k, v)| string_attr(k, v)).collect() + }; + let l = rule.derive(&attrs(&left)).expect("resolves"); + let r = rule.derive(&attrs(&right)).expect("resolves"); + proptest::prop_assert_eq!(left == right, l == r); + } + } } diff --git a/crates/ourios-server/src/config/file.rs b/crates/ourios-server/src/config/file.rs index e514660ff..707545705 100644 --- a/crates/ourios-server/src/config/file.rs +++ b/crates/ourios-server/src/config/file.rs @@ -332,6 +332,27 @@ pub struct ReceiverSection { /// (`receiver.encode_workers`; default: the host's available cores). #[serde(deserialize_with = "scalar_opt")] pub encode_workers: Option, + /// RFC 0045 §3.1 — tenant derivation (`receiver.tenant.*`). + pub tenant: TenantSection, +} + +/// `receiver.tenant.*` — the RFC 0045 §3.1 tenant-derivation rule and +/// divergence watch. Raw string leaves; validation (non-empty, no +/// duplicates, capacity ≥ 1) lives in the resolver, the single path. +#[derive(Debug, Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct TenantSection { + /// Ordered resource-attribute keys joined into the tenant id. `None` + /// (absent) is the `[service.name]` default; an explicit `[]` is a + /// startup error (RFC0045.1), so the two are kept apart here. + #[serde(deserialize_with = "scalar_vec_opt")] + pub rule: Option>, + /// Keys watched for divergence (§3.4). `None` = `[k8s.cluster.name]`. + #[serde(deserialize_with = "scalar_vec_opt")] + pub watch: Option>, + /// Upper bound on remembered (tenant, key) pairs (§3.4). + #[serde(deserialize_with = "scalar_opt")] + pub watch_capacity: Option, } /// One `*_tls` block (RFC 0030 §3.1). Raw string leaves — the §3.1 @@ -672,7 +693,20 @@ impl ReceiverSection { substitute(&mut self.http_addr, lookup)?; self.http_tls.substitute(lookup)?; substitute(&mut self.wal_root, lookup)?; - substitute(&mut self.encode_workers, lookup) + substitute(&mut self.encode_workers, lookup)?; + self.tenant.substitute(lookup) + } +} + +impl TenantSection { + fn substitute( + &mut self, + lookup: &dyn Fn(&str) -> Option, + ) -> Result<(), MalformedReference> { + for key in self.rule.iter_mut().chain(self.watch.iter_mut()).flatten() { + *key = env_subst::resolve(key, lookup)?; + } + substitute(&mut self.watch_capacity, lookup) } } @@ -727,6 +761,16 @@ where .collect()) } +/// [`scalar_vec`] for a key whose absence means "default" while an +/// explicit empty sequence is a value in its own right. +fn scalar_vec_opt<'de, D>(deserializer: D) -> Result>, D::Error> +where + D: serde::Deserializer<'de>, +{ + Ok(Option::>::deserialize(deserializer)? + .map(|scalars| scalars.into_iter().map(|s| s.0).collect())) +} + /// A YAML scalar captured as its string form (see [`scalar_opt`]). struct Scalar(String); @@ -959,6 +1003,38 @@ storage: assert!(matches!(err, FileConfigError::Schema(_)), "got {err:?}"); } + /// RFC0045.1 — `receiver.tenant.*` parses as raw leaves: absent + /// `rule`/`watch` stay `None` (the resolver applies the defaults), an + /// explicit `[]` is preserved as an empty list (the resolver rejects + /// it), elements ride `${env:…}`, and unknown keys are a schema error. + #[test] + fn receiver_tenant_section_parses_and_substitutes() { + let lookup = env(&[("CLUSTER_KEY", "k8s.cluster.name")]); + let cfg = parse("receiver:\n enabled: true\n", &lookup).expect("valid"); + assert!(cfg.receiver.tenant.rule.is_none()); + assert!(cfg.receiver.tenant.watch.is_none()); + assert!(cfg.receiver.tenant.watch_capacity.is_none()); + + let cfg = parse( + "receiver:\n tenant:\n rule: [\"${env:CLUSTER_KEY}\", service.name]\n watch: []\n watch_capacity: 42\n", + &lookup, + ) + .expect("valid"); + assert_eq!( + cfg.receiver.tenant.rule.as_deref(), + Some(&["k8s.cluster.name".to_owned(), "service.name".to_owned()][..]) + ); + assert_eq!(cfg.receiver.tenant.watch.as_deref(), Some(&[][..])); + assert_eq!(cfg.receiver.tenant.watch_capacity.as_deref(), Some("42")); + + let err = parse("receiver:\n tenant:\n rules: [a]\n", &lookup) + .expect_err("unknown key inside receiver.tenant"); + assert!(matches!(err, FileConfigError::Schema(_)), "got {err:?}"); + let err = parse("receiver:\n tenant:\n rule: service.name\n", &lookup) + .expect_err("scalar where a list is expected"); + assert!(matches!(err, FileConfigError::Schema(_)), "got {err:?}"); + } + /// RFC 0022 §3.2 — a bare scalar where a key *list* is expected is a /// schema error (the `scalar_vec` shape rule), mirroring the /// scalar-where-structure rule the other leaves enforce. diff --git a/crates/ourios-server/src/main.rs b/crates/ourios-server/src/main.rs index 3f6fd6a66..b6d373939 100644 --- a/crates/ourios-server/src/main.rs +++ b/crates/ourios-server/src/main.rs @@ -42,10 +42,11 @@ use std::time::Duration; use clap::Parser; use ourios_ingester::Compactor; use ourios_ingester::receiver::tls::TlsSettings; +use ourios_ingester::receiver::{TenantDerivation, TenantRule}; use ourios_parquet::{ CompactionPolicy, ParquetAuditSink, PromotedAttributes, S3Config, StoreConfig, }; -use ourios_server::config::file::{FileConfig, PromotedEntry, TlsSection}; +use ourios_server::config::file::{FileConfig, PromotedEntry, TenantSection, TlsSection}; use ourios_telemetry::TelemetryConfig; use ourios_wal::WalConfig; @@ -125,6 +126,9 @@ struct ReceiverParams { /// (`receiver.encode_workers` / `OURIOS_RECEIVER_ENCODE_WORKERS`; /// default: the host's available cores, validated ≥ 1). encode_workers: usize, + /// RFC 0045 §3.1 — tenant derivation rule + divergence watch + /// (`receiver.tenant.*`, config-file only; default `[service.name]`). + tenant: TenantDerivation, } /// Resolve [`ServerConfig`] from the environment: @@ -277,6 +281,7 @@ fn server_config_from_file(file: &FileConfig) -> Result { if let Some(receiver) = config.receiver.as_mut() { receiver.grpc_tls = tls_settings("receiver.grpc_tls", &file.receiver.grpc_tls)?; receiver.http_tls = tls_settings("receiver.http_tls", &file.receiver.http_tls)?; + receiver.tenant = tenant_derivation(&file.receiver.tenant)?; } config.querier = build_querier_config( file.querier.enabled.as_deref(), @@ -451,9 +456,39 @@ fn build_receiver_config( http_tls: None, wal_root, encode_workers, + tenant: TenantDerivation::default(), })) } +/// Resolve `receiver.tenant.*` (RFC 0045 §3.1 / RFC0045.1): absent keys +/// take the defaults; an empty or duplicate-key `rule` and a +/// `watch_capacity` below 1 are configuration errors. +fn tenant_derivation(section: &TenantSection) -> Result { + let defaults = TenantDerivation::default(); + let rule = match §ion.rule { + Some(keys) => TenantRule::from_keys(keys.iter().cloned()) + .map_err(|e| format!("receiver.tenant.rule: {e}"))?, + None => defaults.rule, + }; + let watch = section.watch.clone().unwrap_or(defaults.watch); + let watch_capacity = match section.watch_capacity.as_deref().map(str::trim) { + Some(raw) if !raw.is_empty() => match raw.parse::() { + Ok(n) if n >= 1 => n, + _ => { + return Err(format!( + "receiver.tenant.watch_capacity must be an integer ≥ 1, got {raw:?}" + )); + } + }, + _ => defaults.watch_capacity, + }; + Ok(TenantDerivation { + rule, + watch, + watch_capacity, + }) +} + /// Parse the RFC 0035 encode-pool worker count: ≥ 1 when set, else the /// host's available cores (min 1 — `available_parallelism` can fail in /// constrained environments, and the pool needs at least one worker). @@ -879,6 +914,7 @@ async fn main() -> Result<(), Box> { promoted: config.promoted.clone(), auth: resolver.clone().expect("resolver built for enabled roles"), encode_workers: params.encode_workers, + tenant: params.tenant.clone(), }) .await?; println!("receiver gRPC listening on {}", handle.grpc_addr); @@ -1726,6 +1762,67 @@ auth: ); } + /// RFC0045.1 — `receiver.tenant` resolution: absent → the + /// `[service.name]` default (byte-identical to the pre-RFC rule), + /// `rule: []` and a duplicate key are configuration errors, and the + /// composite rule + watch settings resolve in order. + #[test] + fn tenant_derivation_defaults_and_validates() { + let lookup = |_: &str| None; + let absent = parse("receiver:\n enabled: true\n", &lookup).expect("valid"); + assert_eq!( + tenant_derivation(&absent.receiver.tenant).expect("default"), + TenantDerivation::default() + ); + assert_eq!( + TenantDerivation::default().rule, + TenantRule::service_name() + ); + + let empty = parse("receiver:\n tenant:\n rule: []\n", &lookup).expect("valid yaml"); + assert!( + tenant_derivation(&empty.receiver.tenant) + .unwrap_err() + .contains("receiver.tenant.rule") + ); + + let dup = parse( + "receiver:\n tenant:\n rule: [service.name, service.name]\n", + &lookup, + ) + .expect("valid yaml"); + assert!( + tenant_derivation(&dup.receiver.tenant) + .unwrap_err() + .contains("service.name") + ); + + let composite = parse( + "receiver:\n tenant:\n rule: [k8s.cluster.name, service.name]\n watch: [cloud.region]\n watch_capacity: 7\n", + &lookup, + ) + .expect("valid yaml"); + let resolved = tenant_derivation(&composite.receiver.tenant).expect("resolves"); + assert_eq!( + resolved.rule, + TenantRule::from_keys(["k8s.cluster.name", "service.name"]).expect("rule") + ); + assert_eq!(resolved.watch, ["cloud.region"]); + assert_eq!(resolved.watch_capacity, 7); + + for bad in ["0", "-1", "many"] { + let cfg = parse( + &format!("receiver:\n tenant:\n watch_capacity: {bad}\n"), + &lookup, + ) + .expect("valid yaml"); + assert!( + tenant_derivation(&cfg.receiver.tenant).is_err(), + "watch_capacity = {bad:?} is rejected" + ); + } + } + #[test] fn build_receiver_config_encode_workers_defaults_and_validates() { // RFC 0035: unset → available cores (≥ 1); explicit values parse; diff --git a/crates/ourios-server/src/receiver.rs b/crates/ourios-server/src/receiver.rs index 92d269cbc..c6f05ab84 100644 --- a/crates/ourios-server/src/receiver.rs +++ b/crates/ourios-server/src/receiver.rs @@ -28,7 +28,9 @@ use ourios_ingester::receiver::tls::{ALPN_GRPC, ALPN_HTTP, TlsSettings}; use ourios_ingester::receiver::tls_serve::{ LISTENER_GRPC, LISTENER_HTTP, TlsListener, reloading_acceptor, tls_incoming, }; -use ourios_ingester::receiver::{CommitCoordinator, IngestPipeline, SharedPipeline, TenantRule}; +use ourios_ingester::receiver::{ + CommitCoordinator, IngestPipeline, SharedPipeline, TenantDerivation, +}; use ourios_ingester::record_sink::{FlushConfig, ParquetRecordSink, SharedParquetSink}; use ourios_ingester::recovery; use ourios_miner::cluster::MinerCluster; @@ -274,6 +276,8 @@ pub struct ReceiverConfig { /// (`receiver.encode_workers`; the config layer validates ≥ 1 and /// defaults to the host's available cores). pub encode_workers: usize, + /// RFC 0045 §3.1 — the tenant derivation rule and divergence watch. + pub tenant: TenantDerivation, } /// A running receiver role: the **resolved** bound addresses (so a `:0` @@ -478,7 +482,7 @@ pub async fn serve(config: ReceiverConfig) -> Result { let mut miner = MinerCluster::with_audit_sink(MinerConfig::default(), Box::new(audit_sink.clone())) .with_record_sink(Box::new(sink.clone())); - let rule = TenantRule::service_name(); + let rule = config.tenant.rule.clone(); let report = recovery::recover(&mut wal, &snapshots_root, &mut miner, &rule) .map_err(|e| format!("startup recovery: {e}"))?; @@ -666,6 +670,7 @@ mod tests { use ourios_core::audit::{AuditSink, ParamType}; use ourios_core::record::{BodyKind, MinedRecord, Param, RecordSink}; use ourios_core::tenant::TenantId; + use ourios_ingester::receiver::TenantRule; use super::*; @@ -888,6 +893,7 @@ mod tests { promoted: PromotedAttributes::default(), auth: AuthResolver::static_only(None), encode_workers: 2, + tenant: TenantDerivation::default(), }) .await .expect("serve"); @@ -919,6 +925,7 @@ mod tests { promoted: PromotedAttributes::default(), auth: AuthResolver::static_only(None), encode_workers: 2, + tenant: TenantDerivation::default(), }) .await .expect("serve"); @@ -1049,6 +1056,7 @@ mod tests { promoted: PromotedAttributes::default(), auth: AuthResolver::static_only(None), encode_workers: 2, + tenant: TenantDerivation::default(), }) .await .expect("serve"); diff --git a/docs/guides/configuration.md b/docs/guides/configuration.md index fc974f844..ae48aeba0 100644 --- a/docs/guides/configuration.md +++ b/docs/guides/configuration.md @@ -47,6 +47,17 @@ receiver: wal_root: /var/lib/ourios/wal # RFC 0035: concurrent Parquet-encode workers (default: all cores). encode_workers: 4 + # RFC 0045: how a tenant id is derived from each ResourceLogs group. + # Default rule is [service.name]. List several keys when the same + # service.name runs in more than one cluster — every key is required, + # values join with "/" (cluster1/fluxcd). Changing the rule affects + # newly ingested data only; stored tenant ids never change. + tenant: + rule: [k8s.cluster.name, service.name] + # Keys watched for the "one tenant spans several clusters" signal + # (a warning + counter, never a rejection). Default: [k8s.cluster.name]. + watch: [k8s.cluster.name] + watch_capacity: 10000 querier: enabled: true From 6ee835866767e99a05e0cd08111129a1b34d6839 Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Mon, 17 Aug 2026 01:59:27 +0200 Subject: [PATCH 02/10] feat(ingester): tenant rule-epoch log so WAL replay derives under the acked rule (RFC0045.10) Startup recovery re-fans-out every surviving frame; a changed rule would abort on frames lacking a new key or silently re-tenant acknowledged data. RuleEpochs (tenant_rule_epochs.json in the WAL root, atomic write, absent = implicit [service.name] epoch) maps a frame's offset to the rule it was acknowledged under; the server advances the log after replay when the configured rule differs. Malformed logs abort startup. Signed-off-by: Jens Holdgaard Pedersen --- crates/ourios-ingester/src/lib.rs | 1 + crates/ourios-ingester/src/recovery.rs | 22 +- crates/ourios-ingester/src/rule_epochs.rs | 341 ++++++++++++++++++ crates/ourios-ingester/tests/it/main.rs | 1 + .../tests/it/rfc0001_3_5_snapshot_restore.rs | 51 ++- .../tests/it/rfc0014_5_crash_no_loss.rs | 4 +- .../tests/it/rfc0035_2_encode_barrier.rs | 9 +- .../tests/it/rfc0035_2_sweep_crash.rs | 4 +- .../tests/it/rfc0045_10_wal_tail_epoch.rs | 163 +++++++++ crates/ourios-server/src/main.rs | 5 +- crates/ourios-server/src/receiver.rs | 18 +- 11 files changed, 585 insertions(+), 34 deletions(-) create mode 100644 crates/ourios-ingester/src/rule_epochs.rs create mode 100644 crates/ourios-ingester/tests/it/rfc0045_10_wal_tail_epoch.rs diff --git a/crates/ourios-ingester/src/lib.rs b/crates/ourios-ingester/src/lib.rs index 3eea7fe0f..e3820ff4f 100644 --- a/crates/ourios-ingester/src/lib.rs +++ b/crates/ourios-ingester/src/lib.rs @@ -40,6 +40,7 @@ pub mod publish; pub mod receiver; pub mod record_sink; pub mod recovery; +pub mod rule_epochs; pub mod snapshot_store; pub use compactor::{Compactor, IngestError, SweepReport, run_sweep, run_sweep_with_promoted}; diff --git a/crates/ourios-ingester/src/recovery.rs b/crates/ourios-ingester/src/recovery.rs index 5cd514318..0004e8ee8 100644 --- a/crates/ourios-ingester/src/recovery.rs +++ b/crates/ourios-ingester/src/recovery.rs @@ -24,7 +24,8 @@ use ourios_miner::snapshot::{RecoveryOutcome, WalHighWater}; use ourios_wal::{FrameKind, FrameSink, RecoveryError, Wal, WalOffset}; use prost::Message; -use crate::receiver::tenant::{TenantRule, fan_out}; +use crate::receiver::tenant::fan_out; +use crate::rule_epochs::RuleEpochs; use crate::snapshot_store::{self, SnapshotStoreError}; /// What recovery did, for the caller to log and for the @@ -114,7 +115,7 @@ pub fn recover( wal: &mut Wal, snapshots_root: &Path, miner: &mut MinerCluster, - rule: &TenantRule, + epochs: &RuleEpochs, ) -> Result { let parquet_horizon = wal.last_checkpoint(); let artefacts = snapshot_store::load_all(snapshots_root).map_err(RecoveryDriverError::Store)?; @@ -152,7 +153,7 @@ pub fn recover( let mut sink = DriverSink { miner, - rule, + epochs, horizons: &horizons, frames_delivered: 0, records_fed: 0, @@ -243,11 +244,12 @@ fn parse_high_water(high_water: Option<&WalHighWater>) -> Option { } /// The §6.6 [`FrameSink`]: per `OtlpBatch` frame, decode → -/// [`fan_out`] → feed each record to the miner iff the frame offset -/// is above that record's tenant horizon. +/// [`fan_out`] under the frame's rule epoch (RFC 0045 §3.3) → feed each +/// record to the miner iff the frame offset is above that record's +/// tenant horizon. struct DriverSink<'a> { miner: &'a mut MinerCluster, - rule: &'a TenantRule, + epochs: &'a RuleEpochs, horizons: &'a HashMap, frames_delivered: u64, records_fed: u64, @@ -273,7 +275,8 @@ impl FrameSink for DriverSink<'_> { FrameKind::OtlpBatch => { let request = ExportLogsServiceRequest::decode(payload).map_err(|e| reject(offset, &e))?; - let records = fan_out(request, self.rule).map_err(|e| reject(offset, &e))?; + let records = fan_out(request, self.epochs.rule_for(offset)) + .map_err(|e| reject(offset, &e))?; for record in &records { let feed = match self.horizons.get(&record.tenant_id) { Some(horizon) => offset > *horizon, @@ -375,11 +378,12 @@ mod tests { #[test] fn sink_rejects_a_malformed_payload_naming_the_offset() { let mut miner = MinerCluster::new(MinerConfig::default()); - let rule = TenantRule::service_name(); + let dir = tempfile::tempdir().expect("tempdir"); + let epochs = RuleEpochs::load(dir.path()).expect("implicit epoch"); let horizons = HashMap::new(); let mut sink = DriverSink { miner: &mut miner, - rule: &rule, + epochs: &epochs, horizons: &horizons, frames_delivered: 0, records_fed: 0, diff --git a/crates/ourios-ingester/src/rule_epochs.rs b/crates/ourios-ingester/src/rule_epochs.rs new file mode 100644 index 000000000..1972e325a --- /dev/null +++ b/crates/ourios-ingester/src/rule_epochs.rs @@ -0,0 +1,341 @@ +//! The tenant rule-epoch log (RFC 0045 §3.3): which [`TenantRule`] each +//! WAL frame was acknowledged under, so startup replay derives a frame's +//! tenant exactly as ingest did even after the operator changed the rule. +//! +//! A sidecar file in the WAL root (`tenant_rule_epochs.json`), never a +//! WAL frame kind: an ordered list of `{rule, after}` entries meaning +//! "frames with offset > `after` derive under `rule`" (`after: null` = +//! from the beginning). An absent file is the single implicit epoch +//! `{[service.name], null}` — every pre-RFC WAL — so upgrading needs no +//! migration. A file that exists but does not parse aborts startup, the +//! same class as a corrupt segment header. + +use std::fs::File; +use std::io::Write; +use std::path::{Path, PathBuf}; + +use ourios_wal::WalOffset; +use serde_json::{Value, json}; + +use crate::receiver::tenant::TenantRule; + +/// The sidecar's file name inside the WAL root. +pub const FILE_NAME: &str = "tenant_rule_epochs.json"; + +/// One epoch: `rule` applies to frames strictly after `after`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RuleEpoch { + pub rule: TenantRule, + pub after: Option, +} + +/// The loaded epoch log, in append order. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RuleEpochs { + path: PathBuf, + epochs: Vec, +} + +impl RuleEpochs { + /// Load `/tenant_rule_epochs.json`, or the implicit + /// `[service.name]` epoch when the file is absent. + /// + /// # Errors + /// + /// [`RuleEpochsError`] on an I/O failure other than not-found, or a + /// file that is not the documented shape. + pub fn load(wal_root: &Path) -> Result { + let path = wal_root.join(FILE_NAME); + let bytes = match std::fs::read(&path) { + Ok(bytes) => bytes, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + return Ok(Self { + path, + epochs: vec![RuleEpoch { + rule: TenantRule::service_name(), + after: None, + }], + }); + } + Err(source) => { + return Err(RuleEpochsError::Io { + op: "read(tenant rule epochs)", + path, + source, + }); + } + }; + let epochs = parse(&bytes).map_err(|detail| RuleEpochsError::Malformed { + path: path.clone(), + detail, + })?; + Ok(Self { path, epochs }) + } + + /// The epochs, oldest first. + #[must_use] + pub fn epochs(&self) -> &[RuleEpoch] { + &self.epochs + } + + /// The rule the newest epoch derives under. + #[must_use] + pub fn current(&self) -> &TenantRule { + // `load` and `advance` keep the list non-empty. + self.epochs + .last() + .map_or_else(|| unreachable!("epoch log is never empty"), |e| &e.rule) + } + + /// The rule a frame at `offset` was acknowledged under: the newest + /// epoch whose `after` lies strictly below `offset`. + #[must_use] + pub fn rule_for(&self, offset: WalOffset) -> &TenantRule { + self.epochs + .iter() + .rev() + .find(|epoch| epoch.after.is_none_or(|after| offset > after)) + .map_or_else(|| self.current(), |epoch| &epoch.rule) + } + + /// Make `rule` the current epoch for frames after `after` (the highest + /// offset replay delivered), persisting the log if the rule differs + /// from the newest epoch's. Returns whether an epoch was appended. + /// + /// # Errors + /// + /// [`RuleEpochsError::Io`] if the sidecar cannot be written durably. + pub fn advance( + &mut self, + rule: &TenantRule, + after: Option, + ) -> Result { + if self.current() == rule { + return Ok(false); + } + self.epochs.push(RuleEpoch { + rule: rule.clone(), + after, + }); + self.persist()?; + Ok(true) + } + + fn persist(&self) -> Result<(), RuleEpochsError> { + let io = |op: &'static str, path: &Path| { + let path = path.to_path_buf(); + move |source| RuleEpochsError::Io { op, path, source } + }; + let bytes = serde_json::to_vec_pretty(&render(&self.epochs)).map_err(|e| { + RuleEpochsError::Malformed { + path: self.path.clone(), + detail: e.to_string(), + } + })?; + let tmp = self.path.with_extension("json.tmp"); + let mut file = File::create(&tmp).map_err(io("create(tenant rule epochs tmp)", &tmp))?; + file.write_all(&bytes) + .map_err(io("write(tenant rule epochs tmp)", &tmp))?; + file.sync_all() + .map_err(io("fsync(tenant rule epochs tmp)", &tmp))?; + std::fs::rename(&tmp, &self.path) + .map_err(io("rename(tenant rule epochs tmp -> live)", &self.path))?; + if let Some(dir) = self.path.parent() { + File::open(dir) + .and_then(|d| d.sync_all()) + .map_err(io("fsync(wal root after tenant rule epochs)", dir))?; + } + Ok(()) + } +} + +fn render(epochs: &[RuleEpoch]) -> Value { + json!({ + "epochs": epochs + .iter() + .map(|epoch| { + json!({ + "rule": epoch.rule.keys(), + "after": epoch.after.map(|o| json!({ + "segment": o.segment.to_string(), + "byte": o.byte, + })), + }) + }) + .collect::>(), + }) +} + +fn parse(bytes: &[u8]) -> Result, String> { + let root: Value = serde_json::from_slice(bytes).map_err(|e| e.to_string())?; + let entries = root + .get("epochs") + .and_then(Value::as_array) + .ok_or("missing `epochs` array")?; + if entries.is_empty() { + return Err("`epochs` is empty".to_owned()); + } + let mut epochs = Vec::with_capacity(entries.len()); + let mut previous: Option = None; + for (index, entry) in entries.iter().enumerate() { + let keys = entry + .get("rule") + .and_then(Value::as_array) + .ok_or_else(|| format!("epochs[{index}].rule is not an array"))? + .iter() + .map(|k| { + k.as_str() + .map(str::to_owned) + .ok_or_else(|| format!("epochs[{index}].rule holds a non-string key")) + }) + .collect::, _>>()?; + let rule = TenantRule::from_keys(keys).map_err(|e| format!("epochs[{index}].rule: {e}"))?; + let after = match entry.get("after") { + None | Some(Value::Null) => None, + Some(after) => { + let segment = after + .get("segment") + .and_then(Value::as_str) + .and_then(|s| s.parse::().ok()) + .ok_or_else(|| format!("epochs[{index}].after.segment is not a UUID"))?; + let byte = after + .get("byte") + .and_then(Value::as_u64) + .ok_or_else(|| format!("epochs[{index}].after.byte is not a u64"))?; + Some(WalOffset { segment, byte }) + } + }; + if let (Some(prev), Some(cur)) = (previous, after) + && cur < prev + { + return Err(format!("epochs[{index}].after precedes the previous epoch")); + } + previous = after.or(previous); + epochs.push(RuleEpoch { rule, after }); + } + Ok(epochs) +} + +/// Failure loading or persisting the epoch log. +#[derive(Debug)] +#[non_exhaustive] +pub enum RuleEpochsError { + Io { + op: &'static str, + path: PathBuf, + source: std::io::Error, + }, + /// The file exists but is not the documented shape — corruption class, + /// surfaced loudly rather than guessed at. + Malformed { path: PathBuf, detail: String }, +} + +impl std::fmt::Display for RuleEpochsError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Io { op, path, source } => { + write!(f, "tenant rule epochs {op} {}: {source}", path.display()) + } + Self::Malformed { path, detail } => { + write!( + f, + "tenant rule epochs {} is malformed: {detail}", + path.display() + ) + } + } + } +} + +impl std::error::Error for RuleEpochsError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Io { source, .. } => Some(source), + Self::Malformed { .. } => None, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn offset(byte: u64) -> WalOffset { + WalOffset { + segment: uuid::Uuid::now_v7(), + byte, + } + } + + // RFC0045.10 — an absent log is the implicit [service.name] epoch. + #[test] + fn absent_log_is_the_service_name_epoch() { + let dir = tempfile::tempdir().expect("tempdir"); + let epochs = RuleEpochs::load(dir.path()).expect("loads"); + assert_eq!(epochs.current(), &TenantRule::service_name()); + assert_eq!(epochs.rule_for(offset(0)), &TenantRule::service_name()); + assert!(!dir.path().join(FILE_NAME).exists(), "load never writes"); + } + + // RFC0045.10 — advancing persists; reload sees the same log; frames at or + // below the boundary keep the old rule, frames above take the new one. + #[test] + fn advance_persists_and_lookup_is_by_offset() { + let dir = tempfile::tempdir().expect("tempdir"); + let composite = TenantRule::from_keys(["k8s.cluster.name", "service.name"]).expect("valid"); + let boundary = offset(100); + + let mut epochs = RuleEpochs::load(dir.path()).expect("loads"); + assert!( + !epochs + .advance(&TenantRule::service_name(), Some(boundary)) + .expect("no-op") + ); + assert!( + !dir.path().join(FILE_NAME).exists(), + "unchanged rule never writes" + ); + assert!(epochs.advance(&composite, Some(boundary)).expect("appends")); + + let reloaded = RuleEpochs::load(dir.path()).expect("reloads"); + assert_eq!(reloaded.epochs(), epochs.epochs()); + assert_eq!(reloaded.current(), &composite); + let earlier = WalOffset { + segment: boundary.segment, + byte: 100, + }; + let later = WalOffset { + segment: boundary.segment, + byte: 101, + }; + assert_eq!(reloaded.rule_for(earlier), &TenantRule::service_name()); + assert_eq!(reloaded.rule_for(later), &composite); + assert_eq!(reloaded.rule_for(offset(0)), &composite, "a newer segment"); + } + + // RFC0045.10 — an unparseable log aborts loudly, naming the file. + #[test] + fn malformed_log_is_an_error_naming_the_file() { + let dir = tempfile::tempdir().expect("tempdir"); + std::fs::write(dir.path().join(FILE_NAME), b"{\"epochs\": []}").expect("write"); + let err = RuleEpochs::load(dir.path()).unwrap_err(); + assert!(matches!(err, RuleEpochsError::Malformed { .. }), "{err:?}"); + assert!(err.to_string().contains(FILE_NAME)); + + std::fs::write(dir.path().join(FILE_NAME), b"not json").expect("write"); + assert!(matches!( + RuleEpochs::load(dir.path()).unwrap_err(), + RuleEpochsError::Malformed { .. } + )); + + std::fs::write( + dir.path().join(FILE_NAME), + b"{\"epochs\": [{\"rule\": [\"a\", \"a\"], \"after\": null}]}", + ) + .expect("write"); + assert!(matches!( + RuleEpochs::load(dir.path()).unwrap_err(), + RuleEpochsError::Malformed { .. } + )); + } +} diff --git a/crates/ourios-ingester/tests/it/main.rs b/crates/ourios-ingester/tests/it/main.rs index 38202b4df..07dfd5c20 100644 --- a/crates/ourios-ingester/tests/it/main.rs +++ b/crates/ourios-ingester/tests/it/main.rs @@ -50,3 +50,4 @@ mod rfc0035_f2_miner_panic_salvage; mod rfc0038_2_ingest_batch_span; mod rfc0043_6_event_keyed_templating; mod rfc0043_event_name_derivation; +mod rfc0045_10_wal_tail_epoch; diff --git a/crates/ourios-ingester/tests/it/rfc0001_3_5_snapshot_restore.rs b/crates/ourios-ingester/tests/it/rfc0001_3_5_snapshot_restore.rs index cda1266fb..9394fc06b 100644 --- a/crates/ourios-ingester/tests/it/rfc0001_3_5_snapshot_restore.rs +++ b/crates/ourios-ingester/tests/it/rfc0001_3_5_snapshot_restore.rs @@ -16,6 +16,7 @@ use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest; use ourios_config::MinerConfig; use ourios_ingester::receiver::{TenantRule, fan_out}; use ourios_ingester::recovery; +use ourios_ingester::rule_epochs::RuleEpochs; use ourios_miner::cluster::MinerCluster; use ourios_miner::snapshot::RecoveryOutcome; use ourios_wal::{FrameKind, Wal, WalOffset}; @@ -60,7 +61,6 @@ async fn rfc0001_3_5_3_restore_plus_tail_replay_equals_full_rebuild() { let tmp = tempfile::TempDir::new().expect("temp"); let root = tmp.path(); let snapshots_root = root.join("snapshots"); - let rule = TenantRule::service_name(); let pre = [ request(vec![ @@ -105,8 +105,13 @@ async fn rfc0001_3_5_3_restore_plus_tail_replay_equals_full_rebuild() { // Act: recover into a fresh miner over the same WAL + snapshots. let mut wal = Wal::open(wal_config(root)).expect("reopen WAL"); let mut recovered = MinerCluster::new(MinerConfig::default()); - let report = - recovery::recover(&mut wal, &snapshots_root, &mut recovered, &rule).expect("recover"); + let report = recovery::recover( + &mut wal, + &snapshots_root, + &mut recovered, + &RuleEpochs::load(root).expect("epochs"), + ) + .expect("recover"); // Assert (a): restored + tail-replayed state equals the // from-scratch control, per tenant. @@ -137,7 +142,6 @@ async fn rfc0001_3_5_2_corrupt_version_discards_and_full_replays() { let tmp = tempfile::TempDir::new().expect("temp"); let root = tmp.path(); let snapshots_root = root.join("snapshots"); - let rule = TenantRule::service_name(); let batches = [ request(vec![resource_logs("checkout", &["user 1 logged in"])]), @@ -159,8 +163,13 @@ async fn rfc0001_3_5_2_corrupt_version_discards_and_full_replays() { // Act let mut wal = Wal::open(wal_config(root)).expect("reopen WAL"); let mut recovered = MinerCluster::new(MinerConfig::default()); - let report = - recovery::recover(&mut wal, &snapshots_root, &mut recovered, &rule).expect("recover"); + let report = recovery::recover( + &mut wal, + &snapshots_root, + &mut recovered, + &RuleEpochs::load(root).expect("epochs"), + ) + .expect("recover"); // Assert: artefact discarded, nothing suppressed, full-replay // state equals the control. @@ -187,7 +196,6 @@ async fn rfc0001_3_5_snapshot_without_a_horizon_discards_and_full_replays() { let tmp = tempfile::TempDir::new().expect("temp"); let root = tmp.path(); let snapshots_root = root.join("snapshots"); - let rule = TenantRule::service_name(); let batches = [ request(vec![resource_logs("checkout", &["user 1 logged in"])]), @@ -208,8 +216,13 @@ async fn rfc0001_3_5_snapshot_without_a_horizon_discards_and_full_replays() { // Act let mut wal = Wal::open(wal_config(root)).expect("reopen WAL"); let mut recovered = MinerCluster::new(MinerConfig::default()); - let report = - recovery::recover(&mut wal, &snapshots_root, &mut recovered, &rule).expect("recover"); + let report = recovery::recover( + &mut wal, + &snapshots_root, + &mut recovered, + &RuleEpochs::load(root).expect("epochs"), + ) + .expect("recover"); // Assert: discarded (not restored without suppression), nothing // suppressed, full-replay state equals the control. @@ -278,7 +291,6 @@ fn rfc0001_3_5_4_externally_truncated_wal_flags_a_stale_gap() { let tmp = tempfile::TempDir::new().expect("temp"); let root = tmp.path(); let snapshots_root = root.join("snapshots"); - let rule = TenantRule::service_name(); let seg1_batches = [ request(vec![resource_logs("checkout", &["user 1 logged in"])]), @@ -315,8 +327,13 @@ fn rfc0001_3_5_4_externally_truncated_wal_flags_a_stale_gap() { // Act let mut wal = Wal::open(wal_config(root)).expect("reopen WAL"); let mut recovered = MinerCluster::new(MinerConfig::default()); - let report = - recovery::recover(&mut wal, &snapshots_root, &mut recovered, &rule).expect("recover"); + let report = recovery::recover( + &mut wal, + &snapshots_root, + &mut recovered, + &RuleEpochs::load(root).expect("epochs"), + ) + .expect("recover"); // Assert: restored + flagged, surviving frames folded, no error. assert_eq!(report.tenants.len(), 1); @@ -338,7 +355,6 @@ async fn rfc0001_3_5_cold_start_without_snapshots_full_replays() { // Arrange: a WAL with batches and no snapshots dir at all. let tmp = tempfile::TempDir::new().expect("temp"); let root = tmp.path(); - let rule = TenantRule::service_name(); let batches = [ request(vec![resource_logs("checkout", &["user 1 logged in"])]), @@ -356,8 +372,13 @@ async fn rfc0001_3_5_cold_start_without_snapshots_full_replays() { // Act let mut wal = Wal::open(wal_config(root)).expect("reopen WAL"); let mut recovered = MinerCluster::new(MinerConfig::default()); - let report = recovery::recover(&mut wal, &root.join("snapshots"), &mut recovered, &rule) - .expect("recover"); + let report = recovery::recover( + &mut wal, + &root.join("snapshots"), + &mut recovered, + &RuleEpochs::load(root).expect("epochs"), + ) + .expect("recover"); // Assert assert!(report.tenants.is_empty(), "no artefacts, no outcomes"); diff --git a/crates/ourios-ingester/tests/it/rfc0014_5_crash_no_loss.rs b/crates/ourios-ingester/tests/it/rfc0014_5_crash_no_loss.rs index 4f3c52743..13ede486d 100644 --- a/crates/ourios-ingester/tests/it/rfc0014_5_crash_no_loss.rs +++ b/crates/ourios-ingester/tests/it/rfc0014_5_crash_no_loss.rs @@ -22,9 +22,9 @@ use std::time::Duration; use ourios_config::MinerConfig; use ourios_core::record::MinedRecord; -use ourios_ingester::receiver::TenantRule; use ourios_ingester::record_sink::{FlushConfig, ParquetRecordSink, SharedParquetSink}; use ourios_ingester::recovery; +use ourios_ingester::rule_epochs::RuleEpochs; use ourios_miner::cluster::MinerCluster; use ourios_parquet::{Reader, Store}; use ourios_wal::{Wal, WalConfig}; @@ -131,7 +131,7 @@ fn rfc0014_5_no_acknowledged_data_loss() { &mut wal, &snapshots_root, &mut miner, - &TenantRule::service_name(), + &RuleEpochs::load(&wal_root).expect("epochs"), ) .expect("startup recovery"); assert_eq!( diff --git a/crates/ourios-ingester/tests/it/rfc0035_2_encode_barrier.rs b/crates/ourios-ingester/tests/it/rfc0035_2_encode_barrier.rs index ce95704c2..89cc22ae8 100644 --- a/crates/ourios-ingester/tests/it/rfc0035_2_encode_barrier.rs +++ b/crates/ourios-ingester/tests/it/rfc0035_2_encode_barrier.rs @@ -26,6 +26,7 @@ use ourios_ingester::encode_pool::EncodePool; use ourios_ingester::receiver::{IngestPipeline, TenantRule, fan_out}; use ourios_ingester::record_sink::{FlushConfig, ParquetRecordSink, SharedParquetSink}; use ourios_ingester::recovery; +use ourios_ingester::rule_epochs::RuleEpochs; use ourios_miner::cluster::MinerCluster; use ourios_parquet::{Reader, Store}; use ourios_wal::{FrameKind, Wal, WalConfig}; @@ -159,7 +160,13 @@ async fn rfc0035_2_high_water_is_stamped_only_after_drain_and_flush() { ..wal_config(&wal_root) }) .expect("reopen WAL"); - recovery::recover(&mut wal, &snapshots_root, &mut recovered, &rule).expect("recover"); + recovery::recover( + &mut wal, + &snapshots_root, + &mut recovered, + &RuleEpochs::load(&wal_root).expect("epochs"), + ) + .expect("recover"); drop(wal); let mut control = MinerCluster::new(MinerConfig::default()); diff --git a/crates/ourios-ingester/tests/it/rfc0035_2_sweep_crash.rs b/crates/ourios-ingester/tests/it/rfc0035_2_sweep_crash.rs index 744cd6ad0..ba8f8e5d5 100644 --- a/crates/ourios-ingester/tests/it/rfc0035_2_sweep_crash.rs +++ b/crates/ourios-ingester/tests/it/rfc0035_2_sweep_crash.rs @@ -18,9 +18,9 @@ use std::process::{Command, Stdio}; use ourios_config::MinerConfig; use ourios_core::record::MinedRecord; -use ourios_ingester::receiver::TenantRule; use ourios_ingester::record_sink::{FlushConfig, ParquetRecordSink, SharedParquetSink}; use ourios_ingester::recovery; +use ourios_ingester::rule_epochs::RuleEpochs; use ourios_miner::cluster::MinerCluster; use ourios_parquet::{Reader, Store}; use ourios_wal::Wal; @@ -111,7 +111,7 @@ fn rfc0035_2_crash_during_the_sweeps_in_flight_publish_replays_the_records() { &mut wal, &snapshots_root, &mut miner, - &TenantRule::service_name(), + &RuleEpochs::load(&wal_root).expect("epochs"), ) .expect("startup recovery"); assert_eq!( diff --git a/crates/ourios-ingester/tests/it/rfc0045_10_wal_tail_epoch.rs b/crates/ourios-ingester/tests/it/rfc0045_10_wal_tail_epoch.rs new file mode 100644 index 000000000..a181d6e74 --- /dev/null +++ b/crates/ourios-ingester/tests/it/rfc0045_10_wal_tail_epoch.rs @@ -0,0 +1,163 @@ +//! RFC0045.10 — The WAL tail keeps its rule epoch. See +//! `docs/rfcs/0045-composite-tenant-derivation.md` §3.3 / §5. +//! +//! Reuses the RFC0014.5 crash fixture: it acknowledges a `service.name: +//! checkout` batch (no `k8s.cluster.name`) under the default rule and is +//! `SIGKILL`ed before any flush, so the frames survive only in the WAL. The +//! "restart" then runs recovery with the composite rule configured. Under +//! RFC 0045 §3.3 the frames derive under the rule they were acknowledged +//! under: startup succeeds although the frames lack the new key, the rows +//! land only in `checkout`, no composite tenant appears, and the epoch log +//! gains one entry — which a second restart honours. + +use std::io::{BufRead, BufReader}; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::Duration; + +use ourios_config::MinerConfig; +use ourios_core::record::MinedRecord; +use ourios_ingester::receiver::TenantRule; +use ourios_ingester::record_sink::{FlushConfig, ParquetRecordSink, SharedParquetSink}; +use ourios_ingester::recovery; +use ourios_ingester::rule_epochs::{FILE_NAME, RuleEpochs}; +use ourios_miner::cluster::MinerCluster; +use ourios_parquet::{Reader, Store}; +use ourios_wal::{Wal, WalConfig}; + +fn wal_config(root: &Path) -> WalConfig { + WalConfig { + root: root.to_path_buf(), + batch_window_ms: 100, + segment_size_bytes: 128 * 1024 * 1024, + segment_age_secs: 600, + housekeeping_secs: 60, + macos_full_fsync: false, + } +} + +fn never_flush() -> FlushConfig { + FlushConfig { + target_bytes: usize::MAX, + max_buffer_age: Duration::from_secs(86_400), + ceiling_bytes: usize::MAX, + } +} + +fn all_rows(root: &Path) -> Vec { + let mut rows = Vec::new(); + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + let Ok(entries) = std::fs::read_dir(&dir) else { + continue; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + stack.push(path); + } else if path.extension().is_some_and(|x| x == "parquet") { + rows.extend( + Reader::open_file(&path) + .expect("open_file") + .read_all() + .expect("read_all"), + ); + } + } + } + rows +} + +/// One "restart": recover the WAL into a fresh miner + sink under the +/// configured `rule`, flush, and return the store's rows. +fn restart_with(wal_root: &Path, bucket_root: &Path, rule: &TenantRule) -> Vec { + let mut wal = Wal::open(wal_config(wal_root)).expect("reopen WAL"); + let store = Store::local(bucket_root).expect("store"); + let sink = SharedParquetSink::new(ParquetRecordSink::new(store, never_flush())); + let mut miner = + MinerCluster::new(MinerConfig::default()).with_record_sink(Box::new(sink.clone())); + let mut epochs = RuleEpochs::load(wal_root).expect("epoch log loads"); + let report = recovery::recover(&mut wal, &wal_root.join("snapshots"), &mut miner, &epochs) + .expect("startup recovery succeeds under the acknowledged-under rule"); + epochs + .advance(rule, report.max_delivered) + .expect("epoch log persists"); + sink.flush_all(); + all_rows(bucket_root) +} + +/// Scenario RFC0045.10 — WAL tail keeps its epoch. +/// See `docs/rfcs/0045-composite-tenant-derivation.md` §5. +#[test] +fn rfc0045_10_wal_tail_keeps_its_epoch() { + // Arrange + let tmp = tempfile::TempDir::new().expect("temp"); + let wal_root = tmp.path().join("wal"); + let bucket_root = tmp.path().join("store"); + std::fs::create_dir_all(&bucket_root).expect("create store root"); + let composite = TenantRule::from_keys(["k8s.cluster.name", "service.name"]).expect("rule"); + + let mut child = Command::new(env!("CARGO_BIN_EXE_receiver_sink_crash_fixture")) + .arg(&wal_root) + .arg(&bucket_root) + .stdout(Stdio::piped()) + .spawn() + .expect("spawn sink crash fixture"); + let stdout = child.stdout.take().expect("fixture stdout piped"); + let mut line = String::new(); + BufReader::new(stdout) + .read_line(&mut line) + .expect("read fixture READY"); + assert_eq!( + line.trim(), + "READY", + "fixture must signal READY (got {line:?})" + ); + child.kill().expect("SIGKILL fixture"); + child.wait().expect("reap fixture"); + assert!( + !wal_root.join(FILE_NAME).exists(), + "a pre-RFC WAL has no epoch log — the implicit [service.name] epoch" + ); + + // Act: restart under the composite rule. + let rows = restart_with(&wal_root, &bucket_root, &composite); + + // Assert: the acknowledged frames derived under [service.name] — only + // `checkout`, nothing composite, nothing lost, nothing duplicated. + let tenants: Vec<&str> = rows.iter().map(|r| r.tenant_id.as_str()).collect(); + assert_eq!( + rows.len(), + 2, + "both acknowledged records recovered: {tenants:?}" + ); + assert!( + tenants.iter().all(|t| *t == "checkout"), + "old-epoch frames stay in their original tenant, got {tenants:?}" + ); + let epochs = RuleEpochs::load(&wal_root).expect("epoch log"); + assert_eq!(epochs.epochs().len(), 2, "one epoch appended"); + assert_eq!(epochs.current(), &composite); + assert!( + epochs.epochs()[1].after.is_some(), + "the boundary is the highest replayed offset" + ); + + // Act again: a second restart under the same composite rule replays the + // same frames — the persisted epoch log keeps them under [service.name]. + let bucket_root_2 = tmp.path().join("store2"); + std::fs::create_dir_all(&bucket_root_2).expect("create store root"); + let rows = restart_with(&wal_root, &bucket_root_2, &composite); + assert!( + rows.iter().all(|r| r.tenant_id.as_str() == "checkout"), + "the persisted epoch log keeps old frames in their epoch" + ); + assert_eq!( + RuleEpochs::load(&wal_root) + .expect("epoch log") + .epochs() + .len(), + 2, + "an unchanged rule appends nothing" + ); +} diff --git a/crates/ourios-server/src/main.rs b/crates/ourios-server/src/main.rs index b6d373939..196d55ace 100644 --- a/crates/ourios-server/src/main.rs +++ b/crates/ourios-server/src/main.rs @@ -1774,10 +1774,7 @@ auth: tenant_derivation(&absent.receiver.tenant).expect("default"), TenantDerivation::default() ); - assert_eq!( - TenantDerivation::default().rule, - TenantRule::service_name() - ); + assert_eq!(TenantDerivation::default().rule, TenantRule::service_name()); let empty = parse("receiver:\n tenant:\n rule: []\n", &lookup).expect("valid yaml"); assert!( diff --git a/crates/ourios-server/src/receiver.rs b/crates/ourios-server/src/receiver.rs index c6f05ab84..ac8147089 100644 --- a/crates/ourios-server/src/receiver.rs +++ b/crates/ourios-server/src/receiver.rs @@ -33,6 +33,7 @@ use ourios_ingester::receiver::{ }; use ourios_ingester::record_sink::{FlushConfig, ParquetRecordSink, SharedParquetSink}; use ourios_ingester::recovery; +use ourios_ingester::rule_epochs::RuleEpochs; use ourios_miner::cluster::MinerCluster; use ourios_parquet::{PromotedAttributes, Store}; use ourios_wal::{Wal, WalConfig, WalOffset}; @@ -466,6 +467,11 @@ async fn bind_listeners( #[allow(clippy::too_many_lines)] pub async fn serve(config: ReceiverConfig) -> Result { let snapshots_root = config.wal.root.join(SNAPSHOTS_DIR); + // RFC 0045 §3.3: replay derives each frame under the rule it was + // acknowledged under; the configured rule takes over for frames appended + // after this start. + let mut epochs = + RuleEpochs::load(&config.wal.root).map_err(|e| format!("startup recovery: {e}"))?; // The §3.4 group-commit knobs, captured before `config.wal` is moved // into `Wal::open`: the batch window and the segment-fill early-cut. let batch_window = Duration::from_millis(config.wal.batch_window_ms); @@ -484,8 +490,18 @@ pub async fn serve(config: ReceiverConfig) -> Result { .with_record_sink(Box::new(sink.clone())); let rule = config.tenant.rule.clone(); - let report = recovery::recover(&mut wal, &snapshots_root, &mut miner, &rule) + let report = recovery::recover(&mut wal, &snapshots_root, &mut miner, &epochs) .map_err(|e| format!("startup recovery: {e}"))?; + if epochs + .advance(&rule, report.max_delivered) + .map_err(|e| format!("startup recovery: {e}"))? + { + tracing::info!( + keys = ?rule.keys(), + "tenant derivation rule changed; frames from here on derive under the new rule, \ + stored tenant ids are unchanged (RFC 0045 §3.3)" + ); + } for tenant in report.tenants.iter().filter(|t| t.stale_gap) { tracing::warn!( name: ourios_semconv::EVENT_OURIOS_RECEIVER_WAL_TRUNCATED, From a35cd8b22aa5e8c0350fdc2f94b59f74e1801925 Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Mon, 17 Aug 2026 02:06:51 +0200 Subject: [PATCH 03/10] feat(receiver): tenant divergence detector + ourios.receiver.tenant.divergences (RFC0045.7/.9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DivergenceWatch remembers the first value per (tenant, watch key) and warns (rate-limited per pair, values bounded to 128 bytes at a UTF-8 boundary) + counts when a later group diverges — the two-clusters-in-one- tenant signal, observed never enforced. Admission-capped at receiver.tenant.watch_capacity with one saturation warning per process. Counter, event, and ourios.tenant.watch.* attributes minted through the semconv registry + weaver; fan_out gains a per-group observer hook. Signed-off-by: Jens Holdgaard Pedersen --- crates/ourios-ingester/src/receiver.rs | 6 +- .../ourios-ingester/src/receiver/pipeline.rs | 21 +- crates/ourios-ingester/src/receiver/tenant.rs | 19 + crates/ourios-ingester/src/receiver/watch.rs | 341 ++++++++++++++++++ crates/ourios-ingester/tests/README.md | 3 + .../tests/it/ingest_support/mod.rs | 29 +- .../tests/rfc0045_divergence_telemetry.rs | 274 ++++++++++++++ crates/ourios-semconv/src/lib.rs | 19 + crates/ourios-server/src/receiver.rs | 3 +- semconv/registry/attributes.yaml | 23 ++ semconv/registry/events.yaml | 24 ++ semconv/registry/metrics.yaml | 17 + 12 files changed, 774 insertions(+), 5 deletions(-) create mode 100644 crates/ourios-ingester/src/receiver/watch.rs create mode 100644 crates/ourios-ingester/tests/rfc0045_divergence_telemetry.rs diff --git a/crates/ourios-ingester/src/receiver.rs b/crates/ourios-ingester/src/receiver.rs index 004a3903b..6af3c80a2 100644 --- a/crates/ourios-ingester/src/receiver.rs +++ b/crates/ourios-ingester/src/receiver.rs @@ -42,6 +42,7 @@ pub mod propagation; pub mod tenant; pub mod tls; pub mod tls_serve; +pub mod watch; pub use auth::{AuthBinding, AuthResolver, Unauthenticated, authenticate_bearer}; pub use commit::CommitCoordinator; @@ -51,4 +52,7 @@ pub use pipeline::{IngestPipeline, Journal, ReceiveError, SharedPipeline}; pub use propagation::{ HeaderExtractor, MetadataExtractor, extract_context, extract_context_from_metadata, }; -pub use tenant::{TenantDerivation, TenantResolutionError, TenantRule, TenantRuleError, fan_out}; +pub use tenant::{ + TenantDerivation, TenantResolutionError, TenantRule, TenantRuleError, fan_out, fan_out_observed, +}; +pub use watch::DivergenceWatch; diff --git a/crates/ourios-ingester/src/receiver/pipeline.rs b/crates/ourios-ingester/src/receiver/pipeline.rs index 3eabc6adb..cd5a4b905 100644 --- a/crates/ourios-ingester/src/receiver/pipeline.rs +++ b/crates/ourios-ingester/src/receiver/pipeline.rs @@ -29,7 +29,8 @@ use tracing::Instrument as _; use crate::metrics::IngestMetrics; use crate::receiver::commit::CommitCoordinator; -use crate::receiver::tenant::{TenantResolutionError, TenantRule, fan_out}; +use crate::receiver::tenant::{TenantResolutionError, TenantRule, fan_out_observed}; +use crate::receiver::watch::DivergenceWatch; /// The §6.9 rotation-cadence callback: receives the miner as it /// stands and the rotation-point high-water mark. See @@ -121,6 +122,9 @@ pub struct IngestPipeline { coordinator: Arc, miner: Mutex, rule: TenantRule, + /// RFC 0045 §3.4 — the divergence detector, when any watch key is + /// configured beyond the rule's own keys. + watch: Option, /// The durable high-water mark after the most recent acked batch (or /// the startup seed). Behind a mutex: concurrent acks update it, and /// the rotation-detection read-then-write must see a consistent value. @@ -164,6 +168,7 @@ impl IngestPipeline { coordinator, miner: Mutex::new(miner), rule, + watch: None, last_durable: Mutex::new(None), rotation_hook: Mutex::new(None), encode_pool: None, @@ -172,6 +177,14 @@ impl IngestPipeline { } } + /// Attach the RFC 0045 §3.4 divergence detector; every derived group + /// is observed after fan-out. + #[must_use] + pub fn with_tenant_watch(mut self, watch: Option) -> Self { + self.watch = watch; + self + } + /// Enable the RFC 0035 §3.1 ordered/concurrent ingest split: the /// gated section runs only Drain match + template-id assignment /// (+ audit), and the Parquet-sink emit runs on `pool`. The pool's @@ -337,7 +350,11 @@ impl IngestPipeline { // Steps 1–2: fan out per tenant. An unresolvable Resource rejects // the entire batch here, before any WAL write (RFC0003.4). - let records = fan_out(request, &self.rule)?; + let records = fan_out_observed(request, &self.rule, |tenant, attributes| { + if let Some(watch) = &self.watch { + watch.observe(tenant, attributes); + } + })?; // Empty fast path (RFC0003.12): no records → success, no WAL // frame, miner untouched. diff --git a/crates/ourios-ingester/src/receiver/tenant.rs b/crates/ourios-ingester/src/receiver/tenant.rs index bedf18cd5..8d1f6d862 100644 --- a/crates/ourios-ingester/src/receiver/tenant.rs +++ b/crates/ourios-ingester/src/receiver/tenant.rs @@ -290,12 +290,31 @@ impl std::error::Error for TenantResolutionError {} pub fn fan_out( request: ExportLogsServiceRequest, rule: &TenantRule, +) -> Result, TenantResolutionError> { + fan_out_observed(request, rule, |_, _| {}) +} + +/// [`fan_out`] with a per-group observer, called with each group's derived +/// tenant and its `Resource.attributes` after derivation succeeds — the +/// RFC 0045 §3.4 divergence detector's hook. Groups after a failing one +/// are never observed (the export is rejected whole). +/// +/// # Errors +/// +/// As [`fan_out`]. +pub fn fan_out_observed( + request: ExportLogsServiceRequest, + rule: &TenantRule, + mut observe: impl FnMut(&TenantId, &[KeyValue]), ) -> Result, TenantResolutionError> { let mut records = Vec::new(); for (index, resource_logs) in request.resource_logs.into_iter().enumerate() { // Derived before `resource_logs` is moved into // `materialize_resource_logs`. let tenant_id = derive_for_group(&resource_logs, index, rule)?; + if let Some(resource) = &resource_logs.resource { + observe(&tenant_id, &resource.attributes); + } records.extend(materialize_resource_logs(resource_logs, &tenant_id)); } Ok(records) diff --git a/crates/ourios-ingester/src/receiver/watch.rs b/crates/ourios-ingester/src/receiver/watch.rs new file mode 100644 index 000000000..f38ae9144 --- /dev/null +++ b/crates/ourios-ingester/src/receiver/watch.rs @@ -0,0 +1,341 @@ +//! The tenant divergence detector (RFC 0045 §3.4). +//! +//! For each configured `watch` key that is not part of the derivation +//! rule, the detector remembers the first value seen per (tenant, key) +//! and announces — warning + `ourios.receiver.tenant.divergences` — when +//! a later `ResourceLogs` group for the same tenant carries a different +//! value. That is the shape of the misconfiguration the RFC exists to +//! surface: two clusters merging into one tenant under a single-key +//! rule. The detector observes, it never rejects. +//! +//! State is bounded (`watch_capacity` entries, first-come admission, one +//! saturation warning per process), values are bounded (128 bytes, +//! UTF-8-safe truncation), and the warning is rate-limited per +//! (tenant, key). Everything resets on restart by design. + +use std::collections::HashMap; +use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant}; + +use opentelemetry::metrics::Counter; +use opentelemetry::{KeyValue as OtelKeyValue, global}; +use opentelemetry_proto::tonic::common::v1::KeyValue; +use opentelemetry_proto::tonic::common::v1::any_value::Value; +use ourios_core::tenant::TenantId; +use ourios_semconv as semconv; + +use super::tenant::TenantDerivation; + +/// Longest value the detector remembers or logs, in bytes (RFC 0045 +/// §3.4 *Value representation*). +pub const MAX_VALUE_BYTES: usize = 128; + +/// Minimum interval between two warnings for the same (tenant, key). +pub const WARN_INTERVAL: Duration = Duration::from_secs(60); + +struct Entry { + first: String, + last_warned: Option, +} + +/// The detector: build one per pipeline from the resolved +/// [`TenantDerivation`]; call [`observe`](Self::observe) once per +/// derived `ResourceLogs` group. +pub struct DivergenceWatch { + keys: Vec, + capacity: usize, + state: Mutex>, + saturated: AtomicBool, + divergences: Counter, +} + +impl std::fmt::Debug for DivergenceWatch { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DivergenceWatch") + .field("keys", &self.keys) + .field("capacity", &self.capacity) + .finish_non_exhaustive() + } +} + +impl DivergenceWatch { + /// The detector for `derivation`, watching every `watch` key the rule + /// does not already join on. `None` when nothing is left to watch. + #[must_use] + pub fn from_derivation(derivation: &TenantDerivation) -> Option { + let keys: Vec = derivation + .watch + .iter() + .filter(|key| !derivation.rule.contains(key)) + .cloned() + .collect(); + if keys.is_empty() { + return None; + } + Some(Self::new(keys, derivation.watch_capacity)) + } + + fn new(keys: Vec, capacity: usize) -> Self { + let divergences = global::meter("ourios.receiver") + .u64_counter(semconv::OURIOS_RECEIVER_TENANT_DIVERGENCES) + .build(); + Self { + keys, + capacity, + state: Mutex::new(HashMap::new()), + saturated: AtomicBool::new(false), + divergences, + } + } + + /// The watched keys (the configured set minus the rule's keys). + #[must_use] + pub fn keys(&self) -> &[String] { + &self.keys + } + + /// Observe one derived group. A key the resource lacks (or carries as + /// a non-string or empty string) is skipped for that group. + pub fn observe(&self, tenant: &TenantId, resource_attributes: &[KeyValue]) { + for key in &self.keys { + let Some(value) = string_attribute(resource_attributes, key) else { + continue; + }; + self.observe_one(tenant, key, value); + } + } + + fn observe_one(&self, tenant: &TenantId, key: &str, value: &str) { + let mut state = self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let slot = (tenant.clone(), key.to_owned()); + let Some(entry) = state.get_mut(&slot) else { + if state.len() >= self.capacity { + if !self.saturated.swap(true, Ordering::Relaxed) { + tracing::warn!( + name: semconv::EVENT_OURIOS_RECEIVER_TENANT_WATCH_SATURATED, + "tenant divergence watch is full ({} entries); further (tenant, key) \ + pairs are not watched — raise receiver.tenant.watch_capacity if this \ + matters", + self.capacity, + ); + } + return; + } + state.insert( + slot, + Entry { + first: bound(value).into_owned(), + last_warned: None, + }, + ); + return; + }; + let seen = bound(value); + if entry.first == seen { + return; + } + self.divergences.add( + 1, + &[OtelKeyValue::new( + semconv::OURIOS_TENANT_WATCH_KEY, + key.to_owned(), + )], + ); + let now = Instant::now(); + if entry + .last_warned + .is_some_and(|last| now.duration_since(last) < WARN_INTERVAL) + { + return; + } + entry.last_warned = Some(now); + tracing::event!( + name: semconv::EVENT_OURIOS_RECEIVER_TENANT_DIVERGENCE, + tracing::Level::WARN, + ourios.tenant = tenant.as_str(), + ourios.tenant.watch.key = key, + ourios.tenant.watch.first_value = entry.first.as_str(), + ourios.tenant.watch.value = seen.as_ref(), + "tenant spans more than one value of a watched resource attribute — if these are \ + different producers, add the key to receiver.tenant.rule (RFC 0045 §3.4)" + ); + } +} + +fn string_attribute<'a>(attributes: &'a [KeyValue], key: &str) -> Option<&'a str> { + attributes + .iter() + .find(|kv| kv.key == key) + .and_then(|kv| kv.value.as_ref()) + .and_then(|value| match value.value.as_ref() { + Some(Value::StringValue(s)) if !s.is_empty() => Some(s.as_str()), + _ => None, + }) +} + +/// `value` truncated to [`MAX_VALUE_BYTES`] at a UTF-8 boundary with a +/// trailing `…`, or borrowed unchanged when it already fits. +fn bound(value: &str) -> std::borrow::Cow<'_, str> { + if value.len() <= MAX_VALUE_BYTES { + return std::borrow::Cow::Borrowed(value); + } + let mut end = MAX_VALUE_BYTES; + while !value.is_char_boundary(end) { + end -= 1; + } + std::borrow::Cow::Owned(format!("{}…", &value[..end])) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::receiver::tenant::TenantRule; + use opentelemetry_proto::tonic::common::v1::AnyValue; + + fn attr(key: &str, value: &str) -> KeyValue { + KeyValue { + key: key.to_owned(), + value: Some(AnyValue { + value: Some(Value::StringValue(value.to_owned())), + }), + ..Default::default() + } + } + + fn watch(capacity: usize) -> DivergenceWatch { + DivergenceWatch::from_derivation(&TenantDerivation { + rule: TenantRule::service_name(), + watch: vec!["k8s.cluster.name".to_owned()], + watch_capacity: capacity, + }) + .expect("one key to watch") + } + + fn entries(watch: &DivergenceWatch) -> usize { + watch.state.lock().expect("lock").len() + } + + // The tracing field literals above must be the registry names — + // `tracing` needs literals, the registry needs them registered. + #[test] + fn warning_field_names_are_the_registered_attributes() { + assert_eq!(semconv::OURIOS_TENANT, "ourios.tenant"); + assert_eq!(semconv::OURIOS_TENANT_WATCH_KEY, "ourios.tenant.watch.key"); + assert_eq!( + semconv::OURIOS_TENANT_WATCH_FIRST_VALUE, + "ourios.tenant.watch.first_value" + ); + assert_eq!( + semconv::OURIOS_TENANT_WATCH_VALUE, + "ourios.tenant.watch.value" + ); + } + + // §3.1 — a watch key that is also a rule key is not watched. + #[test] + fn keys_in_the_rule_are_not_watched() { + let derivation = TenantDerivation { + rule: TenantRule::from_keys(["k8s.cluster.name", "service.name"]).expect("rule"), + watch: vec!["k8s.cluster.name".to_owned()], + watch_capacity: 10, + }; + assert!(DivergenceWatch::from_derivation(&derivation).is_none()); + let derivation = TenantDerivation { + watch: vec!["k8s.cluster.name".to_owned(), "cloud.region".to_owned()], + ..derivation + }; + assert_eq!( + DivergenceWatch::from_derivation(&derivation) + .expect("cloud.region remains") + .keys(), + ["cloud.region"] + ); + } + + // RFC0045.7 — first value remembered; the same value is not a + // divergence; a group lacking the key (or non-string / empty) is skipped. + #[test] + fn remembers_first_value_and_skips_absent_keys() { + let w = watch(10); + let tenant = TenantId::new("fluxcd"); + w.observe(&tenant, &[attr("k8s.cluster.name", "cluster1")]); + w.observe(&tenant, &[attr("k8s.cluster.name", "cluster1")]); + w.observe(&tenant, &[attr("service.name", "fluxcd")]); + w.observe(&tenant, &[attr("k8s.cluster.name", "")]); + w.observe( + &tenant, + &[KeyValue { + key: "k8s.cluster.name".to_owned(), + value: Some(AnyValue { + value: Some(Value::IntValue(3)), + }), + ..Default::default() + }], + ); + assert_eq!(entries(&w), 1); + let state = w.state.lock().expect("lock"); + let entry = state + .get(&(tenant, "k8s.cluster.name".to_owned())) + .expect("entry"); + assert_eq!(entry.first, "cluster1"); + assert!(entry.last_warned.is_none(), "no divergence, no warning"); + } + + // RFC0045.7 — a different value is a divergence (warned once per + // interval); the first value never changes. + #[test] + fn divergent_value_warns_once_per_interval() { + let w = watch(10); + let tenant = TenantId::new("fluxcd"); + w.observe(&tenant, &[attr("k8s.cluster.name", "cluster1")]); + w.observe(&tenant, &[attr("k8s.cluster.name", "cluster2")]); + let first_warn = { + let state = w.state.lock().expect("lock"); + let entry = state + .get(&(tenant.clone(), "k8s.cluster.name".to_owned())) + .expect("entry"); + assert_eq!(entry.first, "cluster1"); + entry.last_warned.expect("warned") + }; + w.observe(&tenant, &[attr("k8s.cluster.name", "cluster3")]); + let state = w.state.lock().expect("lock"); + let entry = state + .get(&(tenant, "k8s.cluster.name".to_owned())) + .expect("entry"); + assert_eq!(entry.last_warned, Some(first_warn), "rate-limited"); + } + + // RFC0045.9 — capacity: first-come admission, saturation announced once. + #[test] + fn capacity_bounds_admission() { + let w = watch(1); + w.observe(&TenantId::new("a"), &[attr("k8s.cluster.name", "c1")]); + w.observe(&TenantId::new("b"), &[attr("k8s.cluster.name", "c1")]); + assert_eq!(entries(&w), 1); + assert!(w.saturated.load(Ordering::Relaxed)); + // The admitted tenant is still watched. + w.observe(&TenantId::new("a"), &[attr("k8s.cluster.name", "c2")]); + let state = w.state.lock().expect("lock"); + assert!( + state[&(TenantId::new("a"), "k8s.cluster.name".to_owned())] + .last_warned + .is_some() + ); + } + + // RFC0045.7 — values are bounded at a UTF-8 boundary with a trailing `…`. + #[test] + fn values_are_bounded_at_a_char_boundary() { + let short = "x".repeat(MAX_VALUE_BYTES); + assert_eq!(bound(&short), short); + // 'é' is two bytes; 127 ASCII bytes + 'é' straddles the boundary. + let long = format!("{}é{}", "x".repeat(MAX_VALUE_BYTES - 1), "tail"); + let bounded = bound(&long); + assert_eq!(bounded, format!("{}…", "x".repeat(MAX_VALUE_BYTES - 1))); + assert!(bounded.len() <= MAX_VALUE_BYTES + '…'.len_utf8()); + } +} diff --git a/crates/ourios-ingester/tests/README.md b/crates/ourios-ingester/tests/README.md index f6dc36239..a7bbe5d22 100644 --- a/crates/ourios-ingester/tests/README.md +++ b/crates/ourios-ingester/tests/README.md @@ -22,6 +22,9 @@ one-per-binary: global in-memory provider. - `rfc0026_telemetry.rs` — the RFC0026.7 rejection-telemetry arm installs the global in-memory provider. +- `rfc0045_divergence_telemetry.rs` — the RFC0045.7/.9 divergence-detector + arm reads `ourios.receiver.tenant.divergences` through the global + in-memory provider. - `rfc0038_3_spawn_boundary.rs` — installs the global in-memory **tracer**; a global (not scoped) tracer is required to capture the `ingest logs` / `sweep partitions` spans across the receiver's `tokio::spawn` and the diff --git a/crates/ourios-ingester/tests/it/ingest_support/mod.rs b/crates/ourios-ingester/tests/it/ingest_support/mod.rs index 8ed777936..d486edf10 100644 --- a/crates/ourios-ingester/tests/it/ingest_support/mod.rs +++ b/crates/ourios-ingester/tests/it/ingest_support/mod.rs @@ -20,7 +20,8 @@ use ourios_wal::{ }; use ourios_ingester::receiver::{ - CommitCoordinator, IngestPipeline, Journal, ReceiveError, TenantRule, + CommitCoordinator, DivergenceWatch, IngestPipeline, Journal, ReceiveError, TenantDerivation, + TenantRule, }; pub fn wal_config(root: &Path) -> WalConfig { @@ -58,6 +59,14 @@ pub fn open_pipeline(root: &Path) -> IngestPipeline { ) } +/// [`open_pipeline`] under an RFC 0045 tenant derivation (rule + watch). +pub fn open_pipeline_with_derivation(root: &Path, derivation: &TenantDerivation) -> IngestPipeline { + let wal = Wal::open(wal_config(root)).expect("open WAL"); + let miner = MinerCluster::new(MinerConfig::default()); + IngestPipeline::new(coordinator(Box::new(wal)), miner, derivation.rule.clone()) + .with_tenant_watch(DivergenceWatch::from_derivation(derivation)) +} + /// One observed `Journal` call, in order. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum JournalCall { @@ -249,6 +258,24 @@ pub fn resource_logs(service: &str, bodies: &[&str]) -> ResourceLogs { } } +/// A `ResourceLogs` whose `Resource` carries the given string attributes +/// (in order) and one scope with one record per body. +pub fn resource_logs_with_attrs(attrs: &[(&str, &str)], bodies: &[&str]) -> ResourceLogs { + let mut group = resource_logs("", bodies); + group.resource = Some(Resource { + attributes: attrs + .iter() + .map(|(key, value)| KeyValue { + key: (*key).to_owned(), + value: Some(string_value(value)), + ..Default::default() + }) + .collect(), + ..Default::default() + }); + group +} + /// A `ResourceLogs` for `service` with **no** `ScopeLogs` at all. pub fn resource_logs_without_scopes(service: &str) -> ResourceLogs { ResourceLogs { diff --git a/crates/ourios-ingester/tests/rfc0045_divergence_telemetry.rs b/crates/ourios-ingester/tests/rfc0045_divergence_telemetry.rs new file mode 100644 index 000000000..8d7d2e46e --- /dev/null +++ b/crates/ourios-ingester/tests/rfc0045_divergence_telemetry.rs @@ -0,0 +1,274 @@ +//! Scenarios RFC0045.7 (divergence detector) and RFC0045.9 (watch state +//! bound) through the ingest pipeline: the warning event and the +//! `ourios.receiver.tenant.divergences` counter. +//! +//! Harness-exempt (RFC0028.2, see `tests/README.md`): installs the +//! **process-global** `OTel` meter provider (`init_in_memory`) once, so +//! both scenarios share this binary and read the same exporter. +//! +//! See `docs/rfcs/0045-composite-tenant-derivation.md` §5. + +#[path = "it/ingest_support/mod.rs"] +mod ingest_support; + +use std::sync::{Arc, Mutex}; + +use ingest_support::{open_pipeline_with_derivation, request, resource_logs_with_attrs}; +use opentelemetry_sdk::metrics::data::{AggregatedMetrics, MetricData, ResourceMetrics}; +use ourios_ingester::receiver::watch::MAX_VALUE_BYTES; +use ourios_ingester::receiver::{TenantDerivation, TenantRule}; +use tracing::field::{Field, Visit}; +use tracing_subscriber::layer::{Context, Layer}; +use tracing_subscriber::prelude::*; + +/// One captured event: its `name` plus stringified fields. +#[derive(Debug, Clone)] +struct Captured { + name: String, + fields: Vec<(String, String)>, +} + +#[derive(Default)] +struct Fields(Vec<(String, String)>); + +impl Visit for Fields { + fn record_debug(&mut self, field: &Field, value: &dyn std::fmt::Debug) { + self.0.push((field.name().to_owned(), format!("{value:?}"))); + } + fn record_str(&mut self, field: &Field, value: &str) { + self.0.push((field.name().to_owned(), value.to_owned())); + } +} + +#[derive(Clone, Default)] +struct Capture(Arc>>); + +impl Layer for Capture { + fn on_event(&self, event: &tracing::Event<'_>, _ctx: Context<'_, S>) { + let mut fields = Fields::default(); + event.record(&mut fields); + self.0.lock().expect("capture lock").push(Captured { + name: event.metadata().name().to_owned(), + fields: fields.0, + }); + } +} + +impl Capture { + fn named(&self, name: &str) -> Vec { + self.0 + .lock() + .expect("capture lock") + .iter() + .filter(|e| e.name == name) + .cloned() + .collect() + } +} + +fn field<'a>(event: &'a Captured, key: &str) -> Option<&'a str> { + event + .fields + .iter() + .find(|(k, _)| k == key) + .map(|(_, v)| v.as_str()) +} + +/// The exported `ourios.receiver.tenant.divergences` total for `key`. +fn divergences(rms: &[ResourceMetrics], key: &str) -> u64 { + rms.iter() + .flat_map(ResourceMetrics::scope_metrics) + .flat_map(opentelemetry_sdk::metrics::data::ScopeMetrics::metrics) + .filter(|m| m.name() == ourios_semconv::OURIOS_RECEIVER_TENANT_DIVERGENCES) + .filter_map(|m| match m.data() { + AggregatedMetrics::U64(MetricData::Sum(sum)) => Some(sum), + _ => None, + }) + .flat_map(opentelemetry_sdk::metrics::data::Sum::data_points) + .filter(|dp| { + dp.attributes().any(|kv| { + kv.key.as_str() == ourios_semconv::OURIOS_TENANT_WATCH_KEY + && kv.value.as_str() == key + }) + }) + .map(opentelemetry_sdk::metrics::data::SumDataPoint::value) + .sum() +} + +fn cluster_group( + service: &str, + cluster: &str, + body: &str, +) -> opentelemetry_proto::tonic::logs::v1::ResourceLogs { + resource_logs_with_attrs( + &[("service.name", service), ("k8s.cluster.name", cluster)], + &[body], + ) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn rfc0045_7_and_9_divergence_detector() { + let (guard, exporter) = ourios_telemetry::init_in_memory("ourios-test-rfc0045"); + let capture = Capture::default(); + let subscriber = tracing_subscriber::registry().with(capture.clone()); + let _guard = tracing::subscriber::set_default(subscriber); + + scenario_7_detector(&capture).await; + scenario_9_capacity(&capture).await; + + // The counter: fluxcd (1) + payments (1) on k8s.cluster.name; a (1) on + // cloud.region. + guard.force_flush().expect("flush"); + let rms = exporter.get_finished_metrics().expect("collect"); + assert_eq!(divergences(&rms, "k8s.cluster.name"), 2); + assert_eq!(divergences(&rms, "cloud.region"), 1); +} + +/// RFC0045.7 — default rule + default watch: two exports sharing +/// service.name but differing in k8s.cluster.name. +async fn scenario_7_detector(capture: &Capture) { + let tmp = tempfile::TempDir::new().expect("temp"); + let pipeline = open_pipeline_with_derivation(tmp.path(), &TenantDerivation::default()); + for cluster in ["cluster1", "cluster1", "cluster2"] { + pipeline + .ingest_bound( + request(vec![cluster_group("fluxcd", cluster, "line")]), + None, + false, + ) + .await + .expect("accepted — the detector never rejects"); + } + let warnings = capture.named(ourios_semconv::EVENT_OURIOS_RECEIVER_TENANT_DIVERGENCE); + assert_eq!( + warnings.len(), + 1, + "one warning for the first divergent batch: {warnings:?}" + ); + let w = &warnings[0]; + assert_eq!(field(w, ourios_semconv::OURIOS_TENANT), Some("fluxcd")); + assert_eq!( + field(w, ourios_semconv::OURIOS_TENANT_WATCH_KEY), + Some("k8s.cluster.name") + ); + assert_eq!( + field(w, ourios_semconv::OURIOS_TENANT_WATCH_FIRST_VALUE), + Some("cluster1") + ); + assert_eq!( + field(w, ourios_semconv::OURIOS_TENANT_WATCH_VALUE), + Some("cluster2") + ); + + // Uniform values: a second tenant that never diverges → nothing more. + for _ in 0..2 { + pipeline + .ingest_bound( + request(vec![cluster_group("checkout", "cluster1", "line")]), + None, + false, + ) + .await + .expect("accepted"); + } + // A group lacking the watch key (or empty / non-string) is accepted and + // not observed. + pipeline + .ingest_bound( + request(vec![resource_logs_with_attrs( + &[("service.name", "fluxcd")], + &["line"], + )]), + None, + false, + ) + .await + .expect("accepted without the watch key"); + pipeline + .ingest_bound( + request(vec![resource_logs_with_attrs( + &[("service.name", "fluxcd"), ("k8s.cluster.name", "")], + &["line"], + )]), + None, + false, + ) + .await + .expect("accepted with an empty watch value"); + assert_eq!( + capture + .named(ourios_semconv::EVENT_OURIOS_RECEIVER_TENANT_DIVERGENCE) + .len(), + 1, + "uniform / absent / empty values add no warning" + ); + + // A value longer than 128 bytes is truncated at a UTF-8 boundary with `…`. + let long = format!("{}é{}", "x".repeat(MAX_VALUE_BYTES - 1), "tail"); + pipeline + .ingest_bound( + request(vec![cluster_group("payments", "c1", "line")]), + None, + false, + ) + .await + .expect("accepted"); + pipeline + .ingest_bound( + request(vec![cluster_group("payments", &long, "line")]), + None, + false, + ) + .await + .expect("accepted"); + let warnings = capture.named(ourios_semconv::EVENT_OURIOS_RECEIVER_TENANT_DIVERGENCE); + let payments = warnings + .iter() + .find(|w| field(w, ourios_semconv::OURIOS_TENANT) == Some("payments")) + .expect("payments diverged"); + assert_eq!( + field(payments, ourios_semconv::OURIOS_TENANT_WATCH_VALUE), + Some(format!("{}…", "x".repeat(MAX_VALUE_BYTES - 1)).as_str()) + ); +} + +/// RFC0045.9 — `watch_capacity: 1`; two tenants each later diverge. +async fn scenario_9_capacity(capture: &Capture) { + let tmp2 = tempfile::TempDir::new().expect("temp"); + let bounded = open_pipeline_with_derivation( + tmp2.path(), + &TenantDerivation { + rule: TenantRule::service_name(), + watch: vec!["cloud.region".to_owned()], + watch_capacity: 1, + }, + ); + let region = |service: &str, region: &str| { + request(vec![resource_logs_with_attrs( + &[("service.name", service), ("cloud.region", region)], + &["line"], + )]) + }; + for (service, r) in [("a", "eu"), ("b", "eu"), ("a", "us"), ("b", "us")] { + bounded + .ingest_bound(region(service, r), None, false) + .await + .expect("every export is accepted"); + } + let saturated = capture.named(ourios_semconv::EVENT_OURIOS_RECEIVER_TENANT_WATCH_SATURATED); + assert_eq!(saturated.len(), 1, "saturation announced exactly once"); + let region_warnings: Vec<_> = capture + .named(ourios_semconv::EVENT_OURIOS_RECEIVER_TENANT_DIVERGENCE) + .into_iter() + .filter(|w| field(w, ourios_semconv::OURIOS_TENANT_WATCH_KEY) == Some("cloud.region")) + .collect(); + assert_eq!( + region_warnings.len(), + 1, + "only the admitted tenant is watched" + ); + assert_eq!( + field(®ion_warnings[0], ourios_semconv::OURIOS_TENANT), + Some("a") + ); +} diff --git a/crates/ourios-semconv/src/lib.rs b/crates/ourios-semconv/src/lib.rs index e684db8dc..b95d12aff 100644 --- a/crates/ourios-semconv/src/lib.rs +++ b/crates/ourios-semconv/src/lib.rs @@ -124,6 +124,9 @@ pub const OURIOS_QUERY_DURATION: &str = "ourios.query.duration"; /// `ourios.query.row_groups` (counter, unit `{row_group}`). pub const OURIOS_QUERY_ROW_GROUPS: &str = "ourios.query.row_groups"; +/// `ourios.receiver.tenant.divergences` (counter, unit `{divergence}`). +pub const OURIOS_RECEIVER_TENANT_DIVERGENCES: &str = "ourios.receiver.tenant.divergences"; + /// `ourios.receiver.tls.handshake_failures` (counter, unit `{failure}`). pub const OURIOS_RECEIVER_TLS_HANDSHAKE_FAILURES: &str = "ourios.receiver.tls.handshake_failures"; @@ -216,6 +219,15 @@ pub const OURIOS_TEMPLATE_MAP_PUBLISH_OUTCOME: &str = "ourios.template_map.publi /// `ourios.tenant` attribute key. pub const OURIOS_TENANT: &str = "ourios.tenant"; +/// `ourios.tenant.watch.first_value` attribute key. +pub const OURIOS_TENANT_WATCH_FIRST_VALUE: &str = "ourios.tenant.watch.first_value"; + +/// `ourios.tenant.watch.key` attribute key. +pub const OURIOS_TENANT_WATCH_KEY: &str = "ourios.tenant.watch.key"; + +/// `ourios.tenant.watch.value` attribute key. +pub const OURIOS_TENANT_WATCH_VALUE: &str = "ourios.tenant.watch.value"; + /// `ourios.tls.failure` attribute key. pub const OURIOS_TLS_FAILURE: &str = "ourios.tls.failure"; @@ -247,6 +259,13 @@ pub const EVENT_OURIOS_RECEIVER_SINK_RETAINED: &str = "ourios.receiver.sink.reta /// `ourios.receiver.snapshot.error` log event name. pub const EVENT_OURIOS_RECEIVER_SNAPSHOT_ERROR: &str = "ourios.receiver.snapshot.error"; +/// `ourios.receiver.tenant.divergence` log event name. +pub const EVENT_OURIOS_RECEIVER_TENANT_DIVERGENCE: &str = "ourios.receiver.tenant.divergence"; + +/// `ourios.receiver.tenant.watch_saturated` log event name. +pub const EVENT_OURIOS_RECEIVER_TENANT_WATCH_SATURATED: &str = + "ourios.receiver.tenant.watch_saturated"; + /// `ourios.receiver.wal.truncated` log event name. pub const EVENT_OURIOS_RECEIVER_WAL_TRUNCATED: &str = "ourios.receiver.wal.truncated"; diff --git a/crates/ourios-server/src/receiver.rs b/crates/ourios-server/src/receiver.rs index ac8147089..16c981d1b 100644 --- a/crates/ourios-server/src/receiver.rs +++ b/crates/ourios-server/src/receiver.rs @@ -29,7 +29,7 @@ use ourios_ingester::receiver::tls_serve::{ LISTENER_GRPC, LISTENER_HTTP, TlsListener, reloading_acceptor, tls_incoming, }; use ourios_ingester::receiver::{ - CommitCoordinator, IngestPipeline, SharedPipeline, TenantDerivation, + CommitCoordinator, DivergenceWatch, IngestPipeline, SharedPipeline, TenantDerivation, }; use ourios_ingester::record_sink::{FlushConfig, ParquetRecordSink, SharedParquetSink}; use ourios_ingester::recovery; @@ -538,6 +538,7 @@ pub async fn serve(config: ReceiverConfig) -> Result { let coordinator = CommitCoordinator::new(Box::new(wal), batch_window, segment_size_bytes); let pipeline: SharedPipeline = Arc::new( IngestPipeline::new(coordinator, miner, rule) + .with_tenant_watch(DivergenceWatch::from_derivation(&config.tenant)) // RFC 0026 §3.4: tenant-binding denials emit `ingest_denied` // through the same durable audit sink as every other event. .with_denial_audit_sink(Box::new(audit_sink.clone())) diff --git a/semconv/registry/attributes.yaml b/semconv/registry/attributes.yaml index 1e8e7cbd1..c6afd5b28 100644 --- a/semconv/registry/attributes.yaml +++ b/semconv/registry/attributes.yaml @@ -267,6 +267,29 @@ groups: The disposition of a failed audit partition flush (issue #302): whether the batch was retained for retry or dropped. + - id: ourios.tenant.watch.key + type: string + stability: development + brief: >- + The resource-attribute key the RFC 0045 §3.4 tenant divergence + detector watched (`receiver.tenant.watch`). + examples: ["k8s.cluster.name", "cloud.region"] + - id: ourios.tenant.watch.first_value + type: string + stability: development + brief: >- + The first value of the watched key observed for the tenant + (bounded to 128 bytes, truncated with a trailing `…`). + examples: ["cluster1"] + - id: ourios.tenant.watch.value + type: string + stability: development + brief: >- + The divergent value of the watched key a later group carried + for the same tenant (bounded to 128 bytes, truncated with a + trailing `…`). + examples: ["cluster2"] + - id: ourios.tls.listener type: members: diff --git a/semconv/registry/events.yaml b/semconv/registry/events.yaml index 0d5af7227..b744508c6 100644 --- a/semconv/registry/events.yaml +++ b/semconv/registry/events.yaml @@ -88,3 +88,27 @@ groups: The WAL was truncated past a tenant's snapshot high-water mark (external mutation); templates first seen in the gap may re-mint — drift is observable via the RFC 0010 drift query. + - id: event.ourios.receiver.tenant.divergence + type: event + name: ourios.receiver.tenant.divergence + stability: development + brief: > + A tenant spans more than one value of a watched resource attribute + (RFC 0045 §3.4): the first observed value and the divergent one are + attached, bounded. Rate-limited per (tenant, key). Not a rejection. + attributes: + - ref: ourios.tenant + requirement_level: required + - ref: ourios.tenant.watch.key + requirement_level: required + - ref: ourios.tenant.watch.first_value + requirement_level: required + - ref: ourios.tenant.watch.value + requirement_level: required + - id: event.ourios.receiver.tenant.watch_saturated + type: event + name: ourios.receiver.tenant.watch_saturated + stability: development + brief: > + The tenant divergence watch reached `receiver.tenant.watch_capacity`; + further (tenant, key) pairs are not watched. Emitted once per process. diff --git a/semconv/registry/metrics.yaml b/semconv/registry/metrics.yaml index 14abb7e9b..493b092d4 100644 --- a/semconv/registry/metrics.yaml +++ b/semconv/registry/metrics.yaml @@ -489,6 +489,23 @@ groups: # fold of the audit stream. Lookup outcomes carry the corruption / # forward-compat signals (`torn` / `unknown_version`); publish outcomes # surface the best-effort write-through's failures. + - id: metric.ourios.receiver.tenant.divergences + type: metric + metric_name: ourios.receiver.tenant.divergences + stability: development + brief: >- + ResourceLogs groups whose watched resource attribute differed from + the value first observed for their tenant (RFC 0045 §3.4) — the + signal that one tenant spans what are probably several producers + (two clusters merged under a single-key rule). Observation only; + ingest is never rejected. The tenant id rides the accompanying + warning event, not this counter. + instrument: counter + unit: "{divergence}" + attributes: + - ref: ourios.tenant.watch.key + requirement_level: required + - id: metric.ourios.template_map.lookups type: metric metric_name: ourios.template_map.lookups From a145614f5fd938edaa007e1107cd2a03e7710417 Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Mon, 17 Aug 2026 02:17:37 +0200 Subject: [PATCH 04/10] fix(parquet)!: store keys are parsed, not re-encoded; rfc0045 end-to-end (RFC0045.2/.3/.4/.5/.8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Store::resolve used ObjectPath::from, which escaped the % of an already RFC 0005-encoded tenant a second time (a%2Fb -> a%252Fb) — the local querier's tenant_id= join and percent_decode_tenant then missed the object. Invisible for plain tenant ids, fatal for composite ones. ObjectPath::parse stores the key verbatim on both backends; invalid keys surface as StoreError::Backend. Regression test in ourios-parquet. The served-binary test drives three server lifetimes over one store + WAL (default rule → composite → composite + token) through OTLP/HTTP, SIGTERM flush and the querier: S2 pair isolated, missing-key 400, injectivity pair distinct, phase-1 files byte-untouched across the rule change, old-epoch tenant still answers, and the RFC 0026 binding rejects the wrong composite tenant with 403. BREAKING CHANGE: objects of a tenant whose id contains any character outside the RFC 0005 unreserved set (`/`, `%`, `=`, `:`, space, …) were written under a doubly-encoded `tenant_id=` key. Such tenants were unreadable on the local backend and mis-attributed by compaction; on S3 they were readable only through the same double encoding. After this fix they are addressed under the once-encoded key. Migrate by renaming the `tenant_id=` prefix to `tenant_id=` (an object copy on S3, a directory rename locally). Tenant ids that are unreserved throughout — every plain service.name — have identical keys before and after (RFC 0045 §3.2). Signed-off-by: Jens Holdgaard Pedersen Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01A6zqjWChsuUiahj3WB5s3H --- crates/ourios-parquet/src/store.rs | 91 ++++- crates/ourios-server/tests/it/main.rs | 1 + .../tests/it/rfc0045_composite_tenant.rs | 371 ++++++++++++++++++ 3 files changed, 448 insertions(+), 15 deletions(-) create mode 100644 crates/ourios-server/tests/it/rfc0045_composite_tenant.rs diff --git a/crates/ourios-parquet/src/store.rs b/crates/ourios-parquet/src/store.rs index bf3a7dd1a..d19f5cdd4 100644 --- a/crates/ourios-parquet/src/store.rs +++ b/crates/ourios-parquet/src/store.rs @@ -506,13 +506,28 @@ impl Store { } /// Resolve a `/`-delimited `key` to an absolute object path under the - /// store prefix. At `red` the prefix is empty, so this is just the key; - /// once prefix scoping is wired (RFC0013.5) the prefix segments lead. - fn resolve(&self, key: &str) -> ObjectPath { - self.prefix - .parts() - .chain(ObjectPath::from(key).parts()) - .collect() + /// store prefix. + /// + /// Keys are already path-safe by construction (RFC 0005 §3.4 + /// `percent_encode_tenant`, fixed partition names, UUID file names), so + /// they are *parsed* — stored verbatim as the object key and, on the + /// local backend, as the directory name — rather than re-encoded. + /// `ObjectPath::from` would escape the `%` of an encoded tenant a second + /// time (`a%2Fb` → `a%252Fb`), putting the object where neither the + /// local querier's `tenant_id=` join nor `percent_decode_tenant` + /// would find it — invisible for plain tenant ids, fatal for the RFC 0045 + /// composite ones. + /// + /// # Errors + /// + /// [`StoreError::Backend`] if `key` is not a valid object path (an + /// empty segment, `.`/`..`, a control character or raw `/` inside a + /// segment) — a programming error at the call site, surfaced rather + /// than silently re-encoded. + fn resolve(&self, key: &str) -> Result { + let path = ObjectPath::parse(key) + .map_err(|source| StoreError::Backend(object_store::Error::InvalidPath { source }))?; + Ok(self.prefix.parts().chain(path.parts()).collect()) } /// Write `bytes` to `key`. @@ -521,7 +536,7 @@ impl Store { /// [`StoreError::Backend`] if the put fails. pub async fn put(&self, key: &str, bytes: Vec) -> Result<(), StoreError> { self.inner - .put(&self.resolve(key), PutPayload::from(bytes)) + .put(&self.resolve(key)?, PutPayload::from(bytes)) .await .map_err(StoreError::Backend)?; Ok(()) @@ -534,7 +549,7 @@ impl Store { pub async fn get(&self, key: &str) -> Result, StoreError> { let got = self .inner - .get(&self.resolve(key)) + .get(&self.resolve(key)?) .await .map_err(StoreError::Backend)?; let bytes = got.bytes().await.map_err(StoreError::Backend)?; @@ -547,7 +562,7 @@ impl Store { /// [`StoreError::Backend`] if the delete fails. pub async fn delete(&self, key: &str) -> Result<(), StoreError> { self.inner - .delete(&self.resolve(key)) + .delete(&self.resolve(key)?) .await .map_err(StoreError::Backend) } @@ -610,7 +625,10 @@ impl Store { /// per-object `head`. Same tenant-isolation gating and key normalisation as /// [`Self::list`]. async fn list_entries(&self, prefix: Option<&str>) -> Result, StoreError> { - let scoped = prefix.map_or_else(|| self.prefix.clone(), |p| self.resolve(p)); + let scoped = match prefix { + Some(p) => self.resolve(p)?, + None => self.prefix.clone(), + }; let metas: Vec = self .inner .list(Some(&scoped)) @@ -686,7 +704,10 @@ impl Store { /// (RFC0019.5): a string-prefix sibling of the requested prefix is excluded. /// `LocalFileSystem` and S3 both surface subdirectories as common-prefixes. async fn list_common_prefixes(&self, prefix: Option<&str>) -> Result, StoreError> { - let scoped = prefix.map_or_else(|| self.prefix.clone(), |p| self.resolve(p)); + let scoped = match prefix { + Some(p) => self.resolve(p)?, + None => self.prefix.clone(), + }; let result = self .inner .list_with_delimiter(Some(&scoped)) @@ -755,7 +776,7 @@ impl Store { pub async fn put_if_absent(&self, key: &str, bytes: Vec) -> Result<(), StoreError> { self.inner .put_opts( - &self.resolve(key), + &self.resolve(key)?, PutPayload::from(bytes), PutOptions::from(PutMode::Create), ) @@ -796,7 +817,7 @@ impl Store { pub async fn get_with_etag(&self, key: &str) -> Result { let got = self .inner - .get(&self.resolve(key)) + .get(&self.resolve(key)?) .await .map_err(StoreError::Backend)?; let e_tag = got.meta.e_tag.clone(); @@ -825,7 +846,7 @@ impl Store { version: None, })); self.inner - .put_opts(&self.resolve(key), PutPayload::from(bytes), opts) + .put_opts(&self.resolve(key)?, PutPayload::from(bytes), opts) .await .map_err(StoreError::Backend)?; Ok(()) @@ -998,6 +1019,46 @@ mod tests { ); } + /// RFC 0005 §3.4 / RFC 0045 §3.2 — a percent-encoded tenant key is stored + /// verbatim: on the local backend the directory is `tenant_id=` + /// exactly (what the local querier joins and `percent_decode_tenant` + /// inverts), and listing returns the same key `put` took. + #[test] + fn encoded_tenant_keys_are_stored_verbatim_and_round_trip() { + let dir = tempfile::TempDir::new().expect("temp dir"); + let store = Store::local(dir.path()).expect("local store"); + let enc = crate::percent_encode_tenant("cluster1/flux%cd"); + assert_eq!(enc, "cluster1%2Fflux%25cd"); + let key = format!("data/tenant_id={enc}/year=2026/x.parquet"); + store.put_blocking(&key, b"row".to_vec()).expect("put"); + + assert!( + dir.path() + .join("data") + .join(format!("tenant_id={enc}")) + .join("year=2026") + .join("x.parquet") + .is_file(), + "the on-disk directory is the once-encoded tenant" + ); + assert_eq!( + store.list_blocking(Some("data/")).expect("list"), + vec![key.clone()], + "listing returns the key put took" + ); + assert_eq!( + store + .list_common_prefixes_blocking(Some("data/")) + .expect("prefixes"), + vec![format!("data/tenant_id={enc}")] + ); + assert_eq!(store.get_blocking(&key).expect("get"), b"row"); + assert!(matches!( + store.put_blocking("data/../x", Vec::new()), + Err(StoreError::Backend(_)) + )); + } + /// `list_blocking` enumerates keys under a prefix recursively, in /// lexicographic order, returning store-relative keys (the same key space /// as `get`/`put`) — the seam the querier/compactor walk instead of diff --git a/crates/ourios-server/tests/it/main.rs b/crates/ourios-server/tests/it/main.rs index 2e98f2e52..ba8ba2ee3 100644 --- a/crates/ourios-server/tests/it/main.rs +++ b/crates/ourios-server/tests/it/main.rs @@ -32,3 +32,4 @@ mod rfc0038_1_request_spans; mod rfc0039_1_query_propagation; mod rfc0039_4_sampling; mod rfc0043_5_event_name_query; +mod rfc0045_composite_tenant; diff --git a/crates/ourios-server/tests/it/rfc0045_composite_tenant.rs b/crates/ourios-server/tests/it/rfc0045_composite_tenant.rs new file mode 100644 index 000000000..3f008f643 --- /dev/null +++ b/crates/ourios-server/tests/it/rfc0045_composite_tenant.rs @@ -0,0 +1,371 @@ +//! RFC 0045 — composite tenant derivation at the process boundary: +//! RFC0045.2 (the S2 scenario end-to-end), .3 (strict missing-key +//! rejection), .4 (join injectivity), .5 (epoch semantics across a rule +//! change), and .8 (auth binding unchanged). +//! +//! Three server lifetimes over one store + WAL root, each driven through +//! `--config`, OTLP/HTTP export, SIGTERM (which flushes to Parquet), and +//! the querier: +//! +//! 1. default rule — one `fluxcd` export lands in tenant `fluxcd`; +//! 2. composite rule `[k8s.cluster.name, service.name]` — the S2 pair, +//! the injectivity pair, and a missing-key rejection; then queries prove +//! every tenant sees only its own rows and the phase-1 files are +//! byte-untouched; +//! 3. composite rule + a static token bound to `cluster1/fluxcd` — a +//! `cluster2/fluxcd` export under that token is refused (403), a +//! `cluster1/fluxcd` one is accepted. +//! +//! Unix-only: shutdown is driven with `kill -TERM` (as in +//! `rfc0003_16_served_binary`). +#![cfg(unix)] + +use std::collections::BTreeMap; +use std::net::SocketAddr; +use std::path::{Path, PathBuf}; +use std::process::Stdio; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest; +use opentelemetry_proto::tonic::common::v1::any_value::Value; +use opentelemetry_proto::tonic::common::v1::{AnyValue, KeyValue}; +use opentelemetry_proto::tonic::logs::v1::{LogRecord, ResourceLogs, ScopeLogs}; +use opentelemetry_proto::tonic::resource::v1::Resource; +use prost::Message; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; +use tokio::net::TcpStream; +use tokio::process::{Child, Command}; +use tokio::time::timeout; + +fn now_ns() -> u64 { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |d| d.as_nanos()); + u64::try_from(nanos).unwrap_or(0) +} + +fn string_value(s: &str) -> AnyValue { + AnyValue { + value: Some(Value::StringValue(s.to_owned())), + } +} + +/// One-record export whose `Resource` carries `attrs`, stamped now so the +/// querier's default look-back window includes it. +fn export(attrs: &[(&str, &str)], body: &str) -> Vec { + ExportLogsServiceRequest { + resource_logs: vec![ResourceLogs { + resource: Some(Resource { + attributes: attrs + .iter() + .map(|(key, value)| KeyValue { + key: (*key).to_owned(), + value: Some(string_value(value)), + ..Default::default() + }) + .collect(), + ..Default::default() + }), + scope_logs: vec![ScopeLogs { + log_records: vec![LogRecord { + time_unix_nano: now_ns(), + body: Some(string_value(body)), + ..Default::default() + }], + ..Default::default() + }], + ..Default::default() + }], + } + .encode_to_vec() +} + +struct Server { + child: Child, + http: SocketAddr, + querier: SocketAddr, +} + +/// Spawn `ourios-server --config` with receiver + querier on ephemeral +/// ports over `tmp` (`store/`, `wal/`), returning the announced addresses. +async fn start(tmp: &Path, tenant_yaml: &str, auth_yaml: &str) -> Server { + let config = format!( + "storage:\n local:\n bucket_root: {store}\n\ + receiver:\n enabled: true\n grpc_addr: 127.0.0.1:0\n http_addr: 127.0.0.1:0\n\ + \x20\x20wal_root: {wal}\n{tenant_yaml}\ + querier:\n enabled: true\n http_addr: 127.0.0.1:0\n{auth_yaml}", + store = tmp.join("store").display(), + wal = tmp.join("wal").display(), + ); + let path = tmp.join(format!("config-{}.yaml", now_ns())); + std::fs::write(&path, config).expect("write config"); + let mut child = Command::new(env!("CARGO_BIN_EXE_ourios-server")) + .arg("--config") + .arg(&path) + .env("RFC0045_TOKEN", "tok-cluster-one") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .spawn() + .expect("spawn ourios-server"); + let stdout = child.stdout.take().expect("stdout piped"); + let mut stderr = child.stderr.take().expect("stderr piped"); + let mut lines = BufReader::new(stdout).lines(); + let mut http = None; + let mut querier = None; + let read = async { + while http.is_none() || querier.is_none() { + let Some(line) = lines.next_line().await.expect("read stdout") else { + let mut err = String::new(); + stderr.read_to_string(&mut err).await.ok(); + panic!("server exited before announcing its addresses; stderr:\n{err}"); + }; + if let Some(rest) = line.strip_prefix("receiver HTTP listening on ") { + http = Some(rest.trim().parse().expect("http addr")); + } else if let Some(rest) = line.strip_prefix("querier HTTP listening on ") { + querier = Some(rest.trim().parse().expect("querier addr")); + } + } + }; + timeout(Duration::from_secs(20), read) + .await + .expect("server announces its addresses"); + Server { + child, + http: http.expect("http"), + querier: querier.expect("querier"), + } +} + +async fn stop(mut server: Server) { + let pid = server.child.id().expect("pid"); + Command::new("kill") + .arg("-TERM") + .arg(pid.to_string()) + .status() + .await + .expect("kill -TERM"); + let status = timeout(Duration::from_secs(20), server.child.wait()) + .await + .expect("exit before timeout") + .expect("await exit"); + assert!(status.success(), "clean shutdown, got {status:?}"); +} + +async fn raw_post(addr: SocketAddr, head: String, body: &[u8]) -> String { + let mut stream = TcpStream::connect(addr).await.expect("connect"); + stream.write_all(head.as_bytes()).await.expect("write head"); + stream.write_all(body).await.expect("write body"); + stream.flush().await.ok(); + let mut response = Vec::new(); + stream.read_to_end(&mut response).await.expect("read"); + String::from_utf8_lossy(&response).into_owned() +} + +/// OTLP/HTTP export; returns the HTTP status code. +async fn post_logs(addr: SocketAddr, body: &[u8], bearer: Option<&str>) -> u16 { + let auth = bearer.map_or(String::new(), |t| format!("Authorization: Bearer {t}\r\n")); + let head = format!( + "POST /v1/logs HTTP/1.1\r\nHost: {addr}\r\nContent-Type: application/x-protobuf\r\n\ + {auth}Content-Length: {}\r\nConnection: close\r\n\r\n", + body.len(), + ); + status_of(&raw_post(addr, head, body).await) +} + +fn status_of(response: &str) -> u16 { + response + .split_whitespace() + .nth(1) + .and_then(|s| s.parse().ok()) + .unwrap_or_else(|| panic!("no status line in {response:?}")) +} + +/// `true` over `tenant` — the row count the querier reports. +async fn rows_for(addr: SocketAddr, tenant: &str) -> u64 { + let dsl = "true"; + let head = format!( + "POST /v1/query HTTP/1.1\r\nHost: {addr}\r\nX-Ourios-Tenant: {tenant}\r\n\ + Content-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + dsl.len(), + ); + let response = raw_post(addr, head, dsl.as_bytes()).await; + assert_eq!(status_of(&response), 200, "query {tenant}: {response}"); + let body = response.split("\r\n\r\n").nth(1).expect("body"); + let json: serde_json::Value = serde_json::from_str(body).expect("json"); + json["rows"].as_u64().expect("rows") +} + +/// Every Parquet object under `root` with its size + mtime — the "was +/// anything rewritten" fingerprint. +fn parquet_fingerprint(root: &Path) -> BTreeMap { + let mut out = BTreeMap::new(); + let mut stack = vec![root.to_path_buf()]; + while let Some(dir) = stack.pop() { + let Ok(entries) = std::fs::read_dir(&dir) else { + continue; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + stack.push(path); + } else if path.extension().is_some_and(|x| x == "parquet") { + let meta = entry.metadata().expect("metadata"); + out.insert(path, (meta.len(), meta.modified().expect("mtime"))); + } + } + } + out +} + +const COMPOSITE: &str = " tenant:\n rule: [k8s.cluster.name, service.name]\n"; + +/// Scenarios RFC0045.2 / .3 / .4 / .5 / .8. +/// See `docs/rfcs/0045-composite-tenant-derivation.md` §5. +#[tokio::test] +async fn rfc0045_composite_tenant_end_to_end() { + let tmp = tempfile::TempDir::new().expect("temp"); + std::fs::create_dir_all(tmp.path().join("store")).expect("store root"); + + // Phase 1 — default rule: fluxcd (with a cluster attribute the default + // rule ignores) lands in tenant `fluxcd`. + let s1 = start(tmp.path(), "", "").await; + assert_eq!( + post_logs( + s1.http, + &export( + &[("service.name", "fluxcd"), ("k8s.cluster.name", "cluster1")], + "epoch one" + ), + None + ) + .await, + 200 + ); + stop(s1).await; + let before = parquet_fingerprint(&tmp.path().join("store")); + assert!(!before.is_empty(), "phase 1 flushed to Parquet on shutdown"); + + // Phase 2 — composite rule. + let s2 = start(tmp.path(), COMPOSITE, "").await; + let cases: [(&str, &str, &str); 4] = [ + // RFC0045.2 — the S2 pair. + ("cluster1", "fluxcd", "from cluster one"), + ("cluster2", "fluxcd", "from cluster two"), + // RFC0045.4 — the injectivity pair. + ("a", "b/c", "left tuple"), + ("a/b", "c", "right tuple"), + ]; + for (cluster, service, body) in cases { + assert_eq!( + post_logs( + s2.http, + &export( + &[("service.name", service), ("k8s.cluster.name", cluster)], + body + ), + None + ) + .await, + 200, + "{cluster}/{service} accepted" + ); + } + // RFC0045.3 — a group lacking / emptying the cluster key rejects the + // whole export, same posture as a missing service.name. + for attrs in [ + vec![("service.name", "fluxcd")], + vec![("service.name", "fluxcd"), ("k8s.cluster.name", "")], + ] { + assert_eq!( + post_logs(s2.http, &export(&attrs, "must not land"), None).await, + 400, + "missing/empty rule key is rejected: {attrs:?}" + ); + } + stop(s2).await; + + // RFC0045.5 — phase-1 files are untouched (nothing rewritten, no + // repartitioning); the old-epoch tenant still answers. + let after = parquet_fingerprint(&tmp.path().join("store")); + for (path, fingerprint) in &before { + assert_eq!( + after.get(path), + Some(fingerprint), + "{} was rewritten or removed by the rule change", + path.display() + ); + } + assert!(after.len() > before.len(), "phase 2 added its own files"); + + // Query every tenant: each sees only its own rows. + let s3 = start(tmp.path(), COMPOSITE, "").await; + let expected = [ + ("fluxcd", 1), + ("cluster1/fluxcd", 1), + ("cluster2/fluxcd", 1), + ("a/b%2Fc", 1), + ("a%2Fb/c", 1), + ("a/b/c", 0), + ("cluster1", 0), + ]; + for (tenant, rows) in expected { + assert_eq!( + rows_for(s3.querier, tenant).await, + rows, + "tenant {tenant}; store: {:#?}", + parquet_fingerprint(&tmp.path().join("store")) + .keys() + .collect::>() + ); + } + stop(s3).await; + + auth_binding_unchanged(tmp.path()).await; +} + +/// Phase 3 / RFC0045.8 — composite rule + a token bound to +/// `cluster1/fluxcd`: a `cluster2/fluxcd` export under it is refused whole, +/// a `cluster1/fluxcd` export is accepted, and no token is still 401. +async fn auth_binding_unchanged(tmp: &Path) { + let auth = "auth:\n tokens:\n - name: cluster-one\n token: ${env:RFC0045_TOKEN}\n\ + \x20\x20\x20\x20\x20\x20tenants: [cluster1/fluxcd]\n"; + let s4 = start(tmp, COMPOSITE, auth).await; + assert_eq!( + post_logs( + s4.http, + &export( + &[("service.name", "fluxcd"), ("k8s.cluster.name", "cluster2")], + "wrong cluster" + ), + Some("tok-cluster-one") + ) + .await, + 403 + ); + assert_eq!( + post_logs( + s4.http, + &export( + &[("service.name", "fluxcd"), ("k8s.cluster.name", "cluster1")], + "right cluster" + ), + Some("tok-cluster-one") + ) + .await, + 200 + ); + assert_eq!( + post_logs( + s4.http, + &export( + &[("service.name", "fluxcd"), ("k8s.cluster.name", "cluster1")], + "no token" + ), + None + ) + .await, + 401 + ); + stop(s4).await; +} From 6e39be71cd8537ac56ba39c67988915d3cf8b79a Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Mon, 17 Aug 2026 02:20:03 +0200 Subject: [PATCH 05/10] feat(helm): receiver.tenant passthrough (RFC 0045) Rendered verbatim under receiver: in the config file when set; empty by default so the chart's rendered config is unchanged. Render test covers both. Signed-off-by: Jens Holdgaard Pedersen --- deploy/helm/ourios/README.md | 1 + deploy/helm/ourios/templates/_helpers.tpl | 4 ++++ deploy/helm/ourios/values.yaml | 9 +++++++++ deploy/helm/render-tests.sh | 23 +++++++++++++++++++++++ 4 files changed, 37 insertions(+) diff --git a/deploy/helm/ourios/README.md b/deploy/helm/ourios/README.md index 6b43c2a35..8f28a306b 100644 --- a/deploy/helm/ourios/README.md +++ b/deploy/helm/ourios/README.md @@ -358,6 +358,7 @@ is intentionally out of scope. Tune the cadence via `compactor.intervalSecs`. | `receiver.replicas` | `1` | Receiver replicas (each gets its own WAL PVC). | | `receiver.wal.size` | `2Gi` | WAL PVC size (`OURIOS_WAL_ROOT`, always local). | | `receiver.wal.storageClassName` | `""` | WAL StorageClass (`""` = cluster default). | +| `receiver.tenant` | `{}` | Tenant derivation (RFC 0045), rendered verbatim as `receiver.tenant` in the config: `rule` (ordered resource-attribute keys joined into the tenant id; default `[service.name]`), `watch`, `watch_capacity`. Set `rule: [k8s.cluster.name, service.name]` when one `service.name` runs in several clusters. | | `querier.enabled` | `true` | Querier Deployment (HTTP `:4319`). | | `querier.replicas` | `2` | Querier replicas (scales independently, no PVC). | | `querier.defaultWindowSecs` | `3600` | Default look-back for a query with no `range(...)`. | diff --git a/deploy/helm/ourios/templates/_helpers.tpl b/deploy/helm/ourios/templates/_helpers.tpl index cb6dfbd2a..8553ce531 100644 --- a/deploy/helm/ourios/templates/_helpers.tpl +++ b/deploy/helm/ourios/templates/_helpers.tpl @@ -152,6 +152,10 @@ receiver: grpc_addr: "0.0.0.0:4317" http_addr: "0.0.0.0:4318" wal_root: {{ $.Values.receiver.wal.mountPath | quote }} +{{- with $.Values.receiver.tenant }} + tenant: +{{ toYaml . | indent 4 }} +{{- end }} compaction: enabled: false {{- else if eq $role "querier" }} diff --git a/deploy/helm/ourios/values.yaml b/deploy/helm/ourios/values.yaml index 2ec83b7bf..66fbaad7b 100644 --- a/deploy/helm/ourios/values.yaml +++ b/deploy/helm/ourios/values.yaml @@ -127,6 +127,15 @@ receiver: extraEnv: [] # - name: OTEL_RESOURCE_ATTRIBUTES # value: deployment.environment.name=production,service.namespace=ourios + # Tenant derivation (RFC 0045): rendered verbatim as `receiver.tenant` in the + # config file. Empty = the [service.name] default. When the same service.name + # runs in several clusters, list the cluster key first — every key is + # required, values join with "/" (cluster1/fluxcd). Changing the rule affects + # newly ingested data only (stored tenant ids never change). + tenant: {} + # rule: [k8s.cluster.name, service.name] + # watch: [k8s.cluster.name] + # watch_capacity: 10000 # Querier role (RFC 0016): a stateless Deployment that reads the configured store # (local or s3) and scales freely. diff --git a/deploy/helm/render-tests.sh b/deploy/helm/render-tests.sh index fb517ad9c..213d4ae5b 100755 --- a/deploy/helm/render-tests.sh +++ b/deploy/helm/render-tests.sh @@ -114,6 +114,29 @@ check "s3 existingSecret envFrom is untouched" \ --set 'receiver.extraEnv[0].name=OTEL_RESOURCE_ATTRIBUTES' \ --set 'receiver.extraEnv[0].value=x=y')" +# --- receiver.tenant passthrough (RFC 0045) --------------------------------- + +# The receiver's rendered config document (the ConfigMap's `receiver.yaml`). +receiver_config() { + helm template t "$CHART" --show-only templates/configmap.yaml "$@" \ + | sed -n '/^ receiver.yaml: |/,/^ [a-z]*.yaml: |/p' +} + +check "defaults render no receiver.tenant block" "" \ + "$(receiver_config | grep '^ tenant:' || true)" +check "receiver.tenant renders verbatim under receiver" \ + " tenant: + rule: + - k8s.cluster.name + - service.name + watch: + - cloud.region" \ + "$(receiver_config \ + --set 'receiver.tenant.rule[0]=k8s.cluster.name' \ + --set 'receiver.tenant.rule[1]=service.name' \ + --set 'receiver.tenant.watch[0]=cloud.region' \ + | sed -n '/^ tenant:/,/^ [a-z_]*:$/p' | sed '$d')" + if ((failures)); then printf '\n%d assertion(s) failed\n' "$failures" >&2 exit 1 From 51bd8ec1836ca2d9f99dfdd0b986eed86b031330 Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Mon, 17 Aug 2026 02:28:03 +0200 Subject: [PATCH 06/10] fix(receiver): divergence detector compares digest + length, previews stay 128 B (RFC0045.7) Comparing the truncated preview would miss two values sharing their first 128 bytes. Epoch-log load also gains the backwards-boundary rejection test (RFC0045.10). Signed-off-by: Jens Holdgaard Pedersen --- crates/ourios-ingester/src/receiver/watch.rs | 53 +++++++++++++++++--- crates/ourios-ingester/src/rule_epochs.rs | 38 ++++++++++++++ 2 files changed, 84 insertions(+), 7 deletions(-) diff --git a/crates/ourios-ingester/src/receiver/watch.rs b/crates/ourios-ingester/src/receiver/watch.rs index f38ae9144..2139ca3c2 100644 --- a/crates/ourios-ingester/src/receiver/watch.rs +++ b/crates/ourios-ingester/src/receiver/watch.rs @@ -14,6 +14,7 @@ //! (tenant, key). Everything resets on restart by design. use std::collections::HashMap; +use std::hash::{Hash, Hasher}; use std::sync::Mutex; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; @@ -35,10 +36,22 @@ pub const MAX_VALUE_BYTES: usize = 128; pub const WARN_INTERVAL: Duration = Duration::from_secs(60); struct Entry { - first: String, + /// Exact identity of the first value: digest + byte length. Comparison + /// never uses the preview, so values sharing a 128-byte prefix are + /// still told apart. + first_digest: u64, + first_len: usize, + /// The bounded rendering of the first value, for the warning only. + first_preview: String, last_warned: Option, } +fn digest(value: &str) -> u64 { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + value.hash(&mut hasher); + hasher.finish() +} + /// The detector: build one per pipeline from the resolved /// [`TenantDerivation`]; call [`observe`](Self::observe) once per /// derived `ResourceLogs` group. @@ -128,16 +141,18 @@ impl DivergenceWatch { state.insert( slot, Entry { - first: bound(value).into_owned(), + first_digest: digest(value), + first_len: value.len(), + first_preview: bound(value).into_owned(), last_warned: None, }, ); return; }; - let seen = bound(value); - if entry.first == seen { + if entry.first_len == value.len() && entry.first_digest == digest(value) { return; } + let seen = bound(value); self.divergences.add( 1, &[OtelKeyValue::new( @@ -158,7 +173,7 @@ impl DivergenceWatch { tracing::Level::WARN, ourios.tenant = tenant.as_str(), ourios.tenant.watch.key = key, - ourios.tenant.watch.first_value = entry.first.as_str(), + ourios.tenant.watch.first_value = entry.first_preview.as_str(), ourios.tenant.watch.value = seen.as_ref(), "tenant spans more than one value of a watched resource attribute — if these are \ different producers, add the key to receiver.tenant.rule (RFC 0045 §3.4)" @@ -281,7 +296,7 @@ mod tests { let entry = state .get(&(tenant, "k8s.cluster.name".to_owned())) .expect("entry"); - assert_eq!(entry.first, "cluster1"); + assert_eq!(entry.first_preview, "cluster1"); assert!(entry.last_warned.is_none(), "no divergence, no warning"); } @@ -298,7 +313,7 @@ mod tests { let entry = state .get(&(tenant.clone(), "k8s.cluster.name".to_owned())) .expect("entry"); - assert_eq!(entry.first, "cluster1"); + assert_eq!(entry.first_preview, "cluster1"); entry.last_warned.expect("warned") }; w.observe(&tenant, &[attr("k8s.cluster.name", "cluster3")]); @@ -327,6 +342,30 @@ mod tests { ); } + // RFC0045.7 — two values sharing their first 128 bytes still diverge: + // comparison is digest + length, the preview is display only. + #[test] + fn shared_prefix_values_still_diverge() { + let w = watch(10); + let tenant = TenantId::new("t"); + let prefix = "p".repeat(MAX_VALUE_BYTES); + w.observe( + &tenant, + &[attr("k8s.cluster.name", &format!("{prefix}-one"))], + ); + w.observe( + &tenant, + &[attr("k8s.cluster.name", &format!("{prefix}-two"))], + ); + let state = w.state.lock().expect("lock"); + let entry = &state[&(tenant, "k8s.cluster.name".to_owned())]; + assert!( + entry.last_warned.is_some(), + "divergence detected past the preview bound" + ); + assert_eq!(entry.first_preview, format!("{prefix}…")); + } + // RFC0045.7 — values are bounded at a UTF-8 boundary with a trailing `…`. #[test] fn values_are_bounded_at_a_char_boundary() { diff --git a/crates/ourios-ingester/src/rule_epochs.rs b/crates/ourios-ingester/src/rule_epochs.rs index 1972e325a..f22343812 100644 --- a/crates/ourios-ingester/src/rule_epochs.rs +++ b/crates/ourios-ingester/src/rule_epochs.rs @@ -337,5 +337,43 @@ mod tests { RuleEpochs::load(dir.path()).unwrap_err(), RuleEpochsError::Malformed { .. } )); + + // Boundaries that go backwards are rejected; an equal boundary is + // append order and accepted. + let later = offset(5); + let earlier = WalOffset { + segment: later.segment, + byte: 4, + }; + let entry = |keys: &str, o: WalOffset| { + format!( + "{{\"rule\": [{keys}], \"after\": {{\"segment\": \"{}\", \"byte\": {}}}}}", + o.segment, o.byte + ) + }; + std::fs::write( + dir.path().join(FILE_NAME), + format!( + "{{\"epochs\": [{}, {}]}}", + entry("\"a\"", later), + entry("\"b\"", earlier) + ), + ) + .expect("write"); + assert!(matches!( + RuleEpochs::load(dir.path()).unwrap_err(), + RuleEpochsError::Malformed { .. } + )); + std::fs::write( + dir.path().join(FILE_NAME), + format!( + "{{\"epochs\": [{}, {}]}}", + entry("\"a\"", later), + entry("\"b\"", later) + ), + ) + .expect("write"); + let epochs = RuleEpochs::load(dir.path()).expect("equal boundary is append order"); + assert_eq!(epochs.current().keys(), ["b"]); } } From 3a12672360a430c4f96ffba04fe38c7f5112840c Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Mon, 17 Aug 2026 03:43:20 +0200 Subject: [PATCH 07/10] fix(receiver): epoch-log strict shape, ellipsis within the 128 B bound, drain server pipes in test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - RuleEpochs: only the first entry may be unbounded; an empty-WAL rule change collapses the log to one entry (load rejects a later null) - DivergenceWatch::bound keeps the rendering ≤ 128 bytes including the … - rfc0045 served-binary test keeps stdout/stderr drained for the process lifetime (a dropped pipe made the server's later println! panic on the Linux runner) and reports stderr on an unclean exit Signed-off-by: Jens Holdgaard Pedersen --- crates/ourios-ingester/src/receiver/watch.rs | 24 ++++++--- crates/ourios-ingester/src/rule_epochs.rs | 49 +++++++++++++++++-- .../tests/rfc0045_divergence_telemetry.rs | 4 +- .../tests/it/rfc0045_composite_tenant.rs | 31 ++++++++++-- 4 files changed, 91 insertions(+), 17 deletions(-) diff --git a/crates/ourios-ingester/src/receiver/watch.rs b/crates/ourios-ingester/src/receiver/watch.rs index 2139ca3c2..5330a05a8 100644 --- a/crates/ourios-ingester/src/receiver/watch.rs +++ b/crates/ourios-ingester/src/receiver/watch.rs @@ -192,13 +192,14 @@ fn string_attribute<'a>(attributes: &'a [KeyValue], key: &str) -> Option<&'a str }) } -/// `value` truncated to [`MAX_VALUE_BYTES`] at a UTF-8 boundary with a -/// trailing `…`, or borrowed unchanged when it already fits. +/// `value` bounded to [`MAX_VALUE_BYTES`] *including* the trailing `…` +/// that marks a truncation (cut at a UTF-8 boundary), or borrowed unchanged +/// when it already fits. fn bound(value: &str) -> std::borrow::Cow<'_, str> { if value.len() <= MAX_VALUE_BYTES { return std::borrow::Cow::Borrowed(value); } - let mut end = MAX_VALUE_BYTES; + let mut end = MAX_VALUE_BYTES - '…'.len_utf8(); while !value.is_char_boundary(end) { end -= 1; } @@ -363,7 +364,10 @@ mod tests { entry.last_warned.is_some(), "divergence detected past the preview bound" ); - assert_eq!(entry.first_preview, format!("{prefix}…")); + assert_eq!( + entry.first_preview, + format!("{}…", "p".repeat(MAX_VALUE_BYTES - '…'.len_utf8())) + ); } // RFC0045.7 — values are bounded at a UTF-8 boundary with a trailing `…`. @@ -371,10 +375,14 @@ mod tests { fn values_are_bounded_at_a_char_boundary() { let short = "x".repeat(MAX_VALUE_BYTES); assert_eq!(bound(&short), short); - // 'é' is two bytes; 127 ASCII bytes + 'é' straddles the boundary. - let long = format!("{}é{}", "x".repeat(MAX_VALUE_BYTES - 1), "tail"); + // 'é' is two bytes and straddles the cut point (125 bytes into a + // 128-byte budget with a 3-byte ellipsis): the cut backs off to the + // char boundary before it. + let long = format!("{}é{}", "x".repeat(MAX_VALUE_BYTES - 4), "tail"); let bounded = bound(&long); - assert_eq!(bounded, format!("{}…", "x".repeat(MAX_VALUE_BYTES - 1))); - assert!(bounded.len() <= MAX_VALUE_BYTES + '…'.len_utf8()); + assert_eq!(bounded, format!("{}…", "x".repeat(MAX_VALUE_BYTES - 4))); + assert!(bounded.len() <= MAX_VALUE_BYTES); + let ascii = "y".repeat(MAX_VALUE_BYTES + 10); + assert_eq!(bound(&ascii).len(), MAX_VALUE_BYTES); } } diff --git a/crates/ourios-ingester/src/rule_epochs.rs b/crates/ourios-ingester/src/rule_epochs.rs index f22343812..42ee87510 100644 --- a/crates/ourios-ingester/src/rule_epochs.rs +++ b/crates/ourios-ingester/src/rule_epochs.rs @@ -100,7 +100,13 @@ impl RuleEpochs { /// Make `rule` the current epoch for frames after `after` (the highest /// offset replay delivered), persisting the log if the rule differs - /// from the newest epoch's. Returns whether an epoch was appended. + /// from the newest epoch's. Returns whether the log changed. + /// + /// `after: None` means replay delivered nothing — the WAL holds no + /// frames — so there is nothing to attribute to earlier epochs and the + /// log collapses to the single entry `{rule, null}`; every entry after + /// the first therefore always carries a boundary (the shape `load` + /// enforces). /// /// # Errors /// @@ -113,10 +119,14 @@ impl RuleEpochs { if self.current() == rule { return Ok(false); } - self.epochs.push(RuleEpoch { + let epoch = RuleEpoch { rule: rule.clone(), after, - }); + }; + match after { + Some(_) => self.epochs.push(epoch), + None => self.epochs = vec![epoch], + } self.persist()?; Ok(true) } @@ -191,7 +201,12 @@ fn parse(bytes: &[u8]) -> Result, String> { .collect::, _>>()?; let rule = TenantRule::from_keys(keys).map_err(|e| format!("epochs[{index}].rule: {e}"))?; let after = match entry.get("after") { - None | Some(Value::Null) => None, + None | Some(Value::Null) if index == 0 => None, + None | Some(Value::Null) => { + return Err(format!( + "epochs[{index}].after is null; only the first epoch is unbounded" + )); + } Some(after) => { let segment = after .get("segment") @@ -313,6 +328,21 @@ mod tests { assert_eq!(reloaded.rule_for(offset(0)), &composite, "a newer segment"); } + // RFC0045.10 — with no frames delivered (an empty WAL) a rule change + // collapses the log to one unbounded entry. + #[test] + fn advance_without_frames_collapses_to_a_single_epoch() { + let dir = tempfile::tempdir().expect("tempdir"); + let composite = TenantRule::from_keys(["a", "b"]).expect("valid"); + let mut epochs = RuleEpochs::load(dir.path()).expect("loads"); + assert!(epochs.advance(&composite, None).expect("writes")); + let reloaded = RuleEpochs::load(dir.path()).expect("reloads"); + assert_eq!(reloaded.epochs().len(), 1); + assert_eq!(reloaded.current(), &composite); + assert_eq!(reloaded.epochs()[0].after, None); + assert_eq!(reloaded.rule_for(offset(0)), &composite); + } + // RFC0045.10 — an unparseable log aborts loudly, naming the file. #[test] fn malformed_log_is_an_error_naming_the_file() { @@ -338,6 +368,17 @@ mod tests { RuleEpochsError::Malformed { .. } )); + // A null boundary after the first entry is rejected. + std::fs::write( + dir.path().join(FILE_NAME), + b"{\"epochs\": [{\"rule\": [\"a\"], \"after\": null}, {\"rule\": [\"b\"], \"after\": null}]}", + ) + .expect("write"); + assert!(matches!( + RuleEpochs::load(dir.path()).unwrap_err(), + RuleEpochsError::Malformed { .. } + )); + // Boundaries that go backwards are rejected; an equal boundary is // append order and accepted. let later = offset(5); diff --git a/crates/ourios-ingester/tests/rfc0045_divergence_telemetry.rs b/crates/ourios-ingester/tests/rfc0045_divergence_telemetry.rs index 8d7d2e46e..22e3990e1 100644 --- a/crates/ourios-ingester/tests/rfc0045_divergence_telemetry.rs +++ b/crates/ourios-ingester/tests/rfc0045_divergence_telemetry.rs @@ -204,7 +204,7 @@ async fn scenario_7_detector(capture: &Capture) { ); // A value longer than 128 bytes is truncated at a UTF-8 boundary with `…`. - let long = format!("{}é{}", "x".repeat(MAX_VALUE_BYTES - 1), "tail"); + let long = format!("{}é{}", "x".repeat(MAX_VALUE_BYTES - 4), "tail"); pipeline .ingest_bound( request(vec![cluster_group("payments", "c1", "line")]), @@ -228,7 +228,7 @@ async fn scenario_7_detector(capture: &Capture) { .expect("payments diverged"); assert_eq!( field(payments, ourios_semconv::OURIOS_TENANT_WATCH_VALUE), - Some(format!("{}…", "x".repeat(MAX_VALUE_BYTES - 1)).as_str()) + Some(format!("{}…", "x".repeat(MAX_VALUE_BYTES - 4)).as_str()) ); } diff --git a/crates/ourios-server/tests/it/rfc0045_composite_tenant.rs b/crates/ourios-server/tests/it/rfc0045_composite_tenant.rs index 3f008f643..316e44bbf 100644 --- a/crates/ourios-server/tests/it/rfc0045_composite_tenant.rs +++ b/crates/ourios-server/tests/it/rfc0045_composite_tenant.rs @@ -84,6 +84,9 @@ struct Server { child: Child, http: SocketAddr, querier: SocketAddr, + /// Drains stdout/stderr for the process lifetime — dropping the pipe + /// readers would make the server's later `println!` hit a closed pipe. + drain: tokio::task::JoinHandle, } /// Spawn `ourios-server --config` with receiver + querier on ephemeral @@ -110,14 +113,24 @@ async fn start(tmp: &Path, tenant_yaml: &str, auth_yaml: &str) -> Server { .expect("spawn ourios-server"); let stdout = child.stdout.take().expect("stdout piped"); let mut stderr = child.stderr.take().expect("stderr piped"); + let stderr_drain = + std::sync::Arc::new(tokio::sync::Mutex::new(Some(tokio::spawn(async move { + let mut err = String::new(); + stderr.read_to_string(&mut err).await.ok(); + err + })))); + let stderr_for_panic = std::sync::Arc::clone(&stderr_drain); let mut lines = BufReader::new(stdout).lines(); let mut http = None; let mut querier = None; let read = async { while http.is_none() || querier.is_none() { let Some(line) = lines.next_line().await.expect("read stdout") else { - let mut err = String::new(); - stderr.read_to_string(&mut err).await.ok(); + let handle = stderr_for_panic.lock().await.take(); + let err = match handle { + Some(h) => h.await.unwrap_or_default(), + None => String::new(), + }; panic!("server exited before announcing its addresses; stderr:\n{err}"); }; if let Some(rest) = line.strip_prefix("receiver HTTP listening on ") { @@ -130,10 +143,18 @@ async fn start(tmp: &Path, tenant_yaml: &str, auth_yaml: &str) -> Server { timeout(Duration::from_secs(20), read) .await .expect("server announces its addresses"); + let drain = tokio::spawn(async move { + while let Ok(Some(_)) = lines.next_line().await {} + match stderr_drain.lock().await.take() { + Some(h) => h.await.unwrap_or_default(), + None => String::new(), + } + }); Server { child, http: http.expect("http"), querier: querier.expect("querier"), + drain, } } @@ -149,7 +170,11 @@ async fn stop(mut server: Server) { .await .expect("exit before timeout") .expect("await exit"); - assert!(status.success(), "clean shutdown, got {status:?}"); + let stderr = server.drain.await.expect("drain"); + assert!( + status.success(), + "clean shutdown, got {status:?}; stderr:\n{stderr}" + ); } async fn raw_post(addr: SocketAddr, head: String, body: &[u8]) -> String { From 0e33a4d9399baeb3d1faa7fe93ab70b45a9b0049 Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Mon, 17 Aug 2026 04:08:42 +0200 Subject: [PATCH 08/10] fix(ingester): blank tenant keys rejected, epoch-log oldest fallback + persist-before-commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - TenantRule::from_keys and receiver.tenant.watch reject empty / whitespace keys at startup (RFC0045.1) - RuleEpochs: a bounded first entry is rejected on load, rule_for falls back to the oldest epoch, advance persists the candidate before committing it in memory (a failed write leaves current() honest) - ourios-parquet: property test — any tenant id (reserved, unreserved, multi-byte) round-trips put → list → get once-encoded - config guide: receiver.tenant is file-only; registry: the divergences group sits with the other receiver metrics Signed-off-by: Jens Holdgaard Pedersen --- crates/ourios-ingester/src/receiver/tenant.rs | 14 +++- crates/ourios-ingester/src/rule_epochs.rs | 84 +++++++++++++++---- crates/ourios-parquet/src/store.rs | 37 ++++++++ crates/ourios-server/src/main.rs | 14 ++++ docs/guides/configuration.md | 4 + semconv/registry/metrics.yaml | 8 +- 6 files changed, 142 insertions(+), 19 deletions(-) diff --git a/crates/ourios-ingester/src/receiver/tenant.rs b/crates/ourios-ingester/src/receiver/tenant.rs index 8d1f6d862..020d2491c 100644 --- a/crates/ourios-ingester/src/receiver/tenant.rs +++ b/crates/ourios-ingester/src/receiver/tenant.rs @@ -51,7 +51,8 @@ impl TenantRule { /// /// # Errors /// - /// [`TenantRuleError::Empty`] for no keys; [`TenantRuleError::Duplicate`] + /// [`TenantRuleError::Empty`] for no keys, [`TenantRuleError::BlankKey`] + /// for an empty or whitespace-only key, [`TenantRuleError::Duplicate`] /// naming the first repeated key. pub fn from_keys(keys: I) -> Result where @@ -62,6 +63,9 @@ impl TenantRule { if keys.is_empty() { return Err(TenantRuleError::Empty); } + if keys.iter().any(|key| key.trim().is_empty()) { + return Err(TenantRuleError::BlankKey); + } let mut seen = std::collections::HashSet::new(); if let Some(duplicate) = keys.iter().find(|key| !seen.insert(key.as_str())) { return Err(TenantRuleError::Duplicate { @@ -179,6 +183,9 @@ impl Default for TenantDerivation { pub enum TenantRuleError { /// No keys at all. Empty, + /// An empty or whitespace-only key — it can never match a resource + /// attribute, so every export would fail at request time. + BlankKey, /// The same key listed twice. Duplicate { key: String }, } @@ -190,6 +197,7 @@ impl std::fmt::Display for TenantRuleError { f, "tenant rule must list at least one resource attribute key" ), + Self::BlankKey => write!(f, "tenant rule lists an empty resource attribute key"), Self::Duplicate { key } => { write!( f, @@ -489,6 +497,10 @@ mod tests { TenantRule::from_keys(Vec::::new()).unwrap_err(), TenantRuleError::Empty ); + assert_eq!( + TenantRule::from_keys(["service.name", " "]).unwrap_err(), + TenantRuleError::BlankKey + ); assert_eq!( TenantRule::from_keys(["service.name", "service.name"]).unwrap_err(), TenantRuleError::Duplicate { diff --git a/crates/ourios-ingester/src/rule_epochs.rs b/crates/ourios-ingester/src/rule_epochs.rs index 42ee87510..3a838f18d 100644 --- a/crates/ourios-ingester/src/rule_epochs.rs +++ b/crates/ourios-ingester/src/rule_epochs.rs @@ -88,14 +88,21 @@ impl RuleEpochs { } /// The rule a frame at `offset` was acknowledged under: the newest - /// epoch whose `after` lies strictly below `offset`. + /// epoch whose `after` lies strictly below `offset`. The first epoch is + /// always unbounded (`load` enforces it), so every offset resolves. #[must_use] pub fn rule_for(&self, offset: WalOffset) -> &TenantRule { self.epochs .iter() .rev() .find(|epoch| epoch.after.is_none_or(|after| offset > after)) - .map_or_else(|| self.current(), |epoch| &epoch.rule) + .map_or_else(|| self.oldest(), |epoch| &epoch.rule) + } + + fn oldest(&self) -> &TenantRule { + self.epochs + .first() + .map_or_else(|| unreachable!("epoch log is never empty"), |e| &e.rule) } /// Make `rule` the current epoch for frames after `after` (the highest @@ -123,25 +130,31 @@ impl RuleEpochs { rule: rule.clone(), after, }; - match after { - Some(_) => self.epochs.push(epoch), - None => self.epochs = vec![epoch], - } - self.persist()?; + let candidate = match after { + Some(_) => { + let mut epochs = self.epochs.clone(); + epochs.push(epoch); + epochs + } + None => vec![epoch], + }; + // Durable first: a failed write leaves `self` reporting the rule the + // sidecar actually records. + self.persist(&candidate)?; + self.epochs = candidate; Ok(true) } - fn persist(&self) -> Result<(), RuleEpochsError> { + fn persist(&self, epochs: &[RuleEpoch]) -> Result<(), RuleEpochsError> { let io = |op: &'static str, path: &Path| { let path = path.to_path_buf(); move |source| RuleEpochsError::Io { op, path, source } }; - let bytes = serde_json::to_vec_pretty(&render(&self.epochs)).map_err(|e| { - RuleEpochsError::Malformed { + let bytes = + serde_json::to_vec_pretty(&render(epochs)).map_err(|e| RuleEpochsError::Malformed { path: self.path.clone(), detail: e.to_string(), - } - })?; + })?; let tmp = self.path.with_extension("json.tmp"); let mut file = File::create(&tmp).map_err(io("create(tenant rule epochs tmp)", &tmp))?; file.write_all(&bytes) @@ -207,6 +220,9 @@ fn parse(bytes: &[u8]) -> Result, String> { "epochs[{index}].after is null; only the first epoch is unbounded" )); } + Some(_) if index == 0 => { + return Err("epochs[0].after must be null; the first epoch is unbounded".to_owned()); + } Some(after) => { let segment = after .get("segment") @@ -343,6 +359,24 @@ mod tests { assert_eq!(reloaded.rule_for(offset(0)), &composite); } + // A failed persist leaves the in-memory log on the rule the sidecar + // records. + #[test] + fn failed_persist_does_not_advance_in_memory() { + let dir = tempfile::tempdir().expect("tempdir"); + let mut epochs = RuleEpochs::load(dir.path()).expect("loads"); + // A file where the sidecar's directory should be: the write fails. + let blocked = dir.path().join("blocked"); + std::fs::write(&blocked, b"").expect("write"); + epochs.path = blocked.join(FILE_NAME); + let composite = TenantRule::from_keys(["a", "b"]).expect("valid"); + assert!(matches!( + epochs.advance(&composite, Some(offset(1))), + Err(RuleEpochsError::Io { .. }) + )); + assert_eq!(epochs.current(), &TenantRule::service_name()); + } + // RFC0045.10 — an unparseable log aborts loudly, naming the file. #[test] fn malformed_log_is_an_error_naming_the_file() { @@ -368,6 +402,22 @@ mod tests { RuleEpochsError::Malformed { .. } )); + // A bounded first entry is rejected (a frame below it would have no + // epoch). + let bounded_first = offset(1); + std::fs::write( + dir.path().join(FILE_NAME), + format!( + "{{\"epochs\": [{{\"rule\": [\"a\"], \"after\": {{\"segment\": \"{}\", \"byte\": {}}}}}]}}", + bounded_first.segment, bounded_first.byte + ), + ) + .expect("write"); + assert!(matches!( + RuleEpochs::load(dir.path()).unwrap_err(), + RuleEpochsError::Malformed { .. } + )); + // A null boundary after the first entry is rejected. std::fs::write( dir.path().join(FILE_NAME), @@ -392,10 +442,11 @@ mod tests { o.segment, o.byte ) }; + let first = "{\"rule\": [\"z\"], \"after\": null}"; std::fs::write( dir.path().join(FILE_NAME), format!( - "{{\"epochs\": [{}, {}]}}", + "{{\"epochs\": [{first}, {}, {}]}}", entry("\"a\"", later), entry("\"b\"", earlier) ), @@ -408,7 +459,7 @@ mod tests { std::fs::write( dir.path().join(FILE_NAME), format!( - "{{\"epochs\": [{}, {}]}}", + "{{\"epochs\": [{first}, {}, {}]}}", entry("\"a\"", later), entry("\"b\"", later) ), @@ -416,5 +467,10 @@ mod tests { .expect("write"); let epochs = RuleEpochs::load(dir.path()).expect("equal boundary is append order"); assert_eq!(epochs.current().keys(), ["b"]); + assert_eq!( + epochs.rule_for(earlier).keys(), + ["z"], + "below every boundary → the oldest epoch" + ); } } diff --git a/crates/ourios-parquet/src/store.rs b/crates/ourios-parquet/src/store.rs index d19f5cdd4..6987c364a 100644 --- a/crates/ourios-parquet/src/store.rs +++ b/crates/ourios-parquet/src/store.rs @@ -1059,6 +1059,43 @@ mod tests { )); } + // RFC 0005 §3.4 / RFC 0045 §3.2 in property form: for any tenant id — + // reserved, unreserved and multi-byte UTF-8 characters mixed — the + // once-encoded key survives put → list → get on the local backend and + // the on-disk directory is exactly `tenant_id=`. + proptest::proptest! { + #![proptest_config(proptest::prelude::ProptestConfig::with_cases(64))] + #[test] + fn encoded_tenant_keys_round_trip_for_any_tenant( + tenant in "[a-z0-9._~/%=:+ éß日]{1,12}", + ) { + let dir = tempfile::TempDir::new().expect("temp dir"); + let store = Store::local(dir.path()).expect("local store"); + let enc = crate::percent_encode_tenant(&tenant); + let key = format!("data/tenant_id={enc}/year=2026/x.parquet"); + store.put_blocking(&key, tenant.as_bytes().to_vec()).expect("put"); + let on_disk = dir + .path() + .join("data") + .join(format!("tenant_id={enc}")) + .join("year=2026") + .join("x.parquet"); + proptest::prop_assert!(on_disk.is_file(), "missing {}", on_disk.display()); + proptest::prop_assert_eq!( + store.list_blocking(Some("data/")).expect("list"), + vec![key.clone()] + ); + proptest::prop_assert_eq!( + store.get_blocking(&key).expect("get"), + tenant.as_bytes().to_vec() + ); + proptest::prop_assert_eq!( + crate::percent_decode_tenant(&enc), + Some(tenant.clone()) + ); + } + } + /// `list_blocking` enumerates keys under a prefix recursively, in /// lexicographic order, returning store-relative keys (the same key space /// as `get`/`put`) — the seam the querier/compactor walk instead of diff --git a/crates/ourios-server/src/main.rs b/crates/ourios-server/src/main.rs index 196d55ace..388847661 100644 --- a/crates/ourios-server/src/main.rs +++ b/crates/ourios-server/src/main.rs @@ -471,6 +471,9 @@ fn tenant_derivation(section: &TenantSection) -> Result defaults.rule, }; let watch = section.watch.clone().unwrap_or(defaults.watch); + if watch.iter().any(|key| key.trim().is_empty()) { + return Err("receiver.tenant.watch lists an empty resource attribute key".to_owned()); + } let watch_capacity = match section.watch_capacity.as_deref().map(str::trim) { Some(raw) if !raw.is_empty() => match raw.parse::() { Ok(n) if n >= 1 => n, @@ -1807,6 +1810,17 @@ auth: assert_eq!(resolved.watch, ["cloud.region"]); assert_eq!(resolved.watch_capacity, 7); + for bad_yaml in [ + "receiver:\n tenant:\n rule: [service.name, \"\"]\n", + "receiver:\n tenant:\n watch: [\" \"]\n", + ] { + let cfg = parse(bad_yaml, &lookup).expect("valid yaml"); + assert!( + tenant_derivation(&cfg.receiver.tenant).is_err(), + "blank key rejected: {bad_yaml:?}" + ); + } + for bad in ["0", "-1", "many"] { let cfg = parse( &format!("receiver:\n tenant:\n watch_capacity: {bad}\n"), diff --git a/docs/guides/configuration.md b/docs/guides/configuration.md index ae48aeba0..2cda08f8d 100644 --- a/docs/guides/configuration.md +++ b/docs/guides/configuration.md @@ -98,3 +98,7 @@ auth: Auth configuration is **file-only** — there are deliberately no `OURIOS_AUTH_*` variables; token values reach the file through `${env:…}` references. + +Tenant derivation (`receiver.tenant`, RFC 0045) is likewise **file-only**: +the environment path always runs the `[service.name]` default. A +multi-cluster rule needs `--config`. diff --git a/semconv/registry/metrics.yaml b/semconv/registry/metrics.yaml index 493b092d4..9fc441269 100644 --- a/semconv/registry/metrics.yaml +++ b/semconv/registry/metrics.yaml @@ -485,10 +485,6 @@ groups: - ref: ourios.tls.reload_error requirement_level: required - # Cached template map (RFC 0033 §3.7): the querier's per-tenant cached - # fold of the audit stream. Lookup outcomes carry the corruption / - # forward-compat signals (`torn` / `unknown_version`); publish outcomes - # surface the best-effort write-through's failures. - id: metric.ourios.receiver.tenant.divergences type: metric metric_name: ourios.receiver.tenant.divergences @@ -506,6 +502,10 @@ groups: - ref: ourios.tenant.watch.key requirement_level: required + # Cached template map (RFC 0033 §3.7): the querier's per-tenant cached + # fold of the audit stream. Lookup outcomes carry the corruption / + # forward-compat signals (`torn` / `unknown_version`); publish outcomes + # surface the best-effort write-through's failures. - id: metric.ourios.template_map.lookups type: metric metric_name: ourios.template_map.lookups From aa8d9c33c9b3419020bd6ac6472cac0d10b59306 Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Mon, 17 Aug 2026 04:31:05 +0200 Subject: [PATCH 09/10] refactor(receiver): divergence detector emits telemetry outside the state lock Signed-off-by: Jens Holdgaard Pedersen --- crates/ourios-ingester/src/receiver/watch.rs | 95 ++++++++++++-------- 1 file changed, 60 insertions(+), 35 deletions(-) diff --git a/crates/ourios-ingester/src/receiver/watch.rs b/crates/ourios-ingester/src/receiver/watch.rs index 5330a05a8..d64f01050 100644 --- a/crates/ourios-ingester/src/receiver/watch.rs +++ b/crates/ourios-ingester/src/receiver/watch.rs @@ -120,6 +120,49 @@ impl DivergenceWatch { } fn observe_one(&self, tenant: &TenantId, key: &str, value: &str) { + // Decide under the lock, emit outside it: telemetry can block on a + // subscriber or exporter and must not stall unrelated ingest. + let Some(outcome) = self.classify(tenant, key, value) else { + return; + }; + match outcome { + Outcome::Saturated => tracing::warn!( + name: semconv::EVENT_OURIOS_RECEIVER_TENANT_WATCH_SATURATED, + "tenant divergence watch is full ({} entries); further (tenant, key) pairs \ + are not watched — raise receiver.tenant.watch_capacity if this matters", + self.capacity, + ), + Outcome::Diverged { + first_preview, + warn, + } => { + self.divergences.add( + 1, + &[OtelKeyValue::new( + semconv::OURIOS_TENANT_WATCH_KEY, + key.to_owned(), + )], + ); + if warn { + tracing::event!( + name: semconv::EVENT_OURIOS_RECEIVER_TENANT_DIVERGENCE, + tracing::Level::WARN, + ourios.tenant = tenant.as_str(), + ourios.tenant.watch.key = key, + ourios.tenant.watch.first_value = first_preview.as_str(), + ourios.tenant.watch.value = bound(value).as_ref(), + "tenant spans more than one value of a watched resource attribute — if \ + these are different producers, add the key to receiver.tenant.rule \ + (RFC 0045 §3.4)" + ); + } + } + } + } + + /// The locked half of [`observe_one`](Self::observe_one): admit or + /// compare, and say what (if anything) to emit once the lock is gone. + fn classify(&self, tenant: &TenantId, key: &str, value: &str) -> Option { let mut state = self .state .lock() @@ -127,16 +170,8 @@ impl DivergenceWatch { let slot = (tenant.clone(), key.to_owned()); let Some(entry) = state.get_mut(&slot) else { if state.len() >= self.capacity { - if !self.saturated.swap(true, Ordering::Relaxed) { - tracing::warn!( - name: semconv::EVENT_OURIOS_RECEIVER_TENANT_WATCH_SATURATED, - "tenant divergence watch is full ({} entries); further (tenant, key) \ - pairs are not watched — raise receiver.tenant.watch_capacity if this \ - matters", - self.capacity, - ); - } - return; + return (!self.saturated.swap(true, Ordering::Relaxed)) + .then_some(Outcome::Saturated); } state.insert( slot, @@ -147,40 +182,30 @@ impl DivergenceWatch { last_warned: None, }, ); - return; + return None; }; if entry.first_len == value.len() && entry.first_digest == digest(value) { - return; + return None; } - let seen = bound(value); - self.divergences.add( - 1, - &[OtelKeyValue::new( - semconv::OURIOS_TENANT_WATCH_KEY, - key.to_owned(), - )], - ); let now = Instant::now(); - if entry + let warn = entry .last_warned - .is_some_and(|last| now.duration_since(last) < WARN_INTERVAL) - { - return; + .is_none_or(|last| now.duration_since(last) >= WARN_INTERVAL); + if warn { + entry.last_warned = Some(now); } - entry.last_warned = Some(now); - tracing::event!( - name: semconv::EVENT_OURIOS_RECEIVER_TENANT_DIVERGENCE, - tracing::Level::WARN, - ourios.tenant = tenant.as_str(), - ourios.tenant.watch.key = key, - ourios.tenant.watch.first_value = entry.first_preview.as_str(), - ourios.tenant.watch.value = seen.as_ref(), - "tenant spans more than one value of a watched resource attribute — if these are \ - different producers, add the key to receiver.tenant.rule (RFC 0045 §3.4)" - ); + Some(Outcome::Diverged { + first_preview: entry.first_preview.clone(), + warn, + }) } } +enum Outcome { + Saturated, + Diverged { first_preview: String, warn: bool }, +} + fn string_attribute<'a>(attributes: &'a [KeyValue], key: &str) -> Option<&'a str> { attributes .iter() From 1e61b6965a47167ed9bd0acd499b4eea789182fd Mon Sep 17 00:00:00 2001 From: Jens Holdgaard Pedersen Date: Mon, 17 Aug 2026 04:53:03 +0200 Subject: [PATCH 10/10] fix(server): reject duplicate receiver.tenant.watch keys; fingerprint wording Signed-off-by: Jens Holdgaard Pedersen --- crates/ourios-ingester/src/receiver/watch.rs | 7 ++++--- crates/ourios-server/src/main.rs | 9 ++++++++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/crates/ourios-ingester/src/receiver/watch.rs b/crates/ourios-ingester/src/receiver/watch.rs index d64f01050..67bded2c6 100644 --- a/crates/ourios-ingester/src/receiver/watch.rs +++ b/crates/ourios-ingester/src/receiver/watch.rs @@ -36,9 +36,10 @@ pub const MAX_VALUE_BYTES: usize = 128; pub const WARN_INTERVAL: Duration = Duration::from_secs(60); struct Entry { - /// Exact identity of the first value: digest + byte length. Comparison - /// never uses the preview, so values sharing a 128-byte prefix are - /// still told apart. + /// Fingerprint of the first value: 64-bit digest + byte length (a + /// collision is astronomically unlikely and only ever *hides* a + /// divergence, never invents one). Comparison never uses the preview, + /// so values sharing a 128-byte prefix are still told apart. first_digest: u64, first_len: usize, /// The bounded rendering of the first value, for the warning only. diff --git a/crates/ourios-server/src/main.rs b/crates/ourios-server/src/main.rs index 388847661..0d771fe3c 100644 --- a/crates/ourios-server/src/main.rs +++ b/crates/ourios-server/src/main.rs @@ -474,6 +474,12 @@ fn tenant_derivation(section: &TenantSection) -> Result match raw.parse::() { Ok(n) if n >= 1 => n, @@ -1813,11 +1819,12 @@ auth: for bad_yaml in [ "receiver:\n tenant:\n rule: [service.name, \"\"]\n", "receiver:\n tenant:\n watch: [\" \"]\n", + "receiver:\n tenant:\n watch: [cloud.region, cloud.region]\n", ] { let cfg = parse(bad_yaml, &lookup).expect("valid yaml"); assert!( tenant_derivation(&cfg.receiver.tenant).is_err(), - "blank key rejected: {bad_yaml:?}" + "blank or duplicate key rejected: {bad_yaml:?}" ); }