feat: layer code-driven plugin config over materialized file configs

crates/cli/src/doctor.rs

@@ -14,7 +14,9 @@ use std::process::Stdio;
 use std::time::Duration;
 
 use nemo_relay::observability::plugin_component::OBSERVABILITY_PLUGIN_KIND;
-use nemo_relay::plugin::{DiagnosticLevel, PluginConfig, validate_plugin_config};
+use nemo_relay::plugin::{
+    DiagnosticLevel, PluginConfig, apply_code_driven_plugin_config, validate_plugin_config,
+};
 use nemo_relay_adaptive::plugin_component::register_adaptive_component;
 use serde::Serialize;
 use serde_json::Value;
@@ -571,28 +573,74 @@ async fn probe_version(binary: &Path) -> Option<String> {
 }
 
 async fn collect_observability(gateway: &GatewayConfig) -> Vec<Check> {
+    // Resolve the code-driven layer (if any) merged over the file/CLI config, so doctor validates
+    // and reports exactly what the runtime would activate. If the file config fails to parse we
+    // layer over an empty base here, but the inner function re-parses the raw value and reports the
+    // parse failure before the overlay is ever used.
+    let base = gateway
+        .plugin_config
+        .as_ref()
+        .and_then(|value| serde_json::from_value::<PluginConfig>(value.clone()).ok())
+        .unwrap_or_default();
+    let effective_overlay = apply_code_driven_plugin_config(&base);
+    collect_observability_layered(gateway.plugin_config.as_ref(), effective_overlay.as_ref()).await
+}
+
+// Reports plugin/observability checks for the effective plugin configuration. `effective_overlay` is
+// `Some` only when a code-driven layer is active, and then carries the layer already merged over the
+// file/CLI config — the highest-precedence source, so its values win per-field. When it is `None`,
+// this validates the file config alone, producing output identical to file-only behavior. Sub-checks
+// (exporter directories/endpoints) run against the effective config so doctor reports what the
+// runtime would actually activate, not just the file layer.
+async fn collect_observability_layered(
+    file_value: Option<&Value>,
+    effective_overlay: Option<&PluginConfig>,
+) -> Vec<Check> {
     let mut checks = Vec::new();
 
-    let Some(plugin_value) = &gateway.plugin_config else {
-        checks.push(Check {
-            name: "Plugins",
-            status: Status::Info,
-            details: "plugins.toml not configured".into(),
-        });
-        return checks;
+    let file_config = match file_value {
+        Some(plugin_value) => match serde_json::from_value::<PluginConfig>(plugin_value.clone()) {
+            Ok(config) => Some(config),
+            Err(err) => {
+                checks.push(Check {
+                    name: "Plugins",
+                    status: Status::Fail,
+                    details: format!("invalid plugin config: {err}"),
+                });
+                return checks;
+            }
+        },
+        None => None,
     };
 
-    let plugin_config = match serde_json::from_value::<PluginConfig>(plugin_value.clone()) {
-        Ok(config) => config,
-        Err(err) => {
+    let layer_active = effective_overlay.is_some();
+    let effective_config = match (file_config.as_ref(), effective_overlay) {
+        (None, None) => {
             checks.push(Check {
                 name: "Plugins",
-                status: Status::Fail,
-                details: format!("invalid plugin config: {err}"),
+                status: Status::Info,
+                details: "plugins.toml not configured".into(),
             });
             return checks;
         }
+        (Some(file), None) => file.clone(),
+        (_, Some(overlay)) => overlay.clone(),
     };
+
+    // Surface where the effective config came from so conflicts have a clear winner. Only emitted
+    // when a code-driven layer is active; otherwise the file-only output is left unchanged.
+    if layer_active {
+        checks.push(Check {
+            name: "Plugin config source",
+            status: Status::Info,
+            details: if file_config.is_some() {
+                "code-driven layer active; overrides file/CLI plugin config".into()
+            } else {
+                "code-driven layer active; no file/CLI plugin config".into()
+            },
+        });
+    }
+
     if let Err(error) = register_adaptive_component() {
         checks.push(Check {
             name: "Adaptive plugin",
@@ -601,12 +649,19 @@ async fn collect_observability(gateway: &GatewayConfig) -> Vec<Check> {
         });
         return checks;
     }
-    let report = validate_plugin_config(&plugin_config);
+    // Prefix diagnostics when a layer is active so the reported failure is attributed to the merged
+    // effective config rather than any single file/CLI source.
+    let detail_prefix = if layer_active {
+        "effective plugin config: "
+    } else {
+        ""
+    };
+    let report = validate_plugin_config(&effective_config);
     if report.diagnostics.is_empty() {
         checks.push(Check {
             name: "Plugins",
             status: Status::Pass,
-            details: "validation passed".into(),
+            details: format!("{detail_prefix}validation passed"),
         });
     } else {
         for diagnostic in report.diagnostics {
@@ -617,12 +672,19 @@ async fn collect_observability(gateway: &GatewayConfig) -> Vec<Check> {
                 } else {
                     Status::Warn
                 },
-                details: format!("{}: {}", diagnostic.code, diagnostic.message),
+                details: format!("{detail_prefix}{}: {}", diagnostic.code, diagnostic.message),
             });
         }
     }
 
-    if let Some(config) = observability_component_config(plugin_value) {
+    // Run exporter sub-checks against the effective config. When no layer is active this is the
+    // original file value (byte-identical output); otherwise it is the merged document.
+    let effective_value = if layer_active {
+        serde_json::to_value(&effective_config).unwrap_or(Value::Null)
+    } else {
+        file_value.cloned().unwrap_or(Value::Null)
+    };
+    if let Some(config) = observability_component_config(&effective_value) {
         collect_observability_component_checks(&mut checks, config).await;
     } else {
         checks.push(Check {

crates/cli/tests/coverage/doctor_tests.rs

@@ -657,6 +657,102 @@ async fn collect_observability_registers_adaptive_before_validation() {
     );
 }
 
+#[tokio::test]
+async fn collect_observability_layered_reports_code_driven_source_and_effective_config() {
+    let temp = tempfile::tempdir().unwrap();
+    let missing = temp.path().join("missing-atif");
+
+    // File/CLI-resolved config: observability present, ATIF disabled.
+    let file_value = serde_json::json!({
+        "version": 1,
+        "components": [{
+            "kind": "observability",
+            "enabled": true,
+            "config": { "version": 1, "atif": { "enabled": false } }
+        }]
+    });
+    // Effective config the code-driven layer produces over the file config: the layer is the higher
+    // precedence source and enables ATIF (with a missing output directory), so the effective config
+    // has ATIF enabled. `apply_code_driven_plugin_config` returns this merged result; doctor's inner
+    // function consumes it directly.
+    let effective: PluginConfig = serde_json::from_value(serde_json::json!({
+        "version": 1,
+        "components": [{
+            "kind": "observability",
+            "enabled": true,
+            "config": { "version": 1, "atif": { "enabled": true, "output_directory": missing } }
+        }]
+    }))
+    .unwrap();
+
+    let checks = collect_observability_layered(Some(&file_value), Some(&effective)).await;
+
+    // The effective source is surfaced as the winning layer.
+    let source = checks
+        .iter()
+        .find(|check| check.name == "Plugin config source")
+        .expect("a plugin config source check");
+    assert_eq!(source.status, Status::Info);
+    assert!(
+        source.details.contains("code-driven layer"),
+        "{:?}",
+        source.details
+    );
+
+    // Sub-checks run against the effective (merged) config: ATIF, disabled in the file, is enabled
+    // by the layer, so the ATIF directory check appears and warns about the missing directory.
+    let atif_check = checks
+        .iter()
+        .find(|check| check.name == "ATIF dir")
+        .expect("ATIF directory check from the effective config");
+    assert_eq!(atif_check.status, Status::Warn);
+    assert!(!missing.exists());
+
+    // Diagnostics/validation are attributed to the effective config when a layer is active.
+    assert!(
+        checks
+            .iter()
+            .any(|check| check.details.contains("effective plugin config")),
+        "expected effective-config attribution: {checks:?}"
+    );
+}
+
+#[tokio::test]
+async fn collect_observability_layered_without_file_config_uses_the_layer() {
+    // No file/CLI plugin config, only a code-driven layer: the layer alone is the effective config,
+    // so doctor reports it as the source and validates it instead of "plugins.toml not configured".
+    let effective: PluginConfig = serde_json::from_value(serde_json::json!({
+        "version": 1,
+        "components": [{ "kind": "observability", "enabled": true, "config": { "version": 1 } }]
+    }))
+    .unwrap();
+
+    let checks = collect_observability_layered(None, Some(&effective)).await;
+
+    assert!(
+        !checks
+            .iter()
+            .any(|check| check.details == "plugins.toml not configured"),
+        "layer-only config must not report 'not configured': {checks:?}"
+    );
+    let source = checks
+        .iter()
+        .find(|check| check.name == "Plugin config source")
+        .expect("a plugin config source check");
+    assert!(
+        source.details.contains("no file/CLI plugin config"),
+        "{:?}",
+        source.details
+    );
+    assert!(
+        checks
+            .iter()
+            .any(|check| check.name == "Plugins"
+                && check.details.contains("effective plugin config")),
+        "expected effective-config validation: {checks:?}"
+    );
+}
+
 #[test]
 fn format_agents_human_lists_supported_and_separates_detected() {
     let agents = vec![

crates/core/Cargo.toml

@@ -118,6 +118,10 @@ path = "tests/integration/subscriber_dispatcher_tests.rs"
 name = "api_surface_integration"
 path = "tests/integration/api_surface_tests.rs"
 
+[[test]]
+name = "plugin_layer_integration"
+path = "tests/integration/plugin_layer_tests.rs"
+
 [[test]]
 name = "atif_storage_integration"
 path = "tests/integration/atif_storage_tests.rs"

crates/core/src/plugin.rs

@@ -44,6 +44,8 @@ type PluginMap = HashMap<String, Arc<dyn Plugin>>;
 static PLUGIN_HANDLERS: LazyLock<RwLock<PluginMap>> = LazyLock::new(|| RwLock::new(HashMap::new()));
 static ACTIVE_PLUGIN_CONFIGURATION: LazyLock<Mutex<Option<ActivePluginConfiguration>>> =
     LazyLock::new(|| Mutex::new(None));
+static PLUGIN_CONFIG_CODE_DRIVEN_LAYER: LazyLock<Mutex<Option<PluginConfig>>> =
+    LazyLock::new(|| Mutex::new(None));
 static BUILTIN_PLUGIN_REGISTRATION: OnceLock<Result<()>> = OnceLock::new();
 
 /// Error type for generic plugin operations.
@@ -899,6 +901,96 @@ pub fn validate_plugin_config(config: &PluginConfig) -> ConfigReport {
     report
 }
 
+/// Sets (or clears with `None`) the process-global code-driven plugin layer.
+///
+/// The layer is the highest-precedence source: [`initialize_plugins`] merges it on
+/// top of the config it is given. This Rust-only API is for embedding hosts and is
+/// not exposed to the language bindings.
+pub fn set_code_driven_plugin_config(config: Option<PluginConfig>) {
+    let mut guard = PLUGIN_CONFIG_CODE_DRIVEN_LAYER
+        .lock()
+        .unwrap_or_else(|err| err.into_inner());
+    *guard = config;
+}
+
+/// Returns the code-driven layer merged over `config`, or [`None`] when no layer
+/// is set (so `config` is already effective). `Some`/`None` also tells callers
+/// whether a layer contributed.
+pub fn apply_code_driven_plugin_config(config: &PluginConfig) -> Option<PluginConfig> {
+    let guard = PLUGIN_CONFIG_CODE_DRIVEN_LAYER
+        .lock()
+        .unwrap_or_else(|err| err.into_inner());
+    guard
+        .as_ref()
+        .map(|layer| merge_plugin_config_layers(config, layer))
+}
+
+/// Merges `overlay` (higher precedence) over `base`.
+///
+/// Components pair by `kind` in order of appearance, so a kind used by a
+/// multi-instance plugin keeps each instance instead of collapsing into the
+/// first. A paired component takes `overlay`'s `enabled` and deep-merges its
+/// `config` (nested objects merge; arrays and scalars are replaced); extra
+/// `overlay` components are appended. Top-level `version`/`policy` come from
+/// `overlay`.
+fn merge_plugin_config_layers(base: &PluginConfig, overlay: &PluginConfig) -> PluginConfig {
+    let mut components = base.components.clone();
+    // Base component positions grouped by kind, so the nth overlay component of a
+    // kind pairs with the nth base component of that kind rather than the first.
+    let mut base_slots: HashMap<String, Vec<usize>> = HashMap::new();
+    for (index, component) in components.iter().enumerate() {
+        base_slots
+            .entry(component.kind.clone())
+            .or_default()
+            .push(index);
+    }
+    let mut consumed: HashMap<String, usize> = HashMap::new();
+    for overlay_component in &overlay.components {
+        let nth = consumed.entry(overlay_component.kind.clone()).or_insert(0);
+        let slot = base_slots
+            .get(&overlay_component.kind)
+            .and_then(|slots| slots.get(*nth))
+            .copied();
+        *nth += 1;
+        match slot {
+            Some(index) => {
+                components[index].enabled = overlay_component.enabled;
+                merge_json_object(
+                    &mut components[index].config,
+                    overlay_component.config.clone(),
+                );
+            }
+            None => components.push(overlay_component.clone()),
+        }
+    }
+    PluginConfig {
+        version: overlay.version,
+        components,
+        policy: overlay.policy,
+    }
+}
+
+/// Recursively merges `overlay` entries into a `base` JSON object.
+///
+/// Nested objects merge key-by-key; arrays and scalar values are replaced by
+/// the `overlay` value. This mirrors the gateway's recursive TOML table merge.
+fn merge_json_object(base: &mut Map<String, Json>, overlay: Map<String, Json>) {
+    for (key, overlay_value) in overlay {
+        match overlay_value {
+            Json::Object(overlay_object) => {
+                if let Some(Json::Object(base_object)) = base.get_mut(&key) {
+                    merge_json_object(base_object, overlay_object);
+                } else {
+                    base.insert(key, Json::Object(overlay_object));
+                }
+            }
+            other => {
+                base.insert(key, other);
+            }
+        }
+    }
+}
+
 /// Returns the JSON Schema for the canonical plugin configuration document.
 #[cfg(feature = "schema")]
 pub fn plugin_config_schema() -> Json {
@@ -926,7 +1018,11 @@ pub fn plugin_config_schema() -> Json {
 /// # Notes
 /// Initialization is replace-with-rollback: the previous active configuration
 /// is removed before the new configuration is activated.
+///
+/// When a code-driven layer is set, it is merged on top of `config` first and the
+/// effective result is what gets validated, activated, and reported.
 pub async fn initialize_plugins(config: PluginConfig) -> Result<ConfigReport> {
+    let config = apply_code_driven_plugin_config(&config).unwrap_or(config);
     let report = validate_plugin_config(&config);
     if report.has_errors() {
         return Err(PluginError::InvalidConfig(join_error_messages(&report)));