diff --git a/components/spider-execution-manager/src/config.rs b/components/spider-execution-manager/src/config.rs index 2e94c2a46..43beac458 100644 --- a/components/spider-execution-manager/src/config.rs +++ b/components/spider-execution-manager/src/config.rs @@ -56,6 +56,7 @@ impl Config { executor_binary_path: self.task_executor.bin_path.clone(), package_dir: self.task_executor.package_dir.clone(), log_dir: self.task_executor.log_dir.clone(), + inherited_env: self.task_executor.inherited_env.clone(), } } } @@ -83,4 +84,10 @@ pub struct TaskExecutorConfig { /// Directory the process pool writes per-executor stderr logs into. pub log_dir: PathBuf, + + /// Names of environment variables forwarded from the execution manager's process into each + /// spawned `spider-task-executor`. Their values are read from this process's environment at + /// spawn time. + #[serde(default)] + pub inherited_env: Vec, } diff --git a/components/spider-execution-manager/src/process_pool.rs b/components/spider-execution-manager/src/process_pool.rs index 90ce5d8ac..6aae75248 100644 --- a/components/spider-execution-manager/src/process_pool.rs +++ b/components/spider-execution-manager/src/process_pool.rs @@ -50,6 +50,11 @@ pub struct ProcessPoolConfig { /// Per-spawn filenames mean each respawn naturally rotates onto a fresh file; a long-lived /// healthy executor accumulates into one file. pub log_dir: PathBuf, + + /// Names of environment variables forwarded from the execution manager's process into each + /// spawned executor. For each key, the value is read from this process's environment at spawn + /// time and set on the child; a key that is unset (or non-Unicode) is skipped with a warning. + pub inherited_env: Vec, } /// Request to execute a task inside the spawned task executor. @@ -208,7 +213,8 @@ impl ProcessPool { /// log file, and wraps the child's stdin/stdout in length-delimited codec frames. /// /// The child's stderr is redirected to `/-.log` in - /// create-or-append mode. + /// create-or-append mode. `RUST_LOG`, if set, is forwarded to the spawned task executor to make + /// the child process' log level match the current execution manager. /// /// # Returns /// @@ -239,6 +245,28 @@ impl ProcessPool { .stdout(Stdio::piped()) .stderr(Stdio::from(log_file)) .kill_on_drop(true); + + if let Ok(rust_log) = std::env::var("RUST_LOG") { + command.env("RUST_LOG", rust_log); + } + + for key in &self.config.inherited_env { + match std::env::var(key) { + Ok(value) => { + command.env(key, value); + } + Err(e) => { + tracing::warn!( + executor_id, + env_key = % key, + err = % e, + "Configured env key could not be read from the execution manager's \ + environment; skipping." + ); + } + } + } + let mut child = command.spawn()?; let stdin = child .stdin diff --git a/components/spider-execution-manager/src/runtime.rs b/components/spider-execution-manager/src/runtime.rs index 1f9f72c79..b3bad11a2 100644 --- a/components/spider-execution-manager/src/runtime.rs +++ b/components/spider-execution-manager/src/runtime.rs @@ -56,6 +56,10 @@ pub struct RuntimeConfig { /// Directory the process pool writes per-executor stderr logs into. pub log_dir: PathBuf, + + /// Names of environment variables forwarded from the execution manager's process into each + /// spawned `spider-task-executor` (values read from this process's environment at spawn time). + pub inherited_env: Vec, } /// Errors returned by [`Runtime`] during bootstrap or the main loop. @@ -150,6 +154,7 @@ impl< executor_binary_path: config.executor_binary_path, package_dir: config.package_dir, log_dir: config.log_dir, + inherited_env: config.inherited_env, })?; let cancellation_token = CancellationToken::new(); diff --git a/tests/huntsman/em-runtime/tests/test_runtime.rs b/tests/huntsman/em-runtime/tests/test_runtime.rs index 31f6d7410..1faca885e 100644 --- a/tests/huntsman/em-runtime/tests/test_runtime.rs +++ b/tests/huntsman/em-runtime/tests/test_runtime.rs @@ -133,6 +133,7 @@ fn runtime_config(heartbeat_interval: Duration) -> RuntimeConfig { executor_binary_path: task_executor_bin(), package_dir: tdl_package_dir(), log_dir, + inherited_env: Vec::new(), } } diff --git a/tests/huntsman/task-executor/tests/test_process_pool.rs b/tests/huntsman/task-executor/tests/test_process_pool.rs index 2b9898438..964bc3da4 100644 --- a/tests/huntsman/task-executor/tests/test_process_pool.rs +++ b/tests/huntsman/task-executor/tests/test_process_pool.rs @@ -63,6 +63,7 @@ fn build_pool() -> ProcessPool { executor_binary_path: task_executor_bin(), package_dir: tdl_package_dir(), log_dir, + inherited_env: Vec::new(), }; ProcessPool::new(config).expect("construct pool") }