feat(spider-execution-manager): Forward RUST_LOG and configured environment variables to spawned task executors.

components/spider-execution-manager/src/config.rs

@@ -56,6 +56,7 @@ impl Config {
             executor_binary_path: self.task_executor.bin_path.clone(),
             package_dir: self.task_executor.package_dir.clone(),
             log_dir: self.task_executor.log_dir.clone(),
+            env_keys: self.task_executor.env_keys.clone(),
         }
     }
 }
@@ -83,4 +84,10 @@ pub struct TaskExecutorConfig {
 
     /// Directory the process pool writes per-executor stderr logs into.
     pub log_dir: PathBuf,
+
+    /// Names of environment variables forwarded from the execution manager's process into each
+    /// spawned `spider-task-executor`. Their values are read from this process's environment at
+    /// spawn time.
+    #[serde(default)]
+    pub env_keys: Vec<String>,
 }

components/spider-execution-manager/src/process_pool.rs

@@ -49,6 +49,11 @@ pub struct ProcessPoolConfig {
     /// Per-spawn filenames mean each respawn naturally rotates onto a fresh file; a long-lived
     /// healthy executor accumulates into one file.
     pub log_dir: PathBuf,
+
+    /// Names of environment variables forwarded from the execution manager's process into each
+    /// spawned executor. For each key, the value is read from this process's environment at spawn
+    /// time and set on the child; a key that is unset (or non-Unicode) is skipped with a warning.
+    pub env_keys: Vec<String>,
 }
 
 /// Request to execute a task inside the spawned task executor.
@@ -203,7 +208,8 @@ impl ProcessPool {
     /// log file, and wraps the child's stdin/stdout in length-delimited codec frames.
     ///
     /// The child's stderr is redirected to `<log_dir>/<em_id>-<executor_id>.log` in
-    /// create-or-append mode.
+    /// create-or-append mode. `RUST_LOG`, if set, is forwarded to the spawned task executor to make
+    /// the child process' log level match the current execution manager.
     ///
     /// # Returns
     ///
@@ -234,6 +240,28 @@ impl ProcessPool {
             .stdout(Stdio::piped())
             .stderr(Stdio::from(log_file))
             .kill_on_drop(true);
+
+        if let Ok(rust_log) = std::env::var("RUST_LOG") {
+            command.env("RUST_LOG", rust_log);
+        }
+
+        for key in &self.config.env_keys {
+            match std::env::var(key) {
+                Ok(value) => {
+                    command.env(key, value);
+                }
+                Err(e) => {
+                    tracing::warn!(
+                        executor_id,
+                        env_key = % key,
+                        err = % e,
+                        "Configured env key could not be read from the execution manager's \
+                         environment; skipping."
+                    );
+                }
+            }
+        }
+
         let mut child = command.spawn()?;
         let stdin = child
             .stdin

components/spider-execution-manager/src/runtime.rs

@@ -56,6 +56,10 @@ pub struct RuntimeConfig {
 
     /// Directory the process pool writes per-executor stderr logs into.
     pub log_dir: PathBuf,
+
+    /// Names of environment variables forwarded from the execution manager's process into each
+    /// spawned `spider-task-executor` (values read from this process's environment at spawn time).
+    pub env_keys: Vec<String>,
 }
 
 /// Errors returned by [`Runtime`] during bootstrap or the main loop.
@@ -150,6 +154,7 @@ impl<
             executor_binary_path: config.executor_binary_path,
             package_dir: config.package_dir,
             log_dir: config.log_dir,
+            env_keys: config.env_keys,
         })?;
 
         let cancellation_token = CancellationToken::new();

tests/huntsman/em-runtime/tests/test_runtime.rs

@@ -133,6 +133,7 @@ fn runtime_config(heartbeat_interval: Duration) -> RuntimeConfig {
         executor_binary_path: task_executor_bin(),
         package_dir: tdl_package_dir(),
         log_dir,
+        env_keys: Vec::new(),
     }
 }
 

tests/huntsman/task-executor/tests/test_process_pool.rs

@@ -63,6 +63,7 @@ fn build_pool() -> ProcessPool {
         executor_binary_path: task_executor_bin(),
         package_dir: tdl_package_dir(),
         log_dir,
+        env_keys: Vec::new(),
     };
     ProcessPool::new(config).expect("construct pool")
 }