From 74ffe05776fbc1e7164ed7cd47777dbb550070c1 Mon Sep 17 00:00:00 2001 From: Hugo Hromic Date: Fri, 11 Aug 2023 18:11:03 +0100 Subject: [PATCH 1/5] feat(exec source): add support for customizing command environment This commit adds support to the `exec` source for: * Setting or updating environment variables for the command before running. * Clearing all environment variables for the command before setting/updating custom ones (if any). The combination of both options allows to create a fully controlled environment for the command. Signed-off-by: Hugo Hromic --- src/sources/exec/mod.rs | 28 +++++++++++++++++++ .../components/sources/base/exec.cue | 25 +++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/src/sources/exec/mod.rs b/src/sources/exec/mod.rs index 7e254ddd05a8b..d4ac894810b69 100644 --- a/src/sources/exec/mod.rs +++ b/src/sources/exec/mod.rs @@ -1,4 +1,5 @@ use std::{ + collections::HashMap, io::{Error, ErrorKind}, path::PathBuf, process::ExitStatus, @@ -61,6 +62,13 @@ pub struct ExecConfig { #[configurable(metadata(docs::examples = "echo", docs::examples = "Hello World!"))] pub command: Vec, + /// Custom environment variables to set or update when running the command. + pub environment: Option>, + + /// Whether or not to clear the environment before applying custom environment variables. + #[serde(default = "default_clear_env")] + pub clear_env: bool, + /// The directory in which to run the command. pub working_directory: Option, @@ -141,6 +149,8 @@ impl Default for ExecConfig { }), streaming: None, command: vec!["echo".to_owned(), "Hello World!".to_owned()], + environment: None, + clear_env: default_clear_env(), working_directory: None, include_stderr: default_include_stderr(), maximum_buffer_size_bytes: default_maximum_buffer_size(), @@ -168,6 +178,10 @@ const fn default_respawn_on_exit() -> bool { true } +const fn default_clear_env() -> bool { + false +} + const fn default_include_stderr() -> bool { true } @@ -610,6 +624,16 @@ fn build_command(config: &ExecConfig) -> Command { command.kill_on_drop(true); + // Clear environment variables if needed + if config.clear_env { + command.env_clear(); + } + + // Configure environment variables if needed + if let Some(envs) = &config.environment { + command.envs(envs); + } + // Explicitly set the current dir if needed if let Some(current_dir) = &config.working_directory { command.current_dir(current_dir); @@ -900,6 +924,8 @@ mod tests { respawn_interval_secs: default_respawn_interval_secs(), }), command: vec!["./runner".to_owned(), "arg1".to_owned(), "arg2".to_owned()], + environment: None, + clear_env: default_clear_env(), working_directory: Some(PathBuf::from("/tmp")), include_stderr: default_include_stderr(), maximum_buffer_size_bytes: default_maximum_buffer_size(), @@ -1112,6 +1138,8 @@ mod tests { respawn_interval_secs: default_respawn_interval_secs(), }), command: vec!["yes".to_owned()], + environment: None, + clear_env: default_clear_env(), working_directory: None, include_stderr: default_include_stderr(), maximum_buffer_size_bytes: default_maximum_buffer_size(), diff --git a/website/cue/reference/components/sources/base/exec.cue b/website/cue/reference/components/sources/base/exec.cue index 5842254670c91..660d0414fe773 100644 --- a/website/cue/reference/components/sources/base/exec.cue +++ b/website/cue/reference/components/sources/base/exec.cue @@ -1,6 +1,11 @@ package metadata base: components: sources: exec: configuration: { + clear_env: { + description: "Whether or not to clear the environment before applying custom environment variables." + required: false + type: bool: default: false + } command: { description: "The command to run, plus any arguments required." required: true @@ -143,6 +148,26 @@ base: components: sources: exec: configuration: { } } } + environment: { + description: """ + Custom environment variables to set or update when running the command. + + If a variable name already exists in the environment, its value is replaced. + """ + required: false + type: object: { + examples: [{ + LANG: "es_ES.UTF-8" + TZ: "Etc/UTC" + PATH: "/bin:/usr/bin:/usr/local/bin" + }] + options: "*": { + description: "An environment variable value." + required: true + type: string: examples: ["foo", "bar"] + } + } + } framing: { description: """ Framing configuration. From 87607dc88bbec4d795f9cd93a8d8d8d89048bd57 Mon Sep 17 00:00:00 2001 From: Hugo Hromic Date: Fri, 11 Aug 2023 23:24:35 +0100 Subject: [PATCH 2/5] Rename `clear_env` option to `clear_environment` --- src/sources/exec/mod.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/sources/exec/mod.rs b/src/sources/exec/mod.rs index d4ac894810b69..60c7c0e0307b4 100644 --- a/src/sources/exec/mod.rs +++ b/src/sources/exec/mod.rs @@ -65,9 +65,9 @@ pub struct ExecConfig { /// Custom environment variables to set or update when running the command. pub environment: Option>, - /// Whether or not to clear the environment before applying custom environment variables. - #[serde(default = "default_clear_env")] - pub clear_env: bool, + /// Whether or not to clear the environment before setting custom environment variables. + #[serde(default = "default_clear_environment")] + pub clear_environment: bool, /// The directory in which to run the command. pub working_directory: Option, @@ -150,7 +150,7 @@ impl Default for ExecConfig { streaming: None, command: vec!["echo".to_owned(), "Hello World!".to_owned()], environment: None, - clear_env: default_clear_env(), + clear_environment: default_clear_environment(), working_directory: None, include_stderr: default_include_stderr(), maximum_buffer_size_bytes: default_maximum_buffer_size(), @@ -178,7 +178,7 @@ const fn default_respawn_on_exit() -> bool { true } -const fn default_clear_env() -> bool { +const fn default_clear_environment() -> bool { false } @@ -625,7 +625,7 @@ fn build_command(config: &ExecConfig) -> Command { command.kill_on_drop(true); // Clear environment variables if needed - if config.clear_env { + if config.clear_environment { command.env_clear(); } @@ -925,7 +925,7 @@ mod tests { }), command: vec!["./runner".to_owned(), "arg1".to_owned(), "arg2".to_owned()], environment: None, - clear_env: default_clear_env(), + clear_environment: default_clear_environment(), working_directory: Some(PathBuf::from("/tmp")), include_stderr: default_include_stderr(), maximum_buffer_size_bytes: default_maximum_buffer_size(), @@ -1139,7 +1139,7 @@ mod tests { }), command: vec!["yes".to_owned()], environment: None, - clear_env: default_clear_env(), + clear_environment: default_clear_environment(), working_directory: None, include_stderr: default_include_stderr(), maximum_buffer_size_bytes: default_maximum_buffer_size(), From dd1e1d5fa932b0c6d28407de49fa6b3aafee5427 Mon Sep 17 00:00:00 2001 From: Hugo Hromic Date: Fri, 11 Aug 2023 23:44:24 +0100 Subject: [PATCH 3/5] Add `configurable` macros for docs generation --- src/sources/exec/mod.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/sources/exec/mod.rs b/src/sources/exec/mod.rs index 60c7c0e0307b4..3953029870e54 100644 --- a/src/sources/exec/mod.rs +++ b/src/sources/exec/mod.rs @@ -63,6 +63,10 @@ pub struct ExecConfig { pub command: Vec, /// Custom environment variables to set or update when running the command. + /// If a variable name already exists in the environment, its value is replaced. + #[serde(default)] + #[configurable(metadata(docs::additional_props_description = "An environment variable."))] + #[configurable(metadata(docs::examples = "environment_examples()"))] pub environment: Option>, /// Whether or not to clear the environment before setting custom environment variables. @@ -186,6 +190,17 @@ const fn default_include_stderr() -> bool { true } +fn environment_examples() -> HashMap { + HashMap::<_, _>::from_iter( + [ + ("LANG".to_owned(), "es_ES.UTF-8".to_owned()), + ("TZ".to_owned(), "Etc/UTC".to_owned()), + ("PATH".to_owned(), "/bin:/usr/bin:/usr/local/bin".to_owned()), + ] + .into_iter(), + ) +} + fn get_hostname() -> Option { crate::get_hostname().ok() } From a9c06485457463a4431a9995f7c20b8e5d18e934 Mon Sep 17 00:00:00 2001 From: Hugo Hromic Date: Sat, 12 Aug 2023 00:11:55 +0100 Subject: [PATCH 4/5] Regenerate docs using vdev build component-docs --- .../cue/reference/components/sources/base/exec.cue | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/website/cue/reference/components/sources/base/exec.cue b/website/cue/reference/components/sources/base/exec.cue index 660d0414fe773..a68980e48d992 100644 --- a/website/cue/reference/components/sources/base/exec.cue +++ b/website/cue/reference/components/sources/base/exec.cue @@ -1,8 +1,8 @@ package metadata base: components: sources: exec: configuration: { - clear_env: { - description: "Whether or not to clear the environment before applying custom environment variables." + clear_environment: { + description: "Whether or not to clear the environment before setting custom environment variables." required: false type: bool: default: false } @@ -151,20 +151,19 @@ base: components: sources: exec: configuration: { environment: { description: """ Custom environment variables to set or update when running the command. - If a variable name already exists in the environment, its value is replaced. """ required: false type: object: { examples: [{ LANG: "es_ES.UTF-8" - TZ: "Etc/UTC" PATH: "/bin:/usr/bin:/usr/local/bin" + TZ: "Etc/UTC" }] options: "*": { - description: "An environment variable value." + description: "An environment variable." required: true - type: string: examples: ["foo", "bar"] + type: string: {} } } } From db792fe971c1c8492a04a1863ed72770b817ece4 Mon Sep 17 00:00:00 2001 From: Hugo Hromic Date: Sat, 12 Aug 2023 01:00:47 +0100 Subject: [PATCH 5/5] Add tests for the new options --- src/sources/exec/mod.rs | 59 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/src/sources/exec/mod.rs b/src/sources/exec/mod.rs index 3953029870e54..1e0502122193d 100644 --- a/src/sources/exec/mod.rs +++ b/src/sources/exec/mod.rs @@ -765,6 +765,7 @@ mod tests { use super::*; use crate::{event::LogEvent, test_util::trace_init}; use bytes::Bytes; + use std::ffi::OsStr; use std::io::Cursor; use vector_core::event::EventMetadata; use vrl::value; @@ -963,6 +964,64 @@ mod tests { assert_eq!(expected_command_string, command_string); } + #[test] + fn test_build_command_custom_environment() { + let config = ExecConfig { + mode: Mode::Streaming, + scheduled: None, + streaming: Some(StreamingConfig { + respawn_on_exit: default_respawn_on_exit(), + respawn_interval_secs: default_respawn_interval_secs(), + }), + command: vec!["./runner".to_owned(), "arg1".to_owned(), "arg2".to_owned()], + environment: Some(HashMap::from([("FOO".to_owned(), "foo".to_owned())])), + clear_environment: default_clear_environment(), + working_directory: Some(PathBuf::from("/tmp")), + include_stderr: default_include_stderr(), + maximum_buffer_size_bytes: default_maximum_buffer_size(), + framing: None, + decoding: default_decoding(), + log_namespace: None, + }; + + let command = build_command(&config); + let cmd = command.as_std(); + + let idx = cmd + .get_envs() + .position(|v| v == (OsStr::new("FOO"), Some(OsStr::new("foo")))); + + assert_ne!(idx, None); + } + + #[test] + fn test_build_command_clear_environment() { + let config = ExecConfig { + mode: Mode::Streaming, + scheduled: None, + streaming: Some(StreamingConfig { + respawn_on_exit: default_respawn_on_exit(), + respawn_interval_secs: default_respawn_interval_secs(), + }), + command: vec!["./runner".to_owned(), "arg1".to_owned(), "arg2".to_owned()], + environment: Some(HashMap::from([("FOO".to_owned(), "foo".to_owned())])), + clear_environment: true, + working_directory: Some(PathBuf::from("/tmp")), + include_stderr: default_include_stderr(), + maximum_buffer_size_bytes: default_maximum_buffer_size(), + framing: None, + decoding: default_decoding(), + log_namespace: None, + }; + + let command = build_command(&config); + let cmd = command.as_std(); + + let envs: Vec<_> = cmd.get_envs().collect(); + + assert_eq!(envs.len(), 1); + } + #[tokio::test] async fn test_spawn_reader_thread() { trace_init();