Merge pull request #255 from microsoft/main

Fork Sync: Update from parent repository

Author: AdamL-Microsoft

docs/unmanaged-nodes.md

@@ -142,3 +142,11 @@ linux
 ```
 docker run --it --rm --entrypoint bash <image_name>
 ```
+
+### mount a local folder in the container
+
+docker allows you to [mount](https://docs.docker.com/storage/bind-mounts/#mount-into-a-non-empty-directory-on-the-container) a local folder when running a container
+
+```
+docker run -it --rm --mount type=bind,source=<local_path>,target=<path_in_container>
+```

src/agent/.rustfmt.toml

@@ -1 +1,2 @@
-edition = "2018"
+edition = "2018"
+newline_style = "Native"

src/agent/Cargo.lock

@@ -4,7 +4,7 @@
 
 use std::collections::{HashMap, HashSet};
 use std::ffi::OsStr;
-use std::io;
+use std::io::{self};
 use std::path::Path;
 use std::process::Command;
 
@@ -37,6 +37,19 @@ pub fn find_missing(mut cmd: Command) -> Result<HashSet<MissingDynamicLibrary>,
     Ok(logs.missing())
 }
 
+pub fn get_linked_library_logs(cmd: &Command) -> Result<std::process::Output, io::Error> {
+    let library_path = explicit_library_path(cmd);
+    let output = LinkedDynamicLibraries::run(cmd.get_program(), library_path)?;
+    Ok(output)
+}
+
+pub fn get_loaded_libraries_logs(cmd: Command) -> Result<std::process::Output, io::Error> {
+    let mut cmd = cmd;
+    cmd.env("LD_DEBUG", "libs");
+    let output = cmd.output()?;
+    Ok(output)
+}
+
 // Compute the `LD_LIBRARY_PATH` value that a `Command` sets, if any.
 //
 // If the command either inherits or unsets the variable, returns `None`.
@@ -253,19 +266,24 @@ impl LinkedDynamicLibraries {
         module: impl AsRef<OsStr>,
         library_path: Option<&OsStr>,
     ) -> Result<Self, io::Error> {
+        let output = Self::run(module, library_path)?;
+        let linked = Self::parse(&*output.stdout);
+        Ok(linked)
+    }
+
+    pub fn run(
+        module: impl AsRef<OsStr>,
+        library_path: Option<&OsStr>,
+    ) -> Result<std::process::Output, io::Error> {
         let mut cmd = Command::new("ldd");
         cmd.arg(module);
-
         if let Some(library_path) = library_path {
             cmd.env(LD_LIBRARY_PATH, library_path);
         } else {
             cmd.env_remove(LD_LIBRARY_PATH);
         }
-
         let output = cmd.output()?;
-        let linked = Self::parse(&*output.stdout);
-
-        Ok(linked)
+        Ok(output)
     }
 
     pub fn parse<R: io::Read>(readable: R) -> Self {

src/agent/dynamic-library/src/linux.rs

@@ -27,17 +27,28 @@ pub enum CheckDynamicLibrariesError {
 pub fn find_missing(
     cmd: Command,
 ) -> Result<Vec<MissingDynamicLibrary>, CheckDynamicLibrariesError> {
-    let image_file = ImageFile::new(cmd.get_program())?;
-    let _sls = image_file.show_loader_snaps()?;
+    let mut handler = LoaderSnapsHandler::default();
+    setup_debugger(cmd, &mut handler)?;
+    Ok(handler.missing_libraries())
+}
 
+pub fn get_logs(cmd: Command) -> Result<Vec<String>, CheckDynamicLibrariesError> {
     let mut handler = LoaderSnapsHandler::default();
+    setup_debugger(cmd, &mut handler)?;
+    Ok(handler.debug_strings)
+}
 
+fn setup_debugger(
+    cmd: Command,
+    handler: &mut LoaderSnapsHandler,
+) -> Result<(), CheckDynamicLibrariesError> {
+    let image_file = ImageFile::new(cmd.get_program())?;
+    let _sls = image_file.show_loader_snaps()?;
     let (mut dbg, _child) =
-        Debugger::init(cmd, &mut handler).map_err(CheckDynamicLibrariesError::Debugger)?;
-
+        Debugger::init(cmd, handler).map_err(CheckDynamicLibrariesError::Debugger)?;
     while !dbg.target().exited() {
         if !dbg
-            .process_event(&mut handler, 1000)
+            .process_event(handler, 1000)
             .map_err(CheckDynamicLibrariesError::Debugger)?
         {
             break;
@@ -47,7 +58,7 @@ pub fn find_missing(
             .map_err(CheckDynamicLibrariesError::Debugger)?;
     }
 
-    Ok(handler.missing_libraries())
+    Ok(())
 }
 
 #[derive(Debug, Error)]

src/agent/dynamic-library/src/windows.rs

@@ -30,6 +30,7 @@ reqwest-retry = { path = "../reqwest-retry" }
 onefuzz-telemetry = { path = "../onefuzz-telemetry" }
 backtrace = "0.3"
 ipc-channel = { git = "https://github.com/servo/ipc-channel", rev = "7f432aa" }
+dynamic-library = {path = "../dynamic-library" }
 
 
 [target.'cfg(target_family = "unix")'.dependencies]

src/agent/onefuzz-agent/Cargo.toml

@@ -40,6 +40,7 @@ pub mod panic;
 pub mod reboot;
 pub mod scheduler;
 pub mod setup;
+pub mod validations;
 pub mod work;
 pub mod worker;
 
@@ -48,6 +49,8 @@ enum Opt {
     Run(RunOpt),
     #[clap(subcommand)]
     Debug(debug::DebugOpt),
+    #[clap(subcommand)]
+    Validate(validations::ValidationCommand),
     Licenses,
     Version,
 }
@@ -84,11 +87,17 @@ fn main() -> Result<()> {
         Opt::Debug(opt) => debug::debug(opt)?,
         Opt::Licenses => licenses()?,
         Opt::Version => version(),
+        Opt::Validate(opt) => validate(opt)?,
     };
 
     Ok(())
 }
 
+fn validate(validation_command: validations::ValidationCommand) -> Result<()> {
+    let rt = tokio::runtime::Runtime::new()?;
+    rt.block_on(async { validations::validate(validation_command).await })
+}
+
 fn version() {
     println!(
         "{} onefuzz:{} git:{}",

src/agent/onefuzz-agent/src/main.rs

@@ -80,10 +80,24 @@ impl SetupRunner {
         onefuzz::fs::set_executable(&setup_dir).await?;
 
         // Create setup container directory symlinks for tasks.
-        for work_unit in &work_set.work_units {
-            create_setup_symlink(&setup_dir, work_unit, self.machine_id).await?;
+        let working_dirs = work_set
+            .work_units
+            .iter()
+            .map(|w| w.working_dir(self.machine_id))
+            .collect::<Vec<_>>()
+            .into_iter()
+            .collect::<Result<Vec<_>>>()?;
+
+        for work_dir in working_dirs {
+            create_setup_symlink(&setup_dir, work_dir).await?;
         }
 
+        Self::run_setup_script(setup_dir).await
+    }
+
+    pub async fn run_setup_script(
+        setup_dir: impl AsRef<Path>,
+    ) -> std::result::Result<Option<Output>, anyhow::Error> {
         // Run setup script, if any.
         let setup_script = SetupScript::new(setup_dir).await?;
 
@@ -111,7 +125,6 @@ impl SetupRunner {
             Some(output)
         } else {
             info!("no setup script to run");
-
             None
         };
 
@@ -120,15 +133,11 @@ impl SetupRunner {
 }
 
 #[cfg(target_family = "windows")]
-async fn create_setup_symlink(
-    setup_dir: &Path,
-    work_unit: &WorkUnit,
-    machine_id: Uuid,
-) -> Result<()> {
+async fn create_setup_symlink(setup_dir: &Path, working_dir: impl AsRef<Path>) -> Result<()> {
     use std::os::windows::fs::symlink_dir;
     use tokio::task::spawn_blocking;
 
-    let working_dir = work_unit.working_dir(machine_id)?;
+    let working_dir = working_dir.as_ref();
 
     let create_work_dir = fs::create_dir_all(&working_dir).await.with_context(|| {
         format!(
@@ -162,14 +171,10 @@ async fn create_setup_symlink(
 }
 
 #[cfg(target_family = "unix")]
-async fn create_setup_symlink(
-    setup_dir: &Path,
-    work_unit: &WorkUnit,
-    machine_id: Uuid,
-) -> Result<()> {
+async fn create_setup_symlink(setup_dir: &Path, working_dir: impl AsRef<Path>) -> Result<()> {
     use tokio::fs::symlink;
 
-    let working_dir = work_unit.working_dir(machine_id)?;
+    let working_dir = working_dir.as_ref();
 
     tokio::fs::create_dir_all(&working_dir)
         .await

src/agent/onefuzz-agent/src/setup.rs

@@ -0,0 +1,145 @@
+use std::path::{Path, PathBuf};
+
+use crate::setup::SetupRunner;
+use anyhow::Result;
+use clap::Parser;
+use onefuzz::{libfuzzer::LibFuzzer, machine_id::MachineIdentity};
+use uuid::Uuid;
+
+#[derive(Parser, Debug)]
+#[clap(rename_all = "snake_case")]
+pub enum ValidationCommand {
+    /// Run the setup script
+    RunSetup { setup_folder: PathBuf },
+    /// Validate the libfuzzer target
+    ValidateLibfuzzer(ValidationConfig),
+    /// Get the execution logs to debug loading issues
+    ExecutionLog(ValidationConfig),
+}
+
+fn parse_key_val<T, U>(
+    s: &str,
+) -> Result<(T, U), Box<dyn std::error::Error + Send + Sync + 'static>>
+where
+    T: std::str::FromStr,
+    T::Err: std::error::Error + Send + Sync + 'static,
+    U: std::str::FromStr,
+    U::Err: std::error::Error + Send + Sync + 'static,
+{
+    let pos = s
+        .find('=')
+        .ok_or_else(|| format!("invalid KEY=value: no `=` found in `{s}`"))?;
+
+    println!("******** pos: {}", pos);
+
+    Ok((s[..pos].parse()?, s[pos + 1..].parse()?))
+}
+
+#[derive(Parser, Debug, Deserialize)]
+#[clap(rename_all = "snake_case")]
+pub struct ValidationConfig {
+    #[clap(long = "seeds")]
+    pub seeds: Option<PathBuf>,
+    #[clap(long = "target_exe")]
+    pub target_exe: PathBuf,
+    #[clap(long = "setup_folder")]
+    pub setup_folder: Option<PathBuf>,
+    #[clap(long = "target_options")]
+    pub target_options: Vec<String>,
+    #[arg(value_parser = parse_key_val::<String, String>, long = "target_env")]
+    pub target_env: Vec<(String, String)>,
+}
+
+pub async fn validate(command: ValidationCommand) -> Result<()> {
+    match command {
+        ValidationCommand::RunSetup { setup_folder } => run_setup(setup_folder).await,
+        ValidationCommand::ValidateLibfuzzer(validation_config) => {
+            validate_libfuzzer(validation_config).await
+        }
+        ValidationCommand::ExecutionLog(validation_config) => get_logs(validation_config).await,
+    }
+}
+
+async fn validate_libfuzzer(config: ValidationConfig) -> Result<()> {
+    let libfuzzer = LibFuzzer::new(
+        &config.target_exe,
+        config.target_options.clone(),
+        config.target_env.into_iter().collect(),
+        config
+            .setup_folder
+            .unwrap_or(config.target_exe.parent().unwrap().to_path_buf()),
+        None::<&PathBuf>,
+        MachineIdentity {
+            machine_id: Uuid::nil(),
+            machine_name: "".to_string(),
+            scaleset_name: None,
+        },
+    );
+
+    if let Some(seeds) = config.seeds {
+        libfuzzer.verify(true, Some(vec![seeds])).await?;
+    }
+
+    Ok(())
+}
+
+async fn run_setup(setup_folder: impl AsRef<Path>) -> Result<()> {
+    let output = SetupRunner::run_setup_script(setup_folder.as_ref()).await?;
+    match output {
+        Some(output) => {
+            if !output.exit_status.success {
+                let error = "error running target setup script".to_owned();
+                bail!("{}", error);
+            }
+        }
+        None => {
+            println!("no setup script to run")
+        }
+    }
+    Ok(())
+}
+
+async fn get_logs(config: ValidationConfig) -> Result<()> {
+    let libfuzzer = LibFuzzer::new(
+        &config.target_exe,
+        config.target_options.clone(),
+        config.target_env.into_iter().collect(),
+        config
+            .setup_folder
+            .unwrap_or(config.target_exe.parent().unwrap().to_path_buf()),
+        None::<&PathBuf>,
+        MachineIdentity {
+            machine_id: Uuid::nil(),
+            machine_name: "".to_string(),
+            scaleset_name: None,
+        },
+    );
+    let cmd = libfuzzer.build_std_command(None, None, None).await?;
+    print_logs(cmd)?;
+    Ok(())
+}
+
+#[cfg(target_os = "windows")]
+fn print_logs(cmd: std::process::Command) -> Result<(), anyhow::Error> {
+    let logs = dynamic_library::windows::get_logs(cmd)?;
+    for log in logs {
+        println!("{log:x?}");
+    }
+
+    Ok(())
+}
+
+#[cfg(target_os = "linux")]
+fn print_logs(cmd: std::process::Command) -> Result<(), anyhow::Error> {
+    let logs = dynamic_library::linux::get_linked_library_logs(&cmd)?;
+    for log in logs.stdout {
+        println!("{log:x?}");
+    }
+
+    let logs2 = dynamic_library::linux::get_loaded_libraries_logs(cmd)?;
+    for log in logs2.stderr {
+        println!("{log:x?}");
+    }
+
+    Ok(())
+}