From b940345ec0c9dc4038f7bb5efb1e7a3d8abbf951 Mon Sep 17 00:00:00 2001 From: Shailesh Gothi Date: Wed, 21 Apr 2021 11:28:24 -0400 Subject: [PATCH 1/2] Supported file pattern for logdog log files Ecs variant awsvpc mode feature's log file has dynamic names, so it was difficult to export such files with current logdog capabilities. This change extends logdog to supporti a new request type `glob`, which takes pattern for file matching. For this case, destination filename and path in tarball will be same as source file. --- sources/Cargo.lock | 8 + sources/logdog/Cargo.toml | 2 + sources/logdog/src/error.rs | 9 ++ sources/logdog/src/log_request.rs | 246 ++++++++++++++++++++++++++++-- 4 files changed, 252 insertions(+), 13 deletions(-) diff --git a/sources/Cargo.lock b/sources/Cargo.lock index 6b242fa5979..04486f22f3d 100644 --- a/sources/Cargo.lock +++ b/sources/Cargo.lock @@ -1356,6 +1356,12 @@ version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6503fe142514ca4799d4c26297c4248239fe8838d827db6bd6065c6ed29a6ce" +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + [[package]] name = "globset" version = "0.4.6" @@ -1815,12 +1821,14 @@ version = "0.1.0" dependencies = [ "cargo-readme", "flate2", + "glob", "reqwest", "shell-words", "snafu", "tar", "tempfile", "url", + "walkdir", ] [[package]] diff --git a/sources/logdog/Cargo.toml b/sources/logdog/Cargo.toml index 3b2efcf87a6..92790dbe5c8 100644 --- a/sources/logdog/Cargo.toml +++ b/sources/logdog/Cargo.toml @@ -10,12 +10,14 @@ exclude = ["README.md"] [dependencies] flate2 = "1.0" +glob = "0.3" reqwest = { version = "0.10.1", default-features = false, features = ["blocking", "rustls-tls"] } shell-words = "1.0.0" snafu = { version = "0.6", features = ["backtraces-impl-backtrace-crate"] } tar = { version = "0.4", default-features = false } tempfile = { version = "3.1.0", default-features = false } url = "2.1.1" +walkdir = "2.3" [build-dependencies] cargo-readme = "3.1" diff --git a/sources/logdog/src/error.rs b/sources/logdog/src/error.rs index 90a37595fb5..827e8c84d94 100644 --- a/sources/logdog/src/error.rs +++ b/sources/logdog/src/error.rs @@ -124,6 +124,15 @@ pub(crate) enum Error { #[snafu(display("Empty command."))] ModeMissing {}, + #[snafu(display("Error parsing glob pattern '{}': {}", pattern, source))] + ParseGlobPattern { + pattern: String, + source: glob::PatternError, + }, + + #[snafu(display("The logdog configuration has a 'glob' line with no glob instructions."))] + PatternMissing {}, + #[snafu(display("Cannot write to / as a file."))] RootAsFile { backtrace: Backtrace }, diff --git a/sources/logdog/src/log_request.rs b/sources/logdog/src/log_request.rs index ec0a46c0796..34639ddbabe 100644 --- a/sources/logdog/src/log_request.rs +++ b/sources/logdog/src/log_request.rs @@ -9,13 +9,16 @@ //! these provide the list of log requests that `logdog` will run. use crate::error::{self, Result}; +use glob::glob; use reqwest::blocking::{Client, Response}; use snafu::{ensure, OptionExt, ResultExt}; +use std::collections::HashSet; use std::fs; use std::fs::File; use std::path::Path; use std::process::{Command, Stdio}; use url::Url; +use walkdir::WalkDir; /// The `logdog` log requests that all variants have in common. const COMMON_REQUESTS: &str = include_str!("../conf/logdog.common.conf"); @@ -33,11 +36,12 @@ pub(crate) fn log_requests() -> Vec<&'static str> { .collect() } -/// A logdog `LogRequest` is in the format `mode filename instructions`. `mode` specifies what type -/// of command it is, e.g. `exec ` for a command or `http` for an HTTP get request. `filename` is -/// the name of the output file. `instructions` is any additional information needed. For example, -/// an `exec` request'ss instructions will include the program and program arguments. An `http` -/// request's instructions will be the URL. +/// A logdog `LogRequest` represents a line from the config file. It starts with a "mode" that +/// specifies what type of request it is, e.g. `exec ` for a command or `http` for an HTTP get +/// request. Some modes then require a `filename` that determines where the data will be saved in +/// the output tarball. The final field is `instructions` which is any additional information +/// needed by the given mode. For example, an `exec` requests' instructions will include the +/// program and program arguments. An `http` request's instructions will be the URL. /// /// # Examples /// @@ -61,11 +65,18 @@ pub(crate) fn log_requests() -> Vec<&'static str> { /// ```text /// file some-conf /etc/some/conf /// ``` +/// +/// This request will copy files with a known prefix into the tarball; this can be useful for dated +/// log files, for example. +/// +/// ```text +/// glob /var/log/my-app.log* +/// ``` #[derive(Debug, Clone)] struct LogRequest<'a> { - /// The log request mode. For example `exec`, `http`, or `file`. + /// The log request mode. For example `exec`, `http`, `file`, or `glob`. mode: &'a str, - /// The filename that the logs will be written to. + /// The filename that the logs will be written to, if appropriate for the mode. filename: &'a str, /// Any additional instructions or commands needed to fulfill the log request. For example, with /// `exec` this will be a program invocation like `echo hello world`. For an `http` request this @@ -91,19 +102,30 @@ where P: AsRef, { let request = request.as_ref(); - // get the first and second token: i.e. mode and output filename, put the remainder of the - // log request into the instructions field (or default to an empty string). let mut iter = request.splitn(3, ' '); - let req = LogRequest { - mode: iter.next().context(error::ModeMissing)?, - filename: iter.next().context(error::FilenameMissing { request })?, - instructions: iter.next().unwrap_or(""), + let mode = iter.next().context(error::ModeMissing)?; + let req = if mode == "glob" { + // for glob request format is "glob " + LogRequest { + mode, + filename: "", + instructions: iter.next().context(error::PatternMissing)?, + } + } else { + // Get the second token (output filename) and put the remainder of the + // log request into the instructions field (or default to an empty string). + LogRequest { + mode, + filename: iter.next().context(error::FilenameMissing { request })?, + instructions: iter.next().unwrap_or(""), + } }; // execute the log request with the correct handler based on the mode field. match req.mode { "exec" => handle_exec_request(&req, tempdir)?, "http" | "https" => handle_http_request(&req, tempdir)?, "file" => handle_file_request(&req, tempdir)?, + "glob" => handle_glob_request(&req, tempdir)?, unmatched => { return Err(error::Error::UnhandledRequest { mode: unmatched.into(), @@ -205,12 +227,106 @@ where Ok(()) } +/// Copies all files matching the glob pattern given by `request.instructions` to the tempdir with filename and path +/// same as source file. +fn handle_glob_request

(request: &LogRequest<'_>, tempdir: P) -> Result<()> +where + P: AsRef, +{ + let mut files = HashSet::new(); + let glob_paths = glob(request.instructions).context(error::ParseGlobPattern { + pattern: request.instructions, + })?; + for entry in glob_paths { + if let Ok(path) = entry { + if path.is_dir() { + // iterate the directory and sub-directory to get all file paths + for candidate in WalkDir::new(&path) { + if let Ok(e) = candidate { + if e.path().is_file() { + files.insert(e.into_path()); + } + } + } + } else { + files.insert(path); + } + } + } + for src_filepath in &files { + // with glob pattern there are chances of multiple targets with same name, therefore + // we maintain source file path and name in destination directory. + // Eg. src file path "/a/b/file" will be converted to "dest_dir/a/b/file" + let relative_path = src_filepath + .strip_prefix("/") + .unwrap_or(src_filepath.as_path()); + let dest_filepath = tempdir.as_ref().join(relative_path); + let dest_dir_path = dest_filepath.parent().context(error::RootAsFile)?; + // create directories in dest file path if it does not exist + fs::create_dir_all(dest_dir_path).context(error::CreateOutputDirectory { + path: dest_dir_path, + })?; + let _ = fs::copy(&src_filepath, &dest_filepath).with_context(|| error::FileCopy { + request: request.to_string(), + from: src_filepath.to_str().unwrap_or(""), + to: &dest_filepath, + })?; + } + Ok(()) +} + #[cfg(test)] mod test { use crate::log_request::handle_log_request; + use std::fs; use std::fs::write; + use std::path::PathBuf; use tempfile::TempDir; + // adds a sub directory and some files to temp directory for file request tests + fn create_source_dir(dir: &TempDir) { + let filenames_content = [ + ("foo.source", "1"), + ("bar.source", "2"), + ("for-bar.log", "3"), + ]; + // add files to temp directory + for entry in filenames_content.iter() { + let filepath = dir.path().join(entry.0); + write(&filepath, entry.1).unwrap(); + } + + let subdir_name_depth1 = "depth1"; + // create sub directory + let subdir_path_depth1 = dir.path().join(subdir_name_depth1); + fs::create_dir(&subdir_path_depth1).unwrap(); + // Add files to sub directory + for entry in filenames_content.iter() { + let filepath = subdir_path_depth1.join(entry.0); + write(&filepath, entry.1).unwrap(); + } + + let subdir_name_depth2 = "depth2"; + // create sub directory + let subdir_path_depth2 = subdir_path_depth1.join(subdir_name_depth2); + fs::create_dir(&subdir_path_depth2).unwrap(); + // Add files to sub directory + for entry in filenames_content.iter() { + let filepath = subdir_path_depth2.join(entry.0); + write(&filepath, entry.1).unwrap(); + } + } + + fn get_dest_filepath(src_dir: &TempDir, filepath: &str) -> PathBuf { + src_dir.path().strip_prefix("/").unwrap().join(filepath) + } + + fn assert_file_match(dest_dir: &TempDir, filepath: PathBuf, want: &str) { + let outfile = dest_dir.path().join(filepath); + let got = std::fs::read_to_string(&outfile).unwrap(); + assert_eq!(got, want); + } + #[test] fn file_request() { let source_dir = TempDir::new().unwrap(); @@ -235,4 +351,108 @@ mod test { let got = std::fs::read_to_string(&outfile).unwrap(); assert_eq!(got, want); } + + #[test] + // ensures single file pattern works + fn glob_single_file_pattern_request() { + let source_dir = TempDir::new().unwrap(); + create_source_dir(&source_dir); + let outdir = TempDir::new().unwrap(); + let request = format!("glob {}/foo.source", source_dir.path().display()); + handle_log_request(&request, outdir.path()).unwrap(); + assert_file_match(&outdir, get_dest_filepath(&source_dir, "foo.source"), "1"); + } + + #[test] + // ensures multiple file pattern works + fn glob_multiple_files_pattern_request() { + let source_dir = TempDir::new().unwrap(); + create_source_dir(&source_dir); + let outdir = TempDir::new().unwrap(); + let request = format!("glob {}/*.source", source_dir.path().display()); + handle_log_request(&request, outdir.path()).unwrap(); + assert_file_match(&outdir, get_dest_filepath(&source_dir, "foo.source"), "1"); + assert_file_match(&outdir, get_dest_filepath(&source_dir, "bar.source"), "2"); + } + + #[test] + // ensures multiple file in nested directory pattern works + fn glob_nested_file_pattern_request() { + let source_dir = TempDir::new().unwrap(); + create_source_dir(&source_dir); + let outdir = TempDir::new().unwrap(); + let request = format!("glob {}/**/*.source", source_dir.path().display()); + handle_log_request(&request, outdir.path()).unwrap(); + assert_file_match(&outdir, get_dest_filepath(&source_dir, "foo.source"), "1"); + assert_file_match(&outdir, get_dest_filepath(&source_dir, "bar.source"), "2"); + assert_file_match( + &outdir, + get_dest_filepath(&source_dir, "depth1/foo.source"), + "1", + ); + assert_file_match( + &outdir, + get_dest_filepath(&source_dir, "depth1/bar.source"), + "2", + ); + assert_file_match( + &outdir, + get_dest_filepath(&source_dir, "depth1/depth2/foo.source"), + "1", + ); + assert_file_match( + &outdir, + get_dest_filepath(&source_dir, "depth1/depth2/bar.source"), + "2", + ); + } + + #[test] + // ensures directory pattern works + fn glob_dir_pattern_request() { + let source_dir = TempDir::new().unwrap(); + create_source_dir(&source_dir); + let outdir = TempDir::new().unwrap(); + let request = format!("glob {}/**/", source_dir.path().display()); + handle_log_request(&request, outdir.path()).unwrap(); + assert_file_match( + &outdir, + get_dest_filepath(&source_dir, "depth1/foo.source"), + "1", + ); + assert_file_match( + &outdir, + get_dest_filepath(&source_dir, "depth1/bar.source"), + "2", + ); + assert_file_match( + &outdir, + get_dest_filepath(&source_dir, "depth1/for-bar.log"), + "3", + ); + assert_file_match( + &outdir, + get_dest_filepath(&source_dir, "depth1/depth2/foo.source"), + "1", + ); + assert_file_match( + &outdir, + get_dest_filepath(&source_dir, "depth1/depth2/bar.source"), + "2", + ); + assert_file_match( + &outdir, + get_dest_filepath(&source_dir, "depth1/depth2/for-bar.log"), + "3", + ); + } + + #[test] + // ensure if pattern is empty it should not panic + fn glob_empty_pattern_request() { + let outdir = TempDir::new().unwrap(); + let request = "glob"; + let err = handle_log_request(&request, outdir.path()).unwrap_err(); + assert!(matches!(err, crate::error::Error::PatternMissing {})); + } } From 6e258a8242765189963dcb009c379c24a5890ab9 Mon Sep 17 00:00:00 2001 From: Shailesh Gothi Date: Thu, 22 Apr 2021 16:26:12 -0400 Subject: [PATCH 2/2] Adds glob pattern in logdog conf for awsvpc log files --- sources/logdog/conf/logdog.aws-ecs-1.conf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sources/logdog/conf/logdog.aws-ecs-1.conf b/sources/logdog/conf/logdog.aws-ecs-1.conf index 4744aebb2b6..8ee6c59c4a8 100644 --- a/sources/logdog/conf/logdog.aws-ecs-1.conf +++ b/sources/logdog/conf/logdog.aws-ecs-1.conf @@ -3,5 +3,6 @@ file docker-daemon.json /etc/docker/daemon.json file ecs-agent-state.json /var/lib/ecs/data/ecs_agent_data.json file ecs-config.json /etc/ecs/ecs.config.json http ecs-tasks http://localhost:51678/v1/tasks -file ecs-cni-bridge-plugin.log /var/log/ecs/ecs-cni-bridge-plugin.log -file ecs-cni-eni-plugin.log /var/log/ecs/ecs-cni-eni-plugin.log +glob /var/log/ecs/ecs-cni-bridge-plugin.log* +glob /var/log/ecs/ecs-cni-eni-plugin.log* +glob /var/log/ecs/vpc-branch-eni.log*