OTA-915: Serve signatures through a new container

Cargo.toml

@@ -5,6 +5,7 @@ members = [
 	"commons",
 	"graph-builder",
 	"policy-engine",
+	"metadata-helper",
 	"prometheus-query",
 	"quay",
 	"e2e",

Justfile

@@ -149,6 +149,28 @@ display-graph:
 
 	jq -cM . | {{invocation_directory()}}/hack/graph.sh | dot -Tsvg > graph.svg; xdg-open graph.svg
 
+run-metadata-helper:
+	#!/usr/bin/env bash
+	export RUST_BACKTRACE=1
+
+	cargo run --package metadata-helper -- -c <(cat <<-EOF
+		verbosity = "vvv"
+
+		[service]
+		scrape_timeout_secs = 300
+		pause_secs = {{pause_secs}}
+		address = "127.0.0.1"
+		port = 8080
+		path_prefix = "{{path_prefix}}"
+		tracing_endpoint = "{{default_tracing_endpoint}}"
+
+		[status]
+		address = "127.0.0.1"
+		port = 9080
+	EOF
+	)
+
+
 run-graph-builder:
 	#!/usr/bin/env bash
 	export RUST_BACKTRACE=1

commons/src/errors.rs

@@ -81,6 +81,10 @@ pub enum GraphError {
     /// Failed to open a file
     #[error("failed to open file: {}", _0)]
     FileOpenError(String),
+
+    /// Resource does not exist
+    #[error("does not exist: {}", _0)]
+    DoesNotExist(String),
 }
 
 impl actix_web::error::ResponseError for GraphError {
@@ -124,6 +128,7 @@ impl GraphError {
             GraphError::InvalidParams(_) => http::StatusCode::BAD_REQUEST,
             GraphError::ArchVersionError(_) => http::StatusCode::INTERNAL_SERVER_ERROR,
             GraphError::FileOpenError(_) => http::StatusCode::INTERNAL_SERVER_ERROR,
+            GraphError::DoesNotExist(_) => http::StatusCode::NOT_FOUND,
         }
     }
 
@@ -140,6 +145,7 @@ impl GraphError {
             GraphError::InvalidParams(_) => "invalid_params",
             GraphError::ArchVersionError(_) => "arch_version_error",
             GraphError::FileOpenError(_) => "file_open_err",
+            GraphError::DoesNotExist(_) => "does_not_exist",
         };
         kind.to_string()
     }

commons/src/lib.rs

@@ -25,6 +25,7 @@ pub mod prelude_errors {
 }
 
 use actix_web::http::header::{HeaderMap, HeaderValue, ACCEPT};
+use actix_web::HttpRequest;
 use flate2::write::GzEncoder;
 use flate2::Compression;
 use std::collections::HashMap;
@@ -184,6 +185,44 @@ pub async fn create_tar(output_path: Box<Path>, data_path: Box<Path>) -> Result<
     Ok(())
 }
 
+/// format the request before logging. Include only details that we need.
+pub fn format_request(req: &HttpRequest) -> String {
+    let no_user_agent = HeaderValue::from_str("user-agent not available").unwrap();
+    let no_accept_type = HeaderValue::from_str("accept value not available").unwrap();
+    let req_type = req.method().as_str();
+    let request = req.path();
+    let query = req.query_string();
+    let user_agent = req
+        .headers()
+        .get("user-agent")
+        .unwrap_or(&no_user_agent)
+        .to_str()
+        .unwrap();
+    let accept_type = req
+        .headers()
+        .get("accept")
+        .unwrap_or(&no_accept_type)
+        .to_str()
+        .unwrap();
+    format!(
+        "Method: '{}', Request: '{}', Query: '{}', User-Agent: '{}', Accept: '{}'",
+        req_type, request, query, user_agent, accept_type
+    )
+}
+
+/// logs api request error
+pub fn api_response_error(req: &HttpRequest, e: GraphError) -> GraphError {
+    log::error!(
+        "Error serving request \"{}\" from '{}': {:?}",
+        format_request(req),
+        req.peer_addr()
+            .map(|addr| addr.to_string())
+            .unwrap_or_else(|| "<not available>".into()),
+        e
+    );
+    e
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

metadata-helper/Cargo.toml

@@ -0,0 +1,41 @@
+[package]
+name = "metadata-helper"
+version = "0.1.0"
+authors = ["Pratik Mahajan <pmahajan@redhat.com>"]
+edition = "2018"
+build = "src/build.rs"
+
+[dependencies]
+actix = "0.13.0"
+actix-cors = "^0.6.1"
+actix-files = "^0.6.2"
+actix-service = "2.0.2"
+actix-web = "^4.0.0-rc.3"
+cincinnati = { path = "../cincinnati" }
+commons = { path = "../commons" }
+custom_debug_derive = "^0.5"
+env_logger = "^0.9"
+futures = "^0.3"
+hyper = "^0.14"
+lazy_static = "^1.2.0"
+log = "^0.4.17"
+opentelemetry = "0.14.0"
+parking_lot = "^0.12"
+prometheus = "0.13"
+semver = { version = "^1.0.16", features = [ "serde" ] }
+serde = "^1.0.152"
+serde_derive = "^1.0.70"
+serde_json = "^1.0.91"
+smart-default = "0.7.1"
+structopt = "^0.3"
+tempfile = "^3.3.0"
+toml = "^0.5"
+url = "^2.3"
+
+[build-dependencies]
+built = { version = "^0.5.1", features = [ "git2" ]}
+
+[dev-dependencies]
+tokio = { version = "1.16", features = [ "rt-multi-thread" ] }
+memchr = "^2.5"
+mockito = "1.0.2"

metadata-helper/src/build.rs

@@ -0,0 +1,5 @@
+extern crate built;
+
+fn main() {
+    built::write_built_file().expect("Failed to acquire build-time information");
+}

metadata-helper/src/config/cli.rs

@@ -0,0 +1,89 @@
+//! Command-line options for metadata-helper.
+
+use super::options;
+use super::AppSettings;
+use commons::prelude_errors::*;
+use commons::MergeOptions;
+
+/// CLI configuration flags, top-level.
+#[derive(Debug, StructOpt)]
+pub struct CliOptions {
+    /// Verbosity level
+    #[structopt(short = "v", parse(from_occurrences))]
+    pub verbosity: u64,
+
+    /// Path to configuration file
+    #[structopt(short = "c")]
+    pub config_path: Option<String>,
+
+    // Main service options
+    #[structopt(flatten)]
+    pub service: options::ServiceOptions,
+
+    // Status service options
+    #[structopt(flatten)]
+    pub status: options::StatusOptions,
+}
+
+impl MergeOptions<CliOptions> for AppSettings {
+    fn try_merge(&mut self, opts: CliOptions) -> Fallible<()> {
+        self.verbosity = match opts.verbosity {
+            0 => self.verbosity,
+            1 => log::LevelFilter::Info,
+            2 => log::LevelFilter::Debug,
+            _ => log::LevelFilter::Trace,
+        };
+
+        self.try_merge(Some(opts.service))?;
+        self.try_merge(Some(opts.status))?;
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::CliOptions;
+    use crate::config::AppSettings;
+    use commons::MergeOptions;
+    use structopt::StructOpt;
+
+    #[test]
+    fn cli_basic() {
+        let no_args = vec!["argv0"];
+        let no_args_cli = CliOptions::from_iter_safe(no_args).unwrap();
+        assert_eq!(no_args_cli.verbosity, 0);
+        assert_eq!(no_args_cli.upstream_method, None);
+
+        let verbose_args = vec!["argv0", "-vvv"];
+        let verbose_cli = CliOptions::from_iter_safe(verbose_args).unwrap();
+        assert_eq!(verbose_cli.verbosity, 3);
+
+        let svc_port_args = vec!["argv0", "--service.port", "9999"];
+        let svc_port_cli = CliOptions::from_iter_safe(svc_port_args).unwrap();
+        assert_eq!(svc_port_cli.service.port, Some(9999));
+    }
+
+    #[test]
+    fn cli_override_toml() {
+        use crate::config::file::FileOptions;
+        use commons::MergeOptions;
+
+        let mut settings = AppSettings::default();
+        assert_eq!(settings.verbosity, log::LevelFilter::Warn);
+
+        let toml_verbosity = r#"verbosity="vvv""#;
+        let file_opts: FileOptions = toml::from_str(toml_verbosity).unwrap();
+        assert_eq!(file_opts.verbosity, Some(log::LevelFilter::Trace));
+
+        settings.try_merge(Some(file_opts)).unwrap();
+        assert_eq!(settings.verbosity, log::LevelFilter::Trace);
+
+        let args = vec!["argv0", "-vv"];
+        let cli_opts = CliOptions::from_iter_safe(args).unwrap();
+        assert_eq!(cli_opts.verbosity, 2);
+
+        settings.try_merge(cli_opts).unwrap();
+        assert_eq!(settings.verbosity, log::LevelFilter::Debug);
+    }
+}

metadata-helper/src/config/file.rs

@@ -0,0 +1,115 @@
+//! TOML file configuration options.
+
+use super::options;
+use super::AppSettings;
+use commons::de::de_loglevel;
+use commons::prelude_errors::*;
+use commons::MergeOptions;
+use std::io::Read;
+use std::{fs, io, path};
+
+/// TOML configuration, top-level.
+#[derive(Debug, Deserialize)]
+pub struct FileOptions {
+    /// Verbosity level.
+    #[serde(default = "Option::default", deserialize_with = "de_loglevel")]
+    pub verbosity: Option<log::LevelFilter>,
+
+    /// Web frontend options.
+    pub service: Option<options::ServiceOptions>,
+
+    /// Status service options.
+    pub status: Option<options::StatusOptions>,
+
+    /// Signatures service options
+    pub signatures: Option<options::SignaturesOptions>,
+}
+
+impl FileOptions {
+    /// Parse a TOML configuration from path.
+    pub fn read_filepath<P>(cfg_path: P) -> Fallible<Self>
+    where
+        P: AsRef<path::Path>,
+    {
+        let cfg_file = fs::File::open(&cfg_path).context(format!(
+            "failed to open config path {:?}",
+            cfg_path.as_ref()
+        ))?;
+        let mut bufrd = io::BufReader::new(cfg_file);
+
+        let mut content = vec![];
+        bufrd.read_to_end(&mut content)?;
+        let cfg = toml::from_slice(&content).context(format!(
+            "failed to parse config file {}:\n{}",
+            cfg_path.as_ref().display(),
+            std::str::from_utf8(&content).unwrap_or("file not decodable")
+        ))?;
+
+        Ok(cfg)
+    }
+}
+
+impl MergeOptions<Option<FileOptions>> for AppSettings {
+    fn try_merge(&mut self, opts: Option<FileOptions>) -> Fallible<()> {
+        if let Some(file) = opts {
+            assign_if_some!(self.verbosity, file.verbosity);
+            self.try_merge(file.service)?;
+            self.try_merge(file.status)?;
+            self.try_merge(file.signatures)?;
+        }
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::FileOptions;
+    use crate::config::AppSettings;
+    use commons::MergeOptions;
+
+    #[test]
+    fn toml_merge_settings() {
+        let mut settings = AppSettings::default();
+        assert_eq!(settings.status_port, 9081);
+
+        let toml_input = "status.port = 2222";
+        let file_opts: FileOptions = toml::from_str(toml_input).unwrap();
+
+        settings.try_merge(Some(file_opts)).unwrap();
+        assert_eq!(settings.status_port, 2222);
+    }
+
+    #[test]
+    fn toml_sample_config() {
+        use super::FileOptions;
+
+        let input_url = hyper::Uri::from_static("0.0.0.0");
+        let opts = {
+            use std::io::Write;
+
+            let sample_config = r#"
+                verbosity = "vvv"
+
+                [service]
+                address = "0.0.0.0"
+                port = 8383
+
+                [status]
+                address = "127.0.0.1"
+            "#;
+
+            let mut config_file = tempfile::NamedTempFile::new().unwrap();
+            config_file
+                .write_fmt(format_args!("{}", sample_config))
+                .unwrap();
+            FileOptions::read_filepath(config_file.path()).unwrap()
+        };
+
+        assert_eq!(opts.verbosity, Some(log::LevelFilter::Trace));
+        assert!(opts.service.is_some());
+
+        let srv = opts.service.unwrap();
+        let srv_url = srv.address.unwrap();
+        assert_eq!(srv_url, input_url);
+    }
+}

metadata-helper/src/config/mod.rs

@@ -0,0 +1,17 @@
+//! Configuration lookup, parsing and validation.
+//!
+//! This module takes care of sourcing configuration options from
+//! multiple inputs (CLI and files), merging, and validating them.
+//! It contains the following entities:
+//!  * "options": configuration fragments (CLI flags, file snippets).
+//!  * "app settings": runtime settings, result of config validation.
+
+mod cli;
+mod file;
+mod options;
+mod settings;
+
+#[cfg(test)]
+pub(crate) use self::file::FileOptions;
+
+pub use self::settings::AppSettings;