diff --git a/Cargo.lock b/Cargo.lock index f8dabf5d2..82e5d20c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -118,7 +118,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.24 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.24 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -295,6 +295,7 @@ dependencies = [ "serde_derive 1.0.84 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.34 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", + "toml 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", "try_from 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -580,7 +581,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.24 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.24 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)", "synstructure 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -680,9 +681,11 @@ dependencies = [ "serde 1.0.84 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.84 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.34 (registry+https://github.com/rust-lang/crates.io-index)", + "smart-default 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "structopt 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)", "tar 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", + "toml 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1594,7 +1597,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.24 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.24 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1661,6 +1664,16 @@ dependencies = [ "unreachable 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "smart-default" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 0.4.24 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "socket2" version = "0.3.8" @@ -1709,7 +1722,7 @@ dependencies = [ "heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.24 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.24 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1725,12 +1738,12 @@ dependencies = [ "heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.24 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.24 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "syn" -version = "0.15.24" +version = "0.15.29" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.24 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1745,7 +1758,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 0.4.24 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.24 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-xid 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1999,6 +2012,14 @@ dependencies = [ "tokio-reactor 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "toml" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde 1.0.84 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "tower-service" version = "0.1.0" @@ -2187,7 +2208,7 @@ dependencies = [ "nom 4.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 0.4.24 (registry+https://github.com/rust-lang/crates.io-index)", "quote 0.6.10 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.24 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2494,6 +2515,7 @@ dependencies = [ "checksum siphasher 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "0b8de496cf83d4ed58b6be86c3a275b8602f6ffe98d3024a869e124147a9a3ac" "checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" "checksum smallvec 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "b73ea3738b47563803ef814925e69be00799a8c07420be8b996f8e98fb2336db" +"checksum smart-default 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "fae090012788f95156fc33f43631242480f311cca6bf7f600943eba14ae032ed" "checksum socket2 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "c4d11a52082057d87cb5caa31ad812f4504b97ab44732cd8359df2e9ff9f48e7" "checksum spin 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "ceac490aa12c567115b40b7b7fceca03a6c9d53d5defea066123debc83c5dc1f" "checksum stable_deref_trait 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "dba1a27d3efae4351c8051072d619e3ade2820635c3958d826bfea39d59b54c8" @@ -2503,7 +2525,7 @@ dependencies = [ "checksum structopt-derive 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "ef98172b1a00b0bec738508d3726540edcbd186d50dfd326f2b1febbb3559f04" "checksum strum 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f6b3fc98c482ff9bb37a6db6a6491218c4c82bec368bd5682033e5b96b969143" "checksum strum_macros 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6969d7021d96b53b12b774d4f412026f1debe7f168a0b8c59e93b4c1e850a05f" -"checksum syn 0.15.24 (registry+https://github.com/rust-lang/crates.io-index)" = "734ecc29cd36e8123850d9bf21dfd62ef8300aaa8f879aabaa899721808be37c" +"checksum syn 0.15.29 (registry+https://github.com/rust-lang/crates.io-index)" = "1825685f977249735d510a242a6727b46efe914bb67e38d30c071b1b72b1d5c2" "checksum synstructure 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "73687139bf99285483c96ac0add482c3776528beac1d97d444f6e91f203a2015" "checksum tar 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)" = "a303ba60a099fcd2aaa646b14d2724591a96a75283e4b7ed3d1a1658909d9ae2" "checksum tempdir 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" @@ -2526,6 +2548,7 @@ dependencies = [ "checksum tokio-timer 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "4f37f0111d76cc5da132fe9bc0590b9b9cfd079bc7e75ac3846278430a299ff8" "checksum tokio-udp 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "66268575b80f4a4a710ef83d087fdfeeabdce9b74c797535fbac18a2cb906e92" "checksum tokio-uds 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "037ffc3ba0e12a0ab4aca92e5234e0dedeb48fddf6ccd260f1f150a36a9f2445" +"checksum toml 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "758664fc71a3a69038656bee8b6be6477d2a6c315a6b81f7081f591bffa4111f" "checksum tower-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b32f72af77f1bfe3d3d4da8516a238ebe7039b51dd8637a09841ac7f16d2c987" "checksum trust-dns-proto 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0838272e89f1c693b4df38dc353412e389cf548ceed6f9fd1af5a8d6e0e7cf74" "checksum trust-dns-proto 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e33f29df428f112ffeda24b328b814b61d6916be29aa89f19bc3f684ba5437b8" diff --git a/cincinnati/Cargo.toml b/cincinnati/Cargo.toml index 80d7cbdbc..5bac30c05 100644 --- a/cincinnati/Cargo.toml +++ b/cincinnati/Cargo.toml @@ -15,6 +15,7 @@ url = "^1.7.2" log = "^0.4.3" futures = "0.1" tokio = "0.1" +toml = "^0.4.10" try_from = "0.3.2" [dev-dependencies] diff --git a/dist/openshift/cincinnati.configmap.yaml b/dist/openshift/cincinnati.configmap.yaml index f3f2a9656..d1640195c 100644 --- a/dist/openshift/cincinnati.configmap.yaml +++ b/dist/openshift/cincinnati.configmap.yaml @@ -5,6 +5,7 @@ metadata: type: Opaque data: gb.address: "0.0.0.0" + gb.status.address: "0.0.0.0" gb.registry: "quay.io" gb.repository: "steveej/cincinnati-test" gb.log.verbosity: "vvv" diff --git a/dist/openshift/cincinnati.yaml b/dist/openshift/cincinnati.yaml index 601d77c2b..12e604104 100644 --- a/dist/openshift/cincinnati.yaml +++ b/dist/openshift/cincinnati.yaml @@ -39,6 +39,11 @@ objects: configMapKeyRef: key: gb.address name: cincinnati + - name: STATUS_ADDRESS + valueFrom: + configMapKeyRef: + key: gb.status.address + name: cincinnati - name: REGISTRY valueFrom: configMapKeyRef: @@ -54,7 +59,14 @@ objects: configMapKeyRef: key: gb.log.verbosity name: cincinnati - args: ["-$(GB_LOG_VERBOSITY)", "--address", "$(ADDRESS)", "--port", "${GB_PORT}", "--registry", "$(REGISTRY)", "--repository", "$(REPOSITORY)", "--credentials-file=/etc/secrets/registry-credentials"] + args: ["-$(GB_LOG_VERBOSITY)", + "--service.address", "$(ADDRESS)", + "--service.port", "${GB_PORT}", + "--status.address", "$(STATUS_ADDRESS)", + "--status.port", "${GB_STATUS_PORT}", + "--upstream.registry.url", "$(REGISTRY)", + "--upstream.registry.repository", "$(REPOSITORY)", + "--upstream.registry.credentials_path", "/etc/secrets/registry-credentials"] ports: - name: graph-builder containerPort: ${{GB_PORT}} diff --git a/docs/user/graph-builder-configuration.md b/docs/user/graph-builder-configuration.md new file mode 100644 index 000000000..4ab197c3d --- /dev/null +++ b/docs/user/graph-builder-configuration.md @@ -0,0 +1,25 @@ +# Graph-builder configuration + +Graph-builder can be configured via TOML files and command-line options, with the latter having higher priority. + +## TOML options + +TOML configuration currently supports the following sections and options: + + - `verbosity` (unsigned integer): log verbosity level, from 0 (errors and warnings only) to 3 (all trace messages). Default: 0. + - `service` (section): configuration options related to the main HTTP Cincinnati service. + - `address` (string): local IP for the main service. Default: "127.0.0.1". + - `mandatory_client_parameters` (list of strings): Cincinnati query parameters that must be present in client requests. Default: empty. + - `path_prefix` (string): namespace prefix for all API endpoints. Default: "". + - `port` (unsigned integer): local port for the main service. Default: 8080. + - `status` (section): configuration options related to the HTTP status service. + - `address` (string): local IP for the status service. Default: "127.0.0.1". + - `port` (unsigned integer): local port for the status service. Default: 9080. + - `upstream` (section): configuration options related to upstream release-data provider. + - `method` (string): upstream provider selector. Allowed values: "registry". Default: "registry". + - `registry` (section): configuration for Docker-v2 registry provider. + - `credentials_path` (string): path to file containing registry credentials, in "dockercfg" format. Default: unset. + - `manifestref_key` (string): metadata key where to record the manifest-reference. Default: "com.openshift.upgrades.graph.release.manifestref". + - `pause_secs` (unsigned integer): pause between repository scrapes, in seconds. Default: 30. + - `repository` (string): target image in the registry. Default: "openshift". + - `url` (string): URL for the registry. Default: "http://localhost:5000". diff --git a/graph-builder/Cargo.toml b/graph-builder/Cargo.toml index 252f22f76..37e8c629a 100644 --- a/graph-builder/Cargo.toml +++ b/graph-builder/Cargo.toml @@ -24,9 +24,11 @@ semver = { version = "^0.9.0", features = [ "serde" ] } serde = "^1.0.70" serde_derive = "^1.0.70" serde_json = "^1.0.22" +smart-default = "^0.5.1" structopt = "^0.2.10" tar = "^0.4.16" tokio = "0.1" +toml = "^0.4.10" url = "^1.7.2" [features] diff --git a/graph-builder/src/config.rs b/graph-builder/src/config.rs deleted file mode 100644 index 2523a6e72..000000000 --- a/graph-builder/src/config.rs +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2018 Alex Crawford -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use cincinnati::plugins::internal::metadata_fetch_quay::{ - DEFAULT_QUAY_LABEL_FILTER, DEFAULT_QUAY_MANIFESTREF_KEY, -}; -use commons::{parse_params_set, parse_path_prefix}; -use quay::v1::DEFAULT_API_BASE; -use std::collections::HashSet; -use std::net::IpAddr; -use std::num::ParseIntError; -use std::path::PathBuf; -use std::str::FromStr; -use std::time::Duration; - -#[derive(Debug, StructOpt)] -pub struct Options { - /// Verbosity level - #[structopt(short = "v", parse(from_occurrences))] - pub verbosity: u64, - - /// URL for the container image registry - #[structopt(long = "registry", default_value = "http://localhost:5000")] - pub registry: String, - - /// Name of the container image repository - #[structopt(long = "repository", default_value = "openshift")] - pub repository: String, - - /// Duration of the pause (in seconds) between scans of the registry - #[structopt( - long = "period", - default_value = "30", - parse(try_from_str = "parse_duration") - )] - pub period: Duration, - - /// Address on which the server will listen - #[structopt(long = "address", default_value = "127.0.0.1")] - pub address: IpAddr, - - /// Port to which the server will bind - #[structopt(long = "port", default_value = "8080")] - pub port: u16, - - /// Credentials file for authentication against the image registry - #[structopt(long = "credentials-file", parse(from_os_str))] - pub credentials_path: Option, - - /// Path prefix for all paths. - #[structopt( - long = "path-prefix", - default_value = "", - parse(from_str = "parse_path_prefix") - )] - pub path_prefix: String, - - /// Comma-separated set of mandatory client parameters. - #[structopt( - long = "mandatory-client-parameters", - default_value = "", - parse(from_str = "parse_params_set") - )] - pub mandatory_client_parameters: HashSet, - - /// Whether to disable the fetching and processing metadata from the quay API - #[structopt(long = "disable-quay-api-metadata")] - pub disable_quay_api_metadata: bool, - - /// Base URL to the quay API host - #[structopt( - long = "quay-api-base", - long_help = "API base URL", - raw(default_value = "DEFAULT_API_BASE") - )] - pub quay_api_base: String, - - /// Filter for receiving quay labels - #[structopt( - long = "quay-label-filter", - raw(default_value = "DEFAULT_QUAY_LABEL_FILTER") - )] - pub quay_label_filter: String, - - /// Metadata key where the quay fetcher expects the manifestref - #[structopt( - long = "quay-manifestref-key", - raw(default_value = "DEFAULT_QUAY_MANIFESTREF_KEY") - )] - pub quay_manifestref_key: String, - - /// Credentials file for authentication against API described at https://docs.quay.io/api/ - #[structopt(long = "quay-api-credentials-path", parse(from_os_str))] - pub quay_api_credentials_path: Option, -} - -fn parse_duration(src: &str) -> Result { - Ok(Duration::from_secs(u64::from_str(src)?)) -} diff --git a/graph-builder/src/config/cli.rs b/graph-builder/src/config/cli.rs new file mode 100644 index 000000000..55ee7a49d --- /dev/null +++ b/graph-builder/src/config/cli.rs @@ -0,0 +1,113 @@ +//! Command-line options. + +use super::options; +use super::{AppSettings, MergeOptions}; + +/// CLI configuration flags, top-level. +#[derive(Debug, StructOpt)] +pub struct CliOptions { + /// Verbosity level + #[structopt(short = "v", parse(from_occurrences))] + pub verbosity: u8, + + /// Path to configuration file + #[structopt(short = "c")] + pub config_path: Option, + + // TODO(lucab): drop this when plugins are configurable. + #[structopt(long = "disable_quay_api_metadata")] + pub disable_quay_api_metadata: Option, + + #[structopt(flatten)] + pub service: options::ServiceOptions, + + #[structopt(flatten)] + pub status: options::StatusOptions, + + /// Fetcher method. + #[structopt(long = "upstream.method")] + pub upstream_method: Option, + + #[structopt(flatten)] + pub upstream_registry: options::DockerRegistryOptions, +} + +impl MergeOptions for AppSettings { + fn merge(&mut self, opts: CliOptions) -> () { + self.verbosity = match opts.verbosity { + 0 => self.verbosity, + 1 => log::LevelFilter::Info, + 2 => log::LevelFilter::Debug, + _ => log::LevelFilter::Trace, + }; + self.merge(Some(opts.service)); + self.merge(Some(opts.status)); + self.merge(Some(opts.upstream_registry)); + + // TODO(lucab): drop this when plugins are configurable. + assign_if_some!( + self.disable_quay_api_metadata, + opts.disable_quay_api_metadata + ); + } +} + +#[cfg(test)] +mod tests { + use super::CliOptions; + use crate::config::{AppSettings, MergeOptions}; + use structopt::StructOpt; + + #[test] + fn cli_basic() { + let no_args = vec!["argv0"]; + let no_args_cli = CliOptions::from_iter_safe(no_args).unwrap(); + assert_eq!(no_args_cli.verbosity, 0); + assert_eq!(no_args_cli.upstream_method, None); + + let verbose_args = vec!["argv0", "-vvv"]; + let verbose_cli = CliOptions::from_iter_safe(verbose_args).unwrap(); + assert_eq!(verbose_cli.verbosity, 3); + + let svc_port_args = vec!["argv0", "--service.port", "9999"]; + let svc_port_cli = CliOptions::from_iter_safe(svc_port_args).unwrap(); + assert_eq!(svc_port_cli.service.port, Some(9999)); + } + + #[test] + fn cli_merge_settings() { + let repo = "cincinnati/cli-test"; + + let mut settings = AppSettings::default(); + assert_eq!(settings.repository, "openshift".to_string()); + + let args = vec!["argv0", "--upstream.registry.repository", repo]; + let cli = CliOptions::from_iter_safe(args).unwrap(); + assert_eq!(cli.upstream_registry.repository, Some(repo.to_string())); + + settings.merge(cli); + assert_eq!(settings.repository, repo.to_string()); + } + + #[test] + fn cli_override_toml() { + use crate::config::file::FileOptions; + + let mut settings = AppSettings::default(); + assert_eq!(settings.verbosity, log::LevelFilter::Warn); + + let toml_verbosity = "verbosity=3"; + let file_opts: FileOptions = toml::from_str(toml_verbosity).unwrap(); + assert_eq!(file_opts.verbosity, Some(log::LevelFilter::Trace)); + + settings.merge(Some(file_opts)); + assert_eq!(settings.verbosity, log::LevelFilter::Trace); + + let args = vec!["argv0", "-vv"]; + let cli_opts = CliOptions::from_iter_safe(args).unwrap(); + assert_eq!(cli_opts.verbosity, 2); + + settings.merge(cli_opts); + assert_eq!(settings.verbosity, log::LevelFilter::Debug); + } +} diff --git a/graph-builder/src/config/file.rs b/graph-builder/src/config/file.rs new file mode 100644 index 000000000..19696ed2e --- /dev/null +++ b/graph-builder/src/config/file.rs @@ -0,0 +1,138 @@ +//! TOML file configuration options. + +use super::options; +use super::{AppSettings, MergeOptions}; +use failure::{Fallible, ResultExt}; +use std::io::Read; +use std::{fs, io, path}; + +/// TOML configuration, top-level. +#[derive(Debug, Deserialize)] +pub struct FileOptions { + /// Verbosity level. + #[serde(default = "Option::default", deserialize_with = "de_loglevel")] + pub verbosity: Option, + + /// Upstream options. + pub upstream: Option, + + /// Web frontend options. + pub service: Option, + + /// Status service options. + pub status: Option, +} + +impl FileOptions { + pub fn read_filepath

(cfg_path: P) -> Fallible + where + P: AsRef, + { + let cfg_file = fs::File::open(&cfg_path).context(format!( + "failed to open config path {:?}", + cfg_path.as_ref() + ))?; + let mut bufrd = io::BufReader::new(cfg_file); + + let mut content = vec![]; + bufrd.read_to_end(&mut content)?; + let cfg = toml::from_slice(&content).context(format!( + "failed to read config file {}", + cfg_path.as_ref().display() + ))?; + + Ok(cfg) + } +} + +impl MergeOptions> for AppSettings { + fn merge(&mut self, opts: Option) -> () { + if let Some(file) = opts { + assign_if_some!(self.verbosity, file.verbosity); + self.merge(file.service); + self.merge(file.status); + self.merge(file.upstream); + } + } +} + +/// Options for upstream fetcher. +#[derive(Debug, Deserialize)] +pub struct UpstreamOptions { + /// Fetcher method. + pub method: Option, + + /// Docker-registry-v2 upstream options. + pub registry: Option, +} + +impl MergeOptions> for AppSettings { + fn merge(&mut self, opts: Option) -> () { + if let Some(upstream) = opts { + self.merge(upstream.registry); + } + } +} + +pub fn de_loglevel<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + use serde::Deserialize; + let numlevel = u8::deserialize(deserializer)?; + + let verbosity = match numlevel { + 0 => log::LevelFilter::Warn, + 1 => log::LevelFilter::Info, + 2 => log::LevelFilter::Debug, + _ => log::LevelFilter::Trace, + }; + Ok(Some(verbosity)) +} + +#[cfg(test)] +mod tests { + use super::FileOptions; + use crate::config::{AppSettings, MergeOptions}; + + #[test] + fn toml_basic() { + let toml_input = "[upstream.registry]\npause_secs=25"; + let file_opts: FileOptions = toml::from_str(toml_input).unwrap(); + + let pause = file_opts + .upstream + .unwrap() + .registry + .unwrap() + .pause_secs + .unwrap(); + assert_eq!(pause, std::time::Duration::from_secs(25)); + } + + #[test] + fn toml_merge_settings() { + let mut settings = AppSettings::default(); + assert_eq!(settings.status_port, 9080); + + let toml_input = "status.port = 2222"; + let file_opts: FileOptions = toml::from_str(toml_input).unwrap(); + + settings.merge(Some(file_opts)); + assert_eq!(settings.status_port, 2222); + } + + #[test] + fn toml_sample_config() { + let filepath = "tests/fixtures/sample-config.toml"; + let opts = FileOptions::read_filepath(filepath).unwrap(); + + assert_eq!(opts.verbosity, Some(log::LevelFilter::Trace)); + assert!(opts.service.is_some()); + + let ups_registry = opts.upstream.unwrap().registry.unwrap(); + assert!(ups_registry.credentials_path.is_none()); + let repo = ups_registry.repository.unwrap(); + assert_eq!(repo, "openshift-release-dev/ocp-release"); + } +} diff --git a/graph-builder/src/config/mod.rs b/graph-builder/src/config/mod.rs new file mode 100644 index 000000000..0a3c9cc1b --- /dev/null +++ b/graph-builder/src/config/mod.rs @@ -0,0 +1,32 @@ +//! Configuration lookup, parsing and validation. +//! +//! This module takes care of sourcing configuration options from +//! multiple inputs (CLI and files), merging, and validating them. +//! It contains the following entities: +//! * "options": configuration fragments (CLI flags, file snippets). +//! * "app settings": runtime settings, result of config validation. + +macro_rules! assign_if_some { + ( $dst:expr, $src:expr ) => {{ + if let Some(x) = $src { + $dst = x.into(); + }; + }}; +} + +mod cli; +mod file; +mod options; +mod settings; + +pub use self::settings::AppSettings; + +/// Merge configuration options into runtime settings. +/// +/// This consumes a generic configuration object, merging its options +/// into runtime settings. It only overlays populated values from config, +/// leaving unset ones preserved as-is from existing settings. +trait MergeOptions { + /// MergeOptions values from `options` into current settings. + fn merge(&mut self, options: T); +} diff --git a/graph-builder/src/config/options.rs b/graph-builder/src/config/options.rs new file mode 100644 index 000000000..45a5c87be --- /dev/null +++ b/graph-builder/src/config/options.rs @@ -0,0 +1,128 @@ +//! Options shared by CLI and TOML. + +use super::{AppSettings, MergeOptions}; +use commons::{parse_params_set, parse_path_prefix}; +use std::collections::HashSet; +use std::net::IpAddr; +use std::path::PathBuf; +use std::time::Duration; + +// TODO(lucab): drop all aliases after staging+production deployments +// have been aligned on new flags. + +/// Status service options. +#[derive(Debug, Deserialize, Serialize, StructOpt)] +pub struct StatusOptions { + /// Address on which the status service will listen + #[structopt(name = "status_address", long = "status.address")] + pub address: Option, + + /// Port to which the status service will bind + #[structopt(name = "status_port", long = "status.port")] + pub port: Option, +} + +/// Options for the main Cincinnati service. +#[derive(Debug, Deserialize, Serialize, StructOpt)] +pub struct ServiceOptions { + /// Address on which the server will listen + #[structopt(name = "service_address", long = "service.address", alias = "address")] + pub address: Option, + + /// Port to which the server will bind + #[structopt(name = "service_port", long = "service.port", alias = "port")] + pub port: Option, + + /// Namespace prefix for all service endpoints (e.g. '//v1/graph') + #[structopt(long = "service.path_prefix", parse(from_str = "parse_path_prefix"))] + pub path_prefix: Option, + + /// Comma-separated set of mandatory client parameters + #[structopt( + long = "service.mandatory_client_parameters", + parse(from_str = "parse_params_set") + )] + pub mandatory_client_parameters: Option>, +} + +/// Options for the Docker-registry-v2 fetcher. +#[derive(Debug, Deserialize, Serialize, StructOpt)] +pub struct DockerRegistryOptions { + /// Duration of the pause (in seconds) between registry scans + #[structopt( + long = "upstream.registry.pause_secs", + parse(try_from_str = "duration_from_secs") + )] + #[serde(default = "Option::default", deserialize_with = "de_duration_secs")] + pub pause_secs: Option, + + /// URL for the container image registry + #[structopt(long = "upstream.registry.url", alias = "registry")] + pub url: Option, + + /// Name of the container image repository + #[structopt(long = "upstream.registry.repository", alias = "repository")] + pub repository: Option, + + /// Credentials file (in "dockercfg" format) for authentication against the image registry + #[structopt( + long = "upstream.registry.credentials_path", + alias = "credentials-file" + )] + pub credentials_path: Option, + + /// Metadata key where to record the manifest-reference + #[structopt(long = "upstream.registry.manifestref_key")] + pub manifestref_key: Option, +} + +impl MergeOptions> for AppSettings { + fn merge(&mut self, opts: Option) -> () { + if let Some(service) = opts { + assign_if_some!(self.address, service.address); + assign_if_some!(self.port, service.port); + assign_if_some!(self.path_prefix, service.path_prefix); + if let Some(params) = service.mandatory_client_parameters { + self.mandatory_client_parameters.extend(params); + } + } + } +} + +impl MergeOptions> for AppSettings { + fn merge(&mut self, opts: Option) -> () { + if let Some(status) = opts { + assign_if_some!(self.status_address, status.address); + assign_if_some!(self.status_port, status.port); + } + } +} + +impl MergeOptions> for AppSettings { + fn merge(&mut self, opts: Option) -> () { + if let Some(registry) = opts { + assign_if_some!(self.pause_secs, registry.pause_secs); + assign_if_some!(self.registry, registry.url); + assign_if_some!(self.repository, registry.repository); + assign_if_some!(self.credentials_path, registry.credentials_path); + assign_if_some!(self.manifestref_key, registry.manifestref_key); + } + } +} + +pub fn de_duration_secs<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + use serde::Deserialize; + let secs = u64::deserialize(deserializer)?; + Ok(Some(Duration::from_secs(secs))) +} + +pub fn duration_from_secs(num: S) -> failure::Fallible +where + S: AsRef, +{ + let secs: u64 = num.as_ref().parse()?; + Ok(Duration::from_secs(secs)) +} diff --git a/graph-builder/src/config/settings.rs b/graph-builder/src/config/settings.rs new file mode 100644 index 000000000..1a7559975 --- /dev/null +++ b/graph-builder/src/config/settings.rs @@ -0,0 +1,94 @@ +use super::MergeOptions; +use super::{cli, file}; +use failure::Fallible; +use std::collections::HashSet; +use std::net::{IpAddr, Ipv4Addr}; +use std::path::PathBuf; +use std::time; +use structopt::StructOpt; + +/// Runtime application settings (validated config). +#[derive(Debug, SmartDefault)] +pub struct AppSettings { + /// Listening address for the main service. + #[default(IpAddr::V4(Ipv4Addr::LOCALHOST))] + pub address: IpAddr, + + /// Optional auth secrets for the registry scraper. + pub credentials_path: Option, + + /// Required client parameters for the main service. + pub mandatory_client_parameters: HashSet, + + /// Metadata key where to record the manifest-reference. + #[default("com.openshift.upgrades.graph.release.manifestref")] + pub manifestref_key: String, + + /// Endpoints namespace for the main service. + pub path_prefix: String, + + /// Pause (in seconds) between registry scrapes. + #[default(time::Duration::from_secs(30))] + pub pause_secs: time::Duration, + + /// Listening port for the main service. + #[default(8080)] + pub port: u16, + + // TODO(lucab): split this in (TLS, hostname+port). + /// Target host for the registry scraper. + #[default("http://localhost:5000")] + pub registry: String, + + /// Target image for the registry scraper. + #[default("openshift")] + pub repository: String, + + /// Listening address for the status service. + #[default(IpAddr::V4(Ipv4Addr::LOCALHOST))] + pub status_address: IpAddr, + + /// Listening port for the status service. + #[default(9080)] + pub status_port: u16, + + /// Global log level. + #[default(log::LevelFilter::Warn)] + pub verbosity: log::LevelFilter, + + // TODO(lucab): drop this when plugins are configurable. + /// Disable quay-metadata (Satellite compat). + #[default(false)] + pub disable_quay_api_metadata: bool, +} + +impl AppSettings { + /// Lookup all optional configs, merge them with defaults, and + /// transform into valid runtime settings. + pub fn assemble() -> Fallible { + // Source options. + let cli_opts = cli::CliOptions::from_args(); + let file_opts = match &cli_opts.config_path { + Some(ref path) => Some(file::FileOptions::read_filepath(path)?), + None => None, + }; + let defaults = Self::default(); + + // Combine options into a single config. + let mut cfg = defaults; + cfg.merge(cli_opts); + cfg.merge(file_opts); + + // Validate and convert to settings. + Self::try_validate(cfg) + } + + /// Validate and build runtime settings. + fn try_validate(self) -> Fallible { + if self.pause_secs.as_secs() == 0 { + bail!("unexpected 0s pause"); + } + + Ok(self) + } +} diff --git a/graph-builder/src/graph.rs b/graph-builder/src/graph.rs index 47940b86b..113efd6cb 100644 --- a/graph-builder/src/graph.rs +++ b/graph-builder/src/graph.rs @@ -88,49 +88,52 @@ impl State { } } -pub fn run<'a>(opts: &'a config::Options, state: &State) -> ! { +pub fn run<'a>(settings: &'a config::AppSettings, state: &State) -> ! { // Grow-only cache, mapping tag (hashed layers) to optional release metadata. let mut cache = HashMap::new(); - let registry = Registry::try_from_str(&opts.registry) - .expect(&format!("failed to parse '{}' as Url", &opts.registry)); + let registry = Registry::try_from_str(&settings.registry) + .expect(&format!("failed to parse '{}' as Url", &settings.registry)); // Read the credentials outside the loop to avoid re-reading the file let (username, password) = - registry::read_credentials(opts.credentials_path.as_ref(), ®istry.host) + registry::read_credentials(settings.credentials_path.as_ref(), ®istry.host) .expect("could not read registry credentials"); - let configured_plugins: Vec>> = - if opts.disable_quay_api_metadata { - debug!("Disabling fetching and processing of quay metadata.."); - vec![] - } else { - use cincinnati::plugins::internal::{ - edge_add_remove::EdgeAddRemovePlugin, metadata_fetch_quay::QuayMetadataFetchPlugin, - node_remove::NodeRemovePlugin, - }; - use cincinnati::plugins::InternalPluginWrapper; - - // TODO(steveeJ): actually make this vec configurable - vec![ - Box::new(InternalPluginWrapper( - QuayMetadataFetchPlugin::try_new( - opts.repository.clone(), - opts.quay_label_filter.clone(), - opts.quay_manifestref_key.clone(), - opts.quay_api_credentials_path.as_ref(), - opts.quay_api_base.clone(), - ) - .expect("could not initialize the QuayMetadataPlugin"), - )), - Box::new(InternalPluginWrapper(NodeRemovePlugin { - key_prefix: opts.quay_label_filter.clone(), - })), - Box::new(InternalPluginWrapper(EdgeAddRemovePlugin { - key_prefix: opts.quay_label_filter.clone(), - })), - ] + let mut configured_plugins: Vec>> = { + use cincinnati::plugins::internal::{ + edge_add_remove::EdgeAddRemovePlugin, metadata_fetch_quay::QuayMetadataFetchPlugin, + node_remove::NodeRemovePlugin, }; + use cincinnati::plugins::{internal, InternalPluginWrapper}; + use quay::v1::DEFAULT_API_BASE; + + // TODO(steveeJ): actually make this vec configurable + vec![ + Box::new(InternalPluginWrapper( + // TODO(lucab): source options from plugins config. + QuayMetadataFetchPlugin::try_new( + settings.repository.clone(), + internal::metadata_fetch_quay::DEFAULT_QUAY_LABEL_FILTER.to_string(), + internal::metadata_fetch_quay::DEFAULT_QUAY_MANIFESTREF_KEY.to_string(), + None, + DEFAULT_API_BASE.to_string(), + ) + .expect("could not initialize the QuayMetadataPlugin"), + )), + Box::new(InternalPluginWrapper(NodeRemovePlugin { + key_prefix: internal::metadata_fetch_quay::DEFAULT_QUAY_LABEL_FILTER.to_string(), + })), + Box::new(InternalPluginWrapper(EdgeAddRemovePlugin { + key_prefix: internal::metadata_fetch_quay::DEFAULT_QUAY_LABEL_FILTER.to_string(), + })), + ] + }; + + // TODO(lucab): drop this when plugins are configurable. + if settings.disable_quay_api_metadata { + configured_plugins = vec![]; + } // Don't wait on the first iteration let mut first_iteration = true; @@ -139,18 +142,18 @@ pub fn run<'a>(opts: &'a config::Options, state: &State) -> ! { if first_iteration { first_iteration = false; } else { - thread::sleep(opts.period); + thread::sleep(settings.pause_secs); } debug!("graph update triggered"); let scrape = registry::fetch_releases( ®istry, - &opts.repository, + &settings.repository, username.as_ref().map(String::as_ref), password.as_ref().map(String::as_ref), &mut cache, - &opts.quay_manifestref_key, + &settings.manifestref_key, ); UPSTREAM_SCRAPES.inc(); @@ -160,7 +163,7 @@ pub fn run<'a>(opts: &'a config::Options, state: &State) -> ! { warn!( "could not find any releases in {}/{}", ®istry.host_port_string(), - &opts.repository + &settings.repository ); }; releases diff --git a/graph-builder/src/lib.rs b/graph-builder/src/lib.rs index 16d96ea0b..f34b02f91 100644 --- a/graph-builder/src/lib.rs +++ b/graph-builder/src/lib.rs @@ -23,9 +23,12 @@ extern crate serde; extern crate serde_derive; extern crate serde_json; #[macro_use] +extern crate smart_default; +#[macro_use] extern crate structopt; extern crate tar; extern crate tokio; +extern crate toml; pub mod config; pub mod graph; diff --git a/graph-builder/src/main.rs b/graph-builder/src/main.rs index e5d9e8749..d04f4ab90 100644 --- a/graph-builder/src/main.rs +++ b/graph-builder/src/main.rs @@ -16,6 +16,7 @@ extern crate actix; extern crate actix_web; extern crate failure; extern crate graph_builder; +#[macro_use] extern crate log; extern crate structopt; @@ -23,47 +24,35 @@ use graph_builder::{config, graph, metrics}; use actix_web::{http::Method, middleware::Logger, server, App}; use failure::Error; -use log::LevelFilter; -use std::{net, thread}; -use structopt::StructOpt; +use std::thread; fn main() -> Result<(), Error> { let sys = actix::System::new("graph-builder"); - let opts = config::Options::from_args(); + let settings = config::AppSettings::assemble()?; env_logger::Builder::from_default_env() - .filter( - Some(module_path!()), - match opts.verbosity { - 0 => LevelFilter::Warn, - 1 => LevelFilter::Info, - 2 => LevelFilter::Debug, - _ => LevelFilter::Trace, - }, - ) + .filter(Some(module_path!()), settings.verbosity) .init(); + debug!("application settings:\n{:#?}", &settings); - let state = graph::State::new(opts.mandatory_client_parameters.clone()); - let addr = (opts.address, opts.port); - let app_prefix = opts.path_prefix.clone(); + let state = graph::State::new(settings.mandatory_client_parameters.clone()); + let service_addr = (settings.address, settings.port); + let status_addr = (settings.status_address, settings.status_port); + let app_prefix = settings.path_prefix.clone(); { let state = state.clone(); - thread::spawn(move || graph::run(&opts, &state)); + thread::spawn(move || graph::run(&settings, &state)); } - // TODO(lucab): make these configurable. - let status_address: net::IpAddr = net::Ipv4Addr::UNSPECIFIED.into(); - let status_port = 9080; - // Status service. server::new(|| { App::new() .middleware(Logger::default()) .route("/metrics", Method::GET, metrics::serve) }) - .bind((status_address, status_port))? + .bind(status_addr)? .start(); // Main service. @@ -75,7 +64,7 @@ fn main() -> Result<(), Error> { .prefix(app_prefix) .route("/v1/graph", Method::GET, graph::index) }) - .bind(addr)? + .bind(service_addr)? .start(); sys.run(); diff --git a/graph-builder/tests/fixtures/sample-config.toml b/graph-builder/tests/fixtures/sample-config.toml new file mode 100644 index 000000000..70b0edb07 --- /dev/null +++ b/graph-builder/tests/fixtures/sample-config.toml @@ -0,0 +1,16 @@ +verbosity = 3 + +[upstream] +method = "registry" + +[upstream.registry] +pause_secs = 35 +url = "quay.io" +repository = "openshift-release-dev/ocp-release" + +[service] +address = "0.0.0.0" +port = 8383 + +[status] +address = "127.0.0.1"