diff --git a/Cargo.lock b/Cargo.lock index cf75c9f68..2b7066c09 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -295,6 +295,7 @@ dependencies = [ "serde_derive 1.0.84 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.34 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", + "toml 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", "try_from 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -683,6 +684,7 @@ dependencies = [ "structopt 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)", "tar 0.4.20 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", + "toml 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -2009,6 +2011,14 @@ dependencies = [ "tokio-reactor 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "toml" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "serde 1.0.84 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "tower-service" version = "0.1.0" @@ -2538,6 +2548,7 @@ dependencies = [ "checksum tokio-timer 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "4f37f0111d76cc5da132fe9bc0590b9b9cfd079bc7e75ac3846278430a299ff8" "checksum tokio-udp 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "66268575b80f4a4a710ef83d087fdfeeabdce9b74c797535fbac18a2cb906e92" "checksum tokio-uds 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "037ffc3ba0e12a0ab4aca92e5234e0dedeb48fddf6ccd260f1f150a36a9f2445" +"checksum toml 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)" = "758664fc71a3a69038656bee8b6be6477d2a6c315a6b81f7081f591bffa4111f" "checksum tower-service 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b32f72af77f1bfe3d3d4da8516a238ebe7039b51dd8637a09841ac7f16d2c987" "checksum trust-dns-proto 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0838272e89f1c693b4df38dc353412e389cf548ceed6f9fd1af5a8d6e0e7cf74" "checksum trust-dns-proto 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e33f29df428f112ffeda24b328b814b61d6916be29aa89f19bc3f684ba5437b8" diff --git a/cincinnati/Cargo.toml b/cincinnati/Cargo.toml index 80d7cbdbc..5bac30c05 100644 --- a/cincinnati/Cargo.toml +++ b/cincinnati/Cargo.toml @@ -15,6 +15,7 @@ url = "^1.7.2" log = "^0.4.3" futures = "0.1" tokio = "0.1" +toml = "^0.4.10" try_from = "0.3.2" [dev-dependencies] diff --git a/cincinnati/src/lib.rs b/cincinnati/src/lib.rs index 4df942e55..129202204 100644 --- a/cincinnati/src/lib.rs +++ b/cincinnati/src/lib.rs @@ -22,6 +22,7 @@ extern crate commons; #[macro_use] extern crate log; extern crate protobuf; +extern crate toml; extern crate try_from; extern crate url; diff --git a/cincinnati/src/plugins/internal/edge_add_remove.rs b/cincinnati/src/plugins/internal/edge_add_remove.rs index a38dde378..5530abb75 100644 --- a/cincinnati/src/plugins/internal/edge_add_remove.rs +++ b/cincinnati/src/plugins/internal/edge_add_remove.rs @@ -2,9 +2,12 @@ use crate as cincinnati; use failure::Fallible; -use plugins::{InternalIO, InternalPlugin}; +use plugins::{InternalIO, InternalPlugin, InternalPluginWrapper, Plugin, PluginIO}; +use std::collections::HashMap; use ReleaseId; +static DEFAULT_KEY_FILTER: &str = "com.openshift.upgrades.graph"; + pub struct EdgeAddRemovePlugin { pub key_prefix: String, } @@ -25,6 +28,32 @@ impl InternalPlugin for EdgeAddRemovePlugin { /// /// The labels are assumed to have the syntax `.(previous|next).(remove|add)=(,)*` impl EdgeAddRemovePlugin { + /// Plugin name, for configuration. + pub(crate) const PLUGIN_NAME: &'static str = "edge-add-remove"; + + /// Validate plugin configuration and fill in defaults. + pub fn sanitize_config(cfg: &mut HashMap) -> Fallible<()> { + let name = cfg.get("name").cloned().unwrap_or_default(); + ensure!(name == Self::PLUGIN_NAME, "unexpected plugin name"); + + cfg.entry("key_prefix".to_string()) + .or_insert_with(|| DEFAULT_KEY_FILTER.to_string()); + // TODO(lucab): perform semantic validation. + + Ok(()) + } + + /// Try to build a plugin from configuration. + pub fn from_settings(cfg: &HashMap) -> Fallible>> { + let key_prefix = cfg + .get("key_prefix") + .ok_or_else(|| format_err!("empty key_prefix"))? + .to_string(); + let plugin = Self { key_prefix }; + + Ok(Box::new(InternalPluginWrapper(plugin))) + } + /// Remove next and previous releases from quay labels /// /// The labels are assumed to have the syntax `.(previous|next).remove=(,)*` diff --git a/cincinnati/src/plugins/internal/metadata_fetch_quay.rs b/cincinnati/src/plugins/internal/metadata_fetch_quay.rs index 7e6088bd4..f315f6b78 100644 --- a/cincinnati/src/plugins/internal/metadata_fetch_quay.rs +++ b/cincinnati/src/plugins/internal/metadata_fetch_quay.rs @@ -10,12 +10,14 @@ extern crate tokio; use self::futures::future::Future; use failure::{Fallible, ResultExt}; -use plugins::{InternalIO, InternalPlugin}; -use std::path::PathBuf; +use plugins::{InternalIO, InternalPlugin, InternalPluginWrapper, Plugin, PluginIO}; +use std::collections::HashMap; +use toml::value::Value; use ReleaseId; pub static DEFAULT_QUAY_LABEL_FILTER: &str = "com.openshift.upgrades.graph"; pub static DEFAULT_QUAY_MANIFESTREF_KEY: &str = "com.openshift.upgrades.graph.release.manifestref"; +pub static DEFAULT_QUAY_REPOSITORY: &str = "openshift"; pub struct QuayMetadataFetchPlugin { client: quay::v1::Client, @@ -24,6 +26,82 @@ pub struct QuayMetadataFetchPlugin { manifestref_key: String, } +/// Plugin settings. +#[derive(Deserialize)] +struct PluginSettings { + api_base: String, + api_credentials_path: Option, + repository: String, + label_filter: String, + manifestref_key: String, +} + +impl QuayMetadataFetchPlugin { + /// Plugin name, for configuration. + pub(crate) const PLUGIN_NAME: &'static str = "quay-metadata"; + + /// Validate plugin configuration and fill in defaults. + pub fn sanitize_config(cfg: &mut HashMap) -> Fallible<()> { + let name = cfg.get("name").cloned().unwrap_or_default(); + ensure!(name == Self::PLUGIN_NAME, "unexpected plugin name"); + + cfg.entry("api_base".to_string()) + .or_insert_with(|| quay::v1::DEFAULT_API_BASE.to_string()); + cfg.entry("repository".to_string()) + .or_insert_with(|| DEFAULT_QUAY_REPOSITORY.to_string()); + cfg.entry("label_filter".to_string()) + .or_insert_with(|| DEFAULT_QUAY_LABEL_FILTER.to_string()); + cfg.entry("manifestref_key".to_string()) + .or_insert_with(|| DEFAULT_QUAY_MANIFESTREF_KEY.to_string()); + // TODO(lucab): perform validation. + + Ok(()) + } + + /// Try to build a plugin from settings. + pub fn from_settings(cfg: &HashMap) -> Fallible>> { + let cfg: PluginSettings = Value::try_from(cfg)?.try_into()?; + + let plugin = Self::try_new( + cfg.repository, + cfg.label_filter, + cfg.manifestref_key, + cfg.api_credentials_path, + cfg.api_base, + )?; + Ok(Box::new(InternalPluginWrapper(plugin))) + } + + fn try_new( + repo: String, + label_filter: String, + manifestref_key: String, + api_token_path: Option, + api_base: String, + ) -> Fallible { + let api_token = match api_token_path { + Some(p) => { + let token = + quay::read_credentials(p).context("could not read quay API credentials")?; + Some(token) + } + None => None, + }; + + let client: quay::v1::Client = quay::v1::Client::builder() + .access_token(api_token) + .api_base(Some(api_base)) + .build()?; + + Ok(Self { + client, + repo, + label_filter, + manifestref_key, + }) + } +} + impl InternalPlugin for QuayMetadataFetchPlugin { fn run_internal(&self, io: InternalIO) -> Fallible { let (mut graph, parameters) = (io.graph, io.parameters); @@ -100,28 +178,3 @@ impl InternalPlugin for QuayMetadataFetchPlugin { Ok(InternalIO { graph, parameters }) } } - -impl QuayMetadataFetchPlugin { - pub fn try_new( - repo: String, - label_filter: String, - manifestref_key: String, - api_token_path: Option<&PathBuf>, - api_base: String, - ) -> Fallible { - let api_token = - quay::read_credentials(api_token_path).expect("could not read quay API credentials"); - - let client: quay::v1::Client = quay::v1::Client::builder() - .access_token(api_token.map(|s| s.to_string())) - .api_base(Some(api_base.to_string())) - .build()?; - - Ok(Self { - client, - repo, - label_filter, - manifestref_key, - }) - } -} diff --git a/cincinnati/src/plugins/internal/node_remove.rs b/cincinnati/src/plugins/internal/node_remove.rs index 54ed630a0..6e5c6a812 100644 --- a/cincinnati/src/plugins/internal/node_remove.rs +++ b/cincinnati/src/plugins/internal/node_remove.rs @@ -1,12 +1,43 @@ //! This plugin removes releases according to its metadata use failure::Fallible; -use plugins::{InternalIO, InternalPlugin}; +use plugins::{InternalIO, InternalPlugin, InternalPluginWrapper, Plugin, PluginIO}; +use std::collections::HashMap; + +static DEFAULT_KEY_FILTER: &str = "com.openshift.upgrades.graph"; pub struct NodeRemovePlugin { pub key_prefix: String, } +impl NodeRemovePlugin { + /// Plugin name, for configuration. + pub(crate) const PLUGIN_NAME: &'static str = "node-remove"; + + /// Validate plugin configuration and fill in defaults. + pub fn sanitize_config(cfg: &mut HashMap) -> Fallible<()> { + let name = cfg.get("name").cloned().unwrap_or_default(); + ensure!(name == Self::PLUGIN_NAME, "unexpected plugin name"); + + cfg.entry("key_prefix".to_string()) + .or_insert_with(|| DEFAULT_KEY_FILTER.to_string()); + // TODO(lucab): perform semantic validation. + + Ok(()) + } + + /// Try to build a plugin from settings. + pub fn from_settings(cfg: &HashMap) -> Fallible>> { + let key_prefix = cfg + .get("key_prefix") + .ok_or_else(|| format_err!("empty key_prefix"))? + .to_string(); + let plugin = Self { key_prefix }; + + Ok(Box::new(InternalPluginWrapper(plugin))) + } +} + impl InternalPlugin for NodeRemovePlugin { fn run_internal(&self, io: InternalIO) -> Fallible { let mut graph = io.graph; diff --git a/cincinnati/src/plugins/mod.rs b/cincinnati/src/plugins/mod.rs index dbea534a9..489a2b09a 100644 --- a/cincinnati/src/plugins/mod.rs +++ b/cincinnati/src/plugins/mod.rs @@ -4,7 +4,9 @@ pub mod external; pub mod interface; pub mod internal; +mod registry; +pub use self::registry::{sanitize_config, try_from_settings}; use crate as cincinnati; use failure::{Error, Fallible, ResultExt}; use plugins::interface::{PluginError, PluginExchange}; diff --git a/cincinnati/src/plugins/registry.rs b/cincinnati/src/plugins/registry.rs new file mode 100644 index 000000000..6c06b0c1b --- /dev/null +++ b/cincinnati/src/plugins/registry.rs @@ -0,0 +1,35 @@ +//! Plugin registry. +//! +//! This registry relies on a static list of all available plugin, +//! referenced by name. It is used for configuration purposes. + +#![allow(clippy::implicit_hasher)] + +use super::internal::edge_add_remove::EdgeAddRemovePlugin; +use super::internal::metadata_fetch_quay::QuayMetadataFetchPlugin; +use super::internal::node_remove::NodeRemovePlugin; +use super::{Plugin, PluginIO}; +use failure::Fallible; +use std::collections::HashMap; + +/// Validate configuration for a plugin and fill in defaults. +pub fn sanitize_config(cfg: &mut HashMap) -> Fallible<()> { + let kind = cfg.get("name").cloned().unwrap_or_default(); + match kind.as_str() { + EdgeAddRemovePlugin::PLUGIN_NAME => EdgeAddRemovePlugin::sanitize_config(cfg), + NodeRemovePlugin::PLUGIN_NAME => NodeRemovePlugin::sanitize_config(cfg), + QuayMetadataFetchPlugin::PLUGIN_NAME => QuayMetadataFetchPlugin::sanitize_config(cfg), + x => bail!("unknown plugin '{}'", x), + } +} + +/// Try to build a plugin from runtime settings. +pub fn try_from_settings(settings: &HashMap) -> Fallible>> { + let kind = settings.get("name").cloned().unwrap_or_default(); + match kind.as_str() { + EdgeAddRemovePlugin::PLUGIN_NAME => EdgeAddRemovePlugin::from_settings(settings), + NodeRemovePlugin::PLUGIN_NAME => NodeRemovePlugin::from_settings(settings), + QuayMetadataFetchPlugin::PLUGIN_NAME => QuayMetadataFetchPlugin::from_settings(settings), + x => bail!("unknown plugin '{}'", x), + } +} diff --git a/dist/openshift/cincinnati.configmap.yaml b/dist/openshift/cincinnati.configmap.yaml index f3f2a9656..d1640195c 100644 --- a/dist/openshift/cincinnati.configmap.yaml +++ b/dist/openshift/cincinnati.configmap.yaml @@ -5,6 +5,7 @@ metadata: type: Opaque data: gb.address: "0.0.0.0" + gb.status.address: "0.0.0.0" gb.registry: "quay.io" gb.repository: "steveej/cincinnati-test" gb.log.verbosity: "vvv" diff --git a/dist/openshift/cincinnati.yaml b/dist/openshift/cincinnati.yaml index 601d77c2b..12e604104 100644 --- a/dist/openshift/cincinnati.yaml +++ b/dist/openshift/cincinnati.yaml @@ -39,6 +39,11 @@ objects: configMapKeyRef: key: gb.address name: cincinnati + - name: STATUS_ADDRESS + valueFrom: + configMapKeyRef: + key: gb.status.address + name: cincinnati - name: REGISTRY valueFrom: configMapKeyRef: @@ -54,7 +59,14 @@ objects: configMapKeyRef: key: gb.log.verbosity name: cincinnati - args: ["-$(GB_LOG_VERBOSITY)", "--address", "$(ADDRESS)", "--port", "${GB_PORT}", "--registry", "$(REGISTRY)", "--repository", "$(REPOSITORY)", "--credentials-file=/etc/secrets/registry-credentials"] + args: ["-$(GB_LOG_VERBOSITY)", + "--service.address", "$(ADDRESS)", + "--service.port", "${GB_PORT}", + "--status.address", "$(STATUS_ADDRESS)", + "--status.port", "${GB_STATUS_PORT}", + "--upstream.registry.url", "$(REGISTRY)", + "--upstream.registry.repository", "$(REPOSITORY)", + "--upstream.registry.credentials_path", "/etc/secrets/registry-credentials"] ports: - name: graph-builder containerPort: ${{GB_PORT}} diff --git a/graph-builder/Cargo.toml b/graph-builder/Cargo.toml index 8ae3e7467..c1adc8d03 100644 --- a/graph-builder/Cargo.toml +++ b/graph-builder/Cargo.toml @@ -27,6 +27,7 @@ serde_json = "^1.0.22" structopt = "^0.2.10" tar = "^0.4.16" tokio = "0.1" +toml = "^0.4.10" url = "^1.7.2" [features] diff --git a/graph-builder/src/config.rs b/graph-builder/src/config.rs deleted file mode 100644 index 2523a6e72..000000000 --- a/graph-builder/src/config.rs +++ /dev/null @@ -1,110 +0,0 @@ -// Copyright 2018 Alex Crawford -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use cincinnati::plugins::internal::metadata_fetch_quay::{ - DEFAULT_QUAY_LABEL_FILTER, DEFAULT_QUAY_MANIFESTREF_KEY, -}; -use commons::{parse_params_set, parse_path_prefix}; -use quay::v1::DEFAULT_API_BASE; -use std::collections::HashSet; -use std::net::IpAddr; -use std::num::ParseIntError; -use std::path::PathBuf; -use std::str::FromStr; -use std::time::Duration; - -#[derive(Debug, StructOpt)] -pub struct Options { - /// Verbosity level - #[structopt(short = "v", parse(from_occurrences))] - pub verbosity: u64, - - /// URL for the container image registry - #[structopt(long = "registry", default_value = "http://localhost:5000")] - pub registry: String, - - /// Name of the container image repository - #[structopt(long = "repository", default_value = "openshift")] - pub repository: String, - - /// Duration of the pause (in seconds) between scans of the registry - #[structopt( - long = "period", - default_value = "30", - parse(try_from_str = "parse_duration") - )] - pub period: Duration, - - /// Address on which the server will listen - #[structopt(long = "address", default_value = "127.0.0.1")] - pub address: IpAddr, - - /// Port to which the server will bind - #[structopt(long = "port", default_value = "8080")] - pub port: u16, - - /// Credentials file for authentication against the image registry - #[structopt(long = "credentials-file", parse(from_os_str))] - pub credentials_path: Option, - - /// Path prefix for all paths. - #[structopt( - long = "path-prefix", - default_value = "", - parse(from_str = "parse_path_prefix") - )] - pub path_prefix: String, - - /// Comma-separated set of mandatory client parameters. - #[structopt( - long = "mandatory-client-parameters", - default_value = "", - parse(from_str = "parse_params_set") - )] - pub mandatory_client_parameters: HashSet, - - /// Whether to disable the fetching and processing metadata from the quay API - #[structopt(long = "disable-quay-api-metadata")] - pub disable_quay_api_metadata: bool, - - /// Base URL to the quay API host - #[structopt( - long = "quay-api-base", - long_help = "API base URL", - raw(default_value = "DEFAULT_API_BASE") - )] - pub quay_api_base: String, - - /// Filter for receiving quay labels - #[structopt( - long = "quay-label-filter", - raw(default_value = "DEFAULT_QUAY_LABEL_FILTER") - )] - pub quay_label_filter: String, - - /// Metadata key where the quay fetcher expects the manifestref - #[structopt( - long = "quay-manifestref-key", - raw(default_value = "DEFAULT_QUAY_MANIFESTREF_KEY") - )] - pub quay_manifestref_key: String, - - /// Credentials file for authentication against API described at https://docs.quay.io/api/ - #[structopt(long = "quay-api-credentials-path", parse(from_os_str))] - pub quay_api_credentials_path: Option, -} - -fn parse_duration(src: &str) -> Result { - Ok(Duration::from_secs(u64::from_str(src)?)) -} diff --git a/graph-builder/src/config/cli.rs b/graph-builder/src/config/cli.rs new file mode 100644 index 000000000..c6c2a933c --- /dev/null +++ b/graph-builder/src/config/cli.rs @@ -0,0 +1,83 @@ +//! Command-line options. + +// TODO(lucab): drop all aliases after migration. + +/// CLI configuration flags, top-level. +#[derive(Debug, StructOpt)] +pub struct CliOptions { + /// Verbosity level + #[structopt(long = "verbosity", short = "v", parse(from_occurrences))] + pub verbosity: u8, + + /// Path to configuration file + #[structopt(short = "c")] + pub config_path: Option, + + #[structopt(flatten)] + pub service: ServiceOptions, + + #[structopt(flatten)] + pub status: StatusOptions, + + #[structopt(flatten)] + pub registry: UpstreamRegistryOptions, +} + +/// CLI configuration flags, status service. +#[derive(Debug, StructOpt)] +pub struct StatusOptions { + /// Address on which the status service will listen + #[structopt(long = "status.address")] + pub address: Option, + + /// Port to which the status service will bind + #[structopt(long = "status.port")] + pub port: Option, +} + +/// CLI configuration flags, main Cincinnati service. +#[derive(Debug, StructOpt)] +pub struct ServiceOptions { + /// Address on which the server will listen + #[structopt(long = "service.address", alias = "address")] + pub address: Option, + + /// Port to which the server will bind + #[structopt(long = "service.port", alias = "port")] + pub port: Option, + + /// Namespace prefix for all service endpoints + #[structopt(long = "service.path_prefix")] + pub path_prefix: Option, + + /// Comma-separated set of mandatory client parameters + #[structopt(long = "service.mandatory_client_parameters")] + pub mandatory_client_parameters: Option, +} + +/// CLI configuration flags, Docker-registry fetcher. +#[derive(Debug, StructOpt)] +pub struct UpstreamRegistryOptions { + /// Duration of the pause (in seconds) between registry scans + #[structopt(long = "upstream.registry.period")] + pub period: Option, + + /// URL for the container image registry + #[structopt(long = "upstream.registry.url", alias = "registry")] + pub url: Option, + + /// Name of the container image repository + #[structopt(long = "upstream.registry.repository", alias = "repository")] + pub repository: Option, + + /// Credentials file for authentication against the image registry + #[structopt( + long = "upstream.registry.credentials_path", + alias = "credentials-file" + )] + pub credentials_path: Option, + + /// Metadata key where to record the manifest-reference + #[structopt(long = "upstream.registry.manifestref_key")] + pub manifestref_key: Option, +} diff --git a/graph-builder/src/config/file.rs b/graph-builder/src/config/file.rs new file mode 100644 index 000000000..d699293e8 --- /dev/null +++ b/graph-builder/src/config/file.rs @@ -0,0 +1,99 @@ +//! TOML file configuration options. + +use failure::{Fallible, ResultExt}; +use std::collections::{HashMap, HashSet}; +use std::io::Read; +use std::{fs, io, path}; + +/// TOML configuration, top-level. +#[derive(Debug, Deserialize)] +pub struct FileOptions { + /// Verbosity level. + pub verbosity: Option, + + /// Upstream options. + pub upstream: Option, + + /// Web frontend options. + pub service: Option, + + /// Status service options. + pub status: Option, + + /// Plugins ordering and options. + pub plugins: Option>>, +} + +impl FileOptions { + pub fn read_filepath>(cfg_path: P) -> Fallible { + let cfg_file = fs::File::open(&cfg_path).context(format!( + "failed to open config path {:?}", + cfg_path.as_ref() + ))?; + let mut bufrd = io::BufReader::new(cfg_file); + + let mut content = vec![]; + bufrd.read_to_end(&mut content)?; + let cfg = toml::from_slice(&content).context(format!( + "failed to read config file {}", + cfg_path.as_ref().display() + ))?; + + Ok(cfg) + } +} + +/// TOML configuration, upstream fetcher. +#[derive(Debug, Deserialize)] +pub struct UpstreamOptions { + /// Fetcher method. + pub method: Option, + + /// Docker-registry v2 upstream options. + pub registry: Option, +} + +/// CLI configuration flags, HTTP frontend serving Cincinnati. +#[derive(Debug, Deserialize)] +pub struct ServiceOptions { + /// Address on which the server will listen + pub address: Option, + + /// Port to which the server will bind + pub port: Option, + + /// Path prefix for all paths. + pub path_prefix: Option, + + /// Comma-separated set of mandatory client parameters. + pub mandatory_client_parameters: Option>, +} + +/// TOML configuration, status service. +#[derive(Debug, Deserialize)] +pub struct StatusOptions { + /// Address on which the status service will listen + pub address: Option, + + /// Port to which the status service will bind + pub port: Option, +} + +/// TOML configuration, Docker-v2 registry fetcher. +#[derive(Debug, Deserialize)] +pub struct RegistryOptions { + /// Duration of the pause (in seconds) between registry scans. + pub period: Option, + + /// URL for the container image registry. + pub url: Option, + + /// Name of the container image repository. + pub repository: Option, + + /// Credentials file for authentication against the image registry. + pub credentials_path: Option, + + /// Metadata key where to record the manifest-reference. + pub manifestref_key: Option, +} diff --git a/graph-builder/src/config/mod.rs b/graph-builder/src/config/mod.rs new file mode 100644 index 000000000..e6a586e56 --- /dev/null +++ b/graph-builder/src/config/mod.rs @@ -0,0 +1,15 @@ +//! Configuration lookup, parsing and validation. +//! +//! This module takes care of sourcing configuration options from +//! multiple inputs (CLI and files), merging, and validating them. +//! It contains the following entities: +//! * "options": configuration fragments (CLI flags, file snippets), optional and stringly-typed. +//! * "unified config": configuration document, result of merging all options and defaults. +//! * "app settings": runtime settings, result of config validation. + +mod cli; +mod file; +mod settings; +mod unified; + +pub use self::settings::AppSettings; diff --git a/graph-builder/src/config/settings.rs b/graph-builder/src/config/settings.rs new file mode 100644 index 000000000..4e7c420b2 --- /dev/null +++ b/graph-builder/src/config/settings.rs @@ -0,0 +1,131 @@ +use super::{cli, file, unified}; +use cincinnati::plugins; +use failure::Fallible; +use std::collections::{HashMap, HashSet}; +use std::net::IpAddr; +use std::path::PathBuf; +use std::str::FromStr; + +/// Runtime application settings (validated config). +#[derive(Debug)] +pub struct AppSettings { + /// Listening address for the main service. + pub address: IpAddr, + /// Optional auth secrets for the registry scraper. + pub credentials_path: Option, + /// Required client parameters for the main service. + pub mandatory_client_parameters: HashSet, + /// Metadata key where to record the manifest-reference. + pub manifestref_key: String, + /// Endpoints namespace for the main service. + pub path_prefix: String, + /// Polling period for the registry scraper. + pub period: std::time::Duration, + /// Plugins configuration. + pub plugins: Vec>, + /// Listening port for the main service. + pub port: u16, + // TODO(lucab): split this in (TLS, hostname+port). + /// Target host for the registry scraper. + pub registry: String, + /// Target image for the registry scraper. + pub repository: String, + /// Listening address for the status service. + pub status_address: IpAddr, + /// Listening port for the status service. + pub status_port: u16, + /// Global log level. + pub verbosity: log::LevelFilter, +} + +impl AppSettings { + /// Lookup all optional configs, merge them with defaults, and + /// transform into valid runtime settings. + pub fn assemble() -> Fallible { + use structopt::StructOpt; + + let default_opts = unified::UnifiedConfig::default(); + let cli_opts = cli::CliOptions::from_args(); + + let file_opts = match &cli_opts.config_path { + Some(ref path) => Some(file::FileOptions::read_filepath(path)?), + None => None, + }; + + let merged_config = default_opts + .merge_file_config(file_opts)? + .merge_cli_config(cli_opts)?; + + Self::validate_config(merged_config) + } + + /// Try to validate configuration and build application settings from it. + fn validate_config(cfg: unified::UnifiedConfig) -> Fallible { + let service_address = IpAddr::from_str(&cfg.address)?; + let status_address = IpAddr::from_str(&cfg.status_address)?; + + let credentials_path = match cfg.credentials_path.as_str() { + "" => None, + s => Some(PathBuf::from(s)), + }; + + let verbosity = match cfg.verbosity { + 0 => log::LevelFilter::Warn, + 1 => log::LevelFilter::Info, + 2 => log::LevelFilter::Debug, + _ => log::LevelFilter::Trace, + }; + + let period = match cfg.period { + 0 => bail!("unexpected 0s refresh period"), + period => std::time::Duration::from_secs(period), + }; + + let mut plugins = cfg.plugins; + // TODO(lucab): drop default plugins after migration. + compat_hardcoded_plugins(&mut plugins); + for plugin_cfg in &mut plugins { + plugins::sanitize_config(plugin_cfg)?; + } + + let opts = Self { + address: service_address, + credentials_path, + mandatory_client_parameters: cfg.mandatory_client_parameters, + manifestref_key: cfg.manifestref_key, + path_prefix: cfg.path_prefix, + period, + plugins, + port: cfg.port, + registry: cfg.registry_url, + repository: cfg.repository, + status_address, + status_port: cfg.status_port, + verbosity, + }; + Ok(opts) + } +} + +// Fill-in hardcoded default plugins for retro-compatibility. +fn compat_hardcoded_plugins(plugins: &mut Vec>) { + if !plugins.is_empty() { + return; + } + + let mut quay_meta_fetch = HashMap::new(); + quay_meta_fetch.insert("name".to_string(), "quay-metadata".to_string()); + quay_meta_fetch.insert( + "repository".to_string(), + "openshift-release-dev/ocp-release".to_string(), + ); + plugins.push(quay_meta_fetch); + + let mut node_remove = HashMap::new(); + node_remove.insert("name".to_string(), "node-remove".to_string()); + plugins.push(node_remove); + + let mut edge_add_remove = HashMap::new(); + edge_add_remove.insert("name".to_string(), "edge-add-remove".to_string()); + plugins.push(edge_add_remove); +} diff --git a/graph-builder/src/config/unified.rs b/graph-builder/src/config/unified.rs new file mode 100644 index 000000000..628e64874 --- /dev/null +++ b/graph-builder/src/config/unified.rs @@ -0,0 +1,156 @@ +//! Unified configuration. + +use super::cli::CliOptions; +use super::file::FileOptions; +use failure::Fallible; +use std::collections::{HashMap, HashSet}; + +static DEFAULT_VERBOSITY: u8 = 0; + +static DEFAULT_SERV_ADDR: &str = "127.0.0.1"; +static DEFAULT_SERV_PORT: u16 = 8080; +static DEFAULT_SERV_PREFIX: &str = ""; + +static DEFAULT_REGISTRY_PERIOD: u64 = 30; +static DEFAULT_REGISTRY_CREDENTIALS_PATH: &str = ""; +static DEFAULT_REGISTRY_MANIFESTREF_KEY: &str = "com.openshift.upgrades.graph.release.manifestref"; +static DEFAULT_REGISTRY_URL: &str = "http://localhost:5000"; +static DEFAULT_REGISTRY_REPO: &str = "openshift"; + +static DEFAULT_STATUS_ADDR: &str = "127.0.0.1"; +static DEFAULT_STATUS_PORT: u16 = 9080; + +macro_rules! maybe_assign { + ( $dst:expr, $src:expr ) => {{ + if let Some(x) = $src { + $dst = x; + }; + }}; +} + +/// Top-level configuration (before semantic validation). +#[derive(Debug)] +pub struct UnifiedConfig { + pub address: String, + pub credentials_path: String, + pub mandatory_client_parameters: HashSet, + pub manifestref_key: String, + pub path_prefix: String, + pub period: u64, + pub plugins: Vec>, + pub port: u16, + pub registry_url: String, + pub repository: String, + pub status_address: String, + pub status_port: u16, + pub verbosity: u8, +} + +impl Default for UnifiedConfig { + fn default() -> Self { + Self { + address: DEFAULT_SERV_ADDR.to_string(), + credentials_path: DEFAULT_REGISTRY_CREDENTIALS_PATH.to_string(), + mandatory_client_parameters: vec![].into_iter().collect(), + manifestref_key: DEFAULT_REGISTRY_MANIFESTREF_KEY.to_string(), + path_prefix: DEFAULT_SERV_PREFIX.to_string(), + period: DEFAULT_REGISTRY_PERIOD, + plugins: vec![], + port: DEFAULT_SERV_PORT, + registry_url: DEFAULT_REGISTRY_URL.to_string(), + repository: DEFAULT_REGISTRY_REPO.to_string(), + status_address: DEFAULT_STATUS_ADDR.to_string(), + status_port: DEFAULT_STATUS_PORT, + verbosity: DEFAULT_VERBOSITY, + } + } +} + +impl UnifiedConfig { + /// Merge command-line options into unified configuration. + pub(crate) fn merge_cli_config(self, cfg: CliOptions) -> Fallible { + let mut merged_cfg = self; + + // Top-level options. + if cfg.verbosity > 0 { + merged_cfg.verbosity = cfg.verbosity; + } + + // Main service options. + maybe_assign!(merged_cfg.address, cfg.service.address); + maybe_assign!(merged_cfg.port, cfg.service.port); + maybe_assign!(merged_cfg.path_prefix, cfg.service.path_prefix); + if let Some(params) = cfg.service.mandatory_client_parameters { + merged_cfg.mandatory_client_parameters = commons::parse_params_set(¶ms); + } + + // Status service options. + maybe_assign!(merged_cfg.status_address, cfg.status.address); + maybe_assign!(merged_cfg.status_port, cfg.status.port); + + // Registry upstream scraper options. + maybe_assign!(merged_cfg.period, cfg.registry.period); + maybe_assign!(merged_cfg.registry_url, cfg.registry.url); + maybe_assign!(merged_cfg.repository, cfg.registry.repository); + maybe_assign!(merged_cfg.credentials_path, cfg.registry.credentials_path); + maybe_assign!(merged_cfg.manifestref_key, cfg.registry.manifestref_key); + + Ok(merged_cfg) + } + + /// Merge TOML options into unified configuration. + pub(crate) fn merge_file_config(self, file_cfg: Option) -> Fallible { + let mut merged_cfg = self; + + // Eaarly-return without updates if there is no configuration file. + let cfg = match file_cfg { + Some(c) => c, + None => return Ok(merged_cfg), + }; + + // Top-level options. + maybe_assign!(merged_cfg.verbosity, cfg.verbosity); + + // Main service options. + if let Some(service) = cfg.service { + maybe_assign!(merged_cfg.address, service.address); + maybe_assign!(merged_cfg.port, service.port); + maybe_assign!(merged_cfg.path_prefix, service.path_prefix); + + if let Some(params) = service.mandatory_client_parameters { + merged_cfg.mandatory_client_parameters.extend(params); + } + } + + // Registry upstream scraper options. + if let Some(upstream) = cfg.upstream { + // TODO(lucab): drop once fedora-coreos usptream is implemented. + if let Some(method) = upstream.method { + ensure!(method == "registry", "unknown upstream method"); + } + + if let Some(registry) = upstream.registry { + maybe_assign!(merged_cfg.credentials_path, registry.credentials_path); + maybe_assign!(merged_cfg.period, registry.period); + maybe_assign!(merged_cfg.registry_url, registry.url); + maybe_assign!(merged_cfg.repository, registry.repository); + maybe_assign!(merged_cfg.manifestref_key, registry.manifestref_key); + } + } + + // Status service options. + if let Some(status) = cfg.status { + maybe_assign!(merged_cfg.status_address, status.address); + maybe_assign!(merged_cfg.status_port, status.port); + } + + // Plugins options. Order is relevant too. + if let Some(plugins) = cfg.plugins { + for entry in plugins { + merged_cfg.plugins.push(entry); + } + } + + Ok(merged_cfg) + } +} diff --git a/graph-builder/src/graph.rs b/graph-builder/src/graph.rs index a32f1a4c0..072e571ec 100644 --- a/graph-builder/src/graph.rs +++ b/graph-builder/src/graph.rs @@ -88,7 +88,7 @@ impl State { } } -pub fn run<'a>(opts: &'a config::Options, state: &State) -> ! { +pub fn run<'a>(opts: &'a config::AppSettings, state: &State) -> ! { // Grow-only cache, mapping tag (hashed layers) to optional release metadata. let mut cache = HashMap::new(); @@ -100,37 +100,12 @@ pub fn run<'a>(opts: &'a config::Options, state: &State) -> ! { registry::read_credentials(opts.credentials_path.as_ref(), ®istry.host) .expect("could not read registry credentials"); - let configured_plugins: Vec>> = - if opts.disable_quay_api_metadata { - debug!("Disabling fetching and processing of quay metadata.."); - vec![] - } else { - use cincinnati::plugins::internal::{ - edge_add_remove::EdgeAddRemovePlugin, metadata_fetch_quay::QuayMetadataFetchPlugin, - node_remove::NodeRemovePlugin, - }; - use cincinnati::plugins::InternalPluginWrapper; - - // TODO(steveeJ): actually make this vec configurable - vec![ - Box::new(InternalPluginWrapper( - QuayMetadataFetchPlugin::try_new( - opts.repository.clone(), - opts.quay_label_filter.clone(), - opts.quay_manifestref_key.clone(), - opts.quay_api_credentials_path.as_ref(), - opts.quay_api_base.clone(), - ) - .expect("could not initialize the QuayMetadataPlugin"), - )), - Box::new(InternalPluginWrapper(NodeRemovePlugin { - key_prefix: opts.quay_label_filter.clone(), - })), - Box::new(InternalPluginWrapper(EdgeAddRemovePlugin { - key_prefix: opts.quay_label_filter.clone(), - })), - ] - }; + // Initialize all plugins from runtime settings. + let plugins: Vec<_> = opts + .plugins + .iter() + .map(|cfg| plugins::try_from_settings(cfg).expect("could not initialize plugin")) + .collect(); // Don't wait on the first iteration let mut first_iteration = true; @@ -150,7 +125,7 @@ pub fn run<'a>(opts: &'a config::Options, state: &State) -> ! { username.as_ref().map(String::as_ref), password.as_ref().map(String::as_ref), &mut cache, - &opts.quay_manifestref_key, + &opts.manifestref_key, ); UPSTREAM_SCRAPES.inc(); @@ -183,7 +158,7 @@ pub fn run<'a>(opts: &'a config::Options, state: &State) -> ! { }; let graph = match cincinnati::plugins::process( - &configured_plugins, + &plugins, cincinnati::plugins::InternalIO { graph, // the plugins used in the graph-builder don't expect any parameters yet diff --git a/graph-builder/src/lib.rs b/graph-builder/src/lib.rs index 16d96ea0b..1778575dd 100644 --- a/graph-builder/src/lib.rs +++ b/graph-builder/src/lib.rs @@ -26,6 +26,7 @@ extern crate serde_json; extern crate structopt; extern crate tar; extern crate tokio; +extern crate toml; pub mod config; pub mod graph; diff --git a/graph-builder/src/main.rs b/graph-builder/src/main.rs index e5d9e8749..0501b1e81 100644 --- a/graph-builder/src/main.rs +++ b/graph-builder/src/main.rs @@ -16,6 +16,7 @@ extern crate actix; extern crate actix_web; extern crate failure; extern crate graph_builder; +#[macro_use] extern crate log; extern crate structopt; @@ -23,29 +24,21 @@ use graph_builder::{config, graph, metrics}; use actix_web::{http::Method, middleware::Logger, server, App}; use failure::Error; -use log::LevelFilter; -use std::{net, thread}; -use structopt::StructOpt; +use std::{thread}; fn main() -> Result<(), Error> { let sys = actix::System::new("graph-builder"); - let opts = config::Options::from_args(); + let opts = config::AppSettings::assemble()?; env_logger::Builder::from_default_env() - .filter( - Some(module_path!()), - match opts.verbosity { - 0 => LevelFilter::Warn, - 1 => LevelFilter::Info, - 2 => LevelFilter::Debug, - _ => LevelFilter::Trace, - }, - ) + .filter(Some(module_path!()), opts.verbosity) .init(); + debug!("application settings:\n{:#?}", &opts); let state = graph::State::new(opts.mandatory_client_parameters.clone()); - let addr = (opts.address, opts.port); + let service_addr = (opts.address, opts.port); + let status_addr = (opts.status_address, opts.status_port); let app_prefix = opts.path_prefix.clone(); { @@ -53,17 +46,13 @@ fn main() -> Result<(), Error> { thread::spawn(move || graph::run(&opts, &state)); } - // TODO(lucab): make these configurable. - let status_address: net::IpAddr = net::Ipv4Addr::UNSPECIFIED.into(); - let status_port = 9080; - // Status service. server::new(|| { App::new() .middleware(Logger::default()) .route("/metrics", Method::GET, metrics::serve) }) - .bind((status_address, status_port))? + .bind(status_addr)? .start(); // Main service. @@ -75,7 +64,7 @@ fn main() -> Result<(), Error> { .prefix(app_prefix) .route("/v1/graph", Method::GET, graph::index) }) - .bind(addr)? + .bind(service_addr)? .start(); sys.run(); diff --git a/quay/src/lib.rs b/quay/src/lib.rs index ff4808a3d..fea0f99fc 100644 --- a/quay/src/lib.rs +++ b/quay/src/lib.rs @@ -13,29 +13,27 @@ use failure::Fallible; use failure::ResultExt; use std::fs::File; use std::io::{BufRead, BufReader}; -use std::path::PathBuf; +use std::path::Path; pub mod v1; -pub fn read_credentials(credentials_path: Option<&PathBuf>) -> Fallible> { - match &credentials_path { - Some(pathbuf) => { - let file = - File::open(pathbuf).context(format!("could not open '{}'", &pathbuf.display()))?; +pub fn read_credentials

(path: P) -> Fallible +where + P: AsRef, +{ + let filepath = path.as_ref(); + let file = File::open(filepath).context(format!("could not open '{}'", filepath.display()))?; - let first_line = BufReader::new(file) - .lines() - .nth(0) - .ok_or_else(|| format_err!("empty credentials."))?; + let first_line = BufReader::new(file) + .lines() + .nth(0) + .ok_or_else(|| format_err!("empty credentials."))?; - let token = first_line?.trim_end().to_string(); + let token = first_line?.trim_end().to_string(); - if token.is_empty() { - bail!("found an empty first line in '{}'", &pathbuf.display()) - } - - Ok(Some(token)) - } - None => Ok(None), + if token.is_empty() { + bail!("found an empty first line in '{}'", filepath.display()) } + + Ok(token) }