graph-builder: move plugins to cincinnati crate and catalog, and make plugins configurable

cincinnati/Cargo.toml

@@ -24,17 +24,25 @@ serde = "1.0.70"
 serde_derive = "1.0.70"
 serde_json = "^1.0.22"
 smart-default = "^0.5.2"
-tokio = "0.2"
+tokio = { version = "0.2.11", features = [ "fs", "stream" ] }
 toml = "^0.4.10"
 url = "^1.7.2"
-semver = "^0.9.0"
+semver = { version = "^0.9.0", features = [ "serde" ] }
 async-trait = "^0.1"
+tempfile = "^3.1.0"
+flate2 = "^1.0.1"
+tar = "^0.4.16"
+dkregistry = { git = "https://github.com/camallo/dkregistry-rs.git", rev = "712f7dae50068948e8678af535ce90bb63afd878" }
+itertools = "^0.8.2"
+serde_yaml = "^0.8.11"
 
 [dev-dependencies]
 maplit = "^1.0.2"
 mockito = "^0.20.0"
 serde_json = "1.0.22"
 twoway = "^0.2"
+walkdir = "2.3.1"
+pretty_assertions = "0.6.1"
 
 [build-dependencies]
 protoc-rust = "2.8"

cincinnati/src/lib.rs

@@ -17,17 +17,6 @@ extern crate failure;
 
 #[macro_use]
 extern crate serde_derive;
-#[macro_use]
-extern crate log;
-#[macro_use]
-extern crate lazy_static;
-#[macro_use]
-extern crate smart_default;
-#[macro_use]
-extern crate custom_debug_derive;
-
-#[cfg(test)]
-extern crate tokio;
 
 #[macro_use]
 pub mod plugins;
@@ -40,14 +29,14 @@ use serde::ser::{Serialize, SerializeStruct, Serializer};
 use std::collections::HashMap;
 use std::{collections, fmt};
 
-pub use daggy::WouldCycle;
+pub use daggy::{self, WouldCycle};
 
 pub const CONTENT_TYPE: &str = "application/json";
 const EXPECT_NODE_WEIGHT: &str = "all exisitng nodes to have a weight (release)";
 
 /// Graph type which stores `Release` as node-weights and `Empty` as edge-weights.
 #[derive(Debug, Default)]
-#[cfg_attr(test, derive(Clone))]
+#[cfg_attr(any(test, feature = "clone"), derive(Clone))]
 pub struct Graph {
     dag: Dag<Release, Empty>,
 }

cincinnati/src/plugins/catalog.rs

@@ -3,14 +3,26 @@
 //! This catalog relies on a static list of all available plugins,
 //! referenced by name. It is used for configuration purposes.
 
+use crate as cincinnati;
+
+use self::cincinnati::plugins::BoxedPlugin;
+
 use super::internal::arch_filter::ArchFilterPlugin;
 use super::internal::channel_filter::ChannelFilterPlugin;
 use super::internal::cincinnati_graph_fetch::CincinnatiGraphFetchPlugin;
 use super::internal::edge_add_remove::EdgeAddRemovePlugin;
+use super::internal::github_openshift_secondary_metadata_scraper::{
+    GithubOpenshiftSecondaryMetadataScraperPlugin, GithubOpenshiftSecondaryMetadataScraperSettings,
+};
 use super::internal::metadata_fetch_quay::QuayMetadataFetchPlugin;
 use super::internal::node_remove::NodeRemovePlugin;
-use crate::plugins::BoxedPlugin;
-use failure::Fallible;
+use super::internal::openshift_secondary_metadata_parser::{
+    OpenshiftSecondaryMetadataParserPlugin, OpenshiftSecondaryMetadataParserSettings,
+};
+use super::internal::release_scrape_dockerv2::{
+    ReleaseScrapeDockerv2Plugin, ReleaseScrapeDockerv2Settings,
+};
+use failure::{bail, format_err, Fallible};
 use std::fmt::Debug;
 
 /// Key used to look up plugin-type in a configuration entry.
@@ -40,18 +52,27 @@ pub fn deserialize_config(cfg: toml::Value) -> Fallible<Box<dyn PluginSettings>>
             CincinnatiGraphFetchPlugin::deserialize_config(cfg)
         }
         ArchFilterPlugin::PLUGIN_NAME => ArchFilterPlugin::deserialize_config(cfg),
+        ReleaseScrapeDockerv2Plugin::PLUGIN_NAME => {
+            ReleaseScrapeDockerv2Settings::deserialize_config(cfg)
+        }
+        GithubOpenshiftSecondaryMetadataScraperPlugin::PLUGIN_NAME => {
+            GithubOpenshiftSecondaryMetadataScraperSettings::deserialize_config(cfg)
+        }
+        OpenshiftSecondaryMetadataParserPlugin::PLUGIN_NAME => {
+            OpenshiftSecondaryMetadataParserSettings::deserialize_config(cfg)
+        }
         x => bail!("unknown plugin '{}'", x),
     }
 }
 
 /// Bulid a vector of plugins from PluginSettings
 pub fn build_plugins(
-    policies: &[Box<dyn PluginSettings>],
+    settings: &[Box<dyn PluginSettings>],
     registry: Option<&prometheus::Registry>,
 ) -> Fallible<Vec<BoxedPlugin>> {
-    let mut plugins = Vec::with_capacity(policies.len());
-    for conf in policies {
-        let plugin = conf.build_plugin(registry)?;
+    let mut plugins = Vec::with_capacity(settings.len());
+    for setting in settings {
+        let plugin = setting.build_plugin(registry)?;
         plugins.push(plugin);
     }
 

cincinnati/src/plugins/external/web.rs

@@ -16,11 +16,13 @@ pub struct _WebPluginClient {
 #[cfg(test)]
 mod tests {
     use crate as cincinnati;
-    use crate::plugins::{interface, ExternalIO, ExternalPlugin, InternalIO, PluginResult};
-    use crate::testing::generate_graph;
+
     use async_trait::async_trait;
+    use cincinnati::plugins::{interface, ExternalIO, ExternalPlugin, InternalIO, PluginResult};
+    use cincinnati::testing::generate_graph;
     use commons::testing::init_runtime;
     use failure::Fallible;
+    use log::trace;
     use std::convert::TryInto;
 
     struct DummyWebClient {

cincinnati/src/plugins/internal/arch_filter.rs

@@ -5,13 +5,14 @@
 //! The filtering also removes any architecture suffixes from the version strings
 //! if they are present. The assumption for this is that the architecture would
 //! be encoded as part of the _build_ information according to the SemVer specification.
-use crate::plugins::{
-    BoxedPlugin, InternalIO, InternalPlugin, InternalPluginWrapper, PluginSettings,
-};
-use async_trait::async_trait;
+
+use crate as cincinnati;
+
+use self::cincinnati::plugins::prelude::*;
+use self::cincinnati::plugins::prelude_plugin_impl::*;
+
 use commons::GraphError;
-use failure::{Fallible, ResultExt};
-use prometheus::Registry;
+use lazy_static::lazy_static;
 
 pub static DEFAULT_KEY_FILTER: &str = "io.openshift.upgrades.graph";
 pub static DEFAULT_ARCH_KEY: &str = "release.arch";
@@ -32,12 +33,13 @@ pub struct ArchFilterPlugin {
 }
 
 impl PluginSettings for ArchFilterPlugin {
-    fn build_plugin(&self, _: Option<&Registry>) -> Fallible<BoxedPlugin> {
+    fn build_plugin(&self, _: Option<&prometheus::Registry>) -> Fallible<BoxedPlugin> {
         Ok(new_plugin!(InternalPluginWrapper(self.clone())))
     }
 }
 
 impl ArchFilterPlugin {
+    /// Plugin name, for configuration.
     pub const PLUGIN_NAME: &'static str = "arch-filter";
 
     /// Validate plugin configuration and fill in defaults.
@@ -99,7 +101,7 @@ impl InternalPlugin for ArchFilterPlugin {
             graph
                 .find_by_fn_mut(|release| {
                     match release {
-                        crate::Release::Concrete(concrete_release) => concrete_release
+                        cincinnati::Release::Concrete(concrete_release) => concrete_release
                             .metadata
                             .remove(&format!("{}.{}", self.key_prefix, self.key_suffix))
                             .map_or(true, |values| {
@@ -138,8 +140,8 @@ impl InternalPlugin for ArchFilterPlugin {
                 };
 
                 match &mut release {
-                    crate::Release::Abstract(release) => release.version = version,
-                    crate::Release::Concrete(release) => release.version = version,
+                    cincinnati::Release::Abstract(release) => release.version = version,
+                    cincinnati::Release::Concrete(release) => release.version = version,
                 };
 
                 Ok(())
@@ -156,9 +158,8 @@ impl InternalPlugin for ArchFilterPlugin {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate as cincinnati;
-    use crate::testing::TestMetadata;
     use cincinnati::testing::generate_custom_graph;
+    use cincinnati::testing::TestMetadata;
     use commons::testing::init_runtime;
 
     #[test]

cincinnati/src/plugins/internal/channel_filter.rs

@@ -2,13 +2,13 @@
 //! It reads the requested channel from the parameters value at key "channel",
 //! and the value must match the regex specified at CHANNEL_VALIDATION_REGEX_STR
 
-use crate::plugins::{
-    BoxedPlugin, InternalIO, InternalPlugin, InternalPluginWrapper, PluginSettings,
-};
-use async_trait::async_trait;
+use crate as cincinnati;
+
+use self::cincinnati::plugins::prelude::*;
+use self::cincinnati::plugins::prelude_plugin_impl::*;
+
 use commons::GraphError;
-use failure::Fallible;
-use prometheus::Registry;
+use lazy_static::lazy_static;
 
 static DEFAULT_KEY_FILTER: &str = "io.openshift.upgrades.graph";
 static DEFAULT_CHANNEL_KEY: &str = "release.channels";
@@ -24,7 +24,7 @@ pub struct ChannelFilterPlugin {
 }
 
 impl PluginSettings for ChannelFilterPlugin {
-    fn build_plugin(&self, _: Option<&Registry>) -> Fallible<BoxedPlugin> {
+    fn build_plugin(&self, _: Option<&prometheus::Registry>) -> Fallible<BoxedPlugin> {
         Ok(new_plugin!(InternalPluginWrapper(self.clone())))
     }
 }
@@ -71,7 +71,7 @@ impl InternalPlugin for ChannelFilterPlugin {
             graph
                 .find_by_fn_mut(|release| {
                     match release {
-                        crate::Release::Concrete(concrete_release) => concrete_release
+                        cincinnati::Release::Concrete(concrete_release) => concrete_release
                             .metadata
                             .get_mut(&format!("{}.{}", self.key_prefix, self.key_suffix))
                             .map_or(true, |values| {
@@ -104,7 +104,7 @@ impl InternalPlugin for ChannelFilterPlugin {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::testing::generate_custom_graph;
+    use cincinnati::testing::generate_custom_graph;
     use commons::testing::init_runtime;
     use std::collections::HashMap;
 
@@ -218,8 +218,8 @@ mod tests {
         struct Datum {
             pub description: String,
             pub parameters: HashMap<String, String>,
-            pub input_graph: crate::Graph,
-            pub expected_graph: crate::Graph,
+            pub input_graph: cincinnati::Graph,
+            pub expected_graph: cincinnati::Graph,
         }
 
         let data = vec![

cincinnati/src/plugins/internal/cincinnati_graph_fetch.rs

@@ -3,15 +3,14 @@
 //! Instead of processing the input graph, this plugin fetches a graph from a
 //! remote endpoint, which makes it effectively discard any given input graph.
 
-use crate::plugins::{
-    BoxedPlugin, InternalIO, InternalPlugin, InternalPluginWrapper, PluginSettings,
-};
-use crate::CONTENT_TYPE;
-use async_trait::async_trait;
+use crate as cincinnati;
+
+use self::cincinnati::plugins::prelude::*;
+use self::cincinnati::plugins::prelude_plugin_impl::*;
+use self::cincinnati::CONTENT_TYPE;
+
 use commons::GraphError;
-use failure::Fallible;
-use futures::TryFutureExt;
-use prometheus::{Counter, Registry};
+use prometheus::Counter;
 use reqwest;
 use reqwest::header::{HeaderValue, ACCEPT};
 
@@ -42,7 +41,7 @@ pub struct CincinnatiGraphFetchPlugin {
 }
 
 impl PluginSettings for CincinnatiGraphFetchSettings {
-    fn build_plugin(&self, registry: Option<&Registry>) -> Fallible<BoxedPlugin> {
+    fn build_plugin(&self, registry: Option<&prometheus::Registry>) -> Fallible<BoxedPlugin> {
         let cfg = self.clone();
         let plugin = CincinnatiGraphFetchPlugin::try_new(cfg.upstream, registry)?;
         Ok(new_plugin!(InternalPluginWrapper(plugin)))
@@ -141,7 +140,7 @@ impl InternalPlugin for CincinnatiGraphFetchPlugin {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::testing::generate_custom_graph;
+    use cincinnati::testing::generate_custom_graph;
     use commons::metrics::{self, RegistryWrapper};
     use commons::testing::{self, init_runtime};
     use failure::{bail, Fallible};
@@ -194,8 +193,8 @@ mod tests {
 
     fetch_upstream_success_test!(
         name: fetch_success_empty_graph_fetch,
-        mock_body: &serde_json::to_string(&crate::Graph::default())?,
-        expected_graph: crate::Graph::default(),
+        mock_body: &serde_json::to_string(&cincinnati::Graph::default())?,
+        expected_graph: cincinnati::Graph::default(),
     );
 
     fetch_upstream_success_test!(