graph-builder: add plugin to fetch openshift/cincinnati-graph-data from github

cincinnati/src/lib.rs

@@ -68,6 +68,14 @@ impl Release {
             Release::Concrete(release) => &release.version,
         }
     }
+
+    /// Get a mutable borrow of the release metadata if any
+    pub fn get_metadata_mut(&mut self) -> Option<&mut HashMap<String, String>> {
+        match self {
+            Release::Abstract(_) => None,
+            Release::Concrete(release) => Some(&mut release.metadata),
+        }
+    }
 }
 
 /// Type to represent a Release with all its information.

graph-builder/Cargo.toml

@@ -27,10 +27,11 @@ semver = { version = "^0.9.0", features = [ "serde" ] }
 serde = "^1.0.70"
 serde_derive = "^1.0.70"
 serde_json = "^1.0.22"
+serde_yaml = "^0.8.11"
 smart-default = "^0.5.1"
 structopt = "^0.2.10"
 tar = "^0.4.16"
-tokio = "0.2"
+tokio = { version = "0.2.11", features = [ "fs", "stream" ] }
 toml = "^0.4.10"
 url = "^1.7.2"
 parking_lot = "^0.8.0"
@@ -42,6 +43,7 @@ custom_debug_derive = "^0.1.7"
 built = "^0.3.2"
 
 [dev-dependencies]
+walkdir = "2.3.1"
 twoway = "^0.2"
 assert-json-diff = "1.0.0"
 test-case = "1.0.0"

graph-builder/src/plugins/github_openshift_secondary_metadata_scraper/github_v3.rs

@@ -0,0 +1,76 @@
+//! This is a helper module for accessing the [GitHub API v3][].
+//!
+//! [GitHub API v3]: https://developer.github.com/v3/
+
+/// Commit structure.
+#[derive(Default, Clone, Debug, Deserialize, PartialEq, Eq)]
+pub(crate) struct Commit {
+    pub(crate) sha: String,
+    pub(crate) url: String,
+}
+
+/// Branch structure.
+#[derive(Debug, Deserialize, PartialEq, Eq)]
+pub(crate) struct Branch {
+    pub(crate) name: String,
+    pub(crate) commit: Commit,
+    pub(crate) protected: bool,
+}
+
+/// Format the URL to request branch information.
+pub(crate) fn branches_url(org: &str, repo: &str) -> String {
+    format!(
+        "https://api.github.com/repos/{org}/{repo}/branches",
+        org = &org,
+        repo = &repo,
+    )
+}
+
+/// Format the URL to request a tarball URL.
+pub(crate) fn tarball_url(org: &str, repo: &str, commit: &Commit) -> String {
+    format!(
+        "https://api.github.com/repos/{org}/{repo}/tarball/{sha}",
+        org = org,
+        repo = repo,
+        sha = commit.sha,
+    )
+}
+
+/// Format a subdirectory name for a specific revision's tarball.
+pub(crate) fn archive_entry_directory_name(org: &str, repo: &str, commit: &Commit) -> String {
+    format!("{}-{}-{}", &org, &repo, &commit.sha[0..7],)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn de_serialize_branch() {
+        let json = r#"
+            [
+                {
+                    "name": "master",
+                    "commit": {
+                        "sha": "fef06adb57b9d965bfc9ae0959bd038f3044207e",
+                        "url": "https://api.github.com/repos/openshift/cincinnati-graph-data/commits/fef06adb57b9d965bfc9ae0959bd038f3044207e"
+                    },
+                    "protected": true
+                }
+            ]
+            "#;
+
+        let branches = serde_json::from_str::<Vec<Branch>>(&json).unwrap();
+
+        let branches_expected = vec![Branch {
+                name: "master".to_string(),
+                commit: Commit {
+                        sha: "fef06adb57b9d965bfc9ae0959bd038f3044207e".to_string(),
+                        url: "https://api.github.com/repos/openshift/cincinnati-graph-data/commits/fef06adb57b9d965bfc9ae0959bd038f3044207e".to_string()
+                },
+                protected: true
+            }];
+
+        assert_eq!(branches_expected, branches);
+    }
+}

graph-builder/src/plugins/github_openshift_secondary_metadata_scraper/mod.rs

@@ -0,0 +1,12 @@
+//! This plugin downloads repository content and extracts it to a given output directory.
+//!
+//! It is meant to be included in the plugin chain, preceding other plugins who
+//! rely on the data being in the output directory.
+//! The plugin will only download a tarball if detects a change of revision or on first run.
+
+mod github_v3;
+pub mod plugin;
+
+pub use plugin::{
+    GithubOpenshiftSecondaryMetadataScraperPlugin, GithubOpenshiftSecondaryMetadataScraperSettings,
+};