diff --git a/Cargo.lock b/Cargo.lock index ea65443ea..ea4f8a052 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -923,9 +923,11 @@ version = "0.1.0" dependencies = [ "assert-json-diff 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "cincinnati 0.1.0", "commons 0.1.0", "env_logger 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", "failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "prometheus-query 0.0.0-dev", "reqwest 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.104 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1224,7 +1226,6 @@ version = "0.1.0" dependencies = [ "actix 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "actix-web 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)", - "assert-json-diff 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "async-trait 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", "built 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1251,7 +1252,6 @@ dependencies = [ "structopt 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", "tar 0.4.26 (registry+https://github.com/rust-lang/crates.io-index)", "tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "test-case 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.4.10 (registry+https://github.com/rust-lang/crates.io-index)", "twoway 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/Justfile b/Justfile index 34c08612c..bf7ca4073 100644 --- a/Justfile +++ b/Justfile @@ -1,3 +1,21 @@ +path_prefix := "api/upgrades_info/" + +testdata_dir := "e2e/tests/testdata/" +metadata_revision_file := "metadata_revision" +metadata_reference :='reference_branch = "master"' + +pause_secs := "9999999" +registry := "https://quay.io" +repository := "openshift-release-dev/ocp-release" +credentials_file := "${HOME}/.docker/config.json" + +metadata_reference_e2e: + printf 'reference_revision = "%s"' "$(cat {{testdata_dir}}/{{metadata_revision_file}})" + +metadata_reference_revision: + #!/usr/bin/env bash + read -r var <"{{testdata_dir}}"/metadata_revision; printf $var + format: cargo fmt --all @@ -25,25 +43,77 @@ test: format export RUST_BACKTRACE=1 RUST_LOG="graph-builder=trace,cincinnati=trace,dkregistry=trace" cargo test --all -_test component features='test-net,test-net-private' rustcargs='--ignored': +_test component cargoargs='--features test-net,test-net-private' rustcargs='--ignored': #!/usr/bin/env bash set -xe - (pushd {{component}} && cargo test --features {{features}} -- {{rustcargs}}) + (pushd {{component}} && cargo test {{cargoargs}} -- {{rustcargs}}) test-net-private: #!/usr/bin/env bash set -e - just _test quay "test-net,test-net-private" "" - just _test cincinnati "test-net,test-net-private" "" - just _test graph-builder "test-net,test-net-private" "" + just _test quay "--features test-net,test-net-private" "" + just _test cincinnati "--features test-net,test-net-private" "" + just _test graph-builder "--features test-net,test-net-private" "" run-ci-tests: #!/usr/bin/env bash set -e hack/run-all-tests.sh -path_prefix := "api/upgrades_info/" +# Runs the client part of the e2e test suite. +run-e2e-test-only filter="e2e": + #!/usr/bin/env bash + set -e + export GRAPH_URL='http://127.0.0.1:8081/{{path_prefix}}v1/graph' + export E2E_METADATA_REVISION="$(just metadata_reference_revision)" + + # we need to use the parent-directory here because the test runs in the graph-builder directory + export E2E_TESTDATA_DIR="../{{ testdata_dir }}" + + just _test e2e "" "{{ filter }}" + +# Spawns a Cincinnati stack on localhost and runs the e2e test suite. +run-e2e: + #!/usr/bin/env bash + set -e + + just \ + registry="{{registry}}" repository="{{repository}}" \ + run-daemons-e2e 2>&1 & + DAEMON_PARENTPID=$! + trap "kill $DAEMON_PARENTPID" EXIT + + # give the graph-builder time to scrape + sleep 180 + + for i in `seq 1 100`; do + just run-e2e-test-only && { + echo Test successful. + exit 0 + } || { + echo Attempt failed. Trying again in 10 seconds. + sleep 10 + } + done + + echo Test failed. + exit 1 + +# Capture new e2e fixtures and refresh the metadata revision file. +e2e-fixtures-capture-only: + #!/usr/bin/env bash + set -e + + for base in "stable"; do + for version in "4.2" "4.3"; do + for arch in "amd64" "s390x"; do + for suffix in "-production"; do + just get-graph-pe"${suffix}" "${base}-${version}" "${arch}" > {{testdata_dir}}/"$(just metadata_reference_revision)_${base}-${version}_${arch}${suffix}".json + done + done + done + done # Reads a graph on stdin, creates an SVG out of it and opens it with SVG-associated default viewer. Meant to be combined with one of the `get-graph-*` recipes. display-graph: @@ -58,14 +128,18 @@ display-graph: jq -cM . | {{invocation_directory()}}/hack/graph.sh | dot -Tsvg > graph.svg; xdg-open graph.svg -run-graph-builder registry="https://quay.io" repository="openshift-release-dev/ocp-release" credentials_file="${HOME}/.docker/config.json": +run-graph-builder: #!/usr/bin/env bash export RUST_BACKTRACE=1 - cargo run --package graph-builder -- -c <(cat <<'EOF' + + trap 'rm -rf "$TMPDIR"' EXIT + export TMPDIR=$(mktemp -d) + + cargo run --package graph-builder -- -c <(cat <<-EOF verbosity = "vvv" [service] - pause_secs = 9999999 + pause_secs = {{pause_secs}} address = "127.0.0.1" port = 8080 path_prefix = "{{path_prefix}}" @@ -82,24 +156,35 @@ run-graph-builder registry="https://quay.io" repository="openshift-release-dev/o credentials_file = "{{credentials_file}}" [[plugin_settings]] - name="quay-metadata" - repository="{{repository}}" + name = "github-secondary-metadata-scrape" + github_org = "openshift" + github_repo = "cincinnati-graph-data" + branch = "master" + output_directory = "${TMPDIR}" + {{metadata_reference}} [[plugin_settings]] - name="node-remove" + name = "openshift-secondary-metadata-parse" + data_directory = "${TMPDIR}" [[plugin_settings]] - name="edge-add-remove" + name = "edge-add-remove" EOF ) run-graph-builder-satellite: - just run-graph-builder 'sat-r220-02.lab.eng.rdu2.redhat.com' 'default_organization-custom-ocp' + just registry='sat-r220-02.lab.eng.rdu2.redhat.com' repository='default_organization-custom-ocp' run-graph-builder + +run-graph-builder-e2e: + just \ + registry="{{registry}}" repository="{{repository}}" \ + metadata_reference="$(just metadata_reference_e2e)" \ + run-graph-builder run-policy-engine: #!/usr/bin/env bash export RUST_BACKTRACE=1 RUST_LOG="policy_engine=trace,cincinnati=trace,actix=trace,actix_web=trace" - cargo run --package policy-engine -- -vvvv --service.address 0.0.0.0 --service.path_prefix {{path_prefix}} --upstream.cincinnati.url 'http://localhost:8080/{{path_prefix}}v1/graph' --service.mandatory_client_parameters='channel' + cargo run --package policy-engine -- -vvvv --service.address 0.0.0.0 --service.path_prefix {{path_prefix}} --upstream.cincinnati.url 'http://127.0.0.1:8080/{{path_prefix}}v1/graph' --service.mandatory_client_parameters='channel' kill-daemons: pkill graph-builder @@ -107,16 +192,29 @@ kill-daemons: run-daemons: #!/usr/bin/env bash - just run-graph-builder #"https://quay.io" "redhat/openshift-cincinnati-test-public-manual" ~/.docker/config.json 2>&1 & + just run-graph-builder 2>&1 & + PG_PID=$! + + just run-policy-engine 2>&1 & + PE_PID=$! + + trap "kill $PG_PID $PE_PID" EXIT + sleep infinity + +run-daemons-e2e: + #!/usr/bin/env bash + just \ + registry="{{registry}}" repository="{{repository}}" \ + run-graph-builder-e2e 2>&1 & PG_PID=$! just run-policy-engine 2>&1 & PE_PID=$! trap "kill $PG_PID $PE_PID" EXIT - while true; do sleep 30; done + sleep infinity -get-graph port channel arch host="http://localhost": +get-graph port channel arch host="http://127.0.0.1": curl --header 'Accept:application/json' {{host}}:{{port}}/{{path_prefix}}v1/graph?channel='{{channel}}'\&arch='{{arch}}' | jq . get-graph-gb: diff --git a/cincinnati/Cargo.toml b/cincinnati/Cargo.toml index e9d9a8322..d769396df 100644 --- a/cincinnati/Cargo.toml +++ b/cincinnati/Cargo.toml @@ -44,7 +44,7 @@ twoway = "^0.2" walkdir = "2.3.1" pretty_assertions = "0.6.1" test-case = "1.0.0" -prettydiff = {} +prettydiff = "0.3.1" [build-dependencies] protoc-rust = "2.8" diff --git a/cincinnati/src/lib.rs b/cincinnati/src/lib.rs index a8f970341..edc4f6dc3 100644 --- a/cincinnati/src/lib.rs +++ b/cincinnati/src/lib.rs @@ -450,9 +450,12 @@ impl Graph { .try_for_each(|mut nw| f(&mut nw)) } - /// Get the edges expressed as version -> versions + /// Get the edges expressed as version -> versions; optionally include edges from/to `Release::Abstract`. #[cfg(any(test, feature = "test"))] - pub fn get_edges(&self) -> Result>, Error> { + pub fn get_edges( + &self, + include_abstract: bool, + ) -> Result>, Error> { let mut edges: MapImpl> = Default::default(); self.dag @@ -467,10 +470,17 @@ impl Graph { let source_version = source.version().to_string(); let target_version = target.version().to_string(); - edges - .entry(source_version) - .or_default() - .insert(target_version); + if include_abstract + || match (source, target) { + (Release::Concrete(_), Release::Concrete(_)) => true, + _ => false, + } + { + edges + .entry(source_version) + .or_default() + .insert(target_version); + } Ok(()) } @@ -937,56 +947,48 @@ pub mod testing { let removed_keys_left = remove_release_metadata(&mut left, unwanted_metadata_keys); let removed_keys_right = remove_release_metadata(&mut right, unwanted_metadata_keys); - let releases_left: std::collections::BTreeSet = (&left).into(); - let releases_right: std::collections::BTreeSet = (&right).into(); - - let edges_left = right.get_edges()?; - let edges_right = left.get_edges()?; + if left == right { + return Ok(()); + } let mut output: Vec = vec![]; + output.push("Graphs differ! Showing differences from left to right".into()); + output.push("-----------------------------------------------------".into()); + + let edges_left = right.get_edges(false)?; + let edges_right = left.get_edges(false)?; + output.push("edges: ".into()); + output.push( + prettydiff::diff_lines( + &serde_json::to_string_pretty(&edges_left)?, + &serde_json::to_string_pretty(&edges_right)?, + ) + .format(), + ); - if releases_left != releases_right { - output.push("Graphs differ! Showing differences from left to right".into()); - output.push("-----------------------------------------------------".into()); - output.push("nodes:".into()); - output.push( - prettydiff::diff_lines( - Box::leak(Box::new(serde_json::to_string_pretty(&releases_left)?)), - Box::leak(Box::new(serde_json::to_string_pretty(&releases_right)?)), - ) - .format(), - ); - - if !unwanted_metadata_keys.is_empty() { - output.push("removed metadata:".into()); - output.push( - prettydiff::diff_lines( - &serde_json::to_string_pretty(&removed_keys_left)?, - &serde_json::to_string_pretty(&removed_keys_right)?, - ) - .format(), - ); - } - } + let releases_left: std::collections::BTreeSet = (&left).into(); + let releases_right: std::collections::BTreeSet = (&right).into(); + output.push("nodes:".into()); + output.push( + prettydiff::diff_lines( + Box::leak(Box::new(serde_json::to_string_pretty(&releases_left)?)), + Box::leak(Box::new(serde_json::to_string_pretty(&releases_right)?)), + ) + .format(), + ); - if edges_left != edges_right { - output.push("edges: ".into()); + if !unwanted_metadata_keys.is_empty() { + output.push("removed metadata:".into()); output.push( prettydiff::diff_lines( - &serde_json::to_string_pretty(&edges_left)?, - &serde_json::to_string_pretty(&edges_right)?, + &serde_json::to_string_pretty(&removed_keys_left)?, + &serde_json::to_string_pretty(&removed_keys_right)?, ) .format(), ); } - let output = output.join("\n"); - - if !output.is_empty() { - return Err(output.into()); - } - - Ok(()) + Err(output.join("\n").into()) } /// Removes the metadata given by the keys and returns it. diff --git a/cincinnati/src/plugins/internal/graph_builder/github_openshift_secondary_metadata_scraper/github_v3.rs b/cincinnati/src/plugins/internal/graph_builder/github_openshift_secondary_metadata_scraper/github_v3.rs index 113a1aacb..5b0ef65d5 100644 --- a/cincinnati/src/plugins/internal/graph_builder/github_openshift_secondary_metadata_scraper/github_v3.rs +++ b/cincinnati/src/plugins/internal/graph_builder/github_openshift_secondary_metadata_scraper/github_v3.rs @@ -43,6 +43,14 @@ pub(crate) fn archive_entry_directory_name(org: &str, repo: &str, commit: &Commi format!("{}-{}-{}", &org, &repo, &commit.sha[0..7],) } +/// Format a commit URL +pub(crate) fn commit_url(org: &str, repo: &str, sha: &str) -> String { + format!( + "https://api.github.com/repos/{}/{}/commits/{}", + org, repo, sha + ) +} + #[cfg(test)] mod tests { use super::*; diff --git a/cincinnati/src/plugins/internal/graph_builder/github_openshift_secondary_metadata_scraper/plugin.rs b/cincinnati/src/plugins/internal/graph_builder/github_openshift_secondary_metadata_scraper/plugin.rs index 499157123..4746f7a38 100644 --- a/cincinnati/src/plugins/internal/graph_builder/github_openshift_secondary_metadata_scraper/plugin.rs +++ b/cincinnati/src/plugins/internal/graph_builder/github_openshift_secondary_metadata_scraper/plugin.rs @@ -1,4 +1,5 @@ use super::github_v3; +use std::convert::{TryFrom, TryInto}; use crate as cincinnati; @@ -13,20 +14,77 @@ pub static DEFAULT_OUTPUT_WHITELIST: &[&str] = &[ "raw/metadata.json", ]; +// Defines the key for placing the data directory path in the IO parameters +pub static GRAPH_DATA_DIR_PARAM_KEY: &str = "io.openshift.upgrades.secondary_metadata.directory"; + +lazy_static::lazy_static! { + pub static ref DEFAULT_REFERENCE_BRANCH: Option = Some(String::from("master")); +} + /// Environment variable name for the Oauth token path pub static GITHUB_SCRAPER_TOKEN_PATH_ENV: &str = "CINCINNATI_GITHUB_SCRAPER_OAUTH_TOKEN_PATH"; static USER_AGENT: &str = "openshift/cincinnati"; +/// Models the scrape mode +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "lowercase")] +pub enum Reference { + Branch(String), + Revision(String), +} + +impl Reference { + fn get_inner(&self) -> &String { + match self { + Self::Branch(s) => s, + Self::Revision(s) => s, + } + } +} + +impl TryFrom<(Option<&String>, Option<&String>)> for Reference { + type Error = failure::Error; + + fn try_from(options: (Option<&String>, Option<&String>)) -> Fallible { + let reference = match (options.0, options.1) { + (Some(branch), Some(revision)) => { + bail!( + "only one of reference_branch or reference_revision can be set. got {:?} and {:?}", + branch, + revision, + ); + } + (None, None) => Reference::Branch(DEFAULT_REFERENCE_BRANCH.clone().unwrap()), + (Some(branch), None) => Reference::Branch(branch.to_string()), + (None, Some(revision)) => Reference::Revision(revision.to_string()), + }; + + Ok(reference) + } +} + /// Plugin settings. #[derive(Debug, SmartDefault, Clone, Deserialize)] #[serde(default)] pub struct GithubOpenshiftSecondaryMetadataScraperSettings { github_org: String, github_repo: String, - branch: String, output_directory: PathBuf, + /// Defines the reference branch to be scraped. + reference_branch: Option, + + /// Defines the reference revision to be scraped. + reference_revision: Option, + + /// Defines the reference to be scraped according to the `Reference` enum. + /// + /// For now, this cannot be provided via TOML due to missing support for + /// deserializing a `toml::Value` to enum newtype variants. + #[serde(skip)] + reference: Option, + /// Vector of regular expressions used as a positive output filter. /// An empty vector is regarded as a configuration error. #[default(DEFAULT_OUTPUT_WHITELIST.iter().map(|s| (*s).to_string()).collect())] @@ -37,11 +95,30 @@ pub struct GithubOpenshiftSecondaryMetadataScraperSettings { impl GithubOpenshiftSecondaryMetadataScraperSettings { /// Validate plugin configuration and fill in defaults. pub fn deserialize_config(cfg: toml::Value) -> Fallible> { - let settings: Self = cfg.try_into()?; + let mut settings: Self = cfg + .clone() + .try_into() + .context(format!("Deserializing {:#?}", &cfg))?; ensure!(!settings.github_org.is_empty(), "empty github_org"); ensure!(!settings.github_repo.is_empty(), "empty github_repo"); - ensure!(!settings.branch.is_empty(), "empty branch"); + + let reference: Reference = ( + settings.reference_branch.as_ref(), + settings.reference_revision.as_ref(), + ) + .try_into()?; + ensure!(!reference.get_inner().is_empty(), "empty reference"); + settings.reference = Some(reference); + + ensure!( + !settings + .output_directory + .to_str() + .unwrap_or_default() + .is_empty(), + "empty output_directory" + ); ensure!( !settings.output_whitelist.is_empty(), "empty output_whitelist" @@ -58,16 +135,18 @@ pub struct State { } /// Plugin. -#[derive(Debug, SmartDefault)] +#[derive(Debug)] pub struct GithubOpenshiftSecondaryMetadataScraperPlugin { settings: GithubOpenshiftSecondaryMetadataScraperSettings, output_whitelist: Vec, - #[default(FuturesMutex::new(Default::default()))] + reference: Reference, + state: FuturesMutex, oauth_token: Option, client: reqwest::Client, + data_dir: tempfile::TempDir, } impl GithubOpenshiftSecondaryMetadataScraperPlugin { @@ -103,17 +182,31 @@ impl GithubOpenshiftSecondaryMetadataScraperPlugin { }) .flatten(); + // Create the output directory if it doesn't exist + std::fs::create_dir_all(&settings.output_directory).context(format!( + "Creating directory {:?}", + &settings.output_directory + ))?; + + let data_dir = tempfile::tempdir_in(&settings.output_directory)?; + Ok(Self { + reference: settings + .reference + .clone() + .ok_or_else(|| failure::err_msg("settings don't contain a 'reference'"))?, settings, output_whitelist, oauth_token, + data_dir, - ..Default::default() + state: FuturesMutex::new(State::default()), + client: reqwest::Client::default(), }) } - /// Lookup the latest commit on the given branch and update `self.state.commit_wanted`. - async fn refresh_commit_wanted(&self) -> Fallible { + /// Lookup the latest commit on the given branch. + async fn get_commit_wanted_branch(&self, branch_wanted: &str) -> Fallible { let url = github_v3::branches_url(&self.settings.github_org, &self.settings.github_repo); trace!("Getting branches from {}", &url); @@ -147,7 +240,7 @@ impl GithubOpenshiftSecondaryMetadataScraperPlugin { let latest_commit = branches .iter() .filter_map(|branch| { - if branch.name == self.settings.branch { + if branch.name == branch_wanted { Some(branch.commit.clone()) } else { None @@ -159,27 +252,48 @@ impl GithubOpenshiftSecondaryMetadataScraperPlugin { "{}/{} does not have branch {}: {:#?}", &self.settings.github_org, &self.settings.github_repo, - &self.settings.branch, + &branch_wanted, &branches )) })?; trace!( "Latest commit on branch {}: {:?}", - &self.settings.branch, + &branch_wanted, &latest_commit ); - let mut state = self.state.lock().await; + Ok(latest_commit) + } + + /// Construct a github_v3::Commit from the given revision + async fn get_commit_wanted_revision(&self, revision: &str) -> github_v3::Commit { + github_v3::Commit { + url: github_v3::commit_url( + &self.settings.github_org, + &self.settings.github_repo, + &revision, + ), + sha: revision.to_owned(), + } + } - (*state).commit_wanted = Some(latest_commit.clone()); + /// Refresh `self.state.commit_wanted` and determine if an update is required. + async fn refresh_commit_wanted(&self) -> Fallible { + let commit_wanted = match &self.reference { + Reference::Revision(revision) => self.get_commit_wanted_revision(revision).await, + Reference::Branch(branch) => self.get_commit_wanted_branch(branch).await?, + }; + + let mut state = self.state.lock().await; - let should_update = if let Some(commit_completed) = &state.commit_completed { - commit_completed != &latest_commit - } else { - true + let should_update = match &state.commit_completed { + Some(commit_completed) => commit_completed != &commit_wanted, + None => true, }; + (*state).commit_wanted = Some(commit_wanted); + Ok(should_update) } @@ -199,6 +313,7 @@ impl GithubOpenshiftSecondaryMetadataScraperPlugin { &commit_wanted, ); + trace!("Downloading {:?} from {}", &commit_wanted, &url); reqwest::Client::new() .get(&url) .header(reqwest::header::USER_AGENT, USER_AGENT) @@ -219,10 +334,9 @@ impl GithubOpenshiftSecondaryMetadataScraperPlugin { /// Extract a given blob to the output directory, adhering to the output whitelist, and finally update the completed commit state. async fn extract(&self, commit: github_v3::Commit, bytes: Box<[u8]>) -> Fallible<()> { // Use a tempdir as intermediary extraction target, and later rename to the destination - let tmpdir = tempfile::tempdir()?; + let tmpdir = tempfile::tempdir_in(&self.settings.output_directory)?; { - let settings = self.settings.clone(); let commit = commit.clone(); let output_whitelist = self.output_whitelist.clone(); let tmpdir = tmpdir.path().to_owned(); @@ -265,7 +379,7 @@ impl GithubOpenshiftSecondaryMetadataScraperPlugin { .iter() .any(|whitelist_regex| whitelist_regex.is_match(&path)) { - debug!("Unpacking {:?} to {:?}", &path, &settings.output_directory); + debug!("Unpacking {:?} to {:?}", &path, &tmpdir); entry .unpack_in(&tmpdir) .context(format!("Unpacking {:?} to {:?}", &path, &tmpdir))?; @@ -284,7 +398,24 @@ impl GithubOpenshiftSecondaryMetadataScraperPlugin { &self.settings.github_repo, &commit, )); - let rename_to = &self.settings.output_directory; + + // Append a directory for safety reasons, so we don't wipe the given output directory if it already exists + let rename_to = &self.data_dir; + + // Remove the target directory if it exists + if tokio::fs::OpenOptions::new() + .read(true) + .write(false) + .create(false) + .open(&rename_to) + .await + .is_ok() + { + let msg = format!("Removing pre-existing directory {:?}", &rename_to); + debug!("{}", &msg); + tokio::fs::remove_dir_all(&rename_to).await.context(msg)?; + } + let msg = format!("Renaming {:?} -> {:?}", &rename_from, &rename_to); // Acquire the state lock as we're going to move files from the @@ -313,7 +444,16 @@ impl PluginSettings for GithubOpenshiftSecondaryMetadataScraperSettings { #[async_trait] impl InternalPlugin for GithubOpenshiftSecondaryMetadataScraperPlugin { - async fn run_internal(self: &Self, io: InternalIO) -> Fallible { + async fn run_internal(self: &Self, mut io: InternalIO) -> Fallible { + io.parameters.insert( + GRAPH_DATA_DIR_PARAM_KEY.to_string(), + self.data_dir + .path() + .to_str() + .ok_or_else(|| failure::err_msg("data_dir cannot be converted to str"))? + .to_string(), + ); + let should_update = self .refresh_commit_wanted() .await @@ -346,12 +486,12 @@ mod network_tests { let oauth_token_path = std::env::var(GITHUB_SCRAPER_TOKEN_PATH_ENV)?; - let settings = - toml::from_str::(&format!( + let settings = GithubOpenshiftSecondaryMetadataScraperSettings::deserialize_config( + toml::Value::from_str(&format!( r#" github_org = "openshift" github_repo = "cincinnati-graph-data" - branch = "master" + reference = {{ revision = "6420f7fbf3724e1e5e329ae8d1e2985973f60c14" }} output_whitelist = [ {} ] output_directory = {:?} oauth_token_path = {:?} @@ -363,54 +503,57 @@ mod network_tests { .join(", "), &tmpdir.path(), oauth_token_path, - ))?; + ))?, + )?; debug!("Settings: {:#?}", &settings); - let plugin = Box::new(GithubOpenshiftSecondaryMetadataScraperPlugin::try_new( - settings, - )?); + let plugin = settings.build_plugin(None)?; - let _ = runtime.block_on(plugin.run_internal(InternalIO { - graph: Default::default(), - parameters: Default::default(), - }))?; - - let regexes = DEFAULT_OUTPUT_WHITELIST - .iter() - .map(|s| regex::Regex::new(s).unwrap()) - .collect::>(); - assert!(!regexes.is_empty(), "no regexes compiled"); - - let extracted_paths: HashSet = walkdir::WalkDir::new(tmpdir.path()) - .into_iter() - .map(Result::unwrap) - .filter(|entry| entry.file_type().is_file()) - .filter_map(|file| { - let path = file.path(); - path.to_str().map(str::to_owned) - }) - .collect(); - assert!(!extracted_paths.is_empty(), "no files were extracted"); - - // ensure all files match the configured regexes - extracted_paths.iter().for_each(|path| { - assert!( - regexes.iter().any(|re| re.is_match(&path)), - "{} doesn't match any of the regexes: {:#?}", - path, - regexes - ) - }); - - // ensure every regex matches at least one file - regexes.iter().for_each(|re| { - assert!( - extracted_paths.iter().any(|path| re.is_match(path)), - "regex {} didn't match a file", - &re - ); - }); + for _ in 0..2 { + let _ = runtime.block_on(plugin.run(cincinnati::plugins::PluginIO::InternalIO( + InternalIO { + graph: Default::default(), + parameters: Default::default(), + }, + )))?; + + let regexes = DEFAULT_OUTPUT_WHITELIST + .iter() + .map(|s| regex::Regex::new(s).unwrap()) + .collect::>(); + assert!(!regexes.is_empty(), "no regexes compiled"); + + let extracted_paths: HashSet = walkdir::WalkDir::new(tmpdir.path()) + .into_iter() + .map(Result::unwrap) + .filter(|entry| entry.file_type().is_file()) + .filter_map(|file| { + let path = file.path(); + path.to_str().map(str::to_owned) + }) + .collect(); + assert!(!extracted_paths.is_empty(), "no files were extracted"); + + // ensure all files match the configured regexes + extracted_paths.iter().for_each(|path| { + assert!( + regexes.iter().any(|re| re.is_match(&path)), + "{} doesn't match any of the regexes: {:#?}", + path, + regexes + ) + }); + + // ensure every regex matches at least one file + regexes.iter().for_each(|re| { + assert!( + extracted_paths.iter().any(|path| re.is_match(path)), + "regex {} didn't match a file", + &re + ); + }); + } Ok(()) } diff --git a/cincinnati/src/plugins/internal/graph_builder/openshift_secondary_metadata_parser/plugin.rs b/cincinnati/src/plugins/internal/graph_builder/openshift_secondary_metadata_parser/plugin.rs index 6f120b760..3bcaf52d4 100644 --- a/cincinnati/src/plugins/internal/graph_builder/openshift_secondary_metadata_parser/plugin.rs +++ b/cincinnati/src/plugins/internal/graph_builder/openshift_secondary_metadata_parser/plugin.rs @@ -1,8 +1,11 @@ use crate as cincinnati; +use self::cincinnati::plugins::internal::graph_builder::github_openshift_secondary_metadata_scraper::plugin::GRAPH_DATA_DIR_PARAM_KEY; use self::cincinnati::plugins::prelude::*; use self::cincinnati::plugins::prelude_plugin_impl::*; +pub static DEFAULT_KEY_FILTER: &str = "io.openshift.upgrades.graph"; + mod graph_data_model { //! This module contains the data types corresponding to the graph data files. @@ -100,6 +103,8 @@ pub static DEFAULT_ARCH: &str = "amd64"; #[serde(default)] pub struct OpenshiftSecondaryMetadataParserSettings { data_directory: PathBuf, + + #[default(DEFAULT_KEY_FILTER.to_string())] key_prefix: String, #[default(DEFAULT_ARCH.to_string())] @@ -202,9 +207,20 @@ where impl OpenshiftSecondaryMetadataParserPlugin { pub(crate) const PLUGIN_NAME: &'static str = "openshift-secondary-metadata-parse"; - async fn process_raw_metadata(&self, io: &mut InternalIO) -> Fallible<()> { - let path = self.settings.data_directory.join("raw/metadata.json"); + fn get_data_directory(&self, io: &InternalIO) -> PathBuf { + if let Some(data_dir) = io.parameters.get(GRAPH_DATA_DIR_PARAM_KEY) { + PathBuf::from(data_dir) + } else { + self.settings.data_directory.clone() + } + } + async fn process_raw_metadata( + &self, + graph: &mut cincinnati::Graph, + data_dir: &PathBuf, + ) -> Fallible<()> { + let path = data_dir.join("raw/metadata.json"); let json = tokio::fs::read(&path) .await .context(format!("Reading {:?}", &path))?; @@ -215,7 +231,7 @@ impl OpenshiftSecondaryMetadataParserPlugin { raw_metadata.iter().for_each(|(version, metadata)| { metadata.iter().for_each(|(key, value)| { - io.graph.find_by_fn_mut(|release| { + graph.find_by_fn_mut(|release| { let release_semver = semver::Version::from_str(release.version()) .context(format!("Parsing {} as SemVer", release.version())); if let Err(e) = &release_semver { @@ -251,8 +267,12 @@ impl OpenshiftSecondaryMetadataParserPlugin { Ok(()) } - async fn process_blocked_edges(&self, io: &mut InternalIO) -> Fallible<()> { - let blocked_edges_dir = self.settings.data_directory.join("blocked-edges"); + async fn process_blocked_edges( + &self, + graph: &mut cincinnati::Graph, + data_dir: &PathBuf, + ) -> Fallible<()> { + let blocked_edges_dir = data_dir.join("blocked-edges"); let blocked_edges: Vec = deserialize_directory_files(&blocked_edges_dir, regex::Regex::new("ya+ml")?) .await @@ -268,7 +288,7 @@ impl OpenshiftSecondaryMetadataParserPlugin { let architectures = { let mut collection = std::collections::BTreeSet::>::new(); - let _ = io.graph.find_by_fn_mut(|release| { + let _ = graph.find_by_fn_mut(|release| { match semver::Version::from_str(release.version()) { Ok(version_semver) => { collection.insert(version_semver.build); @@ -321,15 +341,13 @@ impl OpenshiftSecondaryMetadataParserPlugin { // find all versions in the graph target_versions.iter().for_each(|to| { - match io.graph.find_by_version(&to.to_string()) { + match graph.find_by_version(&to.to_string()) { Some(release_id) => { // add metadata to block edge using the `previous.remove_regex` metadata - match &mut io.graph.get_metadata_as_ref_mut(&release_id).context( - format!( - "[blocked_edges] Getting mutable metadata for {} failed.", - &to.to_string() - ), - ) { + match &mut graph.get_metadata_as_ref_mut(&release_id).context(format!( + "[blocked_edges] Getting mutable metadata for {} failed.", + &to.to_string() + )) { Ok(metadata) => { metadata.insert( format!( @@ -354,8 +372,12 @@ impl OpenshiftSecondaryMetadataParserPlugin { Ok(()) } - async fn process_channels(&self, io: &mut InternalIO) -> Fallible<()> { - let channels_dir = self.settings.data_directory.join("channels"); + async fn process_channels( + &self, + graph: &mut cincinnati::Graph, + data_dir: &PathBuf, + ) -> Fallible<()> { + let channels_dir = data_dir.join("channels"); let channels: Vec = deserialize_directory_files(&channels_dir, regex::Regex::new("ya+ml")?) .await @@ -371,7 +393,7 @@ impl OpenshiftSecondaryMetadataParserPlugin { .iter() .collect::>(); - let releases_in_channel = io.graph.find_by_fn_mut(|release| { + let releases_in_channel = graph.find_by_fn_mut(|release| { let release_semver = match semver::Version::from_str(release.version()) .context(format!("Parsing {} as SemVer", release.version())) { @@ -394,8 +416,8 @@ impl OpenshiftSecondaryMetadataParserPlugin { }); for (release_id, version) in releases_in_channel { - let metadata = match io - .graph + let metadata = match + graph .get_metadata_as_ref_mut(&release_id) .context(format!( "[channels] Getting mutable metadata for {}", @@ -419,7 +441,7 @@ impl OpenshiftSecondaryMetadataParserPlugin { // Sort the channels as some tests and consumers might already depend on // the sorted output which existed in the hack util which is replaced by this plugin. - let sorted_releases = io.graph.find_by_fn_mut(|release| { + let sorted_releases = graph.find_by_fn_mut(|release| { release .get_metadata_mut() .map(|metadata| { @@ -450,9 +472,11 @@ impl OpenshiftSecondaryMetadataParserPlugin { #[async_trait] impl InternalPlugin for OpenshiftSecondaryMetadataParserPlugin { async fn run_internal(self: &Self, mut io: InternalIO) -> Fallible { - self.process_raw_metadata(&mut io).await?; - self.process_blocked_edges(&mut io).await?; - self.process_channels(&mut io).await?; + let data_dir = self.get_data_directory(&io); + + self.process_raw_metadata(&mut io.graph, &data_dir).await?; + self.process_blocked_edges(&mut io.graph, &data_dir).await?; + self.process_channels(&mut io.graph, &data_dir).await?; Ok(io) } diff --git a/commons/src/lib.rs b/commons/src/lib.rs index 975c5124b..d884385e2 100644 --- a/commons/src/lib.rs +++ b/commons/src/lib.rs @@ -32,6 +32,16 @@ where format!("/{}", path_prefix.as_ref().to_string().trim_matches('/')) } +/// Deserialize path_prefix +pub fn de_path_prefix<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + use serde::Deserialize; + let path_prefix = String::deserialize(deserializer)?; + Ok(Some(parse_path_prefix(path_prefix))) +} + /// Parse a comma-separated set of client parameters keys. pub fn parse_params_set(params: S) -> HashSet where diff --git a/dist/Dockerfile.e2e/Dockerfile b/dist/Dockerfile.e2e/Dockerfile index 8b6dc96a3..49ebe0e87 100644 --- a/dist/Dockerfile.e2e/Dockerfile +++ b/dist/Dockerfile.e2e/Dockerfile @@ -40,5 +40,7 @@ COPY --from=rust_builder dist/openshift/observability.yaml dist/openshift/ COPY --from=golang_builder /usr/local/bin/vegeta /usr/bin COPY --from=rust_builder hack/load-testing.sh /usr/local/bin/load-testing.sh COPY --from=rust_builder hack/vegeta.targets vegeta.targets +COPY --from=rust_builder e2e/tests/testdata e2e/tests/testdata +ENV E2E_TESTDATA_DIR "e2e/tests/testdata" ENTRYPOINT ["hack/e2e.sh"] diff --git a/dist/openshift/cincinnati.yaml b/dist/openshift/cincinnati.yaml index 38d21597d..e4f0defcc 100644 --- a/dist/openshift/cincinnati.yaml +++ b/dist/openshift/cincinnati.yaml @@ -318,11 +318,15 @@ parameters: credentials_path = "/etc/secrets/registry_credentials_docker.json" [[plugin_settings]] - name = "quay-metadata" - repository = "openshift-release-dev/ocp-release" + name = "github-secondary-metadata-scrape" + github_org = "openshift" + github_repo = "cincinnati-graph-data" + reference_branch = "master" + output_directory = "/tmp/cincinnati/graph-data" [[plugin_settings]] - name = "node-remove" + name = "openshift-secondary-metadata-parse" + data_directory = "/tmp/cincinnati/graph-data" [[plugin_settings]] name = "edge-add-remove" diff --git a/e2e/Cargo.toml b/e2e/Cargo.toml index 322514ed8..a9ae1cc86 100644 --- a/e2e/Cargo.toml +++ b/e2e/Cargo.toml @@ -18,6 +18,8 @@ url = "^1.7.2" commons = { path = "../commons" } tokio = { version = "0.2.11", features = [ "fs", "stream" ] } prometheus-query = { path = "../prometheus-query" } +lazy_static = "^1.2.0" +cincinnati = { path = "../cincinnati", features = ["test"] } [features] test-e2e-prom-query = [] diff --git a/e2e/tests/e2e.rs b/e2e/tests/e2e.rs index 62ac8dd57..3dbbd0558 100644 --- a/e2e/tests/e2e.rs +++ b/e2e/tests/e2e.rs @@ -1,15 +1,28 @@ -use assert_json_diff::assert_json_include; -use commons::testing::sort_json_graph_by_version; +use failure::ResultExt; use reqwest::header::{HeaderValue, ACCEPT}; -use serde_json::Value; use std::env; use test_case::test_case; use url::Url; -#[test_case("a", "amd64", include_str!("./testdata/a-amd64.json"); "channel a amd64")] -#[test_case("b", "amd64", include_str!("./testdata/b-amd64.json"); "channel b amd64")] -#[test_case("test", "amd64", include_str!("./testdata/test-amd64.json"); "channel test amd64")] -fn e2e_channel_success(channel: &'static str, arch: &'static str, testdata: &str) { +lazy_static::lazy_static! { + static ref METADATA_REVISION: String = std::env::var("E2E_METADATA_REVISION").unwrap(); + static ref TESTDATA_DIR: String = std::env::var("E2E_TESTDATA_DIR").unwrap(); +} + +#[test_case("stable-4.2", "amd64")] +#[test_case("stable-4.2", "s390x")] +#[test_case("stable-4.3", "amd64")] +#[test_case("stable-4.3", "s390x")] +fn e2e_channel_success(channel: &'static str, arch: &'static str) { + let file_suffix = "-production"; + let testdata_path = format!( + "{}/{}_{}_{}{}.json", + *TESTDATA_DIR, *METADATA_REVISION, channel, arch, file_suffix, + ); + let testdata = &std::fs::read_to_string(&testdata_path) + .context(format!("reading {}", &testdata_path)) + .unwrap(); + let mut runtime = commons::testing::init_runtime().unwrap(); let graph_base_url = match env::var("GRAPH_URL") { @@ -17,8 +30,7 @@ fn e2e_channel_success(channel: &'static str, arch: &'static str, testdata: &str _ => panic!("GRAPH_URL unset"), }; - let mut expected: Value = serde_json::from_str(testdata).unwrap(); - sort_json_graph_by_version(&mut expected); + let expected: cincinnati::Graph = serde_json::from_str(testdata).unwrap(); let mut graph_url = Url::parse(&graph_base_url).unwrap(); graph_url @@ -41,7 +53,16 @@ fn e2e_channel_success(channel: &'static str, arch: &'static str, testdata: &str .unwrap(); assert_eq!(res.status().is_success(), true); let text = runtime.block_on(res.text()).unwrap(); - let mut actual = serde_json::from_str(&text).unwrap(); - sort_json_graph_by_version(&mut actual); - assert_json_include!(actual: actual, expected: expected) + let actual: cincinnati::Graph = serde_json::from_str(&text).unwrap(); + + if let Err(e) = cincinnati::testing::compare_graphs_verbose( + expected, + actual, + &[ + "io.openshift.upgrades.graph.previous.remove_regex", + "io.openshift.upgrades.graph.previous.remove", + ], + ) { + panic!("{}", e); + } } diff --git a/e2e/tests/testdata/a-amd64.json b/e2e/tests/testdata/a-amd64.json deleted file mode 100644 index ff2568e84..000000000 --- a/e2e/tests/testdata/a-amd64.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "edges": [ - [ - 1, - 0 - ] - ], - "nodes": [ - { - "version": "0.0.1", - "payload": "quay.io/redhat/openshift-cincinnati-test-public-manual@sha256:73df5efa869eaf57d4125f7655e05e1a72b59d05e55fea06d3701ea5b59234ff", - "metadata": { - "io.openshift.upgrades.graph.release.manifestref": "sha256:73df5efa869eaf57d4125f7655e05e1a72b59d05e55fea06d3701ea5b59234ff", - "io.openshift.upgrades.graph.release.channels": "a,b, test", - "kind": "test" - } - }, - { - "version": "0.0.0", - "payload": "quay.io/redhat/openshift-cincinnati-test-public-manual@sha256:a264db3ac5288c9903dc3db269fca03a0b122fe4af80b57fc5087b329995013d", - "metadata": { - "io.openshift.upgrades.graph.release.channels": "a, test", - "io.openshift.upgrades.graph.release.manifestref": "sha256:a264db3ac5288c9903dc3db269fca03a0b122fe4af80b57fc5087b329995013d" - } - } - ] -} diff --git a/e2e/tests/testdata/b-amd64.json b/e2e/tests/testdata/b-amd64.json deleted file mode 100644 index ccd6fe9b2..000000000 --- a/e2e/tests/testdata/b-amd64.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "nodes": [ - { - "version": "0.0.1", - "payload": "quay.io/redhat/openshift-cincinnati-test-public-manual@sha256:73df5efa869eaf57d4125f7655e05e1a72b59d05e55fea06d3701ea5b59234ff", - "metadata": { - "io.openshift.upgrades.graph.release.manifestref": "sha256:73df5efa869eaf57d4125f7655e05e1a72b59d05e55fea06d3701ea5b59234ff", - "kind": "test", - "io.openshift.upgrades.graph.release.channels": "a,b, test" - } - } - ], - "edges": [] -} diff --git a/e2e/tests/testdata/d980578d2e9b1869b6b1e22711c68c03f0751974_stable-4.2_amd64-production.json b/e2e/tests/testdata/d980578d2e9b1869b6b1e22711c68c03f0751974_stable-4.2_amd64-production.json new file mode 100644 index 000000000..6a52a1227 --- /dev/null +++ b/e2e/tests/testdata/d980578d2e9b1869b6b1e22711c68c03f0751974_stable-4.2_amd64-production.json @@ -0,0 +1,774 @@ +{ + "nodes": [ + { + "version": "4.2.9", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:f28cbabd1227352fe704a00df796a4511880174042dece96233036a10ac61639", + "metadata": { + "url": "https://access.redhat.com/errata/RHBA-2019:3953", + "description": "", + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2", + "io.openshift.upgrades.graph.release.manifestref": "sha256:f28cbabd1227352fe704a00df796a4511880174042dece96233036a10ac61639" + } + }, + { + "version": "4.2.21", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:6c57b48ec03382d9d63c529d4665d133969573966400515777f36dd592ad834a", + "metadata": { + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2,candidate-4.3,fast-4.3,stable-4.3", + "description": "", + "url": "https://access.redhat.com/errata/RHBA-2020:0614", + "io.openshift.upgrades.graph.release.manifestref": "sha256:6c57b48ec03382d9d63c529d4665d133969573966400515777f36dd592ad834a" + } + }, + { + "version": "4.1.28", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:5eaaeb251579254e837f361bbda7f5c897e8cbf15ae52f1d78bfb1bcc18aec7b", + "metadata": { + "io.openshift.upgrades.graph.release.manifestref": "sha256:5eaaeb251579254e837f361bbda7f5c897e8cbf15ae52f1d78bfb1bcc18aec7b", + "description": "", + "io.openshift.upgrades.graph.release.channels": "prerelease-4.1,stable-4.1,candidate-4.2,fast-4.2,stable-4.2", + "url": "https://access.redhat.com/errata/RHBA-2019:4186" + } + }, + { + "version": "4.2.11", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:49ee20ee3102b15a7cf4c019fd8875134fda41ccda1dc27b6e4483ded2aa8a5c", + "metadata": { + "io.openshift.upgrades.graph.release.manifestref": "sha256:49ee20ee3102b15a7cf4c019fd8875134fda41ccda1dc27b6e4483ded2aa8a5c", + "url": "https://access.redhat.com/errata/RHBA-2019:4181", + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2", + "io.openshift.upgrades.graph.previous.remove": "*", + "description": "" + } + }, + { + "version": "4.2.10", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:dc2e38fb00085d6b7f722475f8b7b758a0cb3a02ba42d9acf8a8298a6d510d9c", + "metadata": { + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2", + "url": "https://access.redhat.com/errata/RHBA-2019:4093", + "io.openshift.upgrades.graph.release.manifestref": "sha256:dc2e38fb00085d6b7f722475f8b7b758a0cb3a02ba42d9acf8a8298a6d510d9c", + "description": "" + } + }, + { + "version": "4.2.2", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:dc782b44cac3d59101904cc5da2b9d8bdb90e55a07814df50ea7a13071b0f5f0", + "metadata": { + "url": "https://access.redhat.com/errata/RHBA-2019:3151", + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2", + "io.openshift.upgrades.graph.release.manifestref": "sha256:dc782b44cac3d59101904cc5da2b9d8bdb90e55a07814df50ea7a13071b0f5f0", + "description": "" + } + }, + { + "version": "4.1.26", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:3bc5f8bb2df578ea17e3d5ff5b9ae899a40fe16402b393abec0357f31eb85e58", + "metadata": { + "description": "", + "io.openshift.upgrades.graph.release.channels": "prerelease-4.1,stable-4.1,candidate-4.2,fast-4.2,stable-4.2", + "io.openshift.upgrades.graph.release.manifestref": "sha256:3bc5f8bb2df578ea17e3d5ff5b9ae899a40fe16402b393abec0357f31eb85e58", + "url": "https://access.redhat.com/errata/RHBA-2019:3956" + } + }, + { + "version": "4.2.16", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:e5a6e348721c38a78d9299284fbb5c60fb340135a86b674b038500bf190ad514", + "metadata": { + "url": "https://access.redhat.com/errata/RHBA-2020:0107", + "io.openshift.upgrades.graph.release.manifestref": "sha256:e5a6e348721c38a78d9299284fbb5c60fb340135a86b674b038500bf190ad514", + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2,candidate-4.3,fast-4.3", + "description": "" + } + }, + { + "version": "4.1.34", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:d98c0df40e603b4a8ab344e2b4f44e53a67b036803f45be9a99bc0153ebb3f2e", + "metadata": { + "url": "https://access.redhat.com/errata/RHBA-2020:0399", + "io.openshift.upgrades.graph.release.manifestref": "sha256:d98c0df40e603b4a8ab344e2b4f44e53a67b036803f45be9a99bc0153ebb3f2e", + "description": "", + "io.openshift.upgrades.graph.release.channels": "prerelease-4.1,stable-4.1,candidate-4.2,fast-4.2,stable-4.2" + } + }, + { + "version": "4.1.24", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:6f87fb66dfa907db03981e69474ea3069deab66358c18d965f6331bd727ff23f", + "metadata": { + "io.openshift.upgrades.graph.release.manifestref": "sha256:6f87fb66dfa907db03981e69474ea3069deab66358c18d965f6331bd727ff23f", + "description": "", + "io.openshift.upgrades.graph.release.channels": "prerelease-4.1,stable-4.1,candidate-4.2,fast-4.2,stable-4.2", + "url": "https://access.redhat.com/errata/RHBA-2019:3875" + } + }, + { + "version": "4.2.12", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:77ade34c373062c6a6c869e0e56ef93b2faaa373adadaac1430b29484a24d843", + "metadata": { + "description": "", + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2", + "url": "https://access.redhat.com/errata/RHBA-2019:4181", + "io.openshift.upgrades.graph.release.manifestref": "sha256:77ade34c373062c6a6c869e0e56ef93b2faaa373adadaac1430b29484a24d843" + } + }, + { + "version": "4.2.7", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:bac62983757570b9b8f8bc84c740782984a255c16372b3e30cfc8b52c0a187b9", + "metadata": { + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2", + "description": "", + "io.openshift.upgrades.graph.release.manifestref": "sha256:bac62983757570b9b8f8bc84c740782984a255c16372b3e30cfc8b52c0a187b9", + "url": "https://access.redhat.com/errata/RHBA-2019:3869" + } + }, + { + "version": "4.2.13", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:782b41750f3284f3c8ee2c1f8cb896896da074e362cf8a472846356d1617752d", + "metadata": { + "io.openshift.upgrades.graph.release.manifestref": "sha256:782b41750f3284f3c8ee2c1f8cb896896da074e362cf8a472846356d1617752d", + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2", + "description": "", + "url": "https://access.redhat.com/errata/RHBA-2020:0014" + } + }, + { + "version": "4.1.29", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:a30326a62fabe5478a11ed712f38bab4d1b56f688acbf65f21dfe2b8310f9220", + "metadata": { + "url": "https://access.redhat.com/errata/RHBA-2020:0010", + "io.openshift.upgrades.graph.release.manifestref": "sha256:a30326a62fabe5478a11ed712f38bab4d1b56f688acbf65f21dfe2b8310f9220", + "io.openshift.upgrades.graph.release.channels": "prerelease-4.1,stable-4.1,candidate-4.2,fast-4.2,stable-4.2", + "description": "" + } + }, + { + "version": "4.1.27", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:10519d7ba7e028efec0b9a210e5c75d37f2f5fe74bced4e034bd89a323ea833f", + "metadata": { + "io.openshift.upgrades.graph.release.manifestref": "sha256:10519d7ba7e028efec0b9a210e5c75d37f2f5fe74bced4e034bd89a323ea833f", + "url": "https://access.redhat.com/errata/RHBA-2019:4084", + "description": "", + "io.openshift.upgrades.graph.release.channels": "prerelease-4.1,stable-4.1,candidate-4.2,fast-4.2,stable-4.2" + } + }, + { + "version": "4.1.31", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:09bfa02aca6fb3f34e28b955f75efbaf28d2e760f384b01f9ab4a0fc1154962a", + "metadata": { + "io.openshift.upgrades.graph.release.manifestref": "sha256:09bfa02aca6fb3f34e28b955f75efbaf28d2e760f384b01f9ab4a0fc1154962a", + "io.openshift.upgrades.graph.release.channels": "prerelease-4.1,stable-4.1,candidate-4.2,fast-4.2,stable-4.2", + "url": "https://access.redhat.com/errata/RHBA-2020:0115", + "description": "" + } + }, + { + "version": "4.2.0", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:c5337afd85b94c93ec513f21c8545e3f9e36a227f55d41bc1dfb8fcc3f2be129", + "metadata": { + "io.openshift.upgrades.graph.previous.remove": "4.1.20", + "url": "https://access.redhat.com/errata/RHBA-2019:2922", + "description": "", + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2", + "io.openshift.upgrades.graph.release.manifestref": "sha256:c5337afd85b94c93ec513f21c8545e3f9e36a227f55d41bc1dfb8fcc3f2be129" + } + }, + { + "version": "4.2.14", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:3fabe939da31f9a31f509251b9f73d321e367aba2d09ff392c2f452f6433a95a", + "metadata": { + "description": "", + "url": "https://access.redhat.com/errata/RHBA-2020:0066", + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2", + "io.openshift.upgrades.graph.release.manifestref": "sha256:3fabe939da31f9a31f509251b9f73d321e367aba2d09ff392c2f452f6433a95a" + } + }, + { + "version": "4.1.30", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:c811f484faeefa469492b583033da759ca3323e9810471185a579baab187052c", + "metadata": { + "io.openshift.upgrades.graph.release.channels": "prerelease-4.1,stable-4.1,candidate-4.2,fast-4.2,stable-4.2", + "description": "", + "url": "https://access.redhat.com/errata/RHBA-2020:0071", + "io.openshift.upgrades.graph.release.manifestref": "sha256:c811f484faeefa469492b583033da759ca3323e9810471185a579baab187052c" + } + }, + { + "version": "4.2.18", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:283a1625e18e0b6d7f354b1b022a0aeaab5598f2144ec484faf89e1ecb5c7498", + "metadata": { + "io.openshift.upgrades.graph.release.manifestref": "sha256:283a1625e18e0b6d7f354b1b022a0aeaab5598f2144ec484faf89e1ecb5c7498", + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2,candidate-4.3,fast-4.3", + "url": "https://access.redhat.com/errata/RHBA-2020:0395", + "description": "" + } + }, + { + "version": "4.2.19", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:b51a0c316bb0c11686e6b038ec7c9f7ff96763f47a53c3443ac82e8c054bc035", + "metadata": { + "description": "", + "io.openshift.upgrades.graph.release.manifestref": "sha256:b51a0c316bb0c11686e6b038ec7c9f7ff96763f47a53c3443ac82e8c054bc035", + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2,candidate-4.3,fast-4.3", + "url": "https://access.redhat.com/errata/RHBA-2020:0460" + } + }, + { + "version": "4.2.1", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:32f2e138c0c5af6183830a22801f627bedb414591332960b7c8506ba7f6d7bb6", + "metadata": { + "io.openshift.upgrades.graph.release.manifestref": "sha256:32f2e138c0c5af6183830a22801f627bedb414591332960b7c8506ba7f6d7bb6", + "url": "https://access.redhat.com/errata/RHBA-2019:3151", + "io.openshift.upgrades.graph.previous.remove": "*", + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2", + "description": "" + } + }, + { + "version": "4.2.8", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:4bf307b98beba4d42da3316464013eac120c6e5a398646863ef92b0e2c621230", + "metadata": { + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2", + "description": "", + "io.openshift.upgrades.graph.release.manifestref": "sha256:4bf307b98beba4d42da3316464013eac120c6e5a398646863ef92b0e2c621230", + "url": "https://access.redhat.com/errata/RHBA-2019:3919" + } + }, + { + "version": "4.2.4", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:cebce35c054f1fb066a4dc0a518064945087ac1f3637fe23d2ee2b0c433d6ba8", + "metadata": { + "io.openshift.upgrades.graph.release.manifestref": "sha256:cebce35c054f1fb066a4dc0a518064945087ac1f3637fe23d2ee2b0c433d6ba8", + "url": "https://access.redhat.com/errata/RHBA-2019:3303", + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2", + "description": "" + } + }, + { + "version": "4.2.22", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:81154f5c03294534e1eaf0319bef7a601134f891689ccede5d705ef659aa8c92", + "metadata": { + "io.openshift.upgrades.graph.previous.remove": "*", + "description": "", + "io.openshift.upgrades.graph.release.manifestref": "sha256:81154f5c03294534e1eaf0319bef7a601134f891689ccede5d705ef659aa8c92", + "url": "https://access.redhat.com/errata/RHBA-2020:0685", + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2,candidate-4.3,fast-4.3,stable-4.3" + } + }, + { + "version": "4.2.20", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:bd8aa8e0ce08002d4f8e73d6a2f9de5ae535a6a961ff6b8fdf2c52e4a14cc787", + "metadata": { + "url": "https://access.redhat.com/errata/RHBA-2020:0523", + "io.openshift.upgrades.graph.release.manifestref": "sha256:bd8aa8e0ce08002d4f8e73d6a2f9de5ae535a6a961ff6b8fdf2c52e4a14cc787", + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2,candidate-4.3,fast-4.3,stable-4.3", + "description": "" + } + } + ], + "edges": [ + [ + 5, + 4 + ], + [ + 23, + 4 + ], + [ + 11, + 4 + ], + [ + 22, + 4 + ], + [ + 0, + 4 + ], + [ + 22, + 7 + ], + [ + 22, + 19 + ], + [ + 25, + 1 + ], + [ + 11, + 10 + ], + [ + 16, + 4 + ], + [ + 22, + 10 + ], + [ + 20, + 1 + ], + [ + 10, + 1 + ], + [ + 0, + 10 + ], + [ + 23, + 12 + ], + [ + 22, + 12 + ], + [ + 0, + 12 + ], + [ + 19, + 20 + ], + [ + 3, + 7 + ], + [ + 4, + 20 + ], + [ + 10, + 20 + ], + [ + 5, + 10 + ], + [ + 11, + 17 + ], + [ + 0, + 1 + ], + [ + 12, + 1 + ], + [ + 19, + 1 + ], + [ + 13, + 8 + ], + [ + 0, + 19 + ], + [ + 17, + 7 + ], + [ + 14, + 4 + ], + [ + 5, + 23 + ], + [ + 10, + 7 + ], + [ + 12, + 7 + ], + [ + 0, + 25 + ], + [ + 12, + 25 + ], + [ + 4, + 12 + ], + [ + 16, + 12 + ], + [ + 5, + 12 + ], + [ + 13, + 12 + ], + [ + 11, + 12 + ], + [ + 10, + 12 + ], + [ + 4, + 7 + ], + [ + 8, + 20 + ], + [ + 9, + 15 + ], + [ + 8, + 19 + ], + [ + 6, + 15 + ], + [ + 14, + 15 + ], + [ + 2, + 15 + ], + [ + 13, + 15 + ], + [ + 18, + 15 + ], + [ + 23, + 10 + ], + [ + 3, + 17 + ], + [ + 6, + 14 + ], + [ + 10, + 19 + ], + [ + 12, + 19 + ], + [ + 22, + 17 + ], + [ + 0, + 17 + ], + [ + 2, + 13 + ], + [ + 11, + 7 + ], + [ + 17, + 25 + ], + [ + 0, + 7 + ], + [ + 2, + 18 + ], + [ + 13, + 18 + ], + [ + 9, + 13 + ], + [ + 23, + 7 + ], + [ + 8, + 1 + ], + [ + 9, + 14 + ], + [ + 4, + 25 + ], + [ + 14, + 18 + ], + [ + 10, + 25 + ], + [ + 14, + 13 + ], + [ + 7, + 25 + ], + [ + 20, + 25 + ], + [ + 11, + 19 + ], + [ + 6, + 13 + ], + [ + 9, + 11 + ], + [ + 16, + 11 + ], + [ + 21, + 11 + ], + [ + 5, + 11 + ], + [ + 23, + 11 + ], + [ + 4, + 1 + ], + [ + 9, + 6 + ], + [ + 4, + 19 + ], + [ + 7, + 1 + ], + [ + 7, + 20 + ], + [ + 4, + 10 + ], + [ + 5, + 17 + ], + [ + 15, + 7 + ], + [ + 7, + 19 + ], + [ + 9, + 2 + ], + [ + 8, + 25 + ], + [ + 6, + 2 + ], + [ + 6, + 8 + ], + [ + 14, + 8 + ], + [ + 18, + 8 + ], + [ + 15, + 8 + ], + [ + 2, + 8 + ], + [ + 9, + 8 + ], + [ + 9, + 18 + ], + [ + 23, + 17 + ], + [ + 16, + 22 + ], + [ + 5, + 22 + ], + [ + 23, + 22 + ], + [ + 11, + 22 + ], + [ + 6, + 0 + ], + [ + 16, + 0 + ], + [ + 5, + 0 + ], + [ + 23, + 0 + ], + [ + 11, + 0 + ], + [ + 22, + 0 + ], + [ + 18, + 17 + ], + [ + 16, + 5 + ], + [ + 9, + 22 + ], + [ + 6, + 18 + ], + [ + 16, + 10 + ], + [ + 14, + 2 + ], + [ + 12, + 20 + ], + [ + 17, + 19 + ], + [ + 4, + 17 + ], + [ + 19, + 25 + ], + [ + 21, + 5 + ], + [ + 12, + 17 + ], + [ + 17, + 1 + ], + [ + 17, + 20 + ], + [ + 2, + 10 + ], + [ + 10, + 17 + ] + ] +} diff --git a/e2e/tests/testdata/d980578d2e9b1869b6b1e22711c68c03f0751974_stable-4.2_s390x-production.json b/e2e/tests/testdata/d980578d2e9b1869b6b1e22711c68c03f0751974_stable-4.2_s390x-production.json new file mode 100644 index 000000000..027d17a87 --- /dev/null +++ b/e2e/tests/testdata/d980578d2e9b1869b6b1e22711c68c03f0751974_stable-4.2_s390x-production.json @@ -0,0 +1,175 @@ +{ + "nodes": [ + { + "version": "4.2.12-s390x", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:f1ef86861361140c69adca66278185c09eba12af80415208a2b0f9b014723f26", + "metadata": { + "description": "", + "url": "https://access.redhat.com/errata/RHBA-2019:4181", + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2", + "io.openshift.upgrades.graph.release.manifestref": "sha256:f1ef86861361140c69adca66278185c09eba12af80415208a2b0f9b014723f26" + } + }, + { + "version": "4.2.20", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:4183c6acd0cc53f1ad4282cc04b4fda57ee2b504635016b3acb1464ea9775ccd", + "metadata": { + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2", + "description": "", + "url": "https://access.redhat.com/errata/RHBA-2020:0523", + "io.openshift.upgrades.graph.release.manifestref": "sha256:4183c6acd0cc53f1ad4282cc04b4fda57ee2b504635016b3acb1464ea9775ccd" + } + }, + { + "version": "4.2.19", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:0b33c4263909b9e8960d3f3551466ab158d28dfd15fac661f0a717a70b804775", + "metadata": { + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2", + "description": "", + "io.openshift.upgrades.graph.release.manifestref": "sha256:0b33c4263909b9e8960d3f3551466ab158d28dfd15fac661f0a717a70b804775", + "url": "https://access.redhat.com/errata/RHBA-2020:0460" + } + }, + { + "version": "4.2.21", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:40153ac32cb859c25cd69ef25bfc9c9b87f17d990ec9330fdb78ad2b6ec04fd1", + "metadata": { + "io.openshift.upgrades.graph.release.manifestref": "sha256:40153ac32cb859c25cd69ef25bfc9c9b87f17d990ec9330fdb78ad2b6ec04fd1", + "description": "", + "url": "https://access.redhat.com/errata/RHBA-2020:0614", + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2" + } + }, + { + "version": "4.2.16", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:04b79afdbb41579e0f969bf23cdb16483dbfadbc09a6a0fec60b94328d68135a", + "metadata": { + "url": "https://access.redhat.com/errata/RHBA-2020:0107", + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2", + "io.openshift.upgrades.graph.release.manifestref": "sha256:04b79afdbb41579e0f969bf23cdb16483dbfadbc09a6a0fec60b94328d68135a", + "description": "" + } + }, + { + "version": "4.2.11-s390x", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:96ba6acf8cbcb6863425ecdc006495ae43fbe2b7b55f6002f5a4c77ea6183373", + "metadata": { + "description": "", + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2", + "io.openshift.upgrades.graph.previous.remove": "*", + "url": "https://access.redhat.com/errata/RHBA-2019:4181", + "io.openshift.upgrades.graph.release.manifestref": "sha256:96ba6acf8cbcb6863425ecdc006495ae43fbe2b7b55f6002f5a4c77ea6183373" + } + }, + { + "version": "4.2.13-s390x", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:5bbfbaf203e981a5783f1e73bedd41d2a35d69e11cf7d0d2de0becdd51a1fb1d", + "metadata": { + "io.openshift.upgrades.graph.release.manifestref": "sha256:5bbfbaf203e981a5783f1e73bedd41d2a35d69e11cf7d0d2de0becdd51a1fb1d", + "io.openshift.upgrades.graph.previous.remove": "*", + "description": "", + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2", + "url": "https://access.redhat.com/errata/RHBA-2020:0014" + } + }, + { + "version": "4.2.22", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:cc6aa448cccb6f7d15dd38eae0508fa4fcc1cc720a2d99bd79541cd620f7e5be", + "metadata": { + "url": "https://access.redhat.com/errata/RHBA-2020:0685", + "io.openshift.upgrades.graph.release.manifestref": "sha256:cc6aa448cccb6f7d15dd38eae0508fa4fcc1cc720a2d99bd79541cd620f7e5be", + "description": "", + "io.openshift.upgrades.graph.previous.remove": "*", + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2" + } + }, + { + "version": "4.2.18", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:260ada42ef59e8af11a885897f39f293e7e8b95c40d6a393196b7f0e2b826f0b", + "metadata": { + "description": "", + "url": "https://access.redhat.com/errata/RHBA-2020:0395", + "io.openshift.upgrades.graph.release.manifestref": "sha256:260ada42ef59e8af11a885897f39f293e7e8b95c40d6a393196b7f0e2b826f0b", + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2" + } + } + ], + "edges": [ + [ + 5, + 3 + ], + [ + 5, + 2 + ], + [ + 2, + 3 + ], + [ + 4, + 2 + ], + [ + 2, + 1 + ], + [ + 0, + 1 + ], + [ + 6, + 3 + ], + [ + 8, + 3 + ], + [ + 6, + 2 + ], + [ + 1, + 3 + ], + [ + 4, + 1 + ], + [ + 0, + 3 + ], + [ + 5, + 1 + ], + [ + 4, + 3 + ], + [ + 4, + 8 + ], + [ + 8, + 1 + ], + [ + 8, + 2 + ], + [ + 0, + 2 + ], + [ + 6, + 1 + ] + ] +} diff --git a/e2e/tests/testdata/d980578d2e9b1869b6b1e22711c68c03f0751974_stable-4.3_amd64-production.json b/e2e/tests/testdata/d980578d2e9b1869b6b1e22711c68c03f0751974_stable-4.3_amd64-production.json new file mode 100644 index 000000000..49b08caec --- /dev/null +++ b/e2e/tests/testdata/d980578d2e9b1869b6b1e22711c68c03f0751974_stable-4.3_amd64-production.json @@ -0,0 +1,99 @@ +{ + "nodes": [ + { + "version": "4.3.0", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:3a516480dfd68e0f87f702b4d7bdd6f6a0acfdac5cd2e9767b838ceede34d70d", + "metadata": { + "io.openshift.upgrades.graph.release.manifestref": "sha256:3a516480dfd68e0f87f702b4d7bdd6f6a0acfdac5cd2e9767b838ceede34d70d", + "io.openshift.upgrades.graph.previous.remove": "4.2.16", + "description": "", + "io.openshift.upgrades.graph.release.channels": "candidate-4.3,fast-4.3,stable-4.3", + "url": "https://access.redhat.com/errata/RHBA-2020:0062" + } + }, + { + "version": "4.3.1", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:ea7ac3ad42169b39fce07e5e53403a028644810bee9a212e7456074894df40f3", + "metadata": { + "description": "", + "io.openshift.upgrades.graph.release.manifestref": "sha256:ea7ac3ad42169b39fce07e5e53403a028644810bee9a212e7456074894df40f3", + "io.openshift.upgrades.graph.release.channels": "candidate-4.3,fast-4.3,stable-4.3", + "url": "https://access.redhat.com/errata/RHBA-2020:0391" + } + }, + { + "version": "4.2.20", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:bd8aa8e0ce08002d4f8e73d6a2f9de5ae535a6a961ff6b8fdf2c52e4a14cc787", + "metadata": { + "io.openshift.upgrades.graph.release.manifestref": "sha256:bd8aa8e0ce08002d4f8e73d6a2f9de5ae535a6a961ff6b8fdf2c52e4a14cc787", + "url": "https://access.redhat.com/errata/RHBA-2020:0523", + "description": "", + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2,candidate-4.3,fast-4.3,stable-4.3" + } + }, + { + "version": "4.3.3", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:9b8708b67dd9b7720cb7ab3ed6d12c394f689cc8927df0e727c76809ab383f44", + "metadata": { + "io.openshift.upgrades.graph.previous.remove": "*", + "io.openshift.upgrades.graph.release.manifestref": "sha256:9b8708b67dd9b7720cb7ab3ed6d12c394f689cc8927df0e727c76809ab383f44", + "description": "", + "io.openshift.upgrades.graph.release.channels": "candidate-4.3,fast-4.3,stable-4.3", + "url": "https://access.redhat.com/errata/RHBA-2020:0528" + } + }, + { + "version": "4.2.22", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:81154f5c03294534e1eaf0319bef7a601134f891689ccede5d705ef659aa8c92", + "metadata": { + "io.openshift.upgrades.graph.previous.remove": "*", + "url": "https://access.redhat.com/errata/RHBA-2020:0685", + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2,candidate-4.3,fast-4.3,stable-4.3", + "io.openshift.upgrades.graph.release.manifestref": "sha256:81154f5c03294534e1eaf0319bef7a601134f891689ccede5d705ef659aa8c92", + "description": "" + } + }, + { + "version": "4.2.21", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:6c57b48ec03382d9d63c529d4665d133969573966400515777f36dd592ad834a", + "metadata": { + "description": "", + "io.openshift.upgrades.graph.release.channels": "candidate-4.2,fast-4.2,stable-4.2,candidate-4.3,fast-4.3,stable-4.3", + "io.openshift.upgrades.graph.release.manifestref": "sha256:6c57b48ec03382d9d63c529d4665d133969573966400515777f36dd592ad834a", + "url": "https://access.redhat.com/errata/RHBA-2020:0614" + } + }, + { + "version": "4.3.5", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:64320fbf95d968fc6b9863581a92d373bc75f563a13ae1c727af37450579f61a", + "metadata": { + "io.openshift.upgrades.graph.previous.remove": "*", + "description": "", + "io.openshift.upgrades.graph.release.channels": "candidate-4.3,fast-4.3,stable-4.3,candidate-4.4", + "url": "https://access.redhat.com/errata/RHBA-2020:0676", + "io.openshift.upgrades.graph.release.manifestref": "sha256:64320fbf95d968fc6b9863581a92d373bc75f563a13ae1c727af37450579f61a" + } + }, + { + "version": "4.3.2", + "payload": "quay.io/openshift-release-dev/ocp-release@sha256:cadf53e7181639f6cc77d2430339102db2908de330210c1ff8c7a7dc1cb0e550", + "metadata": { + "url": "https://access.redhat.com/errata/RHBA-2020:0492", + "io.openshift.upgrades.graph.release.channels": "candidate-4.3,fast-4.3,stable-4.3", + "description": "", + "io.openshift.upgrades.graph.previous.remove": "*", + "io.openshift.upgrades.graph.release.manifestref": "sha256:cadf53e7181639f6cc77d2430339102db2908de330210c1ff8c7a7dc1cb0e550" + } + } + ], + "edges": [ + [ + 0, + 1 + ], + [ + 2, + 5 + ] + ] +} diff --git a/e2e/tests/testdata/d980578d2e9b1869b6b1e22711c68c03f0751974_stable-4.3_s390x-production.json b/e2e/tests/testdata/d980578d2e9b1869b6b1e22711c68c03f0751974_stable-4.3_s390x-production.json new file mode 100644 index 000000000..ecb371883 --- /dev/null +++ b/e2e/tests/testdata/d980578d2e9b1869b6b1e22711c68c03f0751974_stable-4.3_s390x-production.json @@ -0,0 +1,4 @@ +{ + "nodes": [], + "edges": [] +} diff --git a/e2e/tests/testdata/metadata_revision b/e2e/tests/testdata/metadata_revision new file mode 100644 index 000000000..ff0b0d618 --- /dev/null +++ b/e2e/tests/testdata/metadata_revision @@ -0,0 +1 @@ +d980578d2e9b1869b6b1e22711c68c03f0751974 diff --git a/e2e/tests/testdata/test-amd64.json b/e2e/tests/testdata/test-amd64.json deleted file mode 100644 index 6f16ccc80..000000000 --- a/e2e/tests/testdata/test-amd64.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "nodes": [ - { - "version": "0.0.1", - "payload": "quay.io/redhat/openshift-cincinnati-test-public-manual@sha256:73df5efa869eaf57d4125f7655e05e1a72b59d05e55fea06d3701ea5b59234ff", - "metadata": { - "io.openshift.upgrades.graph.release.channels": "a,b, test", - "io.openshift.upgrades.graph.release.manifestref": "sha256:73df5efa869eaf57d4125f7655e05e1a72b59d05e55fea06d3701ea5b59234ff", - "kind": "test" - } - }, - { - "version": "0.0.0", - "payload": "quay.io/redhat/openshift-cincinnati-test-public-manual@sha256:a264db3ac5288c9903dc3db269fca03a0b122fe4af80b57fc5087b329995013d", - "metadata": { - "io.openshift.upgrades.graph.release.channels": "a, test", - "io.openshift.upgrades.graph.release.manifestref": "sha256:a264db3ac5288c9903dc3db269fca03a0b122fe4af80b57fc5087b329995013d" - } - } - ], - "edges": [ - [ - 1, - 0 - ] - ] -} diff --git a/graph-builder/Cargo.toml b/graph-builder/Cargo.toml index 596a92410..1f2340b69 100644 --- a/graph-builder/Cargo.toml +++ b/graph-builder/Cargo.toml @@ -42,8 +42,6 @@ built = "^0.3.2" [dev-dependencies] twoway = "^0.2" -assert-json-diff = "1.0.0" -test-case = "1.0.0" [features] test-net = [] diff --git a/graph-builder/src/config/options.rs b/graph-builder/src/config/options.rs index 02c0bb1f7..f0bd4c531 100644 --- a/graph-builder/src/config/options.rs +++ b/graph-builder/src/config/options.rs @@ -1,7 +1,7 @@ //! Options shared by CLI and TOML. use super::AppSettings; -use commons::{parse_params_set, parse_path_prefix, MergeOptions}; +use commons::{de_path_prefix, parse_params_set, parse_path_prefix, MergeOptions}; use failure::Fallible; use std::collections::HashSet; use std::net::IpAddr; @@ -44,6 +44,7 @@ pub struct ServiceOptions { /// Namespace prefix for all service endpoints (e.g. '//v1/graph') #[structopt(long = "service.path_prefix", parse(from_str = "parse_path_prefix"))] + #[serde(default = "Option::default", deserialize_with = "de_path_prefix")] pub path_prefix: Option, /// Comma-separated set of mandatory client parameters diff --git a/graph-builder/src/config/settings.rs b/graph-builder/src/config/settings.rs index e7120dff1..c309229a5 100644 --- a/graph-builder/src/config/settings.rs +++ b/graph-builder/src/config/settings.rs @@ -122,8 +122,16 @@ impl AppSettings { } fn default_openshift_plugin_settings(&self) -> Fallible>> { + use failure::ResultExt; + + use cincinnati::plugins::internal::github_openshift_secondary_metadata_scraper::GITHUB_SCRAPER_TOKEN_PATH_ENV; use cincinnati::plugins::prelude::*; + lazy_static! { + static ref GRAPH_DATA_DIR: tempfile::TempDir = + tempfile::tempdir().expect("failed to create tempdir"); + }; + let plugins = vec![ ReleaseScrapeDockerv2Settings::deserialize_config(toml::from_str(&format!( r#" @@ -146,12 +154,30 @@ impl AppSettings { .map(|path| format!("\ncredentials_path = {:?}", path)) .unwrap_or_default() ))?)?, - plugin_config!( - ("name", QuayMetadataFetchPlugin::PLUGIN_NAME), - ("repository", &self.repository), - ("manifestref_key", &self.manifestref_key) + GithubOpenshiftSecondaryMetadataScraperSettings::deserialize_config(toml::from_str( + &format!( + r#" + github_org = "openshift" + github_repo = "cincinnati-graph-data" + branch = "master" + output_directory = {:?} + {} + "#, + &GRAPH_DATA_DIR.path(), + std::env::var(GITHUB_SCRAPER_TOKEN_PATH_ENV) + .map(|path| format!("oauth_token_path = {:?}", path)) + .unwrap_or_default() + ), + )?)?, + OpenshiftSecondaryMetadataParserSettings::deserialize_config( + toml::from_str(&format!( + r#" + data_directory = {:?} + "#, + &GRAPH_DATA_DIR.path(), + )) + .context("Parsing config string to settings")?, )?, - plugin_config!(("name", NodeRemovePlugin::PLUGIN_NAME))?, plugin_config!(("name", EdgeAddRemovePlugin::PLUGIN_NAME))?, ]; diff --git a/hack/e2e.sh b/hack/e2e.sh index 21f06aae7..9ffd84e58 100755 --- a/hack/e2e.sh +++ b/hack/e2e.sh @@ -54,6 +54,12 @@ oc -n openshift-monitoring create configmap cluster-monitoring-config --from-lit # `oc new-app` would stumble on unknown monitoring.coreos.com/v1 objects, so process and create instead oc process -f dist/openshift/observability.yaml -p NAMESPACE="cincinnati-e2e" | oc apply -f - +# Export the e2e test environment variables +E2E_TESTDATA_DIR="${E2E_TESTDATA_DIR:-e2e/tests/testdata}" +export E2E_TESTDATA_DIR +read -r E2E_METADATA_REVISION <"${E2E_TESTDATA_DIR}"/metadata_revision +export E2E_METADATA_REVISION + # Apply oc template oc new-app -f dist/openshift/cincinnati.yaml \ -p IMAGE="${IMAGE}" \ @@ -61,22 +67,27 @@ oc new-app -f dist/openshift/cincinnati.yaml \ -p GB_CPU_REQUEST=50m \ -p PE_CPU_REQUEST=50m \ -p RUST_BACKTRACE="1" \ - -p GB_PLUGIN_SETTINGS=' + -p GB_PLUGIN_SETTINGS="$(cat <<-EOF [[plugin_settings]] name = "release-scrape-dockerv2" - repository = "redhat/openshift-cincinnati-test-public-manual" + repository = "openshift-release-dev/ocp-release" fetch_concurrency = 128 [[plugin_settings]] - name = "quay-metadata" - repository = "redhat/openshift-cincinnati-test-public-manual" + name = "github-secondary-metadata-scrape" + github_org = "openshift" + github_repo = "cincinnati-graph-data" + reference_revision = "${E2E_METADATA_REVISION}" + output_directory = "/tmp/cincinnati-graph-data" [[plugin_settings]] - name = "node-remove" + name = "openshift-secondary-metadata-parse" + data_directory = "/tmp/cincinnati-graph-data" [[plugin_settings]] name = "edge-add-remove" - ' \ +EOF +)" \ -p ENVIRONMENT_SECRETS="{}" \ ; diff --git a/hack/load-testing.sh b/hack/load-testing.sh index 135c72513..89d26288c 100755 --- a/hack/load-testing.sh +++ b/hack/load-testing.sh @@ -13,7 +13,7 @@ TMP_DIR=$(mktemp -d) duration=30s for workers in 10 50 100; do - for rate in 10 100 500 1000; do + for rate in 10 100; do file="${TMP_DIR}/rate-${rate}-workers-${workers}.bin" echo "Testing workers ${workers}, rate ${rate} -> ${file}" sed "s,GRAPH_URL,${GRAPH_URL},g" vegeta.targets | \ diff --git a/policy-engine/src/config/options.rs b/policy-engine/src/config/options.rs index 3a05ecca9..5404ed8fb 100644 --- a/policy-engine/src/config/options.rs +++ b/policy-engine/src/config/options.rs @@ -1,7 +1,7 @@ //! Options shared by CLI and TOML. use super::AppSettings; -use commons::{parse_params_set, parse_path_prefix, MergeOptions}; +use commons::{de_path_prefix, parse_params_set, parse_path_prefix, MergeOptions}; use failure::Fallible; use std::collections::HashSet; use std::net::IpAddr; @@ -41,6 +41,7 @@ pub struct ServiceOptions { /// Namespace prefix for all service endpoints (e.g. '//v1/graph') #[structopt(long = "service.path_prefix", parse(from_str = "parse_path_prefix"))] + #[serde(default = "Option::default", deserialize_with = "de_path_prefix")] pub path_prefix: Option, /// Comma-separated set of mandatory client parameters