diff --git a/Justfile b/Justfile index 09fb2356e..d32e3e28e 100644 --- a/Justfile +++ b/Justfile @@ -15,7 +15,7 @@ coverage: test _coverage test-pwd +args="": #!/usr/bin/env bash set -e - export RUST_BACKTRACE=0 RUST_LOG="graph-builder=trace,cincinnati=trace,dkregistry=trace" + export RUST_BACKTRACE=1 RUST_LOG="graph-builder=trace,cincinnati=trace,dkregistry=trace" pushd {{invocation_directory()}} cargo test {{args}} diff --git a/cincinnati/src/plugins/internal/graph_builder/github_openshift_secondary_metadata_scraper/mod.rs b/cincinnati/src/plugins/internal/graph_builder/github_openshift_secondary_metadata_scraper/mod.rs index 0ef7d7e8e..8f8dd0067 100644 --- a/cincinnati/src/plugins/internal/graph_builder/github_openshift_secondary_metadata_scraper/mod.rs +++ b/cincinnati/src/plugins/internal/graph_builder/github_openshift_secondary_metadata_scraper/mod.rs @@ -9,4 +9,5 @@ pub mod plugin; pub use plugin::{ GithubOpenshiftSecondaryMetadataScraperPlugin, GithubOpenshiftSecondaryMetadataScraperSettings, + GITHUB_SCRAPER_TOKEN_PATH_ENV, }; diff --git a/cincinnati/src/plugins/internal/graph_builder/github_openshift_secondary_metadata_scraper/plugin.rs b/cincinnati/src/plugins/internal/graph_builder/github_openshift_secondary_metadata_scraper/plugin.rs index 8ce87be8c..499157123 100644 --- a/cincinnati/src/plugins/internal/graph_builder/github_openshift_secondary_metadata_scraper/plugin.rs +++ b/cincinnati/src/plugins/internal/graph_builder/github_openshift_secondary_metadata_scraper/plugin.rs @@ -13,6 +13,9 @@ pub static DEFAULT_OUTPUT_WHITELIST: &[&str] = &[ "raw/metadata.json", ]; +/// Environment variable name for the Oauth token path +pub static GITHUB_SCRAPER_TOKEN_PATH_ENV: &str = "CINCINNATI_GITHUB_SCRAPER_OAUTH_TOKEN_PATH"; + static USER_AGENT: &str = "openshift/cincinnati"; /// Plugin settings. @@ -28,6 +31,7 @@ pub struct GithubOpenshiftSecondaryMetadataScraperSettings { /// An empty vector is regarded as a configuration error. #[default(DEFAULT_OUTPUT_WHITELIST.iter().map(|s| (*s).to_string()).collect())] output_whitelist: Vec, + oauth_token_path: Option, } impl GithubOpenshiftSecondaryMetadataScraperSettings { @@ -61,6 +65,9 @@ pub struct GithubOpenshiftSecondaryMetadataScraperPlugin { #[default(FuturesMutex::new(Default::default()))] state: FuturesMutex, + oauth_token: Option, + + client: reqwest::Client, } impl GithubOpenshiftSecondaryMetadataScraperPlugin { @@ -81,9 +88,25 @@ impl GithubOpenshiftSecondaryMetadataScraperPlugin { ) .context("Parsing output whitelist strings as regex")?; + let oauth_token = (&settings.oauth_token_path) + .clone() + .map(|path| { + std::fs::read_to_string(&path) + .context(format!("Reading Oauth token from {:?}", &path)) + }) + .transpose()? + .map(|token| { + token + .lines() + .next() + .map(|first_line| first_line.trim().to_owned()) + }) + .flatten(); + Ok(Self { settings, output_whitelist, + oauth_token, ..Default::default() }) @@ -94,10 +117,21 @@ impl GithubOpenshiftSecondaryMetadataScraperPlugin { let url = github_v3::branches_url(&self.settings.github_org, &self.settings.github_repo); trace!("Getting branches from {}", &url); - let bytes = reqwest::Client::new() - .get(&url) - .header(reqwest::header::USER_AGENT, USER_AGENT) - .header(reqwest::header::ACCEPT, "application/vnd.github.v3+json") + + let request = { + let request = self + .client + .get(&url) + .header(reqwest::header::USER_AGENT, USER_AGENT) + .header(reqwest::header::ACCEPT, "application/vnd.github.v3+json"); + if let Some(token) = &self.oauth_token { + request.header(reqwest::header::AUTHORIZATION, format!("token {}", token)) + } else { + request + } + }; + + let bytes = request .send() .await .context(format!("Getting branches from {}", &url))? @@ -310,6 +344,8 @@ mod network_tests { let tmpdir = tempfile::tempdir()?; + let oauth_token_path = std::env::var(GITHUB_SCRAPER_TOKEN_PATH_ENV)?; + let settings = toml::from_str::(&format!( r#" @@ -318,6 +354,7 @@ mod network_tests { branch = "master" output_whitelist = [ {} ] output_directory = {:?} + oauth_token_path = {:?} "#, DEFAULT_OUTPUT_WHITELIST .iter() @@ -325,6 +362,7 @@ mod network_tests { .collect::>() .join(", "), &tmpdir.path(), + oauth_token_path, ))?; debug!("Settings: {:#?}", &settings); diff --git a/dist/openshift/cincinnati.yaml b/dist/openshift/cincinnati.yaml index 891ea3e55..38d21597d 100644 --- a/dist/openshift/cincinnati.yaml +++ b/dist/openshift/cincinnati.yaml @@ -39,6 +39,9 @@ objects: configMapKeyRef: key: gb.rust_backtrace name: cincinnati + envFrom: + - configMapRef: + name: environment-secrets command: - ${GB_BINARY} args: [ @@ -198,6 +201,11 @@ objects: pe.log.verbosity: ${{PE_LOG_VERBOSITY}} pe.mandatory_client_parameters: "channel" pe.rust_backtrace: "${RUST_BACKTRACE}" + - apiVersion: v1 + kind: ConfigMap + metadata: + name: environment-secrets + data: ${{ENVIRONMENT_SECRETS}} - apiVersion: v1 kind: ConfigMap metadata: @@ -323,3 +331,5 @@ parameters: displayName: Set RUST_BACKTRACE env var - name: GB_CONFIG_PATH value: "/etc/configs/gb.toml" + - name: ENVIRONMENT_SECRETS + value: '{ "CINCINNATI_GITHUB_SCRAPER_OAUTH_TOKEN_PATH": "/etc/secrets/github_token.key" }' diff --git a/hack/e2e.sh b/hack/e2e.sh index a2b3fe368..cd495d7de 100755 --- a/hack/e2e.sh +++ b/hack/e2e.sh @@ -67,6 +67,7 @@ oc new-app -f dist/openshift/cincinnati.yaml \ [[plugin_settings]] name = "edge-add-remove" ' \ + -p ENVIRONMENT_SECRETS="{}" \ ; # Wait for dc to rollout