graph-builder/plugins/github scraper: add token authorization handling

Justfile

@@ -15,7 +15,7 @@ coverage: test _coverage
 test-pwd +args="":
 	#!/usr/bin/env bash
 	set -e
-	export RUST_BACKTRACE=0 RUST_LOG="graph-builder=trace,cincinnati=trace,dkregistry=trace"
+	export RUST_BACKTRACE=1 RUST_LOG="graph-builder=trace,cincinnati=trace,dkregistry=trace"
 	pushd {{invocation_directory()}}
 	cargo test {{args}}
 

cincinnati/src/plugins/internal/graph_builder/github_openshift_secondary_metadata_scraper/mod.rs

@@ -9,4 +9,5 @@ pub mod plugin;
 
 pub use plugin::{
     GithubOpenshiftSecondaryMetadataScraperPlugin, GithubOpenshiftSecondaryMetadataScraperSettings,
+    GITHUB_SCRAPER_TOKEN_PATH_ENV,
 };

cincinnati/src/plugins/internal/graph_builder/github_openshift_secondary_metadata_scraper/plugin.rs

@@ -13,6 +13,9 @@ pub static DEFAULT_OUTPUT_WHITELIST: &[&str] = &[
     "raw/metadata.json",
 ];
 
+/// Environment variable name for the Oauth token path
+pub static GITHUB_SCRAPER_TOKEN_PATH_ENV: &str = "CINCINNATI_GITHUB_SCRAPER_OAUTH_TOKEN_PATH";
+
 static USER_AGENT: &str = "openshift/cincinnati";
 
 /// Plugin settings.
@@ -28,6 +31,7 @@ pub struct GithubOpenshiftSecondaryMetadataScraperSettings {
     /// An empty vector is regarded as a configuration error.
     #[default(DEFAULT_OUTPUT_WHITELIST.iter().map(|s| (*s).to_string()).collect())]
     output_whitelist: Vec<String>,
+    oauth_token_path: Option<PathBuf>,
 }
 
 impl GithubOpenshiftSecondaryMetadataScraperSettings {
@@ -61,6 +65,9 @@ pub struct GithubOpenshiftSecondaryMetadataScraperPlugin {
 
     #[default(FuturesMutex::new(Default::default()))]
     state: FuturesMutex<State>,
+    oauth_token: Option<String>,
+
+    client: reqwest::Client,
 }
 
 impl GithubOpenshiftSecondaryMetadataScraperPlugin {
@@ -81,9 +88,25 @@ impl GithubOpenshiftSecondaryMetadataScraperPlugin {
             )
             .context("Parsing output whitelist strings as regex")?;
 
+        let oauth_token = (&settings.oauth_token_path)
+            .clone()
+            .map(|path| {
+                std::fs::read_to_string(&path)
+                    .context(format!("Reading Oauth token from {:?}", &path))
+            })
+            .transpose()?
+            .map(|token| {
+                token
+                    .lines()
+                    .next()
+                    .map(|first_line| first_line.trim().to_owned())
+            })
+            .flatten();
+
         Ok(Self {
             settings,
             output_whitelist,
+            oauth_token,
 
             ..Default::default()
         })
@@ -94,10 +117,21 @@ impl GithubOpenshiftSecondaryMetadataScraperPlugin {
         let url = github_v3::branches_url(&self.settings.github_org, &self.settings.github_repo);
 
         trace!("Getting branches from {}", &url);
-        let bytes = reqwest::Client::new()
-            .get(&url)
-            .header(reqwest::header::USER_AGENT, USER_AGENT)
-            .header(reqwest::header::ACCEPT, "application/vnd.github.v3+json")
+
+        let request = {
+            let request = self
+                .client
+                .get(&url)
+                .header(reqwest::header::USER_AGENT, USER_AGENT)
+                .header(reqwest::header::ACCEPT, "application/vnd.github.v3+json");
+            if let Some(token) = &self.oauth_token {
+                request.header(reqwest::header::AUTHORIZATION, format!("token {}", token))
+            } else {
+                request
+            }
+        };
+
+        let bytes = request
             .send()
             .await
             .context(format!("Getting branches from {}", &url))?
@@ -310,6 +344,8 @@ mod network_tests {
 
         let tmpdir = tempfile::tempdir()?;
 
+        let oauth_token_path = std::env::var(GITHUB_SCRAPER_TOKEN_PATH_ENV)?;
+
         let settings =
             toml::from_str::<GithubOpenshiftSecondaryMetadataScraperSettings>(&format!(
                 r#"
@@ -318,13 +354,15 @@ mod network_tests {
                     branch = "master"
                     output_whitelist = [ {} ]
                     output_directory = {:?}
+                    oauth_token_path = {:?}
                 "#,
                 DEFAULT_OUTPUT_WHITELIST
                     .iter()
                     .map(|s| format!(r#"{:?}"#, s))
                     .collect::<Vec<_>>()
                     .join(", "),
                 &tmpdir.path(),
+                oauth_token_path,
             ))?;
 
         debug!("Settings: {:#?}", &settings);

dist/openshift/cincinnati.yaml

@@ -39,6 +39,9 @@ objects:
                     configMapKeyRef:
                       key: gb.rust_backtrace
                       name: cincinnati
+              envFrom:
+                - configMapRef:
+                    name: environment-secrets
               command:
                 - ${GB_BINARY}
               args: [
@@ -198,6 +201,11 @@ objects:
       pe.log.verbosity: ${{PE_LOG_VERBOSITY}}
       pe.mandatory_client_parameters: "channel"
       pe.rust_backtrace: "${RUST_BACKTRACE}"
+  - apiVersion: v1
+    kind: ConfigMap
+    metadata:
+      name: environment-secrets
+    data: ${{ENVIRONMENT_SECRETS}}
   - apiVersion: v1
     kind: ConfigMap
     metadata:
@@ -323,3 +331,5 @@ parameters:
     displayName: Set RUST_BACKTRACE env var
   - name: GB_CONFIG_PATH
     value: "/etc/configs/gb.toml"
+  - name: ENVIRONMENT_SECRETS
+    value: '{ "CINCINNATI_GITHUB_SCRAPER_OAUTH_TOKEN_PATH": "/etc/secrets/github_token.key" }'

hack/e2e.sh

@@ -67,6 +67,7 @@ oc new-app -f dist/openshift/cincinnati.yaml \
       [[plugin_settings]]
       name = "edge-add-remove"
   ' \
+  -p ENVIRONMENT_SECRETS="{}" \
   ;
 
 # Wait for dc to rollout