Repository navigation
fix(clp-tdl-package): Resolve AWS credentials via the SDK's default provider chain for default authentication.
#2438
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 2 commits
da325a6
8c4d1d7
20283bf
33abcf1
6052887
4fa33d9
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -10,6 +10,9 @@ use std::process::Command; | |
| use std::process::Stdio; | ||
|
|
||
| use anyhow::Context; | ||
| use aws_config::BehaviorVersion; | ||
| use aws_sdk_s3::config::Credentials; | ||
| use aws_sdk_s3::config::ProvideCredentials; | ||
| use clp_rust_utils::aws::AWS_DEFAULT_REGION; | ||
| use clp_rust_utils::clp_config::AwsAuthentication; | ||
| use clp_rust_utils::clp_config::S3Config; | ||
|
|
@@ -46,6 +49,7 @@ use crate::common::runtime; | |
| /// * A spawned archive finisher panics. | ||
| /// * Forwards [`build_s3_logs_list`]'s return values on failure. | ||
| /// * Forwards [`std::fs::write`]'s return values on failure. | ||
| /// * Forwards [`s3_credential_env`]'s return values on failure. | ||
| /// * Forwards [`extract_s3_output_config`]'s return values on failure. | ||
| /// * Forwards [`prepare_clp_s_input`]'s return values on failure. | ||
| /// * Forwards [`run_clp_s`]'s return values on failure. | ||
|
|
@@ -84,10 +88,16 @@ pub(super) fn compress( | |
| })?; | ||
| tmp_file_deleter.add(list_path.clone()); | ||
|
|
||
| let runtime = runtime(); | ||
| let S3InputSource { | ||
| aws_authentication, .. | ||
| region_code, | ||
| aws_authentication, | ||
| .. | ||
| } = input_source; | ||
| let credential_env = s3_credential_env(&aws_authentication); | ||
| let input_region = region_code | ||
| .as_ref() | ||
| .map_or(AWS_DEFAULT_REGION, NonEmptyString::as_str); | ||
| let credential_env = s3_credential_env(&runtime, input_region, &aws_authentication)?; | ||
|
|
||
| let s3_config = extract_s3_output_config(config)?; | ||
| let archive_dir = config | ||
|
|
@@ -107,7 +117,6 @@ pub(super) fn compress( | |
| &mut tmp_file_deleter, | ||
| )?; | ||
|
|
||
| let runtime = runtime(); | ||
| let client = build_s3_client(&runtime, s3_config); | ||
| let bucket = s3_config.bucket.to_string(); | ||
|
|
||
|
|
@@ -341,26 +350,66 @@ fn build_s3_logs_list(input_source: &S3InputSource) -> anyhow::Result<String> { | |
| /// | ||
| /// # Returns | ||
| /// | ||
| /// * The env-var name-value pairs for [`AwsAuthentication::Credentials`]. | ||
| /// * An empty vector for [`AwsAuthentication::Default`] (which assumes credentials are already in | ||
| /// the ambient env). | ||
| fn s3_credential_env(auth: &AwsAuthentication) -> Vec<(&'static str, String)> { | ||
| /// The env-var name-value pairs containing the resolved access key, secret key, and optional | ||
| /// session token. | ||
|
20001020ycx marked this conversation as resolved.
Resolved
|
||
| /// | ||
| /// # Errors | ||
| /// | ||
| /// Returns an error if: | ||
| /// | ||
| /// * The default AWS SDK credential provider chain has no provider. | ||
| /// * The default AWS SDK credential provider chain fails to resolve credentials. | ||
|
20001020ycx marked this conversation as resolved.
Resolved
|
||
| fn s3_credential_env( | ||
| runtime: &tokio::runtime::Handle, | ||
| region: &str, | ||
| auth: &AwsAuthentication, | ||
| ) -> anyhow::Result<Vec<(&'static str, String)>> { | ||
| /// The env var holding the AWS access key ID. | ||
| const AWS_ACCESS_KEY_ID_ENV_VAR: &str = "AWS_ACCESS_KEY_ID"; | ||
|
|
||
| /// The env var holding the AWS secret access key. | ||
| const AWS_SECRET_ACCESS_KEY_ENV_VAR: &str = "AWS_SECRET_ACCESS_KEY"; | ||
|
|
||
| match auth { | ||
| AwsAuthentication::Credentials { credentials } => vec![ | ||
| (AWS_ACCESS_KEY_ID_ENV_VAR, credentials.access_key_id.clone()), | ||
| ( | ||
| AWS_SECRET_ACCESS_KEY_ENV_VAR, | ||
| credentials.secret_access_key.clone(), | ||
| ), | ||
| ], | ||
| AwsAuthentication::Default => Vec::new(), | ||
| /// The env var holding the AWS session token. | ||
| const AWS_SESSION_TOKEN_ENV_VAR: &str = "AWS_SESSION_TOKEN"; | ||
|
|
||
| let credentials = match auth { | ||
| AwsAuthentication::Credentials { credentials } => Credentials::new( | ||
| credentials.access_key_id.clone(), | ||
| credentials.secret_access_key.clone(), | ||
| credentials.session_token.clone(), | ||
| None, | ||
| "clp-credentials-provider", | ||
| ), | ||
| AwsAuthentication::Default => { | ||
| let sdk_config = runtime.block_on( | ||
| aws_config::defaults(BehaviorVersion::latest()) | ||
| .region(aws_sdk_s3::config::Region::new(region.to_string())) | ||
| .load(), | ||
| ); | ||
| let provider = sdk_config | ||
| .credentials_provider() | ||
| .context("default AWS SDK credential provider is unavailable")?; | ||
| runtime | ||
| .block_on(provider.provide_credentials()) | ||
| .context("failed to resolve credentials from the default AWS SDK provider chain")? | ||
| } | ||
| }; | ||
|
|
||
| let mut env = vec![ | ||
| ( | ||
| AWS_ACCESS_KEY_ID_ENV_VAR, | ||
| credentials.access_key_id().to_string(), | ||
| ), | ||
| ( | ||
| AWS_SECRET_ACCESS_KEY_ENV_VAR, | ||
| credentials.secret_access_key().to_string(), | ||
| ), | ||
| ]; | ||
| if let Some(session_token) = credentials.session_token() { | ||
| env.push((AWS_SESSION_TOKEN_ENV_VAR, session_token.to_string())); | ||
|
20001020ycx marked this conversation as resolved.
Resolved
|
||
| } | ||
| Ok(env) | ||
|
coderabbitai[bot] marked this conversation as resolved.
Resolved
|
||
| } | ||
|
|
||
| /// Parses a single clp-s `--print-archive-stats` stdout line into an [`ArchiveMetadata`]. | ||
|
|
@@ -897,25 +946,47 @@ mod tests { | |
| Ok(()) | ||
| } | ||
|
|
||
| #[test] | ||
| fn s3_credential_env_default() { | ||
| assert_eq!(s3_credential_env(&AwsAuthentication::Default), Vec::new()); | ||
| } | ||
|
|
||
| #[test] | ||
| fn s3_credential_env_credentials() { | ||
| let runtime = tokio::runtime::Runtime::new().expect("failed to create Tokio runtime"); | ||
| let auth = AwsAuthentication::Credentials { | ||
| credentials: AwsCredentials { | ||
| access_key_id: "the-access-key".to_string(), | ||
| secret_access_key: "the-secret-key".to_string(), | ||
| session_token: Some("the-session-token".to_string()), | ||
| }, | ||
| }; | ||
|
|
||
| assert_eq!( | ||
| s3_credential_env(&auth), | ||
| s3_credential_env(runtime.handle(), "us-east-1", &auth) | ||
| .expect("failed to resolve credentials"), | ||
| vec![ | ||
| ("AWS_ACCESS_KEY_ID", "the-access-key".to_string()), | ||
| ("AWS_SECRET_ACCESS_KEY", "the-secret-key".to_string()), | ||
| ("AWS_SESSION_TOKEN", "the-session-token".to_string()), | ||
| ] | ||
| ); | ||
| } | ||
|
coderabbitai[bot] marked this conversation as resolved.
Resolved
|
||
|
|
||
| #[test] | ||
| fn s3_credential_env_default() { | ||
| let runtime = tokio::runtime::Runtime::new().expect("failed to create Tokio runtime"); | ||
| // SAFETY: No other test in this binary reads or writes these env vars, and the env | ||
| // provider is the first source in the SDK's default chain, so the test is deterministic | ||
| // and network-free. | ||
| unsafe { | ||
| std::env::set_var("AWS_ACCESS_KEY_ID", "the-env-access-key"); | ||
| std::env::set_var("AWS_SECRET_ACCESS_KEY", "the-env-secret-key"); | ||
| std::env::set_var("AWS_SESSION_TOKEN", "the-env-session-token"); | ||
| } | ||
|
|
||
| assert_eq!( | ||
| s3_credential_env(runtime.handle(), "us-east-1", &AwsAuthentication::Default) | ||
| .expect("failed to resolve credentials"), | ||
| vec![ | ||
| ("AWS_ACCESS_KEY_ID", "the-env-access-key".to_string()), | ||
| ("AWS_SECRET_ACCESS_KEY", "the-env-secret-key".to_string()), | ||
| ("AWS_SESSION_TOKEN", "the-env-session-token".to_string()), | ||
| ] | ||
| ); | ||
|
20001020ycx marked this conversation as resolved.
Resolved
|
||
| } | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🩺 Stability & Availability | 🟡 Minor | ⚡ Quick win 🧩 Analysis chain🏁 Script executed: #!/bin/bash
set -euo pipefail
echo "== repository files matching compress.rs =="
fd -a 'compress\.rs$' . | sed 's#^\./##'
echo "== relevant test lines =="
file="$(fd 'compress\.rs$' . | head -n1)"
if [ -n "${file:-}" ]; then
wc -l "$file"
sed -n '930,1020p' "$file" | cat -n | sed 's/^/ /'
fi
echo "== tests around default/aws env usage =="
rg -n 's3_credential_env_env|AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_SESSION_TOKEN|AwsAuthentication::Default|s3_credential_env\(' components/clp-tdl-package/src/task/compression/compress.rs || true
echo "== package/test config references to parallel/run order =="
fd -a 'Cargo.toml|clippy.toml|rust-toolchain|rustfmt|deny.toml' . | sed 's#^\./##' | while read -r f; do
echo "--- $f"
rg -n 'test|future|parallel|threads|deny|rustflags|xargo|rust-analyzer' "$f" || true
done
echo "== all AWS env mutations in tests =="
rg -n 'set_var\("AWS_|remove_var\("AWS_|mock\(AWS_|Env::from_raw|AWS_SESSION_TOKEN|AWS_SECRET_ACCESS_KEY|AWS_ACCESS_KEY_ID' components/clp-tdl-package/components/components/clp-tdl-package/src || true
echo "== module test markers =="
rg -n '#\[cfg\(test\)\]|#\[tokio::test|#\[test|#\[serial|#\[serial_test|serial_test' components/clp-tdl-package/src components -g '*.rs' | head -n 200Repository: y-scope/clp Length of output: 13029 🏁 Script executed: #!/bin/bash
set -euo pipefail
echo "== compress.rs s3_credential_env implementation =="
sed -n '340,425p' components/clp-tdl-package/src/task/compression/compress.rs | cat -n
echo "== clp-tdl-package Cargo.toml =="
cat -n components/clp-tdl-package/Cargo.toml | sed -n '1,180p'
echo "== AWS env var read access in clp-tdl-package tests =="
python3 - <<'PY'
from pathlib import Path
import re
for p in Path('components/clp-tdl-package').rglob('*.rs'):
txt = p.read_text(errors='ignore')
# only print relevant files and matches
if any(env in txt for env in ('AWS_ACCESS_KEY_ID','AWS_SECRET_ACCESS_KEY','AWS_SESSION_TOKEN','AWS_DEFAULT_REGION')):
print(f'--- {p}')
for i,line in enumerate(txt.splitlines(),1):
if any(env in line for env in ('AWS_ACCESS_KEY_ID','AWS_SECRET_ACCESS_KEY','AWS_SESSION_TOKEN','AWS_DEFAULT_REGION')):
print(f"{i}: {line.strip()}")
PY
echo "== clp-tdl-package test attributes with env vars =="
python3 - <<'PY'
from pathlib import Path
for p in Path('components/clp-tdl-package').rglob('*.rs'):
txt = p.read_text(errors='ignore')
if any(env in txt for env in ('AWS_ACCESS_KEY_ID','AWS_SECRET_ACCESS_KEY','AWS_SESSION_TOKEN')):
print(f'--- {p}')
lines=txt.splitlines()
for i,l in enumerate(lines,1):
if '#[test]' in l or '#[tokio::test' in l or 'AWS_ACCESS_KEY_ID' in l or 'AWS_SECRET_ACCESS_KEY' in l or 'AWS_SESSION_TOKEN' in l:
start=max(1,i-3); end=min(len(lines),i+10)
for j in range(start,end+1):
print(f"{j}: {lines[j-1]}")
print()
PY
echo "== lockfile/test parallel config references =="
fd 'Cargo.lock|deny.toml|clippy.toml|rust-toolchain|rustfmt.toml' . | sed 's#^\./##' | while read -r f; do
if [ -f "$f" ]; then
echo "--- $f"
rg -n 'threads|parallel|deny|lint|future|auto-tests|autotest|aws|AWS|test' "$f" || true
fi
doneRepository: y-scope/clp Length of output: 28113 Restore the AWS environment after spoofing credentials. This test sets process-global 🤖 Prompt for AI Agents |
||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.