feat: derive default credentials from aws sdk

.github/workflows/e2e-s3-tests.yml

 } 
 #[fixture] 
 fn r2_host() -> String { 
     "https://e1a7cde76f1780ec06bac859036dbaf7.eu.r2.cloudflarestorage.com".into() 
 } 
 #[fixture] 
 fn r2_credentials() -> Option<(String, String)> { 
     let r2_access_key_id = std::env::var("RATTLER_TEST_R2_READONLY_ACCESS_KEY_ID").ok(); 
     let r2_secret_access_key = std::env::var("RATTLER_TEST_R2_READONLY_SECRET_ACCESS_KEY").ok(); 
     if r2_access_key_id.is_none() 
         || r2_access_key_id.clone().unwrap().is_empty() 
         || r2_secret_access_key.is_none() 
         || r2_secret_access_key.clone().unwrap().is_empty() 
     { 
         eprintln!( 
             "Skipping test as RATTLER_TEST_R2_READONLY_ACCESS_KEY_ID or RATTLER_TEST_R2_READONLY_SECRET_ACCESS_KEY is not set" 
         ); 
         None 
     } else { 
         Some((r2_access_key_id.unwrap(), r2_secret_access_key.unwrap())) 
     } 
 } 
@@ -0,0 +1,87 @@
+on:
+  push:
+    branches: [main]
+  pull_request:
+    paths:
+      - crates/rattler-bin/**
+      - crates/rattler_index/**
+      - crates/rattler_upload/**
+      - crates/rattler_networking/**
+      - .github/workflows/e2e-s3-tests.yml
+
+name: E2E S3 Tests
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
+env:
+  RUST_LOG: info
+  RUST_BACKTRACE: 1
+  CARGO_TERM_COLOR: always
+  DEFAULT_FEATURES: s3
+
+jobs:
+  e2e-minio-test:
+    name: E2E Upload/Index/Download [Minio]
+    runs-on: ubuntu-latest
+
+    env:
+      # Enable sccache.
+      #
+      # This environment variable is picked up by pixi build which will then
+      # set up the rust build using sccache.
+      SCCACHE_GHA_ENABLED: "true"
+
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: recursive
+
+      - uses: prefix-dev/setup-pixi@fef5c9568ca6c4ff7707bf840ab0692ba3f08293 # v0.9.0
+        with:
+          environments: minio
+
+      - run: pixi run e2e-s3-minio
+
+  # TODO: add cloudflare R2 integration tests here as well
+  e2e-aws-s3-test:
+    name: E2E Upload/Index/Download [AWS S3]
+    runs-on: ubuntu-latest
+    # Only run on main branch to avoid creating too many test buckets
+    # if: github.ref == 'refs/heads/main'
+    permissions:
+      id-token: write
+      contents: read
+
+    env:
+      # Enable sccache.
+      #
+      # This environment variable is picked up by pixi build which will then
+      # set up the rust build using sccache.
+      SCCACHE_GHA_ENABLED: "true"
+
+      # The region to create buckets it.
+      AWS_REGION: eu-west-1
+
+      # The name of the test bucket to create.
+      BUCKET: tmp-${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}
+
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: recursive
+
+      - name: Configure AWS (OIDC)
+        uses: aws-actions/configure-aws-credentials@7474bc4690e29a8392af63c5b98e7449536d5c3a # v4.3.1
+        with:
+          aws-region: ${{ env.AWS_REGION }}
+          role-to-assume: arn:aws:iam::239378270001:role/conda-rattler-e2e-test
+
+      - uses: prefix-dev/setup-pixi@fef5c9568ca6c4ff7707bf840ab0692ba3f08293 # v0.9.0
+        with:
+          environments: s3
+
+      - run: pixi run e2e-s3-aws

.gitignore

@@ -26,3 +26,6 @@ test-data/channels/conda-forge/
 
 # Visual studio files
 .vs/
+
+# Local environment files
+*.local.*

Cargo.toml

@@ -34,6 +34,7 @@ async-compression = { version = "0.4", features = [
 ] }
 async-fd-lock = "0.2.0"
 fs4 = "0.13.1"
+async-once-cell = "0.5.4"
 async-trait = "0.1.88"
 axum = { version = "0.8.4", default-features = false, features = [
   "tokio",
@@ -51,7 +52,7 @@ chrono = { version = "0.4.41", default-features = false, features = [
   "serde",
   "alloc",
 ] }
-clap = { version = "4.5.38", features = ["derive"] }
+clap = { version = "4.5.38", features = ["derive", "color", "suggestions", "error-context"] }
 clap-verbosity-flag = "3.0.3"
 cmake = "0.1.54"
 console = { version = "0.16.0", features = ["windows-console-colors"] }
@@ -72,17 +73,9 @@ generic-array = "0.14.7"
 getrandom = { version = "0.3.3", default-features = false }
 glob = "0.3.2"
 google-cloud-auth = { version = "0.22.0", default-features = false }
-aws-config = { version = "=1.5.18", default-features = false, features = [
-  "rt-tokio",
-  "rustls",
-  "sso",
-  "credentials-process",
-] }
-aws-sdk-s3 = { version = "1.85.0", default-features = false, features = [
-  "rt-tokio",
-  "rustls",
-  "sigv4a",
-] }
+aws-config = { version = "=1.5.18", default-features = false }
+aws-sdk-s3 = { version = "1.85.0", default-features = false }
+aws-credential-types = { version = "1.2.5", default-features = false }
 hex = "0.4.3"
 hex-literal = "1.0.0"
 http = "1.3"
@@ -197,13 +190,15 @@ rattler_lock = { path = "crates/rattler_lock", version = "=0.24.0", default-feat
 rattler_macros = { path = "crates/rattler_macros", version = "=1.0.11", default-features = false }
 rattler_menuinst = { path = "crates/rattler_menuinst", version = "=0.2.24", default-features = false }
 rattler_networking = { path = "crates/rattler_networking", version = "=0.25.11", default-features = false }
+rattler_package_streaming = { path = "crates/rattler_package_streaming", version = "=0.23.2", default-features = false }
 rattler_pty = { path = "crates/rattler_pty", version = "=0.2.6", default-features = false }
 rattler_redaction = { path = "crates/rattler_redaction", version = "=0.1.12", default-features = false }
-rattler_package_streaming = { path = "crates/rattler_package_streaming", version = "=0.23.2", default-features = false }
 rattler_repodata_gateway = { path = "crates/rattler_repodata_gateway", version = "=0.24.2", default-features = false }
+rattler_s3 = { path = "crates/rattler_s3", version = "=0.1.0", default-features = false }
 rattler_sandbox = { path = "crates/rattler_sandbox", version = "=0.1.10", default-features = false }
 rattler_shell = { path = "crates/rattler_shell", version = "=0.24.10", default-features = false }
 rattler_solve = { path = "crates/rattler_solve", version = "=3.0.2", default-features = false }
+rattler_upload = { path = "crates/rattler_upload", version = "=0.2.1", default-features = false }
 rattler_virtual_packages = { path = "crates/rattler_virtual_packages", version = "=2.1.4", default-features = false }
 
 # This is also a rattler crate, but we only pin it to minor version

crates/rattler-bin/Cargo.toml

@@ -26,6 +26,7 @@ anyhow = { workspace = true }
 clap = { workspace = true, features = ["derive"] }
 console = { workspace = true, features = ["windows-console-colors"] }
 indicatif = { workspace = true }
+miette = { workspace = true }
 once_cell = { workspace = true }
 rattler = { workspace = true, features = ["indicatif", "cli-tools"] }
 rattler_conda_types = { workspace = true, default-features = false }
@@ -35,6 +36,7 @@ rattler_solve = { workspace = true, default-features = false, features = ["resol
 rattler_virtual_packages = { workspace = true, default-features = false }
 rattler_cache = { workspace = true, default-features = false }
 rattler_menuinst = { workspace = true, default-features = false }
+rattler_upload = { workspace = true, features = ["s3"]}
 reqwest = { workspace = true, features = ["stream"] }
 reqwest-middleware = { workspace = true }
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }

crates/rattler-bin/pixi.toml

@@ -0,0 +1,10 @@
+[package]
+name = "rattler"
+
+[package.build.backend]
+name = "pixi-build-rust"
+version = "0.4.*"
+channels = [
+    "https://prefix.dev/pixi-build-backends",
+    "https://prefix.dev/conda-forge"
+]

crates/rattler-bin/src/commands/auth.rs

@@ -1,8 +1,8 @@
-use anyhow::Result;
+use miette::IntoDiagnostic;
 use rattler::cli::auth;
 
 pub type Opt = auth::Args;
 
-pub async fn auth(opt: Opt) -> Result<()> {
-    auth::execute(opt).await.map_err(|e| anyhow::anyhow!(e))
+pub async fn auth(opt: Opt) -> miette::Result<()> {
+    auth::execute(opt).await.into_diagnostic()
 }