From 5e4768d5277921305e97b463f8f69a722cd5b77c Mon Sep 17 00:00:00 2001 From: Bas Zalmstra <4995967+baszalmstra@users.noreply.github.com> Date: Mon, 20 Oct 2025 14:19:39 +0200 Subject: [PATCH 1/3] feat: make s3 support optional --- Cargo.lock | 1 - Cargo.toml | 14 ++++++-------- src/lib.rs | 3 +++ src/opt.rs | 8 +++++++- src/tool_configuration.rs | 34 +++++++++++++++++++++------------- 5 files changed, 37 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 621f12177..ab2e48590 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4226,7 +4226,6 @@ dependencies = [ "miette", "minijinja", "num_cpus", - "opendal", "pathdiff", "petgraph", "pretty_assertions", diff --git a/Cargo.toml b/Cargo.toml index 44f758296..8b3b8c76c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,7 +51,7 @@ default-run = "rattler-build" rust-version = "1.86.0" [features] -default = ['rustls-tls', 'recipe-generation'] +default = ['rustls-tls', 'recipe-generation', 's3'] native-tls = [ 'reqwest/native-tls', 'rattler/native-tls', @@ -75,6 +75,10 @@ tui = [ 'throbber-widgets-tui', 'tui-input', ] +s3 = [ + 'rattler_networking/s3', + 'rattler_upload/s3', +] recipe-generation = ["rattler_build_recipe_generator"] # This feature needs to add a dependency on # clap-markdown = { git = "https://github.com/ruben-arts/clap-markdown", branch = "main" } @@ -171,9 +175,6 @@ reflink-copy = "0.1.26" rayon = "1.11.0" regex = { workspace = true } async-recursion = { workspace = true } -opendal = { version = "0.54.0", default-features = false, features = [ - "services-s3", -] } # Rattler crates rattler_config = { version = "0.2.11" } @@ -186,12 +187,9 @@ rattler_conda_types = { workspace = true, features = ["rayon"] } rattler_digest = { workspace = true } rattler_index = { version = "0.25.5", default-features = false } rattler_networking = { version = "0.25.16", default-features = false, features = [ - "s3", "rattler_config", ] } -rattler_upload = { version = "0.3.4", default-features = false, features = [ - "s3", -] } +rattler_upload = { version = "0.3.4", default-features = false } rattler_redaction = { version = "0.1.12" } rattler_repodata_gateway = { version = "0.24.7", default-features = false, features = [ "gateway", diff --git a/src/lib.rs b/src/lib.rs index b30a73714..b19a0f66e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -135,6 +135,7 @@ pub fn get_tool_config( ) -> miette::Result { let client = tool_configuration::reqwest_client_from_auth_storage( build_data.common.auth_file.clone(), + #[cfg(feature = "s3")] build_data.common.s3_config.clone(), build_data.common.mirror_config.clone(), build_data.common.allow_insecure_host.clone(), @@ -694,6 +695,7 @@ pub async fn run_test( .with_reqwest_client( tool_configuration::reqwest_client_from_auth_storage( test_data.common.auth_file, + #[cfg(feature = "s3")] test_data.common.s3_config, test_data.common.mirror_config, test_data.common.allow_insecure_host.clone(), @@ -752,6 +754,7 @@ pub async fn rebuild( ) -> miette::Result<()> { let reqwest_client = tool_configuration::reqwest_client_from_auth_storage( rebuild_data.common.auth_file, + #[cfg(feature = "s3")] rebuild_data.common.s3_config.clone(), rebuild_data.common.mirror_config.clone(), rebuild_data.common.allow_insecure_host.clone(), diff --git a/src/opt.rs b/src/opt.rs index 212d5c6bc..9eee77581 100644 --- a/src/opt.rs +++ b/src/opt.rs @@ -11,7 +11,9 @@ use rattler_conda_types::{ NamedChannelOrUrl, Platform, compression_level::CompressionLevel, package::ArchiveType, }; use rattler_config::config::build::PackageFormatAndCompression; -use rattler_networking::{mirror_middleware, s3_middleware}; +use rattler_networking::mirror_middleware; +#[cfg(feature = "s3")] +use rattler_networking::s3_middleware; use rattler_solve::ChannelPriority; use rattler_upload::upload::opt::{Config, UploadOpts}; use serde_json::{Value, json}; @@ -231,6 +233,7 @@ pub struct CommonData { pub experimental: bool, pub auth_file: Option, pub channel_priority: ChannelPriority, + #[cfg(feature = "s3")] pub s3_config: HashMap, pub mirror_config: HashMap>, pub allow_insecure_host: Option>, @@ -285,11 +288,14 @@ impl CommonData { mirror_config.insert(ensure_trailing_slash(key), mirrors); } + #[cfg(feature = "s3")] let s3_config = rattler_networking::s3_middleware::compute_s3_config(&config.s3_options.0); + Self { output_dir: output_dir.unwrap_or_else(|| PathBuf::from("./output")), experimental, auth_file, + #[cfg(feature = "s3")] s3_config, mirror_config, channel_priority: channel_priority.unwrap_or(ChannelPriority::Strict), diff --git a/src/tool_configuration.rs b/src/tool_configuration.rs index a016c7371..7853c9836 100644 --- a/src/tool_configuration.rs +++ b/src/tool_configuration.rs @@ -6,10 +6,12 @@ use std::{collections::HashMap, path::PathBuf, sync::Arc}; use clap::ValueEnum; use rattler::package_cache::PackageCache; use rattler_conda_types::{ChannelConfig, Platform}; +#[cfg(feature = "s3")] +use rattler_networking::s3_middleware; use rattler_networking::{ AuthenticationMiddleware, AuthenticationStorage, authentication_storage::{self, AuthenticationStorageError}, - mirror_middleware, s3_middleware, + mirror_middleware, }; use rattler_repodata_gateway::Gateway; use rattler_solve::ChannelPriority; @@ -85,12 +87,13 @@ impl BaseClient { pub fn new( auth_file: Option, allow_insecure_host: Option>, - s3_middleware_config: HashMap, + #[cfg(feature = "s3")] s3_middleware_config: HashMap, mirror_middleware_config: HashMap>, ) -> Result { let auth_storage = get_auth_store(auth_file)?; let timeout = 5 * 60; + #[cfg(feature = "s3")] let s3_middleware = s3_middleware::S3Middleware::new(s3_middleware_config, auth_storage.clone()); let mirror_middleware = @@ -104,20 +107,24 @@ impl BaseClient { .read_timeout(std::time::Duration::from_secs(timeout)) }; - let client = reqwest_middleware::ClientBuilder::new( + let mut client_builder = reqwest_middleware::ClientBuilder::new( common_settings(reqwest::Client::builder()) .build() .expect("failed to create client"), ) - .with(mirror_middleware) - .with(s3_middleware) - .with_arc(Arc::new(AuthenticationMiddleware::from_auth_storage( - auth_storage.clone(), - ))) - .with(RetryTransientMiddleware::new_with_policy( - ExponentialBackoff::builder().build_with_max_retries(3), - )) - .build(); + .with(mirror_middleware); + + #[cfg(feature = "s3")] + let client_builder = client_builder.with(s3_middleware); + + let client = client_builder + .with_arc(Arc::new(AuthenticationMiddleware::from_auth_storage( + auth_storage.clone(), + ))) + .with(RetryTransientMiddleware::new_with_policy( + ExponentialBackoff::builder().build_with_max_retries(3), + )) + .build(); let dangerous_client = reqwest_middleware::ClientBuilder::new( common_settings(reqwest::Client::builder()) @@ -259,13 +266,14 @@ pub fn get_auth_store( /// * `allow_insecure_host` - Optional list of hosts for which to disable SSL certificate verification pub fn reqwest_client_from_auth_storage( auth_file: Option, - s3_middleware_config: HashMap, + #[cfg(feature = "s3")] s3_middleware_config: HashMap, mirror_middleware_config: HashMap>, allow_insecure_host: Option>, ) -> Result { BaseClient::new( auth_file, allow_insecure_host, + #[cfg(feature = "s3")] s3_middleware_config, mirror_middleware_config, ) From ae55ab4f87685cd49fa1f6f9ddd32654dc2d3089 Mon Sep 17 00:00:00 2001 From: Bas Zalmstra <4995967+baszalmstra@users.noreply.github.com> Date: Mon, 20 Oct 2025 14:21:51 +0200 Subject: [PATCH 2/3] fix: lockfile --- py-rattler-build/pixi.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/py-rattler-build/pixi.lock b/py-rattler-build/pixi.lock index 7d4ed15b1..bba6ecacc 100644 --- a/py-rattler-build/pixi.lock +++ b/py-rattler-build/pixi.lock @@ -3786,7 +3786,7 @@ packages: - __glibc >=2.17 license: BSD-3-Clause input: - hash: 38541d6788d5bf0338ad1c5f71a9c116ddb8a6ed9f19e7e42b9288095dfed289 + hash: 750be6b0351f15dd166a306610b756eddddc00df36d1d18019d7430625795e98 globs: - pyproject.toml - conda: . @@ -3801,7 +3801,7 @@ packages: - __osx >=10.13 license: BSD-3-Clause input: - hash: 38541d6788d5bf0338ad1c5f71a9c116ddb8a6ed9f19e7e42b9288095dfed289 + hash: 750be6b0351f15dd166a306610b756eddddc00df36d1d18019d7430625795e98 globs: - pyproject.toml - conda: . @@ -3816,7 +3816,7 @@ packages: - __osx >=11.0 license: BSD-3-Clause input: - hash: 38541d6788d5bf0338ad1c5f71a9c116ddb8a6ed9f19e7e42b9288095dfed289 + hash: 750be6b0351f15dd166a306610b756eddddc00df36d1d18019d7430625795e98 globs: - pyproject.toml - conda: . @@ -3829,7 +3829,7 @@ packages: - python_abi 3.8.* *_cp38 license: BSD-3-Clause input: - hash: 38541d6788d5bf0338ad1c5f71a9c116ddb8a6ed9f19e7e42b9288095dfed289 + hash: 750be6b0351f15dd166a306610b756eddddc00df36d1d18019d7430625795e98 globs: - pyproject.toml - conda: https://conda.anaconda.org/conda-forge/noarch/pycparser-2.22-pyh29332c3_1.conda From d14932f23266c18c291172c7a55fb51ed648bf6c Mon Sep 17 00:00:00 2001 From: Bas Zalmstra <4995967+baszalmstra@users.noreply.github.com> Date: Mon, 20 Oct 2025 14:32:13 +0200 Subject: [PATCH 3/3] fix: clippy --- src/tool_configuration.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tool_configuration.rs b/src/tool_configuration.rs index 7853c9836..35b29e5ac 100644 --- a/src/tool_configuration.rs +++ b/src/tool_configuration.rs @@ -107,7 +107,7 @@ impl BaseClient { .read_timeout(std::time::Duration::from_secs(timeout)) }; - let mut client_builder = reqwest_middleware::ClientBuilder::new( + let client_builder = reqwest_middleware::ClientBuilder::new( common_settings(reqwest::Client::builder()) .build() .expect("failed to create client"),