Bump opendal to 0.58.1 and fix fallout (fixes local GCS cache usage)

Cargo.toml

@@ -80,19 +80,20 @@ memmap2 = "0.9.4"
 mime = "0.3"
 number_prefix = "0.4"
 object = "0.37"
-opendal = { version = "0.55.0", optional = true, default-features = false }
+opendal = { version = "0.58.1", optional = true, default-features = false }
+opendal-http-transport-reqwest = { version = "0.58.1", optional = true }
+opendal-layer-logging = { version = "0.58.1", optional = true }
 openssl = { version = "0.10.75", optional = true }
 rand = "0.8.4"
 regex = "1.10.3"
-reqsign = { version = "0.18.0", optional = true }
-reqwest = { version = "0.12", features = [
+reqwest = { version = "0.13", features = [
   "json",
   "blocking",
   "stream",
-  "rustls-tls",
-  "rustls-tls-native-roots",
-  "trust-dns",
+  "rustls",
+  "hickory-dns",
 ], optional = true }
+rustls-native-certs = { version = "0.8", optional = true }
 semver = "1.0"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = ">=1.0, <1.0.147" # zmij (used in 1.0.147+) requires Rust 1.84+
@@ -179,17 +180,53 @@ all = [
   "oss",
   "cos",
 ]
-azure = ["opendal/services-azblob", "reqsign", "reqwest"]
-cos = ["opendal/services-cos", "reqsign", "reqwest"]
+azure = [
+  "opendal/services-azblob",
+  "opendal-layer-logging",
+  "opendal-http-transport-reqwest",
+  "reqwest",
+]
+cos = [
+  "opendal/services-cos",
+  "opendal-layer-logging",
+  "opendal-http-transport-reqwest",
+  "reqwest",
+]
 default = ["all"]
-gcs = ["opendal/services-gcs", "reqsign", "url", "reqwest"]
-gha = ["opendal/services-ghac", "reqwest"]
-memcached = ["opendal/services-memcached"]
+gcs = [
+  "opendal/services-gcs",
+  "opendal-layer-logging",
+  "opendal-http-transport-reqwest",
+  "url",
+  "reqwest",
+]
+gha = [
+  "opendal/services-ghac",
+  "opendal-layer-logging",
+  "opendal-http-transport-reqwest",
+  "reqwest",
+]
+memcached = ["opendal/services-memcached", "opendal-layer-logging"]
 native-zlib = []
-oss = ["opendal/services-oss", "reqsign", "reqwest"]
-redis = ["url", "opendal/services-redis"]
-s3 = ["opendal/services-s3", "reqsign", "reqwest"]
-webdav = ["opendal/services-webdav", "reqwest"]
+oss = [
+  "opendal/services-oss",
+  "opendal-layer-logging",
+  "opendal-http-transport-reqwest",
+  "reqwest",
+]
+redis = ["url", "opendal/services-redis", "opendal-layer-logging"]
+s3 = [
+  "opendal/services-s3",
+  "opendal-layer-logging",
+  "opendal-http-transport-reqwest",
+  "reqwest",
+]
+webdav = [
+  "opendal/services-webdav",
+  "opendal-layer-logging",
+  "opendal-http-transport-reqwest",
+  "reqwest",
+]
 # Enable features that will build a vendored version of openssl and
 # statically linked with it, instead of linking against the system-wide openssl
 # dynamically or statically.
@@ -205,6 +242,7 @@ dist-client = [
   "reqwest",
   "url",
   "sha2",
+  "rustls-native-certs",
 ]
 # Enables the sccache-dist binary
 dist-server = [
@@ -217,6 +255,7 @@ dist-server = [
   "rouille",
   "syslog",
   "version-compare",
+  "rustls-native-certs",
 ]
 # Enables dist tests with external requirements
 dist-tests = ["dist-client", "dist-server"]

src/cache/azure.rs

@@ -15,8 +15,9 @@
 
 use opendal::Operator;
 
-use opendal::layers::{HttpClientLayer, LoggingLayer};
+use opendal::OperationContext;
 use opendal::services::Azblob;
+use opendal_layer_logging::LoggingLayer;
 
 use crate::errors::*;
 
@@ -31,9 +32,8 @@ impl AzureBlobCache {
             .root(key_prefix);
 
         let op = Operator::new(builder)?
-            .layer(HttpClientLayer::new(set_user_agent()))
-            .layer(LoggingLayer::default())
-            .finish();
+            .with_context(OperationContext::new().with_http_transport(set_user_agent()))
+            .layer(LoggingLayer::default());
         Ok(op)
     }
 }

src/cache/cache_io.rs

@@ -263,12 +263,16 @@ impl CacheWrite {
         .await?
     }
 
-    /// Add an object containing the contents of `from` to this cache entry at `name`.
-    /// If `mode` is `Some`, store the file entry with that mode.
-    pub fn put_object<T>(&mut self, name: &str, from: &mut T, mode: Option<u32>) -> Result<()>
-    where
-        T: Read,
-    {
+    /// The zstd compression level used for the blobs of a cache entry.
+    fn compression_level() -> i32 {
+        std::env::var("SCCACHE_CACHE_ZSTD_LEVEL")
+            .ok()
+            .and_then(|value| value.parse::<i32>().ok())
+            .unwrap_or(3)
+    }
+
+    /// Start a new zip entry at `name`, storing it with `mode` if it is `Some`.
+    fn start_object(&mut self, name: &str, mode: Option<u32>) -> Result<()> {
         // We're going to declare the compression method as "stored",
         // but we're actually going to store zstd-compressed blobs.
         let opts = FileOptions::default().compression_method(CompressionMethod::Stored);
@@ -279,13 +283,17 @@ impl CacheWrite {
         };
         self.zip
             .start_file(name, opts)
-            .context("Failed to start cache entry object")?;
+            .context("Failed to start cache entry object")
+    }
 
-        let compression_level = std::env::var("SCCACHE_CACHE_ZSTD_LEVEL")
-            .ok()
-            .and_then(|value| value.parse::<i32>().ok())
-            .unwrap_or(3);
-        zstd::stream::copy_encode(from, &mut self.zip, compression_level)?;
+    /// Add an object containing the contents of `from` to this cache entry at `name`.
+    /// If `mode` is `Some`, store the file entry with that mode.
+    pub fn put_object<T>(&mut self, name: &str, from: &mut T, mode: Option<u32>) -> Result<()>
+    where
+        T: Read,
+    {
+        self.start_object(name, mode)?;
+        zstd::stream::copy_encode(from, &mut self.zip, Self::compression_level())?;
         Ok(())
     }
 
@@ -298,10 +306,25 @@ impl CacheWrite {
     }
 
     fn put_bytes(&mut self, name: &str, bytes: &[u8]) -> Result<()> {
-        if !bytes.is_empty() {
-            let mut cursor = Cursor::new(bytes);
-            return self.put_object(name, &mut cursor, None);
+        if bytes.is_empty() {
+            return Ok(());
         }
+        self.start_object(name, None)?;
+
+        // The size of the input is known up front here, so tell zstd about it. It then
+        // sizes its compression context for the actual input instead of for the default
+        // window size, which avoids zeroing a multi-megabyte workspace for the handful of
+        // bytes of stdout/stderr that a compilation typically produces.
+        let mut encoder =
+            zstd::stream::write::Encoder::new(&mut self.zip, Self::compression_level())
+                .context("Failed to create cache entry compressor")?;
+        encoder
+            .set_pledged_src_size(Some(bytes.len() as u64))
+            .context("Failed to set cache entry compressor input size")?;
+        encoder.write_all(bytes)?;
+        encoder
+            .finish()
+            .context("Failed to finish cache entry compressor")?;
         Ok(())
     }
 

src/cache/cos.rs

@@ -10,9 +10,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use opendal::OperationContext;
 use opendal::Operator;
-use opendal::layers::{HttpClientLayer, LoggingLayer};
 use opendal::services::Cos;
+use opendal_layer_logging::LoggingLayer;
 
 use crate::errors::*;
 
@@ -30,9 +31,8 @@ impl COSCache {
         }
 
         let op = Operator::new(builder)?
-            .layer(HttpClientLayer::new(set_user_agent()))
-            .layer(LoggingLayer::default())
-            .finish();
+            .with_context(OperationContext::new().with_http_transport(set_user_agent()))
+            .layer(LoggingLayer::default());
         Ok(op)
     }
 }

src/cache/gcs.rs

@@ -16,10 +16,8 @@
 use crate::cache::CacheMode;
 use crate::errors::*;
 use opendal::Operator;
-use opendal::{
-    layers::{HttpClientLayer, LoggingLayer},
-    services::Gcs,
-};
+use opendal::{OperationContext, services::Gcs};
+use opendal_layer_logging::LoggingLayer;
 use reqwest::Client;
 use serde::Deserialize;
 use url::Url;
@@ -74,9 +72,8 @@ impl GCSCache {
         }
 
         let op = Operator::new(builder)?
-            .layer(HttpClientLayer::new(set_user_agent()))
-            .layer(LoggingLayer::default())
-            .finish();
+            .with_context(OperationContext::new().with_http_transport(set_user_agent()))
+            .layer(LoggingLayer::default());
         Ok(op)
     }
 }

src/cache/gha.rs

@@ -12,9 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use opendal::OperationContext;
 use opendal::Operator;
-use opendal::layers::{HttpClientLayer, LoggingLayer};
 use opendal::services::Ghac;
+use opendal_layer_logging::LoggingLayer;
 
 use crate::VERSION;
 use crate::errors::*;
@@ -41,9 +42,8 @@ impl GHACache {
         };
 
         let op = Operator::new(builder)?
-            .layer(HttpClientLayer::new(set_user_agent()))
-            .layer(LoggingLayer::default())
-            .finish();
+            .with_context(OperationContext::new().with_http_transport(set_user_agent()))
+            .layer(LoggingLayer::default());
         Ok(op)
     }
 }

src/cache/http_client.rs

@@ -1,9 +1,11 @@
-use opendal::raw::HttpClient;
+use opendal::HttpTransporter;
+use opendal_http_transport_reqwest::ReqwestTransport;
 use reqwest::ClientBuilder;
 
-/// Set the user agent (helps with monitoring on the server side)
-pub fn set_user_agent() -> HttpClient {
+/// Build an HTTP transport with a custom user agent (helps with monitoring on
+/// the server side).
+pub fn set_user_agent() -> HttpTransporter {
     let user_agent = format!("{}/{}", env!("CARGO_PKG_NAME"), env!("CARGO_PKG_VERSION"));
     let client = ClientBuilder::new().user_agent(user_agent).build().unwrap();
-    HttpClient::with(client)
+    HttpTransporter::new(ReqwestTransport::new(client))
 }

src/cache/memcached.rs

@@ -16,8 +16,8 @@
 use std::time::Duration;
 
 use opendal::Operator;
-use opendal::layers::LoggingLayer;
 use opendal::services::Memcached;
+use opendal_layer_logging::LoggingLayer;
 
 use crate::errors::*;
 
@@ -45,9 +45,7 @@ impl MemcachedCache {
             .root(key_prefix)
             .default_ttl(Duration::from_secs(expiration.into()));
 
-        let op = Operator::new(builder)?
-            .layer(LoggingLayer::default())
-            .finish();
+        let op = Operator::new(builder)?.layer(LoggingLayer::default());
         Ok(op)
     }
 }

src/cache/oss.rs

@@ -10,9 +10,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use opendal::OperationContext;
 use opendal::Operator;
-use opendal::layers::{HttpClientLayer, LoggingLayer};
 use opendal::services::Oss;
+use opendal_layer_logging::LoggingLayer;
 
 use crate::errors::*;
 
@@ -37,13 +38,12 @@ impl OSSCache {
         if no_credentials {
             // Allow anonymous access to OSS so that OpenDAL will not
             // throw error when no credentials are provided.
-            builder = builder.allow_anonymous();
+            builder = builder.skip_signature();
         }
 
         let op = Operator::new(builder)?
-            .layer(HttpClientLayer::new(set_user_agent()))
-            .layer(LoggingLayer::default())
-            .finish();
+            .with_context(OperationContext::new().with_http_transport(set_user_agent()))
+            .layer(LoggingLayer::default());
         Ok(op)
     }
 }

src/cache/redis.rs

@@ -15,8 +15,8 @@
 
 use crate::errors::*;
 use opendal::Operator;
-use opendal::layers::LoggingLayer;
 use opendal::services::Redis;
+use opendal_layer_logging::LoggingLayer;
 use std::collections::HashMap;
 use std::time::Duration;
 use url::Url;
@@ -47,9 +47,7 @@ impl RedisCache {
             .map(|v| v.parse().unwrap_or_default())
             .unwrap_or_default());
 
-        let op = Operator::new(builder)?
-            .layer(LoggingLayer::default())
-            .finish();
+        let op = Operator::new(builder)?.layer(LoggingLayer::default());
         Ok(op)
     }
 
@@ -98,9 +96,7 @@ impl RedisCache {
             builder = builder.default_ttl(Duration::from_secs(ttl));
         }
 
-        let op = Operator::new(builder)?
-            .layer(LoggingLayer::default())
-            .finish();
+        let op = Operator::new(builder)?.layer(LoggingLayer::default());
         Ok(op)
     }
 }