feat: Apache OpenDAL™ compatible backends

Cargo.toml

@@ -304,6 +304,7 @@ num-derive = "0.4.2"
 num-format = "0.4.4"
 num-traits = "0.2"
 numpy = "0.27"
+opendal = {version = "0.55", default-features = false}
 opentelemetry = {version = "0.31", features = ["trace", "metrics", "logs"]}
 opentelemetry-otlp = {version = "0.31", features = ["grpc-tonic", "logs"]}
 opentelemetry_sdk = {version = "0.31", features = ["logs"]}

daft/daft/__init__.pyi

@@ -903,6 +903,7 @@ class IOConfig:
     disable_suffix_range: bool
     tos: TosConfig
     gravitino: GravitinoConfig
+    opendal_backends: dict[str, dict[str, str]]
 
     def __init__(
         self,
@@ -915,6 +916,7 @@ class IOConfig:
         disable_suffix_range: bool | None = None,
         tos: TosConfig | None = None,
         gravitino: GravitinoConfig | None = None,
+        opendal_backends: dict[str, dict[str, str]] | None = None,
     ): ...
     def replace(
         self,
@@ -927,6 +929,7 @@ class IOConfig:
         disable_suffix_range: bool | None = None,
         tos: TosConfig | None = None,
         gravitino: GravitinoConfig | None = None,
+        opendal_backends: dict[str, dict[str, str]] | None = None,
     ) -> IOConfig:
         """Replaces values if provided, returning a new IOConfig."""
         ...

src/common/io-config/src/config.rs

@@ -1,4 +1,7 @@
-use std::fmt::{Display, Formatter};
+use std::{
+    collections::BTreeMap,
+    fmt::{Display, Formatter},
+};
 
 use serde::{Deserialize, Serialize};
 
@@ -18,6 +21,9 @@ pub struct IOConfig {
     /// disable suffix range requests, please use range with offset
     pub disable_suffix_range: bool,
     pub tos: TosConfig,
+    /// Additional backends configured via OpenDAL.
+    /// Keys are scheme names (e.g. "oss", "cos"), values are key-value config maps.
+    pub opendal_backends: BTreeMap<String, BTreeMap<String, String>>,
 }
 
 impl IOConfig {
@@ -60,6 +66,9 @@ impl IOConfig {
             "TOS config = {{ {} }}",
             self.tos.multiline_display().join(", ")
         ));
+        if !self.opendal_backends.is_empty() {
+            res.push(format!("OpenDAL backends = {:?}", self.opendal_backends));
+        }
         res
     }
 }