diff --git a/.env.example b/.env.example index bd6741dc6f41..3fe8c8f45b39 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,8 @@ # fs OPENDAL_FS_ROOT=/path/to/dir OPENDAL_FS_ATOMIC_WRITE_DIR=/path/to/tempdir +# wasi-fs - WASI Filesystem (requires wasm32-wasip2 target + wasmtime) +# OPENDAL_WASI_FS_ROOT=/ # cos OPENDAL_COS_BUCKET=opendal-testing-1318209832 OPENDAL_COS_ENDPOINT=https://cos.ap-singapore.myqcloud.com diff --git a/.github/workflows/test_wasi_fs.yml b/.github/workflows/test_wasi_fs.yml new file mode 100644 index 000000000000..010bc90f2c83 --- /dev/null +++ b/.github/workflows/test_wasi_fs.yml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Check WASI-FS Service + +on: + push: + branches: [main] + paths: + - 'core/services/wasi-fs/**' + - 'core/core/src/**' + - '.github/workflows/test_wasi_fs.yml' + pull_request: + branches: [main] + paths: + - 'core/services/wasi-fs/**' + - 'core/core/src/**' + - '.github/workflows/test_wasi_fs.yml' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - name: Setup Rust toolchain + uses: ./.github/actions/setup + + - name: Add wasm32-wasip2 target + run: rustup target add wasm32-wasip2 + + # Verify the wasi-fs service compiles for the WASI target. + # Note: Full behavior tests cannot run because tokio doesn't support wasm32-wasip2. + # The service uses blocking WASI Preview 2 APIs which work correctly at runtime. + - name: Check wasi-fs compiles + working-directory: core + run: | + cargo check --target wasm32-wasip2 \ + --features services-wasi-fs \ + -p opendal diff --git a/core/Cargo.lock b/core/Cargo.lock index a2f7917e2dc5..0f4e83a297d6 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -5513,6 +5513,7 @@ dependencies = [ "opendal-service-sled", "opendal-service-tikv", "opendal-service-vercel-blob", + "opendal-service-wasi-fs", "opentelemetry", "opentelemetry-otlp", "opentelemetry_sdk", @@ -6059,6 +6060,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "opendal-service-wasi-fs" +version = "0.55.0" +dependencies = [ + "ctor", + "opendal-core", + "serde", + "wasi 0.14.7+wasi-0.2.4", +] + [[package]] name = "openssh" version = "0.11.6" @@ -10182,6 +10193,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasi" +version = "0.14.7+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" +dependencies = [ + "wasip2", +] + [[package]] name = "wasip2" version = "1.0.1+wasi-0.2.4" @@ -10864,6 +10884,9 @@ name = "wit-bindgen" version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" +dependencies = [ + "bitflags 2.10.0", +] [[package]] name = "writeable" diff --git a/core/Cargo.toml b/core/Cargo.toml index 84d9a5775585..81875c97c63a 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -162,6 +162,7 @@ services-vercel-artifacts = ["opendal-core/services-vercel-artifacts"] services-vercel-blob = ["dep:opendal-service-vercel-blob"] services-webdav = ["opendal-core/services-webdav"] services-webhdfs = ["opendal-core/services-webhdfs"] +services-wasi-fs = ["dep:opendal-service-wasi-fs"] services-yandex-disk = ["opendal-core/services-yandex-disk"] tests = ["opendal-core/tests"] @@ -218,6 +219,7 @@ opendal-service-sled = { path = "services/sled", version = "0.55.0", optional = opendal-service-s3 = { path = "services/s3", version = "0.55.0", optional = true, default-features = false } opendal-service-tikv = { path = "services/tikv", version = "0.55.0", optional = true, default-features = false } opendal-service-vercel-blob = { path = "services/vercel-blob", version = "0.55.0", optional = true, default-features = false } +opendal-service-wasi-fs = { path = "services/wasi-fs", version = "0.55.0", optional = true, default-features = false } [dev-dependencies] anyhow = { version = "1.0.100", features = ["std"] } diff --git a/core/core/src/raw/time.rs b/core/core/src/raw/time.rs index b43f61a82f74..0105005432bc 100644 --- a/core/core/src/raw/time.rs +++ b/core/core/src/raw/time.rs @@ -24,9 +24,11 @@ use std::ops::{Add, AddAssign, Sub, SubAssign}; use std::str::FromStr; pub use std::time::{Duration, UNIX_EPOCH}; -#[cfg(not(target_arch = "wasm32"))] +// For native and WASI targets, use std::time +#[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] pub use std::time::{Instant, SystemTime}; -#[cfg(target_arch = "wasm32")] +// For browser wasm (target_os = "unknown"), use web_time +#[cfg(all(target_arch = "wasm32", target_os = "unknown"))] pub use web_time::{Instant, SystemTime}; /// An instant in time represented as the number of nanoseconds since the Unix epoch. @@ -179,12 +181,14 @@ impl From for Timestamp { impl From for SystemTime { fn from(ts: Timestamp) -> Self { - #[cfg(not(target_arch = "wasm32"))] + // For native and WASI targets, use std::time directly + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] { SystemTime::from(ts.0) } - #[cfg(target_arch = "wasm32")] + // For browser wasm (target_os = "unknown"), use web_time + #[cfg(all(target_arch = "wasm32", target_os = "unknown"))] { use std::time::SystemTime as StdSystemTime; @@ -199,12 +203,14 @@ impl TryFrom for Timestamp { fn try_from(t: SystemTime) -> Result { let t = { - #[cfg(not(target_arch = "wasm32"))] + // For native and WASI targets, use std::time directly + #[cfg(not(all(target_arch = "wasm32", target_os = "unknown")))] { t } - #[cfg(target_arch = "wasm32")] + // For browser wasm (target_os = "unknown"), use web_time + #[cfg(all(target_arch = "wasm32", target_os = "unknown"))] { ::to_std(t) } diff --git a/core/scripts/test_wasi_fs.sh b/core/scripts/test_wasi_fs.sh new file mode 100755 index 000000000000..6db9e9fcdad1 --- /dev/null +++ b/core/scripts/test_wasi_fs.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Test runner for wasi-fs service +# Requires: wasmtime, wasm32-wasip2 target + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CORE_DIR="$(dirname "$SCRIPT_DIR")" +TEST_DIR="${TEST_DIR:-/tmp/opendal-wasi-fs-test}" + +echo "=== WASI-FS Behavior Test Runner ===" +echo "Test directory: $TEST_DIR" + +# Create test directory +mkdir -p "$TEST_DIR" + +# Build behavior tests for wasm32-wasip2 +echo "Building behavior tests for wasm32-wasip2..." +cd "$CORE_DIR" +cargo build --tests --target wasm32-wasip2 --features tests,services-wasi-fs + +# Find the test binary +TEST_BINARY=$(find target/wasm32-wasip2/debug/deps -name "behavior-*.wasm" | head -1) + +if [ -z "$TEST_BINARY" ]; then + echo "Error: Could not find behavior test binary" + exit 1 +fi + +echo "Running tests with wasmtime..." +echo "Binary: $TEST_BINARY" + +# Run with wasmtime, granting access to test directory +OPENDAL_TEST=wasi-fs \ +OPENDAL_WASI_FS_ROOT=/ \ +wasmtime run \ + --dir "$TEST_DIR::/" \ + --env "OPENDAL_TEST=wasi-fs" \ + --env "OPENDAL_WASI_FS_ROOT=/" \ + "$TEST_BINARY" + +echo "=== Tests completed ===" diff --git a/core/services/wasi-fs/Cargo.toml b/core/services/wasi-fs/Cargo.toml new file mode 100644 index 000000000000..036d32c714f3 --- /dev/null +++ b/core/services/wasi-fs/Cargo.toml @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +description = "Apache OpenDAL WASI filesystem service implementation" +name = "opendal-service-wasi-fs" + +authors = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +repository = { workspace = true } +rust-version = { workspace = true } +version = { workspace = true } + +[package.metadata.docs.rs] +all-features = true + +[dependencies] +opendal-core = { path = "../../core", version = "0.55.0", default-features = false } + +ctor = { workspace = true } +serde = { workspace = true, features = ["derive"] } + +[target.'cfg(all(target_arch = "wasm32", target_os = "wasi"))'.dependencies] +wasi = "0.14.7" diff --git a/core/services/wasi-fs/src/backend.rs b/core/services/wasi-fs/src/backend.rs new file mode 100644 index 000000000000..8548673ab082 --- /dev/null +++ b/core/services/wasi-fs/src/backend.rs @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::sync::Arc; + +use opendal_core::raw::*; +use opendal_core::*; + +use super::WASI_FS_SCHEME; +use super::config::WasiFsConfig; +use super::core::WasiFsCore; +use super::deleter::WasiFsDeleter; +use super::lister::WasiFsLister; +use super::reader::WasiFsReader; +use super::writer::WasiFsWriter; + +/// WASI Filesystem service support. +#[doc = include_str!("docs.md")] +#[derive(Debug, Default)] +pub struct WasiFsBuilder { + pub(super) config: WasiFsConfig, +} + +impl WasiFsBuilder { + /// Set the root path for the WASI filesystem. + /// + /// The root must be within a preopened directory. + pub fn root(mut self, path: &str) -> Self { + self.config.root = if path.is_empty() { + None + } else { + Some(path.to_string()) + }; + self + } +} + +impl Builder for WasiFsBuilder { + type Config = WasiFsConfig; + + fn build(self) -> Result { + let root = normalize_root(&self.config.root.unwrap_or_default()); + + let core = WasiFsCore::new(&root)?; + + let info = AccessorInfo::default(); + info.set_scheme(WASI_FS_SCHEME) + .set_root(&root) + .set_native_capability(Capability { + stat: true, + read: true, + write: true, + write_can_empty: true, + create_dir: true, + delete: true, + list: true, + rename: true, + shared: false, + ..Default::default() + }); + + Ok(WasiFsBackend { + core: Arc::new(core), + info: Arc::new(info), + }) + } +} + +#[derive(Debug, Clone)] +pub struct WasiFsBackend { + core: Arc, + info: Arc, +} + +impl Access for WasiFsBackend { + type Reader = WasiFsReader; + type Writer = WasiFsWriter; + type Lister = WasiFsLister; + type Deleter = oio::OneShotDeleter; + + fn info(&self) -> Arc { + self.info.clone() + } + + async fn create_dir(&self, path: &str, _: OpCreateDir) -> Result { + self.core.create_dir(path)?; + Ok(RpCreateDir::default()) + } + + async fn stat(&self, path: &str, _: OpStat) -> Result { + let metadata = self.core.stat(path)?; + Ok(RpStat::new(metadata)) + } + + async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> { + let reader = WasiFsReader::new(self.core.clone(), path, args.range())?; + Ok((RpRead::new(), reader)) + } + + async fn write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::Writer)> { + let writer = WasiFsWriter::new(self.core.clone(), path, args)?; + Ok((RpWrite::default(), writer)) + } + + async fn delete(&self) -> Result<(RpDelete, Self::Deleter)> { + let deleter = WasiFsDeleter::new(self.core.clone()); + Ok((RpDelete::default(), oio::OneShotDeleter::new(deleter))) + } + + async fn list(&self, path: &str, _: OpList) -> Result<(RpList, Self::Lister)> { + let lister = WasiFsLister::new(self.core.clone(), path)?; + Ok((RpList::default(), lister)) + } + + async fn rename(&self, from: &str, to: &str, _: OpRename) -> Result { + self.core.rename(from, to)?; + Ok(RpRename::default()) + } +} diff --git a/core/services/wasi-fs/src/config.rs b/core/services/wasi-fs/src/config.rs new file mode 100644 index 000000000000..d5f6de32b4bc --- /dev/null +++ b/core/services/wasi-fs/src/config.rs @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use serde::{Deserialize, Serialize}; +use std::fmt::Debug; + +use super::backend::WasiFsBuilder; + +/// Configuration for WASI filesystem service. +#[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] +#[serde(default)] +#[non_exhaustive] +pub struct WasiFsConfig { + /// Root path within the WASI filesystem. + /// + /// This path must be within or equal to a preopened directory. + /// If not specified, defaults to the first preopened directory. + pub root: Option, +} + +impl Debug for WasiFsConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("WasiFsConfig") + .field("root", &self.root) + .finish_non_exhaustive() + } +} + +impl opendal_core::Configurator for WasiFsConfig { + type Builder = WasiFsBuilder; + + fn from_uri(uri: &opendal_core::OperatorUri) -> opendal_core::Result { + let mut map = uri.options().clone(); + + if let Some(path) = uri.root() { + if !path.is_empty() { + map.entry("root".to_string()) + .or_insert_with(|| format!("/{path}")); + } + } + + Self::from_iter(map) + } + + fn into_builder(self) -> Self::Builder { + WasiFsBuilder { config: self } + } +} diff --git a/core/services/wasi-fs/src/core.rs b/core/services/wasi-fs/src/core.rs new file mode 100644 index 000000000000..76280f0c0678 --- /dev/null +++ b/core/services/wasi-fs/src/core.rs @@ -0,0 +1,197 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use opendal_core::raw::*; +use opendal_core::*; +use wasi::filesystem::preopens::get_directories; +use wasi::filesystem::types::{ + Descriptor, DescriptorFlags, DescriptorStat, DescriptorType, DirectoryEntryStream, OpenFlags, + PathFlags, +}; + +use super::error::parse_wasi_error; + +#[derive(Debug)] +pub struct WasiFsCore { + /// The preopened directory descriptor we're operating within + root_descriptor: Descriptor, + /// The path within the preopened dir + root_path: String, +} + +impl WasiFsCore { + pub fn new(root: &str) -> Result { + let mut preopens = get_directories(); + + if preopens.is_empty() { + return Err(Error::new( + ErrorKind::ConfigInvalid, + "No preopened directories available from WASI runtime", + )); + } + + let (idx, preopen_path) = Self::find_preopened_dir_index(&preopens, root)?; + let (descriptor, _) = preopens.swap_remove(idx); + + let root_path = if root.starts_with(&preopen_path) { + root.strip_prefix(&preopen_path) + .unwrap_or("") + .trim_start_matches('/') + .to_string() + } else { + String::new() + }; + + Ok(Self { + root_descriptor: descriptor, + root_path, + }) + } + + fn find_preopened_dir_index( + preopens: &[(Descriptor, String)], + root: &str, + ) -> Result<(usize, String)> { + for (idx, (_, path)) in preopens.iter().enumerate() { + if root.starts_with(path) || path == "/" || root == "/" || root.is_empty() { + return Ok((idx, path.clone())); + } + } + + // Fall back to first preopened directory + if preopens.is_empty() { + return Err(Error::new( + ErrorKind::ConfigInvalid, + "No preopened directories", + )); + } + + Ok((0, preopens[0].1.clone())) + } + + fn build_path(&self, path: &str) -> String { + let path = path.trim_start_matches('/').trim_end_matches('/'); + if self.root_path.is_empty() { + path.to_string() + } else if path.is_empty() { + self.root_path.clone() + } else { + format!("{}/{}", self.root_path, path) + } + } + + pub fn stat(&self, path: &str) -> Result { + let abs_path = self.build_path(path); + + let stat = if abs_path.is_empty() { + self.root_descriptor.stat().map_err(parse_wasi_error)? + } else { + self.root_descriptor + .stat_at(PathFlags::empty(), &abs_path) + .map_err(parse_wasi_error)? + }; + + Ok(Self::convert_stat(stat)) + } + + fn convert_stat(stat: DescriptorStat) -> Metadata { + let mode = match stat.type_ { + DescriptorType::Directory => EntryMode::DIR, + DescriptorType::RegularFile => EntryMode::FILE, + _ => EntryMode::Unknown, + }; + + let mut metadata = Metadata::new(mode).with_content_length(stat.size); + + if let Some(mtime) = stat.data_modification_timestamp { + // Convert WASI timestamp (seconds + nanoseconds) to Timestamp + if let Ok(ts) = Timestamp::new(mtime.seconds as i64, mtime.nanoseconds as i32) { + metadata = metadata.with_last_modified(ts); + } + } + + metadata + } + + pub fn create_dir(&self, path: &str) -> Result<()> { + let abs_path = self.build_path(path); + self.root_descriptor + .create_directory_at(&abs_path) + .map_err(parse_wasi_error) + } + + pub fn open_file( + &self, + path: &str, + flags: OpenFlags, + desc_flags: DescriptorFlags, + ) -> Result { + let abs_path = self.build_path(path); + self.root_descriptor + .open_at(PathFlags::empty(), &abs_path, flags, desc_flags) + .map_err(parse_wasi_error) + } + + pub fn read_dir(&self, path: &str) -> Result { + let abs_path = self.build_path(path); + + let dir_desc = if abs_path.is_empty() { + // For the root, we need to open the same directory again + // since we can't clone the descriptor + self.root_descriptor + .read_directory() + .map_err(parse_wasi_error)?; + return self + .root_descriptor + .read_directory() + .map_err(parse_wasi_error); + } else { + self.root_descriptor + .open_at( + PathFlags::empty(), + &abs_path, + OpenFlags::DIRECTORY, + DescriptorFlags::empty(), + ) + .map_err(parse_wasi_error)? + }; + + dir_desc.read_directory().map_err(parse_wasi_error) + } + + pub fn delete_file(&self, path: &str) -> Result<()> { + let abs_path = self.build_path(path); + self.root_descriptor + .unlink_file_at(&abs_path) + .map_err(parse_wasi_error) + } + + pub fn delete_dir(&self, path: &str) -> Result<()> { + let abs_path = self.build_path(path); + self.root_descriptor + .remove_directory_at(&abs_path) + .map_err(parse_wasi_error) + } + + pub fn rename(&self, from: &str, to: &str) -> Result<()> { + let from_path = self.build_path(from); + let to_path = self.build_path(to); + self.root_descriptor + .rename_at(&from_path, &self.root_descriptor, &to_path) + .map_err(parse_wasi_error) + } +} diff --git a/core/services/wasi-fs/src/deleter.rs b/core/services/wasi-fs/src/deleter.rs new file mode 100644 index 000000000000..fbb59da48cb3 --- /dev/null +++ b/core/services/wasi-fs/src/deleter.rs @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::sync::Arc; + +use opendal_core::raw::*; +use opendal_core::*; + +use super::core::WasiFsCore; + +pub struct WasiFsDeleter { + core: Arc, +} + +impl WasiFsDeleter { + pub fn new(core: Arc) -> Self { + Self { core } + } +} + +impl oio::OneShotDelete for WasiFsDeleter { + async fn delete_once(&self, path: String, _args: OpDelete) -> Result<()> { + if path.ends_with('/') { + match self.core.delete_dir(&path) { + Ok(()) => Ok(()), + Err(e) if e.kind() == ErrorKind::NotFound => Ok(()), + Err(e) => Err(e), + } + } else { + match self.core.delete_file(&path) { + Ok(()) => Ok(()), + Err(e) if e.kind() == ErrorKind::NotFound => Ok(()), + Err(e) => Err(e), + } + } + } +} diff --git a/core/services/wasi-fs/src/docs.md b/core/services/wasi-fs/src/docs.md new file mode 100644 index 000000000000..6c88aa8c2306 --- /dev/null +++ b/core/services/wasi-fs/src/docs.md @@ -0,0 +1,69 @@ +This service implements filesystem access for WebAssembly components running +in WASI Preview 2 compatible runtimes such as Wasmtime or Wasmer. + +## Capabilities + +This service supports: + +| Capability | Supported | +|----------------|-----------| +| stat | ✅ | +| read | ✅ | +| write | ✅ | +| create_dir | ✅ | +| delete | ✅ | +| list | ✅ | +| copy | ✅ | +| rename | ✅ | +| atomic_write | ❌ | +| append | ✅ | + +## Configuration + +- `root`: Root path within the WASI filesystem. Must be within a preopened directory. + +## Prerequisites + +1. Target must be `wasm32-wasip2` +2. The WASI runtime must provide preopened directories +3. The requested root path must be accessible via preopened directories + +## Example + +```rust,ignore +use opendal::services::WasiFs; +use opendal::Operator; + +let builder = WasiFs::default().root("/data"); +let op = Operator::new(builder)?.finish(); + +// Read a file +let content = op.read("hello.txt").await?; + +// Write a file +op.write("output.txt", "Hello, WASI!").await?; + +// List directory +let entries = op.list("/").await?; +``` + +## Runtime Configuration + +When running WebAssembly components, ensure the host runtime grants filesystem +access via preopened directories: + +```bash +# Wasmtime +wasmtime run --dir /host/path::/guest/path component.wasm + +# Wasmer +wasmer run --dir /host/path::/guest/path component.wasm +``` + +## Limitations + +- **No atomic writes**: The `abort()` method returns an error since WASI doesn't provide atomic file operations +- **No file locking**: File locking is not supported +- **Symbolic links**: Support depends on the runtime +- **Timestamps**: Accuracy depends on runtime support +- **Platform-specific**: Only compiles for `wasm32-wasip2` target diff --git a/core/services/wasi-fs/src/error.rs b/core/services/wasi-fs/src/error.rs new file mode 100644 index 000000000000..7535a791bf1e --- /dev/null +++ b/core/services/wasi-fs/src/error.rs @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use opendal_core::*; +use wasi::filesystem::types::ErrorCode; + +/// Convert WASI filesystem errors to OpenDAL errors +pub fn parse_wasi_error(code: ErrorCode) -> Error { + let (kind, message) = match code { + ErrorCode::Access => (ErrorKind::PermissionDenied, "Access denied"), + ErrorCode::WouldBlock => (ErrorKind::RateLimited, "Operation would block"), + ErrorCode::Already => (ErrorKind::AlreadyExists, "Resource already exists"), + ErrorCode::BadDescriptor => (ErrorKind::Unexpected, "Bad file descriptor"), + ErrorCode::Busy => (ErrorKind::RateLimited, "Resource busy"), + ErrorCode::Deadlock => (ErrorKind::Unexpected, "Deadlock detected"), + ErrorCode::Quota => (ErrorKind::RateLimited, "Quota exceeded"), + ErrorCode::Exist => (ErrorKind::AlreadyExists, "File exists"), + ErrorCode::FileTooLarge => (ErrorKind::Unexpected, "File too large"), + ErrorCode::IllegalByteSequence => (ErrorKind::Unexpected, "Invalid byte sequence"), + ErrorCode::InProgress => (ErrorKind::RateLimited, "Operation in progress"), + ErrorCode::Interrupted => (ErrorKind::Unexpected, "Operation interrupted"), + ErrorCode::Invalid => (ErrorKind::Unexpected, "Invalid argument"), + ErrorCode::Io => (ErrorKind::Unexpected, "I/O error"), + ErrorCode::IsDirectory => (ErrorKind::IsADirectory, "Is a directory"), + ErrorCode::Loop => (ErrorKind::Unexpected, "Too many symbolic links"), + ErrorCode::TooManyLinks => (ErrorKind::Unexpected, "Too many links"), + ErrorCode::MessageSize => (ErrorKind::Unexpected, "Message too large"), + ErrorCode::NameTooLong => (ErrorKind::Unexpected, "Name too long"), + ErrorCode::NoDevice => (ErrorKind::NotFound, "No such device"), + ErrorCode::NoEntry => (ErrorKind::NotFound, "No such file or directory"), + ErrorCode::NoLock => (ErrorKind::Unexpected, "No locks available"), + ErrorCode::InsufficientMemory => (ErrorKind::Unexpected, "Out of memory"), + ErrorCode::InsufficientSpace => (ErrorKind::Unexpected, "No space left on device"), + ErrorCode::NotDirectory => (ErrorKind::NotADirectory, "Not a directory"), + ErrorCode::NotEmpty => (ErrorKind::Unexpected, "Directory not empty"), + ErrorCode::NotRecoverable => (ErrorKind::Unexpected, "State not recoverable"), + ErrorCode::Unsupported => (ErrorKind::Unsupported, "Operation not supported"), + ErrorCode::NoTty => (ErrorKind::Unexpected, "Not a terminal"), + ErrorCode::NoSuchDevice => (ErrorKind::NotFound, "No such device"), + ErrorCode::Overflow => (ErrorKind::Unexpected, "Value overflow"), + ErrorCode::NotPermitted => (ErrorKind::PermissionDenied, "Operation not permitted"), + ErrorCode::Pipe => (ErrorKind::Unexpected, "Broken pipe"), + ErrorCode::ReadOnly => (ErrorKind::PermissionDenied, "Read-only filesystem"), + ErrorCode::InvalidSeek => (ErrorKind::Unexpected, "Invalid seek"), + ErrorCode::TextFileBusy => (ErrorKind::RateLimited, "Text file busy"), + ErrorCode::CrossDevice => (ErrorKind::Unsupported, "Cross-device link"), + }; + + Error::new(kind, message).with_context("source", "wasi-filesystem") +} + +/// Convert WASI I/O stream errors +pub fn parse_stream_error(err: wasi::io::streams::StreamError) -> Error { + match err { + wasi::io::streams::StreamError::LastOperationFailed(e) => { + Error::new(ErrorKind::Unexpected, "Stream operation failed") + .with_context("details", format!("{:?}", e)) + } + wasi::io::streams::StreamError::Closed => { + Error::new(ErrorKind::Unexpected, "Stream closed") + } + } +} diff --git a/core/services/wasi-fs/src/lib.rs b/core/services/wasi-fs/src/lib.rs new file mode 100644 index 000000000000..ff49f6b4432a --- /dev/null +++ b/core/services/wasi-fs/src/lib.rs @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! WASI Filesystem service for OpenDAL +//! +//! This service provides filesystem access for WebAssembly components +//! running in WASI Preview 2 compatible runtimes (wasmtime, wasmer). + +#![cfg(all(target_arch = "wasm32", target_os = "wasi"))] + +/// Default scheme for wasi-fs service. +pub const WASI_FS_SCHEME: &str = "wasi-fs"; + +use opendal_core::DEFAULT_OPERATOR_REGISTRY; + +mod backend; +mod config; +mod core; +mod deleter; +mod error; +mod lister; +mod reader; +mod writer; + +pub use backend::WasiFsBuilder as WasiFs; +pub use config::WasiFsConfig; + +#[ctor::ctor] +fn register_wasi_fs_service() { + DEFAULT_OPERATOR_REGISTRY.register::(WASI_FS_SCHEME); +} diff --git a/core/services/wasi-fs/src/lister.rs b/core/services/wasi-fs/src/lister.rs new file mode 100644 index 000000000000..8b2062d9dcb6 --- /dev/null +++ b/core/services/wasi-fs/src/lister.rs @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::sync::Arc; + +use opendal_core::raw::*; +use opendal_core::*; +use wasi::filesystem::types::{DescriptorType, DirectoryEntryStream}; + +use super::core::WasiFsCore; +use super::error::parse_wasi_error; + +pub struct WasiFsLister { + stream: DirectoryEntryStream, + path: String, + returned_self: bool, +} + +impl WasiFsLister { + pub fn new(core: Arc, path: &str) -> Result { + let stream = core.read_dir(path)?; + + Ok(Self { + stream, + path: path.to_string(), + returned_self: false, + }) + } +} + +/// # Safety +/// +/// WasiFsLister only accesses WASI resources which are single-threaded in WASM. +unsafe impl Sync for WasiFsLister {} + +impl oio::List for WasiFsLister { + async fn next(&mut self) -> Result> { + if !self.returned_self { + self.returned_self = true; + let entry = oio::Entry::new(&self.path, Metadata::new(EntryMode::DIR)); + return Ok(Some(entry)); + } + + match self.stream.read_directory_entry() { + Ok(Some(entry)) => { + let name = entry.name; + + let mode = match entry.type_ { + DescriptorType::Directory => EntryMode::DIR, + DescriptorType::RegularFile => EntryMode::FILE, + _ => EntryMode::Unknown, + }; + + let path = if self.path.ends_with('/') || self.path.is_empty() { + format!("{}{}", self.path, name) + } else { + format!("{}/{}", self.path, name) + }; + + let path = if mode == EntryMode::DIR { + format!("{}/", path) + } else { + path + }; + + Ok(Some(oio::Entry::new(&path, Metadata::new(mode)))) + } + Ok(None) => Ok(None), + Err(e) => Err(parse_wasi_error(e)), + } + } +} diff --git a/core/services/wasi-fs/src/reader.rs b/core/services/wasi-fs/src/reader.rs new file mode 100644 index 000000000000..8e7da5764142 --- /dev/null +++ b/core/services/wasi-fs/src/reader.rs @@ -0,0 +1,75 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::sync::Arc; + +use opendal_core::raw::*; +use opendal_core::*; +use wasi::filesystem::types::{DescriptorFlags, OpenFlags}; +use wasi::io::streams::InputStream; + +use super::core::WasiFsCore; +use super::error::{parse_stream_error, parse_wasi_error}; + +pub struct WasiFsReader { + stream: InputStream, + remaining: u64, +} + +impl WasiFsReader { + pub fn new(core: Arc, path: &str, range: BytesRange) -> Result { + let file = core.open_file(path, OpenFlags::empty(), DescriptorFlags::READ)?; + + let stat = file.stat().map_err(parse_wasi_error)?; + let offset = range.offset(); + let size = range.size().unwrap_or(stat.size - offset); + + let stream = file.read_via_stream(offset).map_err(parse_wasi_error)?; + + Ok(Self { + stream, + remaining: size, + }) + } +} + +/// # Safety +/// +/// WasiFsReader only accesses WASI resources which are single-threaded in WASM. +unsafe impl Sync for WasiFsReader {} + +impl oio::Read for WasiFsReader { + async fn read(&mut self) -> Result { + if self.remaining == 0 { + return Ok(Buffer::new()); + } + + let to_read = self.remaining.min(64 * 1024); + let data = self + .stream + .blocking_read(to_read) + .map_err(parse_stream_error)?; + + if data.is_empty() { + self.remaining = 0; + return Ok(Buffer::new()); + } + + self.remaining -= data.len() as u64; + Ok(Buffer::from(data)) + } +} diff --git a/core/services/wasi-fs/src/writer.rs b/core/services/wasi-fs/src/writer.rs new file mode 100644 index 000000000000..d6b7f525ba8a --- /dev/null +++ b/core/services/wasi-fs/src/writer.rs @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::sync::Arc; + +use opendal_core::raw::*; +use opendal_core::*; +use wasi::filesystem::types::{Descriptor, DescriptorFlags, OpenFlags}; +use wasi::io::streams::OutputStream; + +use super::core::WasiFsCore; +use super::error::{parse_stream_error, parse_wasi_error}; + +pub struct WasiFsWriter { + file: Descriptor, + stream: OutputStream, + offset: u64, +} + +impl WasiFsWriter { + pub fn new(core: Arc, path: &str, args: OpWrite) -> Result { + let mut open_flags = OpenFlags::CREATE; + + if !args.append() { + open_flags |= OpenFlags::TRUNCATE; + } + + let file = core.open_file(path, open_flags, DescriptorFlags::WRITE)?; + + let offset = if args.append() { + let stat = file.stat().map_err(parse_wasi_error)?; + stat.size + } else { + 0 + }; + + let stream = file.write_via_stream(offset).map_err(parse_wasi_error)?; + + Ok(Self { + file, + stream, + offset, + }) + } +} + +/// # Safety +/// +/// WasiFsWriter only accesses WASI resources which are single-threaded in WASM. +unsafe impl Sync for WasiFsWriter {} + +impl oio::Write for WasiFsWriter { + async fn write(&mut self, bs: Buffer) -> Result<()> { + let data: Vec = bs.to_vec(); + + self.stream + .blocking_write_and_flush(&data) + .map_err(parse_stream_error)?; + + self.offset += data.len() as u64; + Ok(()) + } + + async fn close(&mut self) -> Result { + self.stream.blocking_flush().map_err(parse_stream_error)?; + self.file.sync_data().map_err(parse_wasi_error)?; + + let stat = self.file.stat().map_err(parse_wasi_error)?; + + Ok(Metadata::new(EntryMode::FILE).with_content_length(stat.size)) + } + + async fn abort(&mut self) -> Result<()> { + Err(Error::new( + ErrorKind::Unsupported, + "WasiFs doesn't support abort without atomic write support", + )) + } +} diff --git a/core/src/lib.rs b/core/src/lib.rs index 066ccd4426b0..38d93a1fe769 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -61,6 +61,12 @@ pub mod services { pub use opendal_service_tikv::*; #[cfg(feature = "services-vercel-blob")] pub use opendal_service_vercel_blob::*; + #[cfg(all( + feature = "services-wasi-fs", + target_arch = "wasm32", + target_os = "wasi" + ))] + pub use opendal_service_wasi_fs::*; } /// Re-export of layers.