Skip to content

Commit

Permalink
Merge pull request #3920 from wasmerio/fs-union-fix
Browse files Browse the repository at this point in the history
Fix filesystem access when running python with `wasmer run-unstable .`
  • Loading branch information
Michael Bryan authored May 30, 2023
2 parents 3b7af17 + 4205423 commit 554b202
Show file tree
Hide file tree
Showing 19 changed files with 686 additions and 84 deletions.
11 changes: 9 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

46 changes: 26 additions & 20 deletions lib/cli/src/commands/run_unstable.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ use sha2::{Digest, Sha256};
use tempfile::NamedTempFile;
use tokio::runtime::Handle;
use url::Url;
use wapm_targz_to_pirita::FileMap;
use wapm_targz_to_pirita::{FileMap, TransformManifestFunctions};
use wasmer::{
DeserializeError, Engine, Function, Imports, Instance, Module, Store, Type, TypedFunction,
Value,
Expand Down Expand Up @@ -176,7 +176,7 @@ impl RunUnstable {
}
}

#[tracing::instrument(skip_all)]
#[tracing::instrument(level = "debug", skip_all)]
fn load_injected_packages(&self, runtime: &dyn Runtime) -> Result<Vec<BinaryPackage>, Error> {
let mut dependencies = Vec::new();

Expand Down Expand Up @@ -494,26 +494,10 @@ enum ExecutableTarget {
impl ExecutableTarget {
/// Try to load a Wasmer package from a directory containing a `wasmer.toml`
/// file.
#[tracing::instrument(skip_all)]
fn from_dir(dir: &Path, runtime: &dyn Runtime) -> Result<Self, Error> {
let mut files = BTreeMap::new();
load_files_from_disk(&mut files, dir, dir)?;

let wasmer_toml = DirOrFile::File("wasmer.toml".into());
if let Some(toml_data) = files.remove(&wasmer_toml) {
// HACK(Michael-F-Bryan): The version of wapm-targz-to-pirita we are
// using doesn't know we renamed "wapm.toml" to "wasmer.toml", so we
// manually patch things up if people have already migrated their
// projects.
files
.entry(DirOrFile::File("wapm.toml".into()))
.or_insert(toml_data);
}

let functions = wapm_targz_to_pirita::TransformManifestFunctions::default();
let webc = wapm_targz_to_pirita::generate_webc_file(files, dir, None, &functions)?;

let webc = construct_webc_in_memory(dir)?;
let container = Container::from_bytes(webc)?;

let pkg = runtime
.task_manager()
.block_on(BinaryPackage::from_webc(&container, runtime))?;
Expand Down Expand Up @@ -554,6 +538,28 @@ impl ExecutableTarget {
}
}

#[tracing::instrument(level = "debug", skip_all)]
fn construct_webc_in_memory(dir: &Path) -> Result<Vec<u8>, Error> {
let mut files = BTreeMap::new();
load_files_from_disk(&mut files, dir, dir)?;

let wasmer_toml = DirOrFile::File("wasmer.toml".into());
if let Some(toml_data) = files.remove(&wasmer_toml) {
// HACK(Michael-F-Bryan): The version of wapm-targz-to-pirita we are
// using doesn't know we renamed "wapm.toml" to "wasmer.toml", so we
// manually patch things up if people have already migrated their
// projects.
files
.entry(DirOrFile::File("wapm.toml".into()))
.or_insert(toml_data);
}

let functions = TransformManifestFunctions::default();
let webc = wapm_targz_to_pirita::generate_webc_file(files, dir, None, &functions)?;

Ok(webc)
}

fn load_files_from_disk(files: &mut FileMap, dir: &Path, base: &Path) -> Result<(), Error> {
let entries = dir
.read_dir()
Expand Down
2 changes: 1 addition & 1 deletion lib/emscripten/src/env/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ pub fn call_malloc(mut ctx: &mut FunctionEnvMut<EmEnv>, size: u32) -> u32 {
malloc_ref.call(&mut ctx, size).unwrap()
}

#[warn(dead_code)]
#[allow(dead_code)]
pub fn call_malloc_with_cast<T: Copy>(ctx: &mut FunctionEnvMut<EmEnv>, size: u32) -> WasmPtr<T> {
WasmPtr::new(call_malloc(ctx, size))
}
Expand Down
1 change: 1 addition & 0 deletions lib/virtual-fs/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ bytes = "1"
tokio = { version = "1", features = ["io-util", "sync", "macros"], default_features = false }
pin-project-lite = "0.2.9"
indexmap = "1.9.2"
replace_with = "0.1.7"

[target.'cfg(not(all(target_arch = "wasm32", target_os = "unknown")))'.dependencies]
getrandom = { version = "0.2" }
Expand Down
103 changes: 103 additions & 0 deletions lib/virtual-fs/src/buffer_file.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
//! Used for /dev/zero - infinitely returns zero
//! which is useful for commands like `dd if=/dev/zero of=bigfile.img size=1G`
use std::io::{self, *};
use std::pin::Pin;
use std::task::{Context, Poll};

use tokio::io::{AsyncRead, AsyncSeek, AsyncWrite};

use crate::VirtualFile;

#[derive(Debug, Default)]
pub struct BufferFile {
pub(crate) data: Cursor<Vec<u8>>,
}

impl AsyncSeek for BufferFile {
fn start_seek(mut self: Pin<&mut Self>, position: io::SeekFrom) -> io::Result<()> {
let data = Pin::new(&mut self.data);
data.start_seek(position)
}

fn poll_complete(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<u64>> {
let data = Pin::new(&mut self.data);
data.poll_complete(cx)
}
}

impl AsyncWrite for BufferFile {
fn poll_write(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &[u8],
) -> Poll<io::Result<usize>> {
let data = Pin::new(&mut self.data);
data.poll_write(cx, buf)
}

fn poll_write_vectored(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
bufs: &[io::IoSlice<'_>],
) -> Poll<io::Result<usize>> {
let data = Pin::new(&mut self.data);
data.poll_write_vectored(cx, bufs)
}

fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
let data = Pin::new(&mut self.data);
data.poll_flush(cx)
}

fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
let data = Pin::new(&mut self.data);
data.poll_shutdown(cx)
}
}

impl AsyncRead for BufferFile {
fn poll_read(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &mut tokio::io::ReadBuf<'_>,
) -> Poll<io::Result<()>> {
let data = Pin::new(&mut self.data);
data.poll_read(cx, buf)
}
}

impl VirtualFile for BufferFile {
fn last_accessed(&self) -> u64 {
0
}
fn last_modified(&self) -> u64 {
0
}
fn created_time(&self) -> u64 {
0
}
fn size(&self) -> u64 {
self.data.get_ref().len() as u64
}
fn set_len(&mut self, new_size: u64) -> crate::Result<()> {
self.data.get_mut().resize(new_size as usize, 0);
Ok(())
}
fn unlink(&mut self) -> crate::Result<()> {
Ok(())
}
fn poll_read_ready(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<usize>> {
let cur = self.data.stream_position().unwrap_or_default();
let len = self.data.seek(SeekFrom::End(0)).unwrap_or_default();
if cur < len {
Poll::Ready(Ok((len - cur) as usize))
} else {
Poll::Ready(Ok(0))
}
}

fn poll_write_ready(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<usize>> {
Poll::Ready(Ok(8192))
}
}
Loading

0 comments on commit 554b202

Please sign in to comment.