Merge pull request #3831 from wasmerio/resolver-abstraction

Introduce a package resolver

.github/workflows/test.yaml

@@ -146,7 +146,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
-      matrix: 
+      matrix:
         build-what: [
           {
             key: capi,
@@ -206,7 +206,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
-      matrix: 
+      matrix:
         build-what: [
           {
             key: capi,
@@ -259,7 +259,7 @@ jobs:
           if-no-files-found: error
           retention-days: 2
 
-  build: 
+  build:
     name: ${{ matrix.build-what.name }} on ${{ matrix.metadata.build }}
     runs-on: ${{ matrix.metadata.os }}
     needs: setup
@@ -279,10 +279,10 @@ jobs:
           }
         ]
         metadata: [
-          { 
-            build: linux-x64, 
-            os: ubuntu-22.04, 
-            target: x86_64-unknown-linux-gnu,  
+          {
+            build: linux-x64,
+            os: ubuntu-22.04,
+            target: x86_64-unknown-linux-gnu,
             llvm_url: 'https://github.com/llvm/llvm-project/releases/download/llvmorg-14.0.0/clang+llvm-14.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz'
           },
           {
@@ -329,6 +329,7 @@ jobs:
       - name: Set up base deps on musl
         if: matrix.metadata.build == 'linux-musl'
         run: |
+            apk update
             apk add build-base bash musl-dev curl tar make libtool libffi-dev gcc automake autoconf git openssl-dev g++
       - name: Set up dependencies for Mac OS
         run: |
@@ -478,7 +479,7 @@ jobs:
           if-no-files-found: ignore
           retention-days: 2
 
-  test: 
+  test:
     name: ${{ matrix.stage.description }} on ${{ matrix.metadata.build }}
     runs-on: ${{ matrix.metadata.os }}
     needs: setup
@@ -487,12 +488,12 @@ jobs:
       matrix:
         stage: [
           {
-            description: 'Run wast test suite for all compilers', 
+            description: 'Run wast test suite for all compilers',
             make: 'test-stage-0-wast',
           },
           {
             description: 'Unit-test packages on std',
-            make: 'test-stage-1-test-all', 
+            make: 'test-stage-1-test-all',
           },
           {
             description: 'Unit-test cranelift on no-std',
@@ -504,19 +505,19 @@ jobs:
           },
           {
             description: 'Unit-test wasmer-cli',
-            make: 'test-stage-4-wasmer-cli', 
+            make: 'test-stage-4-wasmer-cli',
           },
           {
             description: 'Unit-test examples',
-            make: 'test-stage-5-test-examples', 
+            make: 'test-stage-5-test-examples',
           }
         ]
         metadata: [
           # We cannot test on macos-arm since we don't have ARM runners
-          { 
-            build: linux-x64, 
-            os: ubuntu-22.04, 
-            target: x86_64-unknown-linux-gnu,  
+          {
+            build: linux-x64,
+            os: ubuntu-22.04,
+            target: x86_64-unknown-linux-gnu,
             llvm_url: 'https://github.com/llvm/llvm-project/releases/download/llvmorg-14.0.0/clang+llvm-14.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz'
           },
           {
@@ -553,6 +554,7 @@ jobs:
       - name: Set up base deps on musl
         if: matrix.metadata.build == 'linux-musl'
         run: |
+            apk update
             apk add build-base bash musl-dev curl tar make libtool libffi-dev gcc automake autoconf git openssl-dev g++
       - name: Set up dependencies for Mac OS
         run: |
@@ -612,7 +614,7 @@ jobs:
           TARGET_DIR: target/${{ matrix.metadata.target }}/release
           CARGO_TARGET: ${{ matrix.metadata.target }}
 
-  test_integration_cli: 
+  test_integration_cli:
     name: CLI integration tests on ${{ matrix.build }}
     runs-on: ${{ matrix.os }}
     needs: [build, build_linux_aarch64, build_linux_riscv64]
@@ -651,6 +653,7 @@ jobs:
       - name: Set up base deps on musl
         if: matrix.build == 'linux-musl'
         run: |
+            apk update
             apk add build-base bash musl-dev curl tar make libtool libffi-dev gcc automake autoconf git openssl-dev g++
       - uses: actions/download-artifact@v3
         id: download

lib/cli/src/commands/run/wasi.rs

@@ -1,17 +1,22 @@
 use crate::utils::{parse_envvar, parse_mapdir};
-use anyhow::Result;
+use anyhow::{Context, Result};
 use std::{
     collections::{BTreeSet, HashMap},
     path::{Path, PathBuf},
     sync::Arc,
 };
 use virtual_fs::{DeviceFile, FileSystem, PassthruFileSystem, RootFileSystemBuilder};
-use wasmer::{AsStoreMut, Instance, Module, RuntimeError, Value};
+use wasmer::{AsStoreMut, Engine, Instance, Module, RuntimeError, Value};
+use wasmer_registry::WasmerConfig;
 use wasmer_wasix::{
+    bin_factory::BinaryPackage,
     default_fs_backing, get_wasi_versions,
     os::{tty_sys::SysTty, TtyBridge},
     runners::MappedDirectory,
-    runtime::task_manager::tokio::TokioTaskManager,
+    runtime::{
+        resolver::{PackageResolver, RegistryResolver},
+        task_manager::tokio::TokioTaskManager,
+    },
     types::__WASI_STDIN_FILENO,
     PluggableRuntime, WasiEnv, WasiEnvBuilder, WasiError, WasiFunctionEnv, WasiVersion,
 };
@@ -49,10 +54,10 @@ pub struct Wasi {
     #[clap(long = "use", name = "USE")]
     uses: Vec<String>,
 
-    /// List of webc packages that are explicitely included for execution
+    /// List of webc packages that are explicitly included for execution
     /// Note: these packages will be used instead of those in the registry
     #[clap(long = "include-webc", name = "WEBC")]
-    include_webcs: Vec<String>,
+    include_webcs: Vec<PathBuf>,
 
     /// List of injected atoms
     #[clap(long = "map-command", name = "MAPCMD")]
@@ -136,29 +141,17 @@ impl Wasi {
             .map(|(a, b)| (a.to_string(), b.to_string()))
             .collect::<HashMap<_, _>>();
 
-        let mut rt = PluggableRuntime::new(Arc::new(TokioTaskManager::shared()));
-
-        if self.networking {
-            rt.set_networking_implementation(virtual_net::host::LocalNetworking::default());
-        } else {
-            rt.set_networking_implementation(virtual_net::UnsupportedVirtualNetworking::default());
-        }
-
-        if !self.no_tty {
-            let tty = Arc::new(SysTty::default());
-            tty.reset();
-            rt.set_tty(tty)
-        }
-
         let engine = store.as_store_mut().engine().clone();
-        rt.set_engine(Some(engine));
+
+        let rt = self
+            .prepare_runtime(engine)
+            .context("Unable to prepare the wasi runtime")?;
 
         let builder = WasiEnv::builder(program_name)
             .runtime(Arc::new(rt))
             .args(args)
             .envs(self.env_vars.clone())
             .uses(self.uses.clone())
-            .include_webcs(self.include_webcs.clone())
             .map_commands(map_commands);
 
         let mut builder = if wasmer_wasix::is_wasix_module(module) {
@@ -212,6 +205,31 @@ impl Wasi {
         Ok(builder)
     }
 
+    fn prepare_runtime(&self, engine: Engine) -> Result<PluggableRuntime> {
+        let mut rt = PluggableRuntime::new(Arc::new(TokioTaskManager::shared()));
+
+        if self.networking {
+            rt.set_networking_implementation(virtual_net::host::LocalNetworking::default());
+        } else {
+            rt.set_networking_implementation(virtual_net::UnsupportedVirtualNetworking::default());
+        }
+
+        if !self.no_tty {
+            let tty = Arc::new(SysTty::default());
+            tty.reset();
+            rt.set_tty(tty);
+        }
+
+        rt.set_engine(Some(engine));
+
+        let resolver = self
+            .prepare_resolver()
+            .context("Unable to prepare the package resolver")?;
+        rt.set_resolver(resolver);
+
+        Ok(rt)
+    }
+
     /// Helper function for instantiating a module with Wasi imports for the `Run` command.
     pub fn instantiate(
         &self,
@@ -254,4 +272,39 @@ impl Wasi {
             ..Self::default()
         })
     }
+
+    fn prepare_resolver(&self) -> Result<impl PackageResolver> {
+        let mut resolver = wapm_resolver()?;
+
+        for path in &self.include_webcs {
+            let pkg = preload_webc(path)
+                .with_context(|| format!("Unable to load \"{}\"", path.display()))?;
+            resolver.add_preload(pkg);
+        }
+
+        Ok(resolver.with_cache())
+    }
+}
+
+fn wapm_resolver() -> Result<RegistryResolver, anyhow::Error> {
+    // FIXME(Michael-F-Bryan): Ideally, all of this would in the
+    // RegistryResolver::from_env() constructor, but we don't want to add
+    // wasmer-registry as a dependency of wasmer-wasix just yet.
+    let wasmer_home = WasmerConfig::get_wasmer_dir().map_err(anyhow::Error::msg)?;
+    let cache_dir = wasmer_registry::get_webc_dir(&wasmer_home);
+    let config =
+        wasmer_registry::WasmerConfig::from_file(&wasmer_home).map_err(anyhow::Error::msg)?;
+
+    let registry = config.registry.get_graphql_url();
+    let registry = registry
+        .parse()
+        .with_context(|| format!("Unable to parse \"{registry}\" as a URL"))?;
+
+    Ok(RegistryResolver::new(cache_dir, registry))
+}
+
+fn preload_webc(path: &Path) -> Result<BinaryPackage> {
+    let bytes = std::fs::read(path)?;
+    let webc = wasmer_wasix::wapm::parse_static_webc(bytes)?;
+    Ok(webc)
 }

lib/wasi/Cargo.toml

@@ -59,6 +59,8 @@ wcgi = { version = "0.1.2", optional = true }
 wcgi-host = { version = "0.1.2", optional = true }
 tower-http = { version = "0.4.0", features = ["trace", "util", "catch-panic", "cors"], optional = true }
 tower = { version = "0.4.13", features = ["make", "util"], optional = true }
+url = "2.3.1"
+semver = "1.0.17"
 
 [target.'cfg(not(target_arch = "riscv64"))'.dependencies.reqwest]
 version = "0.11"

lib/wasi/src/bin_factory/binary_package.rs

@@ -2,6 +2,7 @@ use std::sync::{Arc, Mutex, RwLock};
 
 use derivative::*;
 use once_cell::sync::OnceCell;
+use semver::Version;
 use virtual_fs::FileSystem;
 use webc::compat::SharedBytes;
 
@@ -57,7 +58,7 @@ pub struct BinaryPackage {
     pub webc_fs: Option<Arc<dyn FileSystem + Send + Sync + 'static>>,
     pub commands: Arc<RwLock<Vec<BinaryPackageCommand>>>,
     pub uses: Vec<String>,
-    pub version: String,
+    pub version: Version,
     pub module_memory_footprint: u64,
     pub file_system_memory_footprint: u64,
 }