From c938c5d847507f4c0b40e49f3d8362518ebf8bee Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Thu, 10 Aug 2023 16:53:03 +0800 Subject: [PATCH 1/3] Add conversions between http::Request and wasmer_wasix::http::HttpRequest --- lib/wasix/src/http/client.rs | 54 ++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/lib/wasix/src/http/client.rs b/lib/wasix/src/http/client.rs index 174709cf8ab..df3ff78b0f3 100644 --- a/lib/wasix/src/http/client.rs +++ b/lib/wasix/src/http/client.rs @@ -65,6 +65,25 @@ pub struct HttpRequest { pub options: HttpRequestOptions, } +impl HttpRequest { + fn from_http_parts(parts: http::request::Parts, body: impl Into>>) -> Self { + let http::request::Parts { + method, + uri, + headers, + .. + } = parts; + + HttpRequest { + url: uri.to_string().parse().unwrap(), + method, + headers, + body: body.into(), + options: HttpRequestOptions::default(), + } + } +} + impl std::fmt::Debug for HttpRequest { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let HttpRequest { @@ -85,6 +104,41 @@ impl std::fmt::Debug for HttpRequest { } } +impl From>>> for HttpRequest { + fn from(value: http::Request>>) -> Self { + let (parts, body) = value.into_parts(); + HttpRequest::from_http_parts(parts, body) + } +} + +impl From>> for HttpRequest { + fn from(value: http::Request>) -> Self { + let (parts, body) = value.into_parts(); + HttpRequest::from_http_parts(parts, body) + } +} + +impl From> for HttpRequest { + fn from(value: http::Request<&str>) -> Self { + value.map(|body| body.to_string()).into() + } +} + +impl From> for HttpRequest { + fn from(value: http::Request) -> Self { + let (parts, body) = value.into_parts(); + HttpRequest::from_http_parts(parts, body.into_bytes()) + } +} + +impl From> for HttpRequest { + fn from(value: http::Request<()>) -> Self { + let (parts, _) = value.into_parts(); + HttpRequest::from_http_parts(parts, None) + } +} + + // TODO: use types from http crate? pub struct HttpResponse { pub body: Option>, From bab9fa620fd49b8c5a7f0651ebd137885e5a7d1b Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Thu, 10 Aug 2023 16:53:31 +0800 Subject: [PATCH 2/3] Introduce a WebHttpClient --- Cargo.lock | 3 + lib/wasix/Cargo.toml | 8 +- lib/wasix/src/http/client.rs | 1 - lib/wasix/src/http/mod.rs | 7 +- lib/wasix/src/http/web_http_client.rs | 272 ++++++++++++++++++++++++ lib/wasix/src/lib.rs | 3 + lib/wasix/src/runtime/resolver/utils.rs | 1 + lib/wasix/src/utils/mod.rs | 16 +- lib/wasix/src/utils/web.rs | 15 ++ 9 files changed, 315 insertions(+), 11 deletions(-) create mode 100644 lib/wasix/src/http/web_http_client.rs create mode 100644 lib/wasix/src/utils/web.rs diff --git a/Cargo.lock b/Cargo.lock index f4e875719fd..53c0ea9f2e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6125,6 +6125,7 @@ dependencies = [ "hex", "http", "hyper", + "js-sys", "lazy_static", "libc", "linked_hash_set", @@ -6161,6 +6162,7 @@ dependencies = [ "wai-bindgen-wasmer", "waker-fn", "wasm-bindgen", + "wasm-bindgen-futures", "wasm-bindgen-test", "wasmer", "wasmer-emscripten", @@ -6168,6 +6170,7 @@ dependencies = [ "wasmer-wasix-types", "wcgi", "wcgi-host", + "web-sys", "webc", "weezl", "winapi", diff --git a/lib/wasix/Cargo.toml b/lib/wasix/Cargo.toml index 44de8219ca4..52dda742cb8 100644 --- a/lib/wasix/Cargo.toml +++ b/lib/wasix/Cargo.toml @@ -68,6 +68,9 @@ url = "2.3.1" petgraph = "0.6.3" rayon = { version = "1.7.0", optional = true } wasm-bindgen = { version = "0.2.87", optional = true } +js-sys = { version = "0.3.64", optional = true } +wasm-bindgen-futures = { version = "0.4.37", optional = true } +web-sys = { version = "0.3.64", features = ["Request", "RequestInit", "Window", "WorkerGlobalScope", "RequestMode", "Response", "Headers"], optional = true } [target.'cfg(not(target_arch = "riscv64"))'.dependencies.reqwest] version = "0.11" @@ -94,6 +97,7 @@ winapi = "0.3" wasmer = { path = "../api", version = "=4.1.1", default-features = false, features = ["wat", "js-serializable-module"] } tokio = { version = "1", features = [ "sync", "macros", "rt" ], default_features = false } pretty_assertions = "1.3.0" +wasm-bindgen-test = "0.3.0" [target.'cfg(target_arch = "wasm32")'.dev-dependencies] wasm-bindgen-test = "0.3.0" @@ -111,7 +115,7 @@ time = ["tokio/time"] webc_runner_rt_wcgi = ["hyper", "wcgi", "wcgi-host", "tower", "tower-http"] webc_runner_rt_emscripten = ["wasmer-emscripten"] -sys = ["webc/mmap", "time"] +sys = ["webc/mmap", "time", "virtual-mio/sys"] sys-default = ["sys", "logging", "host-fs", "sys-poll", "sys-thread", "host-vnet", "host-threads", "host-reqwest"] sys-poll = [] sys-thread = ["tokio/rt", "tokio/time", "tokio/rt-multi-thread", "rayon"] @@ -119,7 +123,7 @@ sys-thread = ["tokio/rt", "tokio/time", "tokio/rt-multi-thread", "rayon"] # Deprecated. Kept it for compatibility compiler = [] -js = ["virtual-fs/no-time", "getrandom/js", "chrono"] +js = ["virtual-fs/no-time", "getrandom/js", "chrono", "js-sys", "wasm-bindgen", "wasm-bindgen-futures", "web-sys"] js-default = ["js"] test-js = ["js", "wasmer/wat"] diff --git a/lib/wasix/src/http/client.rs b/lib/wasix/src/http/client.rs index df3ff78b0f3..824834d5272 100644 --- a/lib/wasix/src/http/client.rs +++ b/lib/wasix/src/http/client.rs @@ -138,7 +138,6 @@ impl From> for HttpRequest { } } - // TODO: use types from http crate? pub struct HttpResponse { pub body: Option>, diff --git a/lib/wasix/src/http/mod.rs b/lib/wasix/src/http/mod.rs index 5d6c02dba31..1afb9ca9cd3 100644 --- a/lib/wasix/src/http/mod.rs +++ b/lib/wasix/src/http/mod.rs @@ -4,7 +4,10 @@ pub mod client_impl; #[cfg(feature = "host-reqwest")] pub mod reqwest; -pub use self::client::*; +#[cfg(feature = "js")] +mod web_http_client; + +pub use self::{client::*, web_http_client::WebHttpClient}; pub(crate) const USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "-", env!("CARGO_PKG_VERSION")); @@ -13,6 +16,8 @@ pub fn default_http_client() -> Option cfg_if::cfg_if! { if #[cfg(feature = "host-reqwest")] { Some(self::reqwest::ReqwestHttpClient::default()) + } else if #[cfg(feature = "js")] { + Some(web_http_client::WebHttpClient::default()) } else { // Note: We need something to use with turbofish otherwise returning // a plain None will complain about not being able to infer the "T" diff --git a/lib/wasix/src/http/web_http_client.rs b/lib/wasix/src/http/web_http_client.rs new file mode 100644 index 00000000000..8d95bf4057e --- /dev/null +++ b/lib/wasix/src/http/web_http_client.rs @@ -0,0 +1,272 @@ +use anyhow::{Context, Error}; +use futures::future::BoxFuture; +use http::header::{HeaderMap, HeaderValue, IntoHeaderName}; +use wasm_bindgen::{JsCast, JsValue}; +use wasm_bindgen_futures::JsFuture; +use web_sys::{RequestInit, RequestMode, Window, WorkerGlobalScope}; + +use crate::{ + http::{HttpClient, HttpRequest, HttpRequestOptions, HttpResponse}, + utils::web::js_error, +}; + +#[derive(Debug, Default, Clone, PartialEq)] +#[non_exhaustive] +pub struct WebHttpClient { + default_headers: HeaderMap, +} + +impl WebHttpClient { + pub fn new() -> Self { + WebHttpClient { + default_headers: HeaderMap::new(), + } + } + + pub fn with_default_header( + &mut self, + name: impl IntoHeaderName, + value: HeaderValue, + ) -> &mut Self { + self.default_headers.insert(name, value); + self + } + + pub fn extend_default_headers(&mut self, map: HeaderMap) -> &mut Self { + self.default_headers.extend(map); + self + } +} + +impl HttpClient for WebHttpClient { + fn request(&self, mut request: HttpRequest) -> BoxFuture<'_, Result> { + let (sender, receiver) = futures::channel::oneshot::channel(); + + for (name, value) in &self.default_headers { + if !request.headers.contains_key(name) { + request.headers.insert(name, value.clone()); + } + } + + // Note: We can't spawn this on our normal thread-pool because + // JavaScript promises are !Send, so we run it on the browser's event + // loop directly. + wasm_bindgen_futures::spawn_local(async move { + let result = fetch(request).await; + let _ = sender.send(result); + }); + + Box::pin(async move { + match receiver.await { + Ok(result) => result, + Err(e) => Err(Error::new(e)), + } + }) + } +} + +/// Send a `fetch()` request using the browser APIs. +async fn fetch(request: HttpRequest) -> Result { + let HttpRequest { + url, + method, + headers, + body, + options: HttpRequestOptions { + gzip: _, + cors_proxy, + }, + } = request; + + let mut opts = RequestInit::new(); + opts.method(method.as_str()); + opts.mode(RequestMode::Cors); + + if let Some(data) = body { + let data_len = data.len(); + let array = js_sys::Uint8Array::new_with_length(data_len as u32); + array.copy_from(&data[..]); + + opts.body(Some(&array)); + } + + let request = { + let request = web_sys::Request::new_with_str_and_init(url.as_str(), &opts) + .map_err(js_error) + .context("Could not construct request object")?; + + let set_headers = request.headers(); + + for (name, val) in headers.iter() { + let val = String::from_utf8_lossy(val.as_bytes()); + set_headers + .set(name.as_str(), &val) + .map_err(js_error) + .with_context(|| format!("could not apply request header: '{name}': '{val}'"))?; + } + request + }; + + let resp_value = match call_fetch(&request).await { + Ok(a) => a, + Err(e) => { + // If the request failed it may be because of CORS so if a cors proxy + // is configured then try again with the cors proxy + let url = if let Some(cors_proxy) = cors_proxy { + format!("https://{}/{}", cors_proxy, url) + } else { + return Err(js_error(e).context(format!("Could not fetch '{url}'"))); + }; + + let request = web_sys::Request::new_with_str_and_init(&url, &opts) + .map_err(js_error) + .with_context(|| format!("Could not construct request for url '{url}'"))?; + + let set_headers = request.headers(); + for (name, val) in headers.iter() { + let value = String::from_utf8_lossy(val.as_bytes()); + set_headers + .set(name.as_str(), &value) + .map_err(js_error) + .with_context(|| { + anyhow::anyhow!("Could not apply request header: '{name}': '{value}'") + })?; + } + + call_fetch(&request) + .await + .map_err(js_error) + .with_context(|| format!("Could not fetch '{url}'"))? + } + }; + + let response = resp_value.dyn_ref().unwrap(); + read_response(response).await +} + +async fn read_response(response: &web_sys::Response) -> Result { + let status = http::StatusCode::from_u16(response.status())?; + let headers = headers(response.headers()).context("Unable to read the headers")?; + let body = get_response_data(response).await?; + + Ok(HttpResponse { + body: Some(body), + redirected: response.redirected(), + status, + headers, + }) +} + +fn headers(headers: web_sys::Headers) -> Result { + let iter = js_sys::try_iter(&headers) + .map_err(js_error)? + .context("Not an iterator")?; + let mut header_map = http::HeaderMap::new(); + + for pair in iter { + let pair = pair.map_err(js_error)?; + let [key, value]: [js_sys::JsString; 2] = + js_array(&pair).context("Unable to unpack the header's key-value pairs")?; + + let key = String::from(key); + let key: http::HeaderName = key.parse()?; + let value = String::from(value); + let value = http::HeaderValue::from_str(&value) + .with_context(|| format!("Invalid header value: {value}"))?; + + header_map.insert(key, value); + } + + Ok(header_map) +} + +fn js_array(value: &JsValue) -> Result<[T; N], anyhow::Error> +where + T: JsCast, +{ + let array: &js_sys::Array = value.dyn_ref().context("Not an array")?; + + let mut items = Vec::new(); + + for value in array.iter() { + let item = value + .dyn_into() + .map_err(|_| anyhow::anyhow!("Unable to cast to a {}", std::any::type_name::()))?; + items.push(item); + } + + <[T; N]>::try_from(items).map_err(|original| { + anyhow::anyhow!( + "Unable to turn a list of {} items into an array of {N} items", + original.len() + ) + }) +} + +pub async fn get_response_data(resp: &web_sys::Response) -> Result, anyhow::Error> { + let buffer = JsFuture::from(resp.array_buffer().unwrap()) + .await + .map_err(js_error) + .with_context(|| "Could not retrieve response body".to_string())?; + + let buffer = js_sys::Uint8Array::new(&buffer); + + Ok(buffer.to_vec()) +} + +fn call_fetch(request: &web_sys::Request) -> JsFuture { + let global = js_sys::global(); + if JsValue::from_str("WorkerGlobalScope").js_in(&global) + && global.is_instance_of::() + { + JsFuture::from( + global + .unchecked_into::() + .fetch_with_request(request), + ) + } else { + JsFuture::from( + global + .unchecked_into::() + .fetch_with_request(request), + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::runtime::resolver::WapmSource; + + #[wasm_bindgen_test::wasm_bindgen_test] + async fn query_the_wasmer_registry_graphql_endpoint() { + let http_client = WebHttpClient::default(); + let query = r#"{ + "query": "{ info { defaultFrontend } }" + }"#; + let request = http::Request::post(WapmSource::WASMER_PROD_ENDPOINT) + .header(http::header::CONTENT_TYPE, "application/json") + .body(query) + .unwrap(); + + let response = http_client.request(request.into()).await.unwrap(); + + assert_eq!( + response + .headers + .get(http::header::CONTENT_TYPE) + .unwrap() + .to_str() + .unwrap(), + "application/json", + ); + let body: serde_json::Value = + serde_json::from_slice(response.body.as_deref().unwrap()).unwrap(); + let frontend_url = body + .pointer("/data/info/defaultFrontend") + .unwrap() + .as_str() + .unwrap(); + assert_eq!(frontend_url, "https://wasmer.io"); + } +} diff --git a/lib/wasix/src/lib.rs b/lib/wasix/src/lib.rs index f3db61da7b3..b273a4f5e89 100644 --- a/lib/wasix/src/lib.rs +++ b/lib/wasix/src/lib.rs @@ -30,6 +30,9 @@ compile_error!( "The `js` feature must be enabled only for the `wasm32` target (either `wasm32-unknown-unknown` or `wasm32-wasi`)." ); +#[cfg(all(test, target_arch = "wasm32"))] +wasm_bindgen_test::wasm_bindgen_test_configure!(run_in_browser); + #[cfg(test)] #[macro_use] extern crate pretty_assertions; diff --git a/lib/wasix/src/runtime/resolver/utils.rs b/lib/wasix/src/runtime/resolver/utils.rs index 34cc5b5d72b..11f83505971 100644 --- a/lib/wasix/src/runtime/resolver/utils.rs +++ b/lib/wasix/src/runtime/resolver/utils.rs @@ -86,6 +86,7 @@ pub(crate) fn file_path_from_url(url: &Url) -> Result { #[cfg(test)] mod tests { + #[allow(unused_imports)] use super::*; #[test] diff --git a/lib/wasix/src/utils/mod.rs b/lib/wasix/src/utils/mod.rs index 572e9e0fa22..6cf471da1d8 100644 --- a/lib/wasix/src/utils/mod.rs +++ b/lib/wasix/src/utils/mod.rs @@ -1,20 +1,22 @@ +mod dummy_waker; mod owned_mutex_guard; pub mod store; mod thread_parker; -mod dummy_waker; -pub use self::dummy_waker::WasiDummyWaker; - -use std::collections::BTreeSet; +#[cfg(feature = "js")] +pub(crate) mod web; -use wasmer::Module; -use wasmer_wasix_types::wasi::Errno; +pub use self::{dummy_waker::WasiDummyWaker, thread_parker::WasiParkingLot}; -pub use self::thread_parker::WasiParkingLot; pub(crate) use owned_mutex_guard::{ read_owned, write_owned, OwnedRwLockReadGuard, OwnedRwLockWriteGuard, }; +use std::collections::BTreeSet; + +use wasmer::Module; +use wasmer_wasix_types::wasi::Errno; + /// Check if a provided module is compiled for some version of WASI. /// Use [`get_wasi_version`] to find out which version of WASI the module is. pub fn is_wasi_module(module: &Module) -> bool { diff --git a/lib/wasix/src/utils/web.rs b/lib/wasix/src/utils/web.rs new file mode 100644 index 00000000000..ee340c1f87f --- /dev/null +++ b/lib/wasix/src/utils/web.rs @@ -0,0 +1,15 @@ +use wasm_bindgen::{JsCast, JsValue}; + +/// Try to extract the most appropriate error message from a [`JsValue`], +/// falling back to a generic error message. +pub fn js_error(value: JsValue) -> anyhow::Error { + if let Some(e) = value.dyn_ref::() { + anyhow::Error::msg(String::from(e.message())) + } else if let Some(obj) = value.dyn_ref::() { + return anyhow::Error::msg(String::from(obj.to_string())); + } else if let Some(s) = value.dyn_ref::() { + return anyhow::Error::msg(String::from(s)); + } else { + anyhow::anyhow!("An unknown error occurred: {value:?}") + } +} From 2d7ba1dc408ee2a63032a50742c18f6ed3543e8d Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Thu, 10 Aug 2023 17:34:33 +0800 Subject: [PATCH 3/3] Updated wasi-web's Cargo.lock and fixed a compilation error --- lib/wasi-web/Cargo.lock | 4 ++++ lib/wasix/src/http/mod.rs | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/wasi-web/Cargo.lock b/lib/wasi-web/Cargo.lock index 71392050ba0..0737fa9a6a6 100644 --- a/lib/wasi-web/Cargo.lock +++ b/lib/wasi-web/Cargo.lock @@ -2544,6 +2544,7 @@ dependencies = [ "heapless", "hex", "http", + "js-sys", "lazy_static", "libc", "linked_hash_set", @@ -2571,9 +2572,12 @@ dependencies = [ "virtual-net", "wai-bindgen-wasmer", "waker-fn", + "wasm-bindgen", + "wasm-bindgen-futures", "wasmer", "wasmer-types", "wasmer-wasix-types", + "web-sys", "webc", "weezl", "winapi", diff --git a/lib/wasix/src/http/mod.rs b/lib/wasix/src/http/mod.rs index 1afb9ca9cd3..b2f6aebce56 100644 --- a/lib/wasix/src/http/mod.rs +++ b/lib/wasix/src/http/mod.rs @@ -7,7 +7,10 @@ pub mod reqwest; #[cfg(feature = "js")] mod web_http_client; -pub use self::{client::*, web_http_client::WebHttpClient}; +#[cfg(feature = "js")] +pub use self::web_http_client::WebHttpClient; + +pub use self::client::*; pub(crate) const USER_AGENT: &str = concat!(env!("CARGO_PKG_NAME"), "-", env!("CARGO_PKG_VERSION"));