diff --git a/lib/wasi/src/http/client.rs b/lib/wasi/src/http/client.rs index b8a332897a5..174da36ac61 100644 --- a/lib/wasi/src/http/client.rs +++ b/lib/wasi/src/http/client.rs @@ -1,6 +1,8 @@ use std::{collections::HashSet, ops::Deref, sync::Arc}; use futures::future::BoxFuture; +use http::{HeaderMap, Method, StatusCode}; +use url::Url; /// Defines http client permissions. #[derive(Clone, Debug)] @@ -47,19 +49,27 @@ pub struct HttpRequestOptions { // TODO: use types from http crate? pub struct HttpRequest { - pub url: String, - pub method: String, - pub headers: Vec<(String, String)>, + pub url: Url, + pub method: Method, + pub headers: HeaderMap, pub body: Option>, pub options: HttpRequestOptions, } impl std::fmt::Debug for HttpRequest { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let HttpRequest { + url, + method, + headers, + body, + options, + } = self; + f.debug_struct("HttpRequest") - .field("url", &self.url) - .field("method", &self.method) - .field("headers", &self.headers) + .field("url", &format_args!("{}", url)) + .field("method", method) + .field("headers", headers) .field("body", &self.body.as_deref().map(String::from_utf8_lossy)) .field("options", &self.options) .finish() @@ -68,25 +78,33 @@ impl std::fmt::Debug for HttpRequest { // TODO: use types from http crate? pub struct HttpResponse { - pub pos: usize, pub body: Option>, - pub ok: bool, pub redirected: bool, - pub status: u16, - pub status_text: String, - pub headers: Vec<(String, String)>, + pub status: StatusCode, + pub headers: HeaderMap, +} + +impl HttpResponse { + pub fn is_ok(&self) -> bool { + !self.status.is_client_error() && !self.status.is_server_error() + } } impl std::fmt::Debug for HttpResponse { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let HttpResponse { + body, + redirected, + status, + headers, + } = self; + f.debug_struct("HttpResponse") - .field("pos", &self.pos) - .field("body", &self.body.as_deref().map(String::from_utf8_lossy)) - .field("ok", &self.ok) + .field("ok", &self.is_ok()) .field("redirected", &self.redirected) .field("status", &self.status) - .field("status_text", &self.status_text) .field("headers", &self.headers) + .field("body", &self.body.as_deref().map(String::from_utf8_lossy)) .finish() } } diff --git a/lib/wasi/src/http/reqwest.rs b/lib/wasi/src/http/reqwest.rs index 935044b876e..5390b5a8fd6 100644 --- a/lib/wasi/src/http/reqwest.rs +++ b/lib/wasi/src/http/reqwest.rs @@ -32,24 +32,15 @@ impl ReqwestHttpClient { let response = client.execute(request).await?; - let status = response.status().as_u16(); - let status_text = response.status().as_str().to_string(); + let status = response.status(); // TODO: prevent redundant header copy. - let headers = response - .headers() - .iter() - .map(|(k, v)| (k.to_string(), v.to_str().unwrap().to_string())) - .collect(); let data = response.bytes().await?.to_vec(); Ok(HttpResponse { - pos: 0usize, - ok: true, status, - status_text, redirected: false, body: Some(data), - headers, + headers: std::mem::take(response.headers_mut()), }) } } diff --git a/lib/wasi/src/runtime/package_loader/builtin_loader.rs b/lib/wasi/src/runtime/package_loader/builtin_loader.rs index 20b4da26a78..80f4b5ebf04 100644 --- a/lib/wasi/src/runtime/package_loader/builtin_loader.rs +++ b/lib/wasi/src/runtime/package_loader/builtin_loader.rs @@ -8,6 +8,7 @@ use std::{ use anyhow::{Context, Error}; use bytes::Bytes; +use http::{HeaderMap, Method}; use tempfile::NamedTempFile; use webc::{ compat::{Container, ContainerError}, @@ -16,7 +17,7 @@ use webc::{ use crate::{ bin_factory::BinaryPackage, - http::{HttpClient, HttpRequest, HttpResponse, USER_AGENT}, + http::{HttpClient, HttpRequest, USER_AGENT}, runtime::{ package_loader::PackageLoader, resolver::{DistributionInfo, PackageSummary, Resolution, WebcHash}, @@ -99,26 +100,19 @@ impl BuiltinPackageLoader { } let request = HttpRequest { - url: dist.webc.to_string(), - method: "GET".to_string(), - headers: vec![ - ("Accept".to_string(), "application/webc".to_string()), - ("User-Agent".to_string(), USER_AGENT.to_string()), - ], + url: dist.webc.clone(), + method: Method::GET, + headers: headers(), body: None, options: Default::default(), }; - let HttpResponse { - body, - ok, - status, - status_text, - .. - } = self.client.request(request).await?; + let response = self.client.request(request).await?; - if !ok { - anyhow::bail!("{status} {status_text}"); + if !response.is_ok() { + let url = &dist.webc; + return Err(crate::runtime::resolver::polyfills::http_error(&response) + .context(format!("The GET request to \"{url}\" failed"))); } let body = body.context("The response didn't contain a body")?; @@ -213,6 +207,13 @@ impl PackageLoader for BuiltinPackageLoader { } } +fn headers() -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert("Accept", "application/webc".parse().unwrap()); + headers.insert("User-Agent", USER_AGENT.parse().unwrap()); + headers +} + fn discover_wasmer_dir() -> Option { // TODO: We should reuse the same logic from the wasmer CLI. std::env::var("WASMER_DIR") @@ -307,6 +308,7 @@ mod tests { use std::{collections::VecDeque, sync::Mutex}; use futures::future::BoxFuture; + use http::{HeaderMap, StatusCode}; use tempfile::TempDir; use crate::{ @@ -348,13 +350,10 @@ mod tests { async fn cache_misses_will_trigger_a_download() { let temp = TempDir::new().unwrap(); let client = Arc::new(DummyClient::with_responses([HttpResponse { - pos: 0, body: Some(PYTHON.to_vec()), - ok: true, redirected: false, - status: 200, - status_text: "OK".to_string(), - headers: Vec::new(), + status: StatusCode::OK, + headers: HeaderMap::new(), }])); let loader = BuiltinPackageLoader::new_with_client(temp.path(), client.clone()); let summary = PackageSummary { @@ -376,15 +375,11 @@ mod tests { // A HTTP request was sent let requests = client.requests.lock().unwrap(); let request = &requests[0]; - assert_eq!(request.url, summary.dist.webc.to_string()); + assert_eq!(request.url, summary.dist.webc); assert_eq!(request.method, "GET"); - assert_eq!( - request.headers, - [ - ("Accept".to_string(), "application/webc".to_string()), - ("User-Agent".to_string(), USER_AGENT.to_string()), - ] - ); + assert_eq!(request.headers.len(), 2); + assert_eq!(request.headers["Accept"], "application/webc"); + assert_eq!(request.headers["User-Agent"], USER_AGENT); // Make sure we got the right package let manifest = container.manifest(); assert_eq!(manifest.entrypoint.as_deref(), Some("python")); diff --git a/lib/wasi/src/runtime/resolver/polyfills.rs b/lib/wasi/src/runtime/resolver/polyfills.rs index 7f7614ab6a1..f4344d4207a 100644 --- a/lib/wasi/src/runtime/resolver/polyfills.rs +++ b/lib/wasi/src/runtime/resolver/polyfills.rs @@ -1,7 +1,11 @@ use std::path::Path; +use anyhow::Error; +use http::{HeaderMap, StatusCode}; use url::Url; +use crate::http::{HttpResponse, USER_AGENT}; + /// Polyfill for [`Url::from_file_path()`] that works on `wasm32-unknown-unknown`. pub(crate) fn url_from_file_path(path: impl AsRef) -> Option { let path = path.as_ref(); @@ -25,6 +29,29 @@ pub(crate) fn url_from_file_path(path: impl AsRef) -> Option { buffer.parse().ok() } +pub(crate) fn webc_headers() -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert("Accept", "application/webc".parse().unwrap()); + headers.insert("User-Agent", USER_AGENT.parse().unwrap()); + headers +} + +pub(crate) fn http_error(response: &HttpResponse) -> Error { + let status = response.status; + + if status == StatusCode::SERVICE_UNAVAILABLE { + if let Some(retry_after) = response + .headers + .get("Retry-After") + .and_then(|retry_after| retry_after.to_str().ok()) + { + anyhow::anyhow!("{status} (Retry After: {retry_after})"); + } + } + + Error::msg(status) +} + #[cfg(test)] mod tests { use super::*; diff --git a/lib/wasi/src/runtime/resolver/wapm_source.rs b/lib/wasi/src/runtime/resolver/wapm_source.rs index d6e80c4345f..83a19da8c9d 100644 --- a/lib/wasi/src/runtime/resolver/wapm_source.rs +++ b/lib/wasi/src/runtime/resolver/wapm_source.rs @@ -1,12 +1,13 @@ use std::sync::Arc; use anyhow::{Context, Error}; +use http::{HeaderMap, Method}; use semver::Version; use url::Url; use webc::metadata::Manifest; use crate::{ - http::{HttpClient, HttpRequest, HttpResponse}, + http::{HttpClient, HttpRequest}, runtime::resolver::{ DistributionInfo, PackageInfo, PackageSpecifier, PackageSummary, Source, WebcHash, }, @@ -52,33 +53,22 @@ impl Source for WapmSource { let body = serde_json::to_string(&body)?; let request = HttpRequest { - url: self.registry_endpoint.to_string(), - method: "POST".to_string(), + url: self.registry_endpoint.clone(), + method: Method::POST, body: Some(body.into_bytes()), - headers: vec![ - ( - "User-Agent".to_string(), - crate::http::USER_AGENT.to_string(), - ), - ("Content-Type".to_string(), "application/json".to_string()), - ], + headers: headers(), options: Default::default(), }; - let HttpResponse { - ok, - status, - status_text, - body, - .. - } = self.client.request(request).await?; + let response = self.client.request(request).await?; - if !ok { + if !response.is_ok() { let url = &self.registry_endpoint; - anyhow::bail!("\"{url}\" replied with {status} {status_text}"); + let status = response.status; + anyhow::bail!("\"{url}\" replied with {status}"); } - let body = body.unwrap_or_default(); + let body = response.body.unwrap_or_default(); let response: WapmWebQuery = serde_json::from_slice(&body).context("Unable to deserialize the response")?; @@ -101,6 +91,13 @@ impl Source for WapmSource { } } +fn headers() -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert("Accept", "application/json".parse().unwrap()); + headers.insert("User-Agent", USER_AGENT.parse().unwrap()); + headers +} + fn decode_summary(pkg_version: WapmWebQueryGetPackageVersion) -> Result { let WapmWebQueryGetPackageVersion { manifest, @@ -180,6 +177,8 @@ pub struct WapmWebQueryGetPackageVersionDistribution { mod tests { use std::collections::HashMap; + use http::{HeaderMap, StatusCode}; + use crate::runtime::resolver::inputs::{DistributionInfo, PackageInfo}; use super::*; @@ -200,12 +199,11 @@ mod tests { // -H "Content-Type: application/json" \ // -X POST \ // -d '@wasmer_pack_cli_request.json' > wasmer_pack_cli_response.json - assert_eq!(request.method, "POST"); - assert_eq!(request.url, WapmSource::WAPM_PROD_ENDPOINT); - let headers: HashMap = request.headers.into_iter().collect(); - assert_eq!(headers.len(), 2); - assert_eq!(headers["User-Agent"], crate::http::USER_AGENT); - assert_eq!(headers["Content-Type"], "application/json"); + assert_eq!(request.method, http::Method::POST); + assert_eq!(request.url.as_str(), WapmSource::WAPM_PROD_ENDPOINT); + assert_eq!(request.headers.len(), 2); + assert_eq!(request.headers["User-Agent"], crate::http::USER_AGENT); + assert_eq!(request.headers["Content-Type"], "application/json"); let body: serde_json::Value = serde_json::from_slice(request.body.as_deref().unwrap()).unwrap(); @@ -215,13 +213,10 @@ mod tests { Box::pin(async { Ok(HttpResponse { - pos: 0, body: Some(WASMER_PACK_CLI_RESPONSE.to_vec()), - ok: true, redirected: false, - status: 200, - status_text: "OK".to_string(), - headers: Vec::new(), + status: StatusCode::OK, + headers: HeaderMap::new(), }) }) } diff --git a/lib/wasi/src/runtime/resolver/web_source.rs b/lib/wasi/src/runtime/resolver/web_source.rs index bc715e33e30..f12777429a7 100644 --- a/lib/wasi/src/runtime/resolver/web_source.rs +++ b/lib/wasi/src/runtime/resolver/web_source.rs @@ -7,13 +7,14 @@ use std::{ }; use anyhow::{Context, Error}; +use http::Method; use sha2::{Digest, Sha256}; use tempfile::NamedTempFile; use url::Url; use webc::compat::Container; use crate::{ - http::{HttpClient, HttpRequest, HttpResponse, USER_AGENT}, + http::{HttpClient, HttpRequest}, runtime::resolver::{ DistributionInfo, PackageInfo, PackageSpecifier, PackageSummary, Source, WebcHash, }, @@ -179,72 +180,57 @@ impl WebSource { async fn get_etag(&self, url: &Url) -> Result { let request = HttpRequest { - url: url.to_string(), - method: "HEAD".to_string(), - headers: headers(), + url: url.clone(), + method: Method::HEAD, + headers: super::polyfills::webc_headers(), body: None, options: Default::default(), }; - let HttpResponse { - ok, - status, - status_text, - headers, - .. - } = self.client.request(request).await?; - if !ok { - anyhow::bail!("HEAD request to \"{url}\" failed with {status} {status_text}"); + let response = self.client.request(request).await?; + + if !response.is_ok() { + return Err(super::polyfills::http_error(&response) + .context(format!("The HEAD request to \"{url}\" failed"))); } - let etag = headers - .into_iter() - .find(|(name, _)| name.to_string().to_lowercase() == "etag") - .map(|(_, value)| value) - .context("The HEAD request didn't contain an ETag header`")?; + let etag = response + .headers + .get("ETag") + .context("The HEAD request didn't contain an ETag header`")? + .to_str() + .context("The ETag wasn't valid UTF-8")?; - Ok(etag) + Ok(etag.to_string()) } async fn fetch(&self, url: &Url) -> Result<(Vec, Option), Error> { let request = HttpRequest { - url: url.to_string(), - method: "GET".to_string(), - headers: headers(), + url: url.clone(), + method: Method::GET, + headers: super::polyfills::webc_headers(), body: None, options: Default::default(), }; - let HttpResponse { - ok, - status, - status_text, - headers, - body, - .. - } = self.client.request(request).await?; + let response = self.client.request(request).await?; - if !ok { - anyhow::bail!("HEAD request to \"{url}\" failed with {status} {status_text}"); + if !response.is_ok() { + return Err(super::polyfills::http_error(&response) + .context(format!("The GET request to \"{url}\" failed"))); } - let body = body.context("Response didn't contain a body")?; + let body = response.body.context("Response didn't contain a body")?; - let etag = headers - .into_iter() - .find(|(name, _)| name.to_string().to_lowercase() == "etag") - .map(|(_, value)| value); + let etag = response + .headers + .get("ETag") + .and_then(|etag| etag.to_str().ok()) + .map(|etag| etag.to_string()); Ok((body, etag)) } } -fn headers() -> Vec<(String, String)> { - vec![ - ("Accept".to_string(), "application/webc".to_string()), - ("User-Agent".to_string(), USER_AGENT.to_string()), - ] -} - #[async_trait::async_trait] impl Source for WebSource { #[tracing::instrument(level = "debug", skip_all, fields(%package))] @@ -388,8 +374,11 @@ mod tests { use std::{collections::VecDeque, sync::Mutex}; use futures::future::BoxFuture; + use http::{HeaderMap, StatusCode}; use tempfile::TempDir; + use crate::http::HttpResponse; + use super::*; const PYTHON: &[u8] = include_bytes!("../../../../c-api/examples/assets/python-0.1.0.wasmer"); @@ -428,19 +417,15 @@ mod tests { impl ResponseBuilder { pub fn new() -> Self { ResponseBuilder(HttpResponse { - pos: 0, body: None, - ok: true, redirected: false, - status: 200, - status_text: "OK".to_string(), - headers: Vec::new(), + status: StatusCode::OK, + headers: HeaderMap::new(), }) } - pub fn with_status(mut self, code: u16, text: impl Into) -> Self { + pub fn with_status(mut self, code: StatusCode) -> Self { self.0.status = code; - self.0.status_text = text.into(); self } @@ -454,7 +439,7 @@ mod tests { } pub fn with_header(mut self, name: impl Into, value: impl Into) -> Self { - self.0.headers.push((name.into(), value.into())); + self.0.headers.insert(name, value.into()); self } @@ -511,7 +496,7 @@ mod tests { async fn fall_back_to_stale_cache_if_request_fails() { let temp = TempDir::new().unwrap(); let client = Arc::new(DummyClient::with_responses([ResponseBuilder::new() - .with_status(500, "Internal Server Error") + .with_status(StatusCode::INTERNAL_SERVER_ERROR) .build()])); // Add something to the cache let python_path = temp.path().join(DUMMY_URL_HASH);