From bb82c3ed4985127b3cb04ba23064c57343370f3b Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Wed, 21 Dec 2022 18:34:07 +0100 Subject: [PATCH 1/2] fix: force raw codec if requested --- iroh-gateway/src/client.rs | 12 +++++++++++- iroh-gateway/src/handlers.rs | 14 +++++++++----- iroh-resolver/src/resolver.rs | 25 +++++++++++++++++++++---- 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/iroh-gateway/src/client.rs b/iroh-gateway/src/client.rs index 45bb2067d0..3873a70a51 100644 --- a/iroh-gateway/src/client.rs +++ b/iroh-gateway/src/client.rs @@ -99,8 +99,18 @@ impl Client { pub async fn retrieve_path_metadata( &self, path: iroh_resolver::resolver::Path, + format: Option, ) -> Result { info!("retrieve path metadata {}", path); + if let Some(f) = format { + if f == ResponseFormat::Raw { + return self + .resolver + .resolve_raw(path) + .await + .map_err(|e| e.to_string()); + } + } self.resolver.resolve(path).await.map_err(|e| e.to_string()) } @@ -116,7 +126,7 @@ impl Client { let path_metadata = if let Some(path_metadata) = path_metadata { path_metadata } else { - self.retrieve_path_metadata(path.clone()).await? + self.retrieve_path_metadata(path.clone(), None).await? }; let metadata = path_metadata.metadata().clone(); record_ttfb_metrics(start_time, &metadata.source); diff --git a/iroh-gateway/src/handlers.rs b/iroh-gateway/src/handlers.rs index 1f0049be91..73343ee2f4 100644 --- a/iroh-gateway/src/handlers.rs +++ b/iroh-gateway/src/handlers.rs @@ -180,7 +180,15 @@ async fn request_preprocessing( } } - let path_metadata = match state.client.retrieve_path_metadata(path.clone()).await { + // parse query params + let format = get_response_format(request_headers, &query_params.format) + .map_err(|err| GatewayError::new(StatusCode::BAD_REQUEST, &err))?; + + let path_metadata = match state + .client + .retrieve_path_metadata(path.clone(), Some(format.clone())) + .await + { Ok(metadata) => metadata, Err(e) => { if e == "offline" { @@ -217,10 +225,6 @@ async fn request_preprocessing( )); } - // parse query params - let format = get_response_format(request_headers, &query_params.format) - .map_err(|err| GatewayError::new(StatusCode::BAD_REQUEST, &err))?; - if let Some(resp) = etag_check(request_headers, &CidOrDomain::Cid(*resolved_cid), &format) { return Ok(RequestPreprocessingResult::RespondImmediately(resp)); } diff --git a/iroh-resolver/src/resolver.rs b/iroh-resolver/src/resolver.rs index 6e13834776..02b60e673c 100644 --- a/iroh-resolver/src/resolver.rs +++ b/iroh-resolver/src/resolver.rs @@ -695,7 +695,7 @@ impl Resolver { let this = self.clone(); self.resolve_recursive_mapped(root, None, move |cid, ctx| { let this = this.clone(); - async move { this.resolve_with_ctx(ctx, Path::from_cid(cid)).await } + async move { this.resolve_with_ctx(ctx, Path::from_cid(cid), false).await } }) } @@ -781,10 +781,24 @@ impl Resolver { pub async fn resolve(&self, path: Path) -> Result { let ctx = LoaderContext::from_path(self.next_id(), self.session_closer.clone()); - self.resolve_with_ctx(ctx, path).await + self.resolve_with_ctx(ctx, path, false).await } - pub async fn resolve_with_ctx(&self, mut ctx: LoaderContext, path: Path) -> Result { + /// Resolves through a given path, returning the [`Cid`] and raw bytes of the final leaf. + /// Forces the RAW codec. + #[tracing::instrument(skip(self))] + pub async fn resolve_raw(&self, path: Path) -> Result { + let ctx = LoaderContext::from_path(self.next_id(), self.session_closer.clone()); + + self.resolve_with_ctx(ctx, path, true).await + } + + pub async fn resolve_with_ctx( + &self, + mut ctx: LoaderContext, + path: Path, + force_raw: bool, + ) -> Result { // Resolve the root block. let (root_cid, loaded_cid) = self.resolve_root(&path, &mut ctx).await?; match loaded_cid.source { @@ -792,7 +806,10 @@ impl Resolver { _ => inc!(ResolverMetrics::CacheMiss), } - let codec = Codec::try_from(root_cid.codec()).context("unknown codec")?; + let codec = match force_raw { + true => Codec::Raw, + false => Codec::try_from(root_cid.codec()).context("unknown codec")?, + }; match codec { Codec::DagPb => { From 6886a9842278d5a4a89f078d3d25bb9bc605a22d Mon Sep 17 00:00:00 2001 From: Asmir Avdicevic Date: Thu, 22 Dec 2022 01:17:04 +0100 Subject: [PATCH 2/2] tests --- iroh-gateway/src/core.rs | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/iroh-gateway/src/core.rs b/iroh-gateway/src/core.rs index f908fb98a1..ea99a8a69e 100644 --- a/iroh-gateway/src/core.rs +++ b/iroh-gateway/src/core.rs @@ -396,6 +396,42 @@ mod tests { assert!(body.starts_with(b"")); } + #[tokio::test] + async fn test_raw_fetch() { + let files = &[( + "hello.txt".to_string(), + Alphanumeric + .sample_string(&mut SmallRng::seed_from_u64(42), 8 * 1024 * 1024) + .bytes() + .collect(), + )]; + let large_file = &files[0].1; + let test_setup = setup_test(false, files).await; + + let res = do_request( + "GET", + &format!("localhost:{}", test_setup.gateway_addr.port()), + &format!("/ipfs/{}/hello.txt", test_setup.root_cid), + None, + ) + .await; + assert_eq!(http::StatusCode::OK, res.status()); + let body = hyper::body::to_bytes(res.into_body()).await.unwrap(); + assert_eq!(&body[..], large_file); + + let res = do_request( + "GET", + &format!("localhost:{}", test_setup.gateway_addr.port()), + &format!("/ipfs/{}/hello.txt?format=raw", test_setup.root_cid), + None, + ) + .await; + assert_eq!(http::StatusCode::OK, res.status()); + let body = hyper::body::to_bytes(res.into_body()).await.unwrap(); + let ufs = iroh_unixfs::unixfs::UnixfsNode::decode(&test_setup.root_cid, body).unwrap(); + assert_eq!(ufs.typ().unwrap(), iroh_unixfs::unixfs::DataType::File); + } + // TODO(b5) - refactor to return anyhow::Result<()> #[tokio::test] async fn test_fetch_car_recursive() {