Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix: force raw codec if requested #644

Merged
merged 3 commits into from
Jan 4, 2023
Merged

fix: force raw codec if requested #644

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bb82c3e
fix: force raw codec if requested
Arqu Dec 21, 2022
6886a98
tests
Arqu Dec 22, 2022
59fb85f
Merge branch 'main' into arqu/verify_hash
Arqu Jan 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion iroh-gateway/src/client.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,8 +99,18 @@ impl<T: ContentLoader + std::marker::Unpin> Client<T> {
pub async fn retrieve_path_metadata(
&self,
path: iroh_resolver::resolver::Path,
format: Option<ResponseFormat>,
) -> Result<Out, String> {
info!("retrieve path metadata {}", path);
if let Some(f) = format {
if f == ResponseFormat::Raw {
return self
.resolver
.resolve_raw(path)
.await
.map_err(|e| e.to_string());
}
}
self.resolver.resolve(path).await.map_err(|e| e.to_string())
}

Expand All @@ -116,7 +126,7 @@ impl<T: ContentLoader + std::marker::Unpin> Client<T> {
let path_metadata = if let Some(path_metadata) = path_metadata {
path_metadata
} else {
self.retrieve_path_metadata(path.clone()).await?
self.retrieve_path_metadata(path.clone(), None).await?
};
let metadata = path_metadata.metadata().clone();
record_ttfb_metrics(start_time, &metadata.source);
Expand Down
36 changes: 36 additions & 0 deletions iroh-gateway/src/core.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -396,6 +396,42 @@ mod tests {
assert!(body.starts_with(b"<!DOCTYPE html>"));
}

#[tokio::test]
async fn test_raw_fetch() {
let files = &[(
"hello.txt".to_string(),
Alphanumeric
.sample_string(&mut SmallRng::seed_from_u64(42), 8 * 1024 * 1024)
.bytes()
.collect(),
)];
let large_file = &files[0].1;
let test_setup = setup_test(false, files).await;

let res = do_request(
"GET",
&format!("localhost:{}", test_setup.gateway_addr.port()),
&format!("/ipfs/{}/hello.txt", test_setup.root_cid),
None,
)
.await;
assert_eq!(http::StatusCode::OK, res.status());
let body = hyper::body::to_bytes(res.into_body()).await.unwrap();
assert_eq!(&body[..], large_file);

let res = do_request(
"GET",
&format!("localhost:{}", test_setup.gateway_addr.port()),
&format!("/ipfs/{}/hello.txt?format=raw", test_setup.root_cid),
None,
)
.await;
assert_eq!(http::StatusCode::OK, res.status());
let body = hyper::body::to_bytes(res.into_body()).await.unwrap();
let ufs = iroh_unixfs::unixfs::UnixfsNode::decode(&test_setup.root_cid, body).unwrap();
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Am slightly confused on how to unpack this further without significantly complicating things.

assert_eq!(ufs.typ().unwrap(), iroh_unixfs::unixfs::DataType::File);
}

// TODO(b5) - refactor to return anyhow::Result<()>
#[tokio::test]
async fn test_fetch_car_recursive() {
Expand Down
14 changes: 9 additions & 5 deletions iroh-gateway/src/handlers.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,15 @@ async fn request_preprocessing<T: ContentLoader + Unpin>(
}
}

let path_metadata = match state.client.retrieve_path_metadata(path.clone()).await {
// parse query params
let format = get_response_format(request_headers, &query_params.format)
.map_err(|err| GatewayError::new(StatusCode::BAD_REQUEST, &err))?;

let path_metadata = match state
.client
.retrieve_path_metadata(path.clone(), Some(format.clone()))
.await
{
Ok(metadata) => metadata,
Err(e) => {
if e == "offline" {
Expand Down Expand Up @@ -217,10 +225,6 @@ async fn request_preprocessing<T: ContentLoader + Unpin>(
));
}

// parse query params
let format = get_response_format(request_headers, &query_params.format)
.map_err(|err| GatewayError::new(StatusCode::BAD_REQUEST, &err))?;

if let Some(resp) = etag_check(request_headers, &CidOrDomain::Cid(*resolved_cid), &format) {
return Ok(RequestPreprocessingResult::RespondImmediately(resp));
}
Expand Down
25 changes: 21 additions & 4 deletions iroh-resolver/src/resolver.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -695,7 +695,7 @@ impl<T: ContentLoader> Resolver<T> {
let this = self.clone();
self.resolve_recursive_mapped(root, None, move |cid, ctx| {
let this = this.clone();
async move { this.resolve_with_ctx(ctx, Path::from_cid(cid)).await }
async move { this.resolve_with_ctx(ctx, Path::from_cid(cid), false).await }
})
}

Expand Down Expand Up @@ -781,18 +781,35 @@ impl<T: ContentLoader> Resolver<T> {
pub async fn resolve(&self, path: Path) -> Result<Out> {
let ctx = LoaderContext::from_path(self.next_id(), self.session_closer.clone());

self.resolve_with_ctx(ctx, path).await
self.resolve_with_ctx(ctx, path, false).await
}

pub async fn resolve_with_ctx(&self, mut ctx: LoaderContext, path: Path) -> Result<Out> {
/// Resolves through a given path, returning the [`Cid`] and raw bytes of the final leaf.
/// Forces the RAW codec.
#[tracing::instrument(skip(self))]
pub async fn resolve_raw(&self, path: Path) -> Result<Out> {
let ctx = LoaderContext::from_path(self.next_id(), self.session_closer.clone());

self.resolve_with_ctx(ctx, path, true).await
}

pub async fn resolve_with_ctx(
&self,
mut ctx: LoaderContext,
path: Path,
force_raw: bool,
) -> Result<Out> {
// Resolve the root block.
let (root_cid, loaded_cid) = self.resolve_root(&path, &mut ctx).await?;
match loaded_cid.source {
Source::Store(_) => inc!(ResolverMetrics::CacheHit),
_ => inc!(ResolverMetrics::CacheMiss),
}

let codec = Codec::try_from(root_cid.codec()).context("unknown codec")?;
let codec = match force_raw {
true => Codec::Raw,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this works, but is really misusing the current setup

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

but I guess it works for now

false => Codec::try_from(root_cid.codec()).context("unknown codec")?,
};

match codec {
Codec::DagPb => {
Expand Down