feat: propagate input loading/collecting errors to top level

README.md

@@ -194,7 +194,6 @@ outdated information.
 | Custom user agent    | ![yes]  | ![no]         | ![no]    | ![yes]                | ![no]        | ![yes]               | ![no]                 | ![no]  |
 | Relative URLs        | ![yes]  | ![yes]        | ![no]    | ![yes]                | ![yes]       | ![yes]               | ![yes]                | ![yes] |
 | Anchors/Fragments    | ![yes]  | ![no]         | ![no]    | ![no]                 | ![no]        | ![yes]               | ![yes]                | ![no]  |
-| Skip relative URLs   | ![yes]  | ![no]         | ![no]    | ![maybe]              | ![no]        | ![no]                | ![no]                 | ![no]  |
 | Include patterns     | ![yes]️ | ![yes]        | ![no]    | ![yes]                | ![no]        | ![no]                | ![no]                 | ![no]  |
 | Exclude patterns     | ![yes]  | ![no]         | ![yes]   | ![yes]                | ![yes]       | ![yes]               | ![yes]                | ![yes] |
 | Handle redirects     | ![yes]  | ![yes]        | ![yes]   | ![yes]                | ![yes]       | ![yes]               | ![yes]                | ![yes] |

examples/collect_links/collect_links.rs

@@ -1,10 +1,10 @@
-use lychee_lib::{Collector, Input, InputSource, Result};
+use lychee_lib::{Collector, Input, InputSource, RequestError};
 use reqwest::Url;
 use std::{collections::HashSet, path::PathBuf};
 use tokio_stream::StreamExt;
 
 #[tokio::main]
-async fn main() -> Result<()> {
+async fn main() -> Result<(), Box<RequestError>> {
     // Collect all links from the following inputs
     let inputs = HashSet::from_iter([
         Input::from_input_source(InputSource::RemoteUrl(Box::new(
@@ -19,7 +19,7 @@ async fn main() -> Result<()> {
         .skip_ignored(false) // skip files that are ignored by git? (default=true)
         .use_html5ever(false) // use html5ever for parsing? (default=false)
         .collect_links(inputs) // base url or directory
-        .collect::<Result<Vec<_>>>()
+        .collect::<Result<Vec<_>, _>>()
         .await?;
 
     dbg!(links);

lychee-bin/src/commands/check.rs

@@ -10,9 +10,10 @@ use reqwest::Url;
 use tokio::sync::mpsc;
 use tokio_stream::wrappers::ReceiverStream;
 
+use lychee_lib::InputSource;
+use lychee_lib::RequestError;
 use lychee_lib::archive::Archive;
 use lychee_lib::{Client, ErrorKind, Request, Response, Uri};
-use lychee_lib::{InputSource, Result};
 use lychee_lib::{ResponseBody, Status};
 
 use crate::formatters::get_response_formatter;
@@ -27,9 +28,9 @@ use super::CommandParams;
 
 pub(crate) async fn check<S>(
     params: CommandParams<S>,
-) -> Result<(ResponseStats, Arc<Cache>, ExitCode)>
+) -> Result<(ResponseStats, Arc<Cache>, ExitCode), ErrorKind>
 where
-    S: futures::Stream<Item = Result<Request>>,
+    S: futures::Stream<Item = Result<Request, RequestError>>,
 {
     // Setup
     let (send_req, recv_req) = mpsc::channel(params.cfg.max_concurrency);
@@ -176,36 +177,36 @@ async fn suggest_archived_links(
 // the show_results_task to finish
 async fn send_inputs_loop<S>(
     requests: S,
-    send_req: mpsc::Sender<Result<Request>>,
+    send_req: mpsc::Sender<Result<Request, RequestError>>,
     bar: Option<ProgressBar>,
-) -> Result<()>
+) -> Result<(), ErrorKind>
 where
-    S: futures::Stream<Item = Result<Request>>,
+    S: futures::Stream<Item = Result<Request, RequestError>>,
 {
     tokio::pin!(requests);
     while let Some(request) = requests.next().await {
-        let request = request?;
         if let Some(pb) = &bar {
             pb.inc_length(1);
-            pb.set_message(request.to_string());
+            match &request {
+                Ok(x) => pb.set_message(x.to_string()),
+                Err(e) => pb.set_message(e.to_string()),
+            }
         }
-        send_req
-            .send(Ok(request))
-            .await
-            .expect("Cannot send request");
+        send_req.send(request).await.expect("Cannot send request");
     }
     Ok(())
 }
 
 /// Reads from the request channel and updates the progress bar status
 async fn progress_bar_task(
-    mut recv_resp: mpsc::Receiver<Response>,
+    mut recv_resp: mpsc::Receiver<Result<Response, ErrorKind>>,
     verbose: Verbosity,
     pb: Option<ProgressBar>,
     formatter: Box<dyn ResponseFormatter>,
     mut stats: ResponseStats,
-) -> Result<(Option<ProgressBar>, ResponseStats)> {
+) -> Result<(Option<ProgressBar>, ResponseStats), ErrorKind> {
     while let Some(response) = recv_resp.recv().await {
+        let response = response?;
         show_progress(
             &mut io::stderr(),
             pb.as_ref(),
@@ -232,8 +233,8 @@ fn init_progress_bar(initial_message: &'static str) -> ProgressBar {
 }
 
 async fn request_channel_task(
-    recv_req: mpsc::Receiver<Result<Request>>,
-    send_resp: mpsc::Sender<Response>,
+    recv_req: mpsc::Receiver<Result<Request, RequestError>>,
+    send_resp: mpsc::Sender<Result<Response, ErrorKind>>,
     max_concurrency: usize,
     client: Client,
     cache: Arc<Cache>,
@@ -243,8 +244,7 @@ async fn request_channel_task(
     StreamExt::for_each_concurrent(
         ReceiverStream::new(recv_req),
         max_concurrency,
-        |request: Result<Request>| async {
-            let request = request.expect("cannot read request");
+        |request: Result<Request, RequestError>| async {
             let response = handle(
                 &client,
                 cache.clone(),
@@ -277,19 +277,32 @@ async fn check_url(client: &Client, request: Request) -> Response {
         Response::new(
             uri.clone(),
             Status::Error(ErrorKind::InvalidURI(uri.clone())),
-            source,
+            source.into(),
         )
     })
 }
 
 /// Handle a single request
+///
+/// # Errors
+///
+/// An Err is returned if and only if there was an error while loading
+/// a *user-provided* input argument. Other errors, including errors in
+/// link resolution and in resolved inputs, will be returned as Ok with
+/// a failed response.
 async fn handle(
     client: &Client,
     cache: Arc<Cache>,
     cache_exclude_status: HashSet<u16>,
-    request: Request,
+    request: Result<Request, RequestError>,
     accept: HashSet<u16>,
-) -> Response {
+) -> Result<Response, ErrorKind> {
+    // Note that the RequestError cases bypass the cache.
+    let request = match request {
+        Ok(x) => x,
+        Err(e) => return e.into_response(),
+    };
+
     let uri = request.uri.clone();
     if let Some(v) = cache.get(&uri) {
         // Found a cached request
@@ -304,7 +317,7 @@ async fn handle(
             // code.
             Status::from_cache_status(v.value().status, &accept)
         };
-        return Response::new(uri.clone(), status, request.source);
+        return Ok(Response::new(uri.clone(), status, request.source.into()));
     }
 
     // Request was not cached; run a normal check
@@ -318,11 +331,11 @@ async fn handle(
     // - Skip caching links for which the status code has been explicitly excluded from the cache.
     let status = response.status();
     if ignore_cache(&uri, status, &cache_exclude_status) {
-        return response;
+        return Ok(response);
     }
 
     cache.insert(uri, status.into());
-    response
+    Ok(response)
 }
 
 /// Returns `true` if the response should be ignored in the cache.
@@ -351,7 +364,7 @@ fn show_progress(
     response: &Response,
     formatter: &dyn ResponseFormatter,
     verbose: &Verbosity,
-) -> Result<()> {
+) -> Result<(), ErrorKind> {
     // In case the log level is set to info, we want to show the detailed
     // response output. Otherwise, we only show the essential information
     // (typically the status code and the URL, but this is dependent on the
@@ -402,7 +415,7 @@ mod tests {
     use crate::{formatters::get_response_formatter, options};
     use http::StatusCode;
     use log::info;
-    use lychee_lib::{CacheStatus, ClientBuilder, ErrorKind, ResolvedInputSource, Uri};
+    use lychee_lib::{CacheStatus, ClientBuilder, ErrorKind, Uri};
 
     use super::*;
 
@@ -412,7 +425,7 @@ mod tests {
         let response = Response::new(
             Uri::try_from("http://127.0.0.1").unwrap(),
             Status::Cached(CacheStatus::Ok(200)),
-            ResolvedInputSource::Stdin,
+            InputSource::Stdin,
         );
         let formatter = get_response_formatter(&options::OutputMode::Plain);
         show_progress(
@@ -434,7 +447,7 @@ mod tests {
         let response = Response::new(
             Uri::try_from("http://127.0.0.1").unwrap(),
             Status::Cached(CacheStatus::Ok(200)),
-            ResolvedInputSource::Stdin,
+            InputSource::Stdin,
         );
         let formatter = get_response_formatter(&options::OutputMode::Plain);
         show_progress(

lychee-bin/src/commands/dump.rs

@@ -1,6 +1,7 @@
 use log::error;
+use log::warn;
 use lychee_lib::Request;
-use lychee_lib::Result;
+use lychee_lib::RequestError;
 use std::fs;
 use std::io::{self, Write};
 use tokio_stream::StreamExt;
@@ -11,9 +12,9 @@ use crate::verbosity::Verbosity;
 use super::CommandParams;
 
 /// Dump all detected links to stdout without checking them
-pub(crate) async fn dump<S>(params: CommandParams<S>) -> Result<ExitCode>
+pub(crate) async fn dump<S>(params: CommandParams<S>) -> lychee_lib::Result<ExitCode>
 where
-    S: futures::Stream<Item = Result<Request>>,
+    S: futures::Stream<Item = Result<Request, RequestError>>,
 {
     let requests = params.requests;
     tokio::pin!(requests);
@@ -25,7 +26,17 @@ where
     let mut writer = super::create_writer(params.cfg.output)?;
 
     while let Some(request) = requests.next().await {
-        let mut request = request?;
+        if let Err(e @ RequestError::UserInputContent { .. }) = request {
+            return Err(e.into_error());
+        }
+
+        let mut request = match request {
+            Ok(x) => x,
+            Err(e) => {
+                warn!("{e}");
+                continue;
+            }
+        };
 
         // Apply URI remappings (if any)
         params.client.remap(&mut request.uri)?;

lychee-bin/src/commands/mod.rs

@@ -14,11 +14,11 @@ use std::sync::Arc;
 
 use crate::cache::Cache;
 use crate::options::Config;
-use lychee_lib::Result;
+use lychee_lib::RequestError;
 use lychee_lib::{Client, Request};
 
 /// Parameters passed to every command
-pub(crate) struct CommandParams<S: futures::Stream<Item = Result<Request>>> {
+pub(crate) struct CommandParams<S: futures::Stream<Item = Result<Request, RequestError>>> {
     pub(crate) client: Client,
     pub(crate) cache: Arc<Cache>,
     pub(crate) requests: S,
@@ -30,7 +30,7 @@ pub(crate) struct CommandParams<S: futures::Stream<Item = Result<Request>>> {
 /// # Errors
 ///
 /// Returns an error if the output file cannot be opened.
-fn create_writer(output: Option<PathBuf>) -> Result<Box<dyn Write>> {
+fn create_writer(output: Option<PathBuf>) -> lychee_lib::Result<Box<dyn Write>> {
     Ok(match output {
         Some(path) => Box::new(fs::OpenOptions::new().append(true).open(path)?),
         None => Box::new(io::stdout().lock()),

lychee-bin/src/formatters/response/color.rs

@@ -20,7 +20,9 @@ impl ColorFormatter {
             | Status::Unsupported(_)
             | Status::Cached(CacheStatus::Excluded | CacheStatus::Unsupported) => &DIM,
             Status::UnknownStatusCode(_) | Status::Timeout(_) => &YELLOW,
-            Status::Error(_) | Status::Cached(CacheStatus::Error(_)) => &PINK,
+            Status::Error(_) | Status::RequestError(_) | Status::Cached(CacheStatus::Error(_)) => {
+                &PINK
+            }
         }
     }
 

lychee-bin/src/formatters/response/emoji.rs

@@ -19,7 +19,9 @@ impl EmojiFormatter {
             | Status::Cached(CacheStatus::Excluded | CacheStatus::Unsupported) => "🚫",
             Status::Redirected(_, _) => "↪️",
             Status::UnknownStatusCode(_) | Status::Timeout(_) => "⚠️",
-            Status::Error(_) | Status::Cached(CacheStatus::Error(_)) => "❌",
+            Status::Error(_) | Status::RequestError(_) | Status::Cached(CacheStatus::Error(_)) => {
+                "❌"
+            }
         }
     }
 }

lychee-bin/src/formatters/stats/markdown.rs

@@ -163,10 +163,7 @@ impl StatsFormatter for Markdown {
 #[cfg(test)]
 mod tests {
     use http::StatusCode;
-    use lychee_lib::{
-        CacheStatus, InputSource, Redirects, ResolvedInputSource, Response, ResponseBody, Status,
-        Uri,
-    };
+    use lychee_lib::{CacheStatus, InputSource, Redirects, Response, ResponseBody, Status, Uri};
     use reqwest::Url;
 
     use crate::formatters::suggestion::Suggestion;
@@ -228,7 +225,7 @@ mod tests {
         stats.add(Response::new(
             Uri::try_from("http://127.0.0.1").unwrap(),
             Status::Cached(CacheStatus::Error(Some(404))),
-            ResolvedInputSource::Stdin,
+            InputSource::Stdin,
         ));
 
         // Add suggestion
@@ -252,7 +249,7 @@ mod tests {
                     Url::parse("http://redirected.dev").unwrap(),
                 ]),
             ),
-            ResolvedInputSource::Stdin,
+            InputSource::Stdin,
         ));
 
         let summary = MarkdownResponseStats(stats);

lychee-bin/src/formatters/stats/mod.rs

@@ -55,14 +55,14 @@ where
 mod tests {
     use super::*;
 
-    use lychee_lib::{ErrorKind, ResolvedInputSource, Response, Status, Uri};
+    use lychee_lib::{ErrorKind, Response, Status, Uri};
     use url::Url;
 
     fn make_test_url(url: &str) -> Url {
         Url::parse(url).expect("Expected valid Website URI")
     }
 
-    fn make_test_response(url_str: &str, source: ResolvedInputSource) -> Response {
+    fn make_test_response(url_str: &str, source: InputSource) -> Response {
         let uri = Uri::from(make_test_url(url_str));
 
         Response::new(uri, Status::Error(ErrorKind::EmptyUrl), source)
@@ -74,18 +74,12 @@ mod tests {
 
         // Sorted list of test sources
         let test_sources = vec![
-            ResolvedInputSource::RemoteUrl(Box::new(make_test_url("https://example.com/404"))),
-            ResolvedInputSource::RemoteUrl(Box::new(make_test_url("https://example.com/home"))),
-            ResolvedInputSource::RemoteUrl(Box::new(make_test_url("https://example.com/page/1"))),
-            ResolvedInputSource::RemoteUrl(Box::new(make_test_url("https://example.com/page/10"))),
+            InputSource::RemoteUrl(Box::new(make_test_url("https://example.com/404"))),
+            InputSource::RemoteUrl(Box::new(make_test_url("https://example.com/home"))),
+            InputSource::RemoteUrl(Box::new(make_test_url("https://example.com/page/1"))),
+            InputSource::RemoteUrl(Box::new(make_test_url("https://example.com/page/10"))),
         ];
 
-        let unresolved_test_sources: Vec<InputSource> = test_sources
-            .iter()
-            .map(Clone::clone)
-            .map(Into::<InputSource>::into)
-            .collect();
-
         // Sorted list of test responses
         let test_response_urls = vec![
             "https://example.com/",
@@ -110,7 +104,7 @@ mod tests {
             .collect();
 
         // Check that the input sources are sorted
-        assert_eq!(unresolved_test_sources, sorted_sources);
+        assert_eq!(test_sources, sorted_sources);
 
         // Check that the responses are sorted
         for (_, response_bodies) in sorted_errors {