Add support for custom headers in input processing

README.md

@@ -458,8 +458,13 @@ Options:
 
           Example: --fallback-extensions html,htm,php,asp,aspx,jsp,cgi
 
-      --header <HEADER>
-          Custom request header
+  -H, --header <HEADER:VALUE>
+          Set custom header for requests
+
+          Some websites require custom headers to be passed in order to return valid responses. 
+          You can specify custom headers in the format 'Name: Value'. For example, 'Accept: text/html'.
+          This is the same format that other tools like curl or wget use. 
+          Multiple headers can be specified by using the flag multiple times.
 
   -a, --accept <ACCEPT>
           A List of accepted status codes for valid links

docs/TROUBLESHOOTING.md

@@ -48,9 +48,9 @@ Some sites expect one or more custom headers to return a valid response. \
 For example, crates.io expects a `Accept: text/html` header or else it \
 will [return a 404](https://github.com/rust-lang/crates.io/issues/788).
 
-To fix that you can pass additional headers like so: `--header "accept=text/html"`. \
+To fix that you can pass additional headers like so: `--header "Accept: text/html"`. \
 You can use that argument multiple times to add more headers. \
-Or, you can accept all content/MIME types: `--header "accept=*/*"`.
+Or, you can accept all content/MIME types: `--header "Accept: */*"`.
 
 See more info about the Accept header
 [over at MDN](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept).

examples/collect_links/collect_links.rs

@@ -1,3 +1,4 @@
+use http::HeaderMap;
 use lychee_lib::{Collector, Input, InputSource, Result};
 use reqwest::Url;
 use std::path::PathBuf;
@@ -13,11 +14,13 @@ async fn main() -> Result<()> {
             )),
             file_type_hint: None,
             excluded_paths: None,
+            headers: HeaderMap::new(),
         },
         Input {
             source: InputSource::FsPath(PathBuf::from("fixtures/TEST.md")),
             file_type_hint: None,
             excluded_paths: None,
+            headers: HeaderMap::new(),
         },
     ];
 

fixtures/configs/headers.toml

@@ -0,0 +1,6 @@
+[header]
+X-Foo = "Bar"
+X-Bar = "Baz"
+
+# Alternative TOML syntax:
+# header = { X-Foo = "Bar", X-Bar = "Baz" }

fixtures/configs/smoketest.toml

@@ -69,7 +69,7 @@ require_https = false
 method = "get"
 
 # Custom request headers
-headers = []
+header = { X-Foo = "Bar", X-Bar = "Baz" }
 
 # Remap URI matching pattern to different URI.
 # This also supports (named) capturing groups.

lychee-bin/Cargo.toml

@@ -28,8 +28,10 @@ env_logger = "0.11.8"
 futures = "0.3.31"
 headers = "0.4.0"
 http = "1.3.1"
+http-serde = "2.1.1"
 humantime = "2.2.0"
 humantime-serde = "1.1.1"
+human-sort = "0.2.2"
 indicatif = "0.17.11"
 log = "0.4.27"
 openssl-sys = { version = "0.9.108", optional = true }
@@ -55,7 +57,7 @@ tokio = { version = "1.45.0", features = ["full"] }
 tokio-stream = "0.1.17"
 toml = "0.8.22"
 url = "2.5.4"
-human-sort = "0.2.2"
+
 
 [dev-dependencies]
 assert_cmd = "2.0.17"

lychee-bin/src/client.rs

@@ -1,7 +1,7 @@
-use crate::options::Config;
-use crate::parse::{parse_duration_secs, parse_headers, parse_remaps};
+use crate::options::{Config, HeaderMapExt};
+use crate::parse::{parse_duration_secs, parse_remaps};
 use anyhow::{Context, Result};
-use http::StatusCode;
+use http::{HeaderMap, StatusCode};
 use lychee_lib::{Client, ClientBuilder};
 use regex::RegexSet;
 use reqwest_cookie_store::CookieStoreMutex;
@@ -10,7 +10,6 @@ use std::{collections::HashSet, str::FromStr};
 
 /// Creates a client according to the command-line config
 pub(crate) fn create(cfg: &Config, cookie_jar: Option<&Arc<CookieStoreMutex>>) -> Result<Client> {
-    let headers = parse_headers(&cfg.header)?;
     let timeout = parse_duration_secs(cfg.timeout);
     let retry_wait_time = parse_duration_secs(cfg.retry_wait_time);
     let method: reqwest::Method = reqwest::Method::from_str(&cfg.method.to_uppercase())?;
@@ -34,6 +33,8 @@ pub(crate) fn create(cfg: &Config, cookie_jar: Option<&Arc<CookieStoreMutex>>) -
         .map(|value| StatusCode::from_u16(*value))
         .collect::<Result<HashSet<_>, _>>()?;
 
+    let headers = HeaderMap::from_header_pairs(&cfg.header)?;
+
     ClientBuilder::builder()
         .remaps(remaps)
         .base(cfg.base_url.clone())