Improve wikilink parsing and checking

README.md

@@ -18,6 +18,7 @@ Available as a command-line utility, a library and a [GitHub Action](https://git
 
 <!-- START doctoc generated TOC please keep comment here to allow auto update -->
 <!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+
 ## Table of Contents
 
 - [Development](#development)
@@ -573,7 +574,7 @@ Options:
           Find links in verbatim sections like `pre`- and `code` blocks
 
       --include-wikilinks
-          Check WikiLinks in Markdown files
+          Check WikiLinks in Markdown files, this requires specifying --base-url
 
       --index-files <INDEX_FILES>
           When checking locally, resolves directory links to a separate index file.

fixtures/wiki/Dash-Usage.md

@@ -0,0 +1 @@
+# Header

fixtures/wiki/Non-existent.md

@@ -0,0 +1,5 @@
+# Links to non-existing Files
+
+[[Does not exist]]
+[[Doesn't exist.md]]
+[[Does_not_exist]]

fixtures/wiki/Space Usage.md

@@ -0,0 +1 @@
+# Header

fixtures/wiki/Underscore_Usage.md

@@ -0,0 +1 @@
+# Header

fixtures/wiki/Usage.md

@@ -0,0 +1 @@
+# Header

fixtures/wiki/obsidian-style-plus-headers.md

@@ -0,0 +1,8 @@
+[[#LocalHeader]]
+
+# LocalHeader
+
+[[Usage#Header|HeaderRenaming]]
+[[Space Usage#Header|HeaderRenaming]]
+[[Space Usage DifferentDirectory#Header|HeaderRenaming]]
+[[DifferentDirectory#Header|HeaderRenaming]]

fixtures/wiki/obsidian-style.md

@@ -0,0 +1,4 @@
+[[Usage]]
+[[Space Usage]]
+[[Space Usage DifferentDirectory]]
+[[DifferentDirectory]]

fixtures/wiki/subdirectory/Different-Directory-Dash.md

@@ -0,0 +1 @@
+# Header

fixtures/wiki/subdirectory/DifferentDirectory.md

@@ -0,0 +1 @@
+# Header

fixtures/wiki/subdirectory/Different_Directory_Underscore.md

@@ -0,0 +1 @@
+# Header

fixtures/wiki/subdirectory/Space Usage DifferentDirectory.md

@@ -0,0 +1 @@
+# Header

fixtures/wiki/wikilink-style.md

@@ -0,0 +1,19 @@
+[[#LocalHeader]]
+
+[[Usage]]
+[[Space Usage]]
+[[Dash Usage]]
+[[Underscore Usage]]
+[[DifferentDirectory]]
+[[Different Directory Dash]]
+[[Different Directory Underscore]]
+
+[[Usage#Header|HeaderRenaming]]
+[[Space Usage#Header|HeaderRenaming]]
+[[Dash Usage#Header|HeaderRenaming]]
+[[Underscore Usage#Header|HeaderRenaming]]
+[[DifferentDirectory#Header|HeaderRenaming]]
+[[Different Directory Dash#Header|HeaderRenaming]]
+[[Different Directory Underscore#Header|HeaderRenaming]]
+
+# LocalHeader

lychee-bin/src/client.rs

@@ -55,6 +55,7 @@ pub(crate) fn create(cfg: &Config, cookie_jar: Option<&Arc<CookieStoreMutex>>) -
         .include_fragments(cfg.include_fragments)
         .fallback_extensions(cfg.fallback_extensions.clone())
         .index_files(cfg.index_files.clone())
+        .include_wikilinks(cfg.include_wikilinks)
         .rate_limit_config(RateLimitConfig::from_options(
             cfg.host_concurrency,
             cfg.host_request_interval,

lychee-bin/src/options.rs

@@ -891,7 +891,8 @@ and existing cookies will be updated."
     pub(crate) cookie_jar: Option<PathBuf>,
 
     #[allow(clippy::doc_markdown)]
-    /// Check WikiLinks in Markdown files
+    /// Check WikiLinks in Markdown files, this requires specifying --base-url
+    #[clap(requires = "base_url")]
     #[arg(long)]
     #[serde(default)]
     pub(crate) include_wikilinks: bool,

lychee-bin/tests/cli.rs

@@ -2586,6 +2586,8 @@ The config file should contain every possible key for documentation purposes."
         cargo_bin_cmd!()
             .arg("--dump")
             .arg("--include-wikilinks")
+            .arg("--base-url")
+            .arg(fixtures_path!())
             .arg(test_path)
             .assert()
             .success()
@@ -3046,6 +3048,74 @@ The config file should contain every possible key for documentation purposes."
             .stdout(contains("https://example.org")); // Should extract the link as plaintext
     }
 
+    #[test]
+    fn test_wikilink_fixture_obsidian_style() {
+        let input = fixtures_path!().join("wiki/obsidian-style.md");
+
+        // testing without fragments should not yield failures
+        cargo_bin_cmd!()
+            .arg(&input)
+            .arg("--include-wikilinks")
+            .arg("--fallback-extensions")
+            .arg("md")
+            .arg("--base-url")
+            .arg(fixtures_path!())
+            .assert()
+            .success()
+            .stdout(contains("4 OK"));
+    }
+
+    #[test]
+    fn test_wikilink_fixture_wikilink_non_existent() {
+        let input = fixtures_path!().join("wiki/Non-existent.md");
+
+        cargo_bin_cmd!()
+            .arg(&input)
+            .arg("--include-wikilinks")
+            .arg("--fallback-extensions")
+            .arg("md")
+            .arg("--base-url")
+            .arg(fixtures_path!())
+            .assert()
+            .failure()
+            .stdout(contains("3 Errors"));
+    }
+
+    #[test]
+    fn test_wikilink_fixture_with_fragments_obsidian_style_fixtures_excluded() {
+        let input = fixtures_path!().join("wiki/obsidian-style-plus-headers.md");
+
+        // fragments should resolve all headers
+        cargo_bin_cmd!()
+            .arg(&input)
+            .arg("--include-wikilinks")
+            .arg("--fallback-extensions")
+            .arg("md")
+            .arg("--base-url")
+            .arg(fixtures_path!())
+            .assert()
+            .success()
+            .stdout(contains("4 OK"));
+    }
+
+    #[test]
+    fn test_wikilink_fixture_with_fragments_obsidian_style() {
+        let input = fixtures_path!().join("wiki/obsidian-style-plus-headers.md");
+
+        // fragments should resolve all headers
+        cargo_bin_cmd!()
+            .arg(&input)
+            .arg("--include-wikilinks")
+            .arg("--include-fragments")
+            .arg("--fallback-extensions")
+            .arg("md")
+            .arg("--base-url")
+            .arg(fixtures_path!())
+            .assert()
+            .success()
+            .stdout(contains("4 OK"));
+    }
+
     /// An input which matches nothing should print a warning and continue.
     #[test]
     fn test_input_matching_nothing_warns() -> Result<()> {

lychee-lib/Cargo.toml

@@ -64,6 +64,7 @@ tokio = { version = "1.48.0", features = ["full"] }
 toml = "0.9.10"
 typed-builder = "0.23.2"
 url = { version = "2.5.7", features = ["serde"] }
+walkdir = "2.5.0"
 
 [dependencies.par-stream]
 version = "0.10.2"

lychee-lib/src/checker/file.rs

@@ -3,8 +3,9 @@ use log::warn;
 use std::borrow::Cow;
 use std::path::{Path, PathBuf};
 
+use crate::checker::wikilink::resolver::WikilinkResolver;
 use crate::{
-    Base, ErrorKind, Status, Uri,
+    Base, ErrorKind, Result, Status, Uri,
     utils::fragment_checker::{FragmentChecker, FragmentInput},
 };
 
@@ -34,6 +35,8 @@ pub(crate) struct FileChecker {
     include_fragments: bool,
     /// Utility for performing fragment checks in HTML files.
     fragment_checker: FragmentChecker,
+    /// Utility for optionally resolving Wikilinks.
+    wikilink_resolver: Option<WikilinkResolver>,
 }
 
 impl FileChecker {
@@ -45,19 +48,35 @@ impl FileChecker {
     /// * `fallback_extensions` - List of extensions to try if the original file is not found.
     /// * `index_files` - Optional list of index file names to search for if the path is a directory.
     /// * `include_fragments` - Whether to check for fragment existence in HTML files.
+    /// * `include_wikilinks` - Whether to check the existence of Wikilinks found in Markdown files .
+    ///
+    /// # Errors
+    ///
+    /// Fails if an invalid `base` is provided when including wikilinks.
     pub(crate) fn new(
         base: Option<Base>,
         fallback_extensions: Vec<String>,
         index_files: Option<Vec<String>>,
         include_fragments: bool,
-    ) -> Self {
-        Self {
+        include_wikilinks: bool,
+    ) -> Result<Self> {
+        let wikilink_resolver = if include_wikilinks {
+            Some(WikilinkResolver::new(
+                base.as_ref(),
+                fallback_extensions.clone(),
+            )?)
+        } else {
+            None
+        };
+
+        Ok(Self {
             base,
             fallback_extensions,
             index_files,
             include_fragments,
             fragment_checker: FragmentChecker::new(),
-        }
+            wikilink_resolver,
+        })
     }
 
     /// Checks the given file URI for existence and validity.
@@ -127,16 +146,20 @@ impl FileChecker {
     /// Returns `Ok` with the resolved path if it is valid, otherwise returns
     /// `Err` with an appropriate error. The returned path, if any, is guaranteed
     /// to exist and may be a file or a directory.
-    fn resolve_local_path<'a>(
-        &self,
-        path: &'a Path,
-        uri: &Uri,
-    ) -> Result<Cow<'a, Path>, ErrorKind> {
+    fn resolve_local_path<'a>(&self, path: &'a Path, uri: &Uri) -> Result<Cow<'a, Path>> {
         let path = match path.metadata() {
             // for non-existing paths, attempt fallback extensions
-            Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
-                self.apply_fallback_extensions(path, uri).map(Cow::Owned)
-            }
+            // if fallback extensions don't help, try wikilinks
+            Err(e) if e.kind() == std::io::ErrorKind::NotFound => self
+                .apply_fallback_extensions(path, uri)
+                .or_else(|_| {
+                    if let Some(resolver) = &self.wikilink_resolver {
+                        resolver.resolve(path, uri)
+                    } else {
+                        Err(ErrorKind::InvalidFilePath(uri.clone()))
+                    }
+                })
+                .map(Cow::Owned),
 
             // other IO errors are unexpected and should fail the check
             Err(e) => Err(ErrorKind::ReadFileInput(e, path.to_path_buf())),
@@ -181,7 +204,7 @@ impl FileChecker {
     ///
     /// Returns `Ok(PathBuf)` with the resolved file path, or `Err` if no valid file is found.
     /// If `Ok` is returned, the contained `PathBuf` is guaranteed to exist and be a file.
-    fn apply_fallback_extensions(&self, path: &Path, uri: &Uri) -> Result<PathBuf, ErrorKind> {
+    fn apply_fallback_extensions(&self, path: &Path, uri: &Uri) -> Result<PathBuf> {
         // If it's already a file, use it directly
         if path.is_file() {
             return Ok(path.to_path_buf());
@@ -221,7 +244,7 @@ impl FileChecker {
     /// is guaranteed to exist. In most cases, the returned path will be a file path.
     ///
     /// If index files are disabled, simply returns `Ok(dir_path)`.
-    fn apply_index_files(&self, dir_path: &Path) -> Result<PathBuf, ErrorKind> {
+    fn apply_index_files(&self, dir_path: &Path) -> Result<PathBuf> {
         // this implements the "disabled" case by treating a directory as its
         // own index file.
         let index_names_to_try = match &self.index_files {
@@ -372,7 +395,7 @@ mod tests {
     #[tokio::test]
     async fn test_default() {
         // default behaviour accepts dir links as long as the directory exists.
-        let checker = FileChecker::new(None, vec![], None, true);
+        let checker = FileChecker::new(None, vec![], None, true, false).unwrap();
 
         assert_filecheck!(&checker, "filechecker/index_dir", Status::Ok(_));
 
@@ -430,7 +453,9 @@ mod tests {
             vec![],
             Some(vec!["index.html".to_owned(), "index.md".to_owned()]),
             true,
-        );
+            false,
+        )
+        .unwrap();
 
         assert_resolves!(
             &checker,
@@ -468,7 +493,9 @@ mod tests {
             vec!["html".to_owned()],
             Some(vec!["index".to_owned()]),
             false,
-        );
+            false,
+        )
+        .unwrap();
 
         // this test case has a subdir 'same_name' and a file 'same_name.html'.
         // this shows that the index file resolving is applied in this case and
@@ -492,7 +519,8 @@ mod tests {
     #[tokio::test]
     async fn test_empty_index_list_corner() {
         // empty index_files list will reject all directory links
-        let checker_no_indexes = FileChecker::new(None, vec![], Some(vec![]), false);
+        let checker_no_indexes =
+            FileChecker::new(None, vec![], Some(vec![]), false, false).unwrap();
         assert_resolves!(
             &checker_no_indexes,
             "filechecker/index_dir",
@@ -516,7 +544,8 @@ mod tests {
             "..".to_owned(),
             "/".to_owned(),
         ];
-        let checker_dir_indexes = FileChecker::new(None, vec![], Some(dir_names), false);
+        let checker_dir_indexes =
+            FileChecker::new(None, vec![], Some(dir_names), false, false).unwrap();
         assert_resolves!(
             &checker_dir_indexes,
             "filechecker/index_dir",
@@ -537,7 +566,9 @@ mod tests {
             vec![],
             Some(vec!["../index_dir/index.html".to_owned()]),
             true,
-        );
+            false,
+        )
+        .unwrap();
         assert_resolves!(
             &checker_dotdot,
             "filechecker/empty_dir#fragment",
@@ -550,7 +581,8 @@ mod tests {
             .to_str()
             .expect("expected utf-8 fixtures path")
             .to_owned();
-        let checker_absolute = FileChecker::new(None, vec![], Some(vec![absolute_html]), true);
+        let checker_absolute =
+            FileChecker::new(None, vec![], Some(vec![absolute_html]), true, false).unwrap();
         assert_resolves!(
             &checker_absolute,
             "filechecker/empty_dir#fragment",
@@ -560,7 +592,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_fallback_extensions_on_directories() {
-        let checker = FileChecker::new(None, vec!["html".to_owned()], None, true);
+        let checker = FileChecker::new(None, vec!["html".to_owned()], None, true, false).unwrap();
 
         // fallback extensions should be applied when directory links are resolved
         // to directories (i.e., the default index_files behavior or if `.`

lychee-lib/src/checker/mod.rs

@@ -5,3 +5,4 @@
 pub(crate) mod file;
 pub(crate) mod mail;
 pub(crate) mod website;
+pub(crate) mod wikilink;