From 5c00189e2e659ce089c20ad503021c1271d48bac Mon Sep 17 00:00:00 2001 From: JayJayArr Date: Mon, 18 Aug 2025 12:41:37 +0200 Subject: [PATCH 1/4] feat: make wikilink extraction and checking opt-in --- README.md | 10 ++++++++-- fixtures/TEST_WIKI.md | 2 ++ lychee-bin/src/main.rs | 3 ++- lychee-bin/src/options.rs | 6 ++++++ lychee-bin/tests/cli.rs | 25 +++++++++++++++++++++++++ lychee-lib/src/collector.rs | 16 +++++++++++++++- lychee-lib/src/extract/markdown.rs | 30 +++++++++++++++++++----------- lychee-lib/src/extract/mod.rs | 16 +++++++++++----- 8 files changed, 88 insertions(+), 20 deletions(-) create mode 100644 fixtures/TEST_WIKI.md diff --git a/README.md b/README.md index dc06536538..781eb17bbf 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ Available as a command-line utility, a library and a [GitHub Action](https://git + ## Table of Contents - [Development](#development) @@ -165,7 +166,7 @@ outdated information. | Language | Rust | Ruby | Go | JS | TypeScript | Python | JS | PHP | | Async/Parallel | ![yes] | ![yes] | ![yes] | ![yes] | ![yes] | ![yes] | ![yes] | ![yes] | | JSON output | ![yes] | ![no] | ![yes] | ![yes] | ![yes] | ![maybe]1 | ![yes] | ![yes] | -| Static binary | ![yes] | ![no] | ![yes] | ![no] | ![no] | ️![no] | ![no] | ![no] | +| Static binary | ![yes] | ![no] | ![yes] | ![no] | ![no] | ️![no] | ![no] | ![no] | | Markdown files | ![yes] | ![yes] | ![no] | ![no] | ![no] | ![yes] | ![yes] | ![no] | | HTML files | ![yes] | ![no] | ![no] | ![yes] | ![yes] | ![no] | ![yes] | ![no] | | Text files | ![yes] | ![no] | ![no] | ![no] | ![no] | ![no] | ![no] | ![no] | @@ -177,7 +178,7 @@ outdated information. | Relative URLs | ![yes] | ![yes] | ![no] | ![yes] | ![yes] | ![yes] | ![yes] | ![yes] | | Anchors/Fragments | ![yes] | ![no] | ![no] | ![no] | ![no] | ![yes] | ![yes] | ![no] | | Skip relative URLs | ![yes] | ![no] | ![no] | ![maybe] | ![no] | ![no] | ![no] | ![no] | -| Include patterns | ![yes]️ | ![yes] | ![no] | ![yes] | ![no] | ![no] | ![no] | ![no] | +| Include patterns | ![yes]️ | ![yes] | ![no] | ![yes] | ![no] | ![no] | ![no] | ![no] | | Exclude patterns | ![yes] | ![no] | ![yes] | ![yes] | ![yes] | ![yes] | ![yes] | ![yes] | | Handle redirects | ![yes] | ![yes] | ![yes] | ![yes] | ![yes] | ![yes] | ![yes] | ![yes] | | Ignore insecure SSL | ![yes] | ![yes] | ![yes] | ![no] | ![no] | ![yes] | ![no] | ![yes] | @@ -555,6 +556,9 @@ Options: --cookie-jar Tell lychee to read cookies from the given file. Cookies will be stored in the cookie jar and sent with requests. New cookies will be stored in the cookie jar and existing cookies will be updated + --include-wikilinks + Check found wikilinks in markdown files + -h, --help Print help (see a summary with '-h') @@ -678,6 +682,7 @@ which includes usage instructions. ## Pre-commit Usage Lychee can also be used as a [pre-commit](https://pre-commit.com/) hook. + ```yaml # .pre-commit-config.yaml repos: @@ -690,6 +695,7 @@ repos: ``` Rather than running on staged-files only, Lychee can be run against an entire repository. + ```yaml - id: lychee args: ["--no-progress", "."] diff --git a/fixtures/TEST_WIKI.md b/fixtures/TEST_WIKI.md new file mode 100644 index 0000000000..4f01004a93 --- /dev/null +++ b/fixtures/TEST_WIKI.md @@ -0,0 +1,2 @@ +Test file including a normal link and a wikilink +[[LycheeWikilink]] diff --git a/lychee-bin/src/main.rs b/lychee-bin/src/main.rs index 18dc6455fa..9e4cd6cca5 100644 --- a/lychee-bin/src/main.rs +++ b/lychee-bin/src/main.rs @@ -324,7 +324,8 @@ async fn run(opts: &LycheeOptions) -> Result { .headers(HeaderMap::from_header_pairs(&opts.config.header)?) .excluded_paths(PathExcludes::new(opts.config.exclude_path.clone())?) // File a bug if you rely on this envvar! It's going to go away eventually. - .use_html5ever(std::env::var("LYCHEE_USE_HTML5EVER").is_ok_and(|x| x == "1")); + .use_html5ever(std::env::var("LYCHEE_USE_HTML5EVER").is_ok_and(|x| x == "1")) + .include_wikilinks(opts.config.include_wikilinks); if opts.config.dump_inputs { let sources = collector.collect_sources(inputs); diff --git a/lychee-bin/src/options.rs b/lychee-bin/src/options.rs index aebf1c888a..17e0b16426 100644 --- a/lychee-bin/src/options.rs +++ b/lychee-bin/src/options.rs @@ -702,6 +702,11 @@ separated list of accepted status codes. This example will accept 200, 201, #[arg(long)] #[serde(default)] pub(crate) cookie_jar: Option, + + /// Check found wikilinks in markdown files + #[arg(long)] + #[serde(default)] + pub(crate) include_wikilinks: bool, } impl Config { @@ -757,6 +762,7 @@ impl Config { include_fragments: false; include_mail: false; include_verbatim: false; + include_wikilinks: false; include: Vec::::new(); insecure: false; max_cache_age: humantime::parse_duration(DEFAULT_MAX_CACHE_AGE).unwrap(); diff --git a/lychee-bin/tests/cli.rs b/lychee-bin/tests/cli.rs index 510b680514..fdd4a88186 100644 --- a/lychee-bin/tests/cli.rs +++ b/lychee-bin/tests/cli.rs @@ -2288,4 +2288,29 @@ mod cli { .success() .stdout(contains("https://www.example.com/smth.")); } + + #[test] + fn test_wikilink_extract_when_specified() { + let test_path = fixtures_path().join("TEST_WIKI.md"); + + let mut cmd = main_command(); + cmd.arg("--dump") + .arg("--include-wikilinks") + .arg(test_path) + .assert() + .success() + .stdout(contains("LycheeWikilink")); + } + + #[test] + fn test_wikilink_dont_extract_when_not_specified() { + let test_path = fixtures_path().join("TEST_WIKI.md"); + + let mut cmd = main_command(); + cmd.arg("--dump") + .arg(test_path) + .assert() + .success() + .stdout(is_empty()); + } } diff --git a/lychee-lib/src/collector.rs b/lychee-lib/src/collector.rs index a895656031..966a1af877 100644 --- a/lychee-lib/src/collector.rs +++ b/lychee-lib/src/collector.rs @@ -26,6 +26,7 @@ pub struct Collector { skip_ignored: bool, skip_hidden: bool, include_verbatim: bool, + include_wikilinks: bool, use_html5ever: bool, root_dir: Option, base: Option, @@ -44,6 +45,7 @@ impl Default for Collector { basic_auth_extractor: None, skip_missing_inputs: false, include_verbatim: false, + include_wikilinks: false, use_html5ever: false, skip_hidden: true, skip_ignored: true, @@ -73,6 +75,7 @@ impl Collector { basic_auth_extractor: None, skip_missing_inputs: false, include_verbatim: false, + include_wikilinks: false, use_html5ever: false, skip_hidden: true, skip_ignored: true, @@ -135,6 +138,13 @@ impl Collector { self } + /// Check found wikilinks in markdown files + #[must_use] + pub const fn include_wikilinks(mut self, yes: bool) -> Self { + self.include_wikilinks = yes; + self + } + /// Pass a [`BasicAuthExtractor`] which is capable to match found /// URIs to basic auth credentials. These credentials get passed to the /// request in question. @@ -221,7 +231,11 @@ impl Collector { let basic_auth_extractor = self.basic_auth_extractor.clone(); async move { let content = content?; - let extractor = Extractor::new(self.use_html5ever, self.include_verbatim); + let extractor = Extractor::new( + self.use_html5ever, + self.include_verbatim, + self.include_wikilinks, + ); let uris: Vec = extractor.extract(&content); let requests = request::create( uris, diff --git a/lychee-lib/src/extract/markdown.rs b/lychee-lib/src/extract/markdown.rs index 41d6b515ff..f0f05d6b52 100644 --- a/lychee-lib/src/extract/markdown.rs +++ b/lychee-lib/src/extract/markdown.rs @@ -14,7 +14,11 @@ fn md_extensions() -> Options { } /// Extract unparsed URL strings from a Markdown string. -pub(crate) fn extract_markdown(input: &str, include_verbatim: bool) -> Vec { +pub(crate) fn extract_markdown( + input: &str, + include_verbatim: bool, + include_wikilinks: bool, +) -> Vec { // In some cases it is undesirable to extract links from within code blocks, // which is why we keep track of entries and exits while traversing the input. let mut inside_code_block = false; @@ -64,6 +68,10 @@ pub(crate) fn extract_markdown(input: &str, include_verbatim: bool) -> Vec { + //Exclude wikilinks if not specifically allowed + if !include_wikilinks { + return None; + } inside_link_block = true; //Ignore gitlab toc notation: https://docs.gitlab.com/user/markdown/#table-of-contents if ["_TOC_".to_string(), "TOC".to_string()].contains(&dest_url.to_string()) { @@ -280,7 +288,7 @@ or inline like `https://bar.org` for instance. }, ]; - let uris = extract_markdown(MD_INPUT, false); + let uris = extract_markdown(MD_INPUT, false, false); assert_eq!(uris, expected); } @@ -309,7 +317,7 @@ or inline like `https://bar.org` for instance. }, ]; - let uris = extract_markdown(MD_INPUT, true); + let uris = extract_markdown(MD_INPUT, true, false); assert_eq!(uris, expected); } @@ -325,7 +333,7 @@ Some pre-formatted http://pre.com let expected = vec![]; - let uris = extract_markdown(input, false); + let uris = extract_markdown(input, false, false); assert_eq!(uris, expected); } @@ -358,7 +366,7 @@ $$ [\psi](\mathbf{L}) $$ "; - let uris = extract_markdown(input, true); + let uris = extract_markdown(input, true, false); assert!(uris.is_empty()); } @@ -366,7 +374,7 @@ $$ fn test_single_word_footnote_is_not_detected_as_link() { let markdown = "This footnote is[^actually] a link.\n\n[^actually]: not"; let expected = vec![]; - let uris = extract_markdown(markdown, true); + let uris = extract_markdown(markdown, true, false); assert_eq!(uris, expected); } @@ -378,7 +386,7 @@ $$ element: None, attribute: None, }]; - let uris = extract_markdown(markdown, true); + let uris = extract_markdown(markdown, true, false); assert_eq!(uris, expected); } @@ -390,7 +398,7 @@ $$ element: None, attribute: None, }]; - let uris = extract_markdown(markdown, true); + let uris = extract_markdown(markdown, true, false); assert_eq!(uris, expected); } @@ -402,7 +410,7 @@ $$ element: Some("a".to_string()), attribute: Some("href".to_string()), }]; - let uris = extract_markdown(markdown, true); + let uris = extract_markdown(markdown, true, true); assert_eq!(uris, expected); } @@ -421,14 +429,14 @@ $$ attribute: Some("href".to_string()), }, ]; - let uris = extract_markdown(markdown, true); + let uris = extract_markdown(markdown, true, true); assert_eq!(uris, expected); } #[test] fn test_ignore_gitlab_toc() { let markdown = r"[[_TOC_]][TOC]"; - let uris = extract_markdown(markdown, true); + let uris = extract_markdown(markdown, true, true); assert!(uris.is_empty()); } } diff --git a/lychee-lib/src/extract/mod.rs b/lychee-lib/src/extract/mod.rs index 91b48078d8..f3a3f783fa 100644 --- a/lychee-lib/src/extract/mod.rs +++ b/lychee-lib/src/extract/mod.rs @@ -14,6 +14,7 @@ use plaintext::extract_raw_uri_from_plaintext; pub struct Extractor { use_html5ever: bool, include_verbatim: bool, + include_wikilinks: bool, } impl Extractor { @@ -30,10 +31,11 @@ impl Extractor { /// For more information, consult the `pulldown_cmark` documentation about code blocks /// [here](https://docs.rs/pulldown-cmark/latest/pulldown_cmark/enum.CodeBlockKind.html) #[must_use] - pub const fn new(use_html5ever: bool, include_verbatim: bool) -> Self { + pub const fn new(use_html5ever: bool, include_verbatim: bool, include_wikilinks: bool) -> Self { Self { use_html5ever, include_verbatim, + include_wikilinks, } } @@ -42,7 +44,11 @@ impl Extractor { #[must_use] pub fn extract(&self, input_content: &InputContent) -> Vec { match input_content.file_type { - FileType::Markdown => extract_markdown(&input_content.content, self.include_verbatim), + FileType::Markdown => extract_markdown( + &input_content.content, + self.include_verbatim, + self.include_wikilinks, + ), FileType::Html => { if self.use_html5ever { html::html5ever::extract_html(&input_content.content, self.include_verbatim) @@ -72,7 +78,7 @@ mod tests { fn extract_uris(input: &str, file_type: FileType) -> HashSet { let input_content = InputContent::from_string(input, file_type); - let extractor = Extractor::new(false, false); + let extractor = Extractor::new(false, false, false); let uris_html5gum: HashSet = extractor .extract(&input_content) .into_iter() @@ -84,7 +90,7 @@ mod tests { uris }; - let extractor = Extractor::new(true, false); + let extractor = Extractor::new(true, false, false); let uris_html5ever: HashSet = extractor .extract(&input_content) .into_iter() @@ -216,7 +222,7 @@ mod tests { }; for use_html5ever in [true, false] { - let extractor = Extractor::new(use_html5ever, false); + let extractor = Extractor::new(use_html5ever, false, false); let links = extractor.extract(input_content); let urls = links From 239c59742e50d08c464f9cc8a410169720f6f8ef Mon Sep 17 00:00:00 2001 From: JayJayArr Date: Mon, 18 Aug 2025 13:22:46 +0200 Subject: [PATCH 2/4] fix merge --- lychee-bin/src/main.rs | 16 +++------------- lychee-bin/tests/cli.rs | 1 + 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/lychee-bin/src/main.rs b/lychee-bin/src/main.rs index 1a81fcbfa4..2dee326e9b 100644 --- a/lychee-bin/src/main.rs +++ b/lychee-bin/src/main.rs @@ -316,18 +316,6 @@ async fn run(opts: &LycheeOptions) -> Result { } }; - - let mut collector = Collector::new(opts.config.root_dir.clone(), base)? - .skip_missing_inputs(opts.config.skip_missing) - .skip_hidden(!opts.config.hidden) - .skip_ignored(!opts.config.no_ignore) - .include_verbatim(opts.config.include_verbatim) - .headers(HeaderMap::from_header_pairs(&opts.config.header)?) - .excluded_paths(PathExcludes::new(opts.config.exclude_path.clone())?) - // File a bug if you rely on this envvar! It's going to go away eventually. - .use_html5ever(std::env::var("LYCHEE_USE_HTML5EVER").is_ok_and(|x| x == "1")) - .include_wikilinks(opts.config.include_wikilinks); - if opts.config.dump_inputs { let exit_code = commands::dump_inputs( inputs, @@ -350,7 +338,9 @@ async fn run(opts: &LycheeOptions) -> Result { .headers(HeaderMap::from_header_pairs(&opts.config.header)?) .excluded_paths(PathExcludes::new(opts.config.exclude_path.clone())?) // File a bug if you rely on this envvar! It's going to go away eventually. - .use_html5ever(std::env::var("LYCHEE_USE_HTML5EVER").is_ok_and(|x| x == "1")); + .use_html5ever(std::env::var("LYCHEE_USE_HTML5EVER").is_ok_and(|x| x == "1")) + .include_wikilinks(opts.config.include_wikilinks); + collector = if let Some(ref basic_auth) = opts.config.basic_auth { collector.basic_auth_extractor(BasicAuthExtractor::new(basic_auth)?) } else { diff --git a/lychee-bin/tests/cli.rs b/lychee-bin/tests/cli.rs index 993717ee6d..9479e80f1f 100644 --- a/lychee-bin/tests/cli.rs +++ b/lychee-bin/tests/cli.rs @@ -2465,6 +2465,7 @@ mod cli { .assert() .success() .stdout(is_empty()); + } #[test] fn test_index_files_default() { From 51aafbade5af975cd46a09b0cf709d4718a0e872 Mon Sep 17 00:00:00 2001 From: JayJayArr Date: Mon, 18 Aug 2025 14:28:26 +0200 Subject: [PATCH 3/4] spellcheck docs --- README.md | 2 +- lychee-bin/src/options.rs | 3 ++- lychee-lib/src/collector.rs | 3 ++- lychee-lib/src/extract/markdown.rs | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 584470721e..25f0b54eb9 100644 --- a/README.md +++ b/README.md @@ -589,7 +589,7 @@ Options: Tell lychee to read cookies from the given file. Cookies will be stored in the cookie jar and sent with requests. New cookies will be stored in the cookie jar and existing cookies will be updated --include-wikilinks - Check found wikilinks in markdown files + Check WikiLinks in Markdown files -h, --help Print help (see a summary with '-h') diff --git a/lychee-bin/src/options.rs b/lychee-bin/src/options.rs index 159d0cff32..780eed6ca4 100644 --- a/lychee-bin/src/options.rs +++ b/lychee-bin/src/options.rs @@ -735,7 +735,8 @@ separated list of accepted status codes. This example will accept 200, 201, #[serde(default)] pub(crate) cookie_jar: Option, - /// Check found wikilinks in markdown files + #[allow(clippy::doc_markdown)] + /// Check WikiLinks in Markdown files #[arg(long)] #[serde(default)] pub(crate) include_wikilinks: bool, diff --git a/lychee-lib/src/collector.rs b/lychee-lib/src/collector.rs index 112bdc02be..1fba31352d 100644 --- a/lychee-lib/src/collector.rs +++ b/lychee-lib/src/collector.rs @@ -141,7 +141,8 @@ impl Collector { self } - /// Check found wikilinks in markdown files + #[allow(clippy::doc_markdown)] + /// Check WikiLinks in Markdown files #[must_use] pub const fn include_wikilinks(mut self, yes: bool) -> Self { self.include_wikilinks = yes; diff --git a/lychee-lib/src/extract/markdown.rs b/lychee-lib/src/extract/markdown.rs index f0f05d6b52..0f06497283 100644 --- a/lychee-lib/src/extract/markdown.rs +++ b/lychee-lib/src/extract/markdown.rs @@ -68,7 +68,7 @@ pub(crate) fn extract_markdown( Some(extract_raw_uri_from_plaintext(&dest_url)), // Wiki URL (`[[http://example.com]]`) LinkType::WikiLink { has_pothole: _ } => { - //Exclude wikilinks if not specifically allowed + // Exclude WikiLinks if not specifically enabled if !include_wikilinks { return None; } From dab15a131efe6373c5aba17a8e474a481721a148 Mon Sep 17 00:00:00 2001 From: JayJayArr Date: Mon, 18 Aug 2025 16:38:59 +0200 Subject: [PATCH 4/4] explicit doc change --- lychee-lib/src/extract/markdown.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lychee-lib/src/extract/markdown.rs b/lychee-lib/src/extract/markdown.rs index 0f06497283..51099097c1 100644 --- a/lychee-lib/src/extract/markdown.rs +++ b/lychee-lib/src/extract/markdown.rs @@ -68,7 +68,7 @@ pub(crate) fn extract_markdown( Some(extract_raw_uri_from_plaintext(&dest_url)), // Wiki URL (`[[http://example.com]]`) LinkType::WikiLink { has_pothole: _ } => { - // Exclude WikiLinks if not specifically enabled + // Exclude WikiLinks if not explicitly enabled if !include_wikilinks { return None; }