diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml index 7e2ffda951..867cd4b8e3 100644 --- a/.github/workflows/release-binary.yml +++ b/.github/workflows/release-binary.yml @@ -85,7 +85,10 @@ jobs: cd target/${{ matrix.target }}/release ${GNU_PREFIX}strip lychee chmod +x lychee - tar -c lychee | gzip > lychee.tar.gz + mkdir docs + cp ../../../{README.md, docs/TROUBLESHOOTING.md} docs + ./lychee --generate man > docs/lychee.1 + tar -c lychee docs/ | gzip > lychee.tar.gz - name: Upload binary uses: actions/upload-release-asset@v1 diff --git a/Cargo.lock b/Cargo.lock index 78b6fb7dc6..9218d1a930 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -44,12 +44,6 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - [[package]] name = "android_system_properties" version = "0.1.5" @@ -662,17 +656,16 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.41" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" +checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" dependencies = [ - "android-tzdata", "iana-time-zone", "js-sys", "num-traits", "serde", "wasm-bindgen", - "windows-link 0.1.3", + "windows-link 0.2.0", ] [[package]] @@ -742,6 +735,16 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" +[[package]] +name = "clap_mangen" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b4c3c54b30f0d9adcb47f25f61fcce35c4dd8916638c6b82fbd5f4fb4179e2" +dependencies = [ + "clap", + "roff", +] + [[package]] name = "client_pool" version = "0.1.0" @@ -2512,7 +2515,9 @@ dependencies = [ "anyhow", "assert-json-diff", "assert_cmd", + "chrono", "clap", + "clap_mangen", "console", "const_format", "cookie_store 0.22.0", @@ -3675,6 +3680,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "roff" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88f8660c1ff60292143c98d08fc6e2f654d722db50410e3f3797d40baaf9d8f3" + [[package]] name = "rstest" version = "0.26.1" diff --git a/README.md b/README.md index 0fa4666a66..0af194522e 100644 --- a/README.md +++ b/README.md @@ -325,10 +325,10 @@ Please follow the [GitHub App Setup][github-app-setup] example. There is an extensive list of command line parameters to customize the behavior. See below for a full list. -```text -A fast, async link checker +```help-message +lychee is a fast, asynchronous link checker which detects broken URLs and mail addresses in local files and websites. It supports Markdown and HTML and works well with many plain text file formats. -Finds broken URLs and mail addresses inside Markdown, HTML, `reStructuredText`, websites and more! +lychee is powered by lychee-lib, the Rust library for link checking. Usage: lychee [OPTIONS] [inputs]... @@ -342,58 +342,63 @@ Arguments: NOTE: Use `--` to separate inputs from options that allow multiple arguments. Options: - --files-from - Read input filenames from the given file or stdin (if path is '-'). + -a, --accept + A List of accepted status codes for valid links - This is useful when you have a large number of inputs that would be - cumbersome to specify on the command line directly. + The following accept range syntax is supported: [start]..[[=]end]|code. Some valid + examples are: - Examples: - lychee --files-from list.txt - find . -name '*.md' | lychee --files-from - - echo 'README.md' | lychee --files-from - + - 200 (accepts the 200 status code only) + - ..204 (accepts any status code < 204) + - ..=204 (accepts any status code <= 204) + - 200..=204 (accepts any status code from 200 to 204 inclusive) + - 200..205 (accepts any status code from 200 to 205 excluding 205, same as 200..=204) - File Format: - Each line should contain one input (file path, URL, or glob pattern). - Lines starting with '#' are treated as comments and ignored. - Empty lines are also ignored. + Use "lychee --accept '200..=204, 429, 500' ..." to provide a comma- + separated list of accepted status codes. This example will accept 200, 201, + 202, 203, 204, 429, and 500 as valid status codes. - -c, --config - Configuration file to use + [default: 100..=103,200..=299] - [default: lychee.toml] + --archive + Specify the use of a specific web archive. Can be used in combination with `--suggest` - -v, --verbose... - Set verbosity level; more output per occurrence (e.g. `-v` or `-vv`) + [possible values: wayback] - -q, --quiet... - Less output per occurrence (e.g. `-q` or `-qq`) + -b, --base-url + Base URL to use when resolving relative URLs in local files. If specified, + relative links in local files are interpreted as being relative to the given + base URL. - -n, --no-progress - Do not show progress bar. - This is recommended for non-interactive shells (e.g. for continuous integration) + For example, given a base URL of `https://example.com/dir/page`, the link `a` + would resolve to `https://example.com/dir/a` and the link `/b` would resolve + to `https://example.com/b`. This behavior is not affected by the filesystem + path of the file containing these links. - --extensions - Test the specified file extensions for URIs when checking files locally. + Note that relative URLs without a leading slash become siblings of the base + URL. If, instead, the base URL ended in a slash, the link would become a child + of the base URL. For example, a base URL of `https://example.com/dir/page/` and + a link of `a` would resolve to `https://example.com/dir/page/a`. - Multiple extensions can be separated by commas. Note that if you want to check filetypes, - which have multiple extensions, e.g. HTML files with both .html and .htm extensions, you need to - specify both extensions explicitly. + Basically, the base URL option resolves links as if the local files were hosted + at the given base URL address. - [default: md,mkd,mdx,mdown,mdwn,mkdn,mkdown,markdown,html,htm,txt] + The provided base URL value must either be a URL (with scheme) or an absolute path. + Note that certain URL schemes cannot be used as a base, e.g., `data` and `mailto`. - --default-extension - Default file extension to treat files without extensions as having. + --base + Deprecated; use `--base-url` instead - This is useful for files without extensions or with unknown extensions. The extension will be used to determine the file type for processing. Examples: --default-extension md, --default-extension html + --basic-auth + Basic authentication support. E.g. `http://example.com username:password` - --cache - Use request cache stored on disk at `.lycheecache` + -c, --config + Configuration file to use - --max-cache-age - Discard all cached requests older than this duration + [default: lychee.toml] - [default: 1d] + --cache + Use request cache stored on disk at `.lycheecache` --cache-exclude-status A list of status codes that will be ignored from the cache @@ -411,7 +416,15 @@ Options: comma-separated list of excluded status codes. This example will not cache results with a status code of 429, 500 and 501. - [default: ] + --cookie-jar + Tell lychee to read cookies from the given file. Cookies will be stored in the + cookie jar and sent with requests. New cookies will be stored in the cookie jar + and existing cookies will be updated. + + --default-extension + Default file extension to treat files without extensions as having. + + This is useful for files without extensions or with unknown extensions. The extension will be used to determine the file type for processing. Examples: --default-extension md, --default-extension html --dump Don't perform any link checking. Instead, dump all the links extracted from inputs that would be checked @@ -419,93 +432,114 @@ Options: --dump-inputs Don't perform any link extraction and checking. Instead, dump all input sources from which links would be collected - --archive - Specify the use of a specific web archive. Can be used in combination with `--suggest` + -E, --exclude-all-private + Exclude all private IPs from checking. + Equivalent to `--exclude-private --exclude-link-local --exclude-loopback` - [possible values: wayback] + --exclude + Exclude URLs and mail addresses from checking. The values are treated as regular expressions - --suggest - Suggest link replacements for broken links, using a web archive. The web archive can be specified with `--archive` + --exclude-file + Deprecated; use `--exclude-path` instead - -m, --max-redirects - Maximum number of allowed redirects + --exclude-link-local + Exclude link-local IP address range from checking - [default: 5] + --exclude-loopback + Exclude loopback IP address range and localhost from checking - --max-retries - Maximum number of retries per request + --exclude-path + Exclude paths from getting checked. The values are treated as regular expressions - [default: 3] + --exclude-private + Exclude private IP address ranges from checking - --min-tls - Minimum accepted TLS Version + --extensions + Test the specified file extensions for URIs when checking files locally. - [possible values: TLSv1_0, TLSv1_1, TLSv1_2, TLSv1_3] + Multiple extensions can be separated by commas. Note that if you want to check filetypes, + which have multiple extensions, e.g. HTML files with both .html and .htm extensions, you need to + specify both extensions explicitly. - --max-concurrency - Maximum number of concurrent network requests + [default: md,mkd,mdx,mdown,mdwn,mkdn,mkdown,markdown,html,htm,txt] - [default: 128] + -f, --format + Output format of final status report - -T, --threads - Number of threads to utilize. Defaults to number of cores available to the system + [default: compact] + [possible values: compact, detailed, json, markdown, raw] - -u, --user-agent - User agent + --fallback-extensions + When checking locally, attempts to locate missing files by trying the given + fallback extensions. Multiple extensions can be separated by commas. Extensions + will be checked in order of appearance. - [default: lychee/x.y.z] + Example: --fallback-extensions html,htm,php,asp,aspx,jsp,cgi - -i, --insecure - Proceed for server connections considered insecure (invalid TLS) + Note: This option takes effect on `file://` URIs which do not exist and on + `file://` URIs pointing to directories which resolve to themself (by the + --index-files logic). - -s, --scheme - Only test links with the given schemes (e.g. https). Omit to check links with - any other scheme. At the moment, we support http, https, file, and mailto. + --files-from + Read input filenames from the given file or stdin (if path is '-'). - --offline - Only check local files and block network requests + This is useful when you have a large number of inputs that would be + cumbersome to specify on the command line directly. - --include - URLs to check (supports regex). Has preference over all excludes + Examples: + lychee --files-from list.txt + find . -name '*.md' | lychee --files-from - + echo 'README.md' | lychee --files-from - - --exclude - Exclude URLs and mail addresses from checking. The values are treated as regular expressions + File Format: + Each line should contain one input (file path, URL, or glob pattern). + Lines starting with '#' are treated as comments and ignored. + Empty lines are also ignored. - --exclude-file - Deprecated; use `--exclude-path` instead + --generate + Generate special output (e.g. the man page) instead of performing link checking - --exclude-path - Exclude paths from getting checked. The values are treated as regular expressions + [possible values: man] - -E, --exclude-all-private - Exclude all private IPs from checking. - Equivalent to `--exclude-private --exclude-link-local --exclude-loopback` + --github-token + GitHub API token to use when checking github.com links, to avoid rate limiting - --exclude-private - Exclude private IP address ranges from checking + [env: GITHUB_TOKEN] - --exclude-link-local - Exclude link-local IP address range from checking + --glob-ignore-case + Ignore case when expanding filesystem path glob inputs - --exclude-loopback - Exclude loopback IP address range and localhost from checking + -h, --help + Print help (see a summary with '-h') - --include-mail - Also check email addresses + -H, --header + Set custom header for requests - --remap - Remap URI matching pattern to different URI + Some websites require custom headers to be passed in order to return valid responses. + You can specify custom headers in the format 'Name: Value'. For example, 'Accept: text/html'. + This is the same format that other tools like curl or wget use. + Multiple headers can be specified by using the flag multiple times. - --fallback-extensions - When checking locally, attempts to locate missing files by trying the given - fallback extensions. Multiple extensions can be separated by commas. Extensions - will be checked in order of appearance. + --hidden + Do not skip hidden directories and files - Example: --fallback-extensions html,htm,php,asp,aspx,jsp,cgi + -i, --insecure + Proceed for server connections considered insecure (invalid TLS) - Note: This option takes effect on `file://` URIs which do not exist and on - `file://` URIs pointing to directories which resolve to themself (by the - --index-files logic). + --include + URLs to check (supports regex). Has preference over all excludes + + --include-fragments + Enable the checking of fragments in links + + --include-mail + Also check email addresses + + --include-verbatim + Find links in verbatim sections like `pre`- and `code` blocks + + --include-wikilinks + Check WikiLinks in Markdown files --index-files When checking locally, resolves directory links to a separate index file. @@ -531,73 +565,63 @@ Options: Note: This option only takes effect on `file://` URIs which exist and point to a directory. - -H, --header - Set custom header for requests + -m, --max-redirects + Maximum number of allowed redirects - Some websites require custom headers to be passed in order to return valid responses. - You can specify custom headers in the format 'Name: Value'. For example, 'Accept: text/html'. - This is the same format that other tools like curl or wget use. - Multiple headers can be specified by using the flag multiple times. + [default: 5] - -a, --accept - A List of accepted status codes for valid links + --max-cache-age + Discard all cached requests older than this duration - The following accept range syntax is supported: [start]..[[=]end]|code. Some valid - examples are: + [default: 1d] - - 200 (accepts the 200 status code only) - - ..204 (accepts any status code < 204) - - ..=204 (accepts any status code <= 204) - - 200..=204 (accepts any status code from 200 to 204 inclusive) - - 200..205 (accepts any status code from 200 to 205 excluding 205, same as 200..=204) + --max-concurrency + Maximum number of concurrent network requests - Use "lychee --accept '200..=204, 429, 500' ..." to provide a comma- - separated list of accepted status codes. This example will accept 200, 201, - 202, 203, 204, 429, and 500 as valid status codes. + [default: 128] - [default: 100..=103,200..=299] + --max-retries + Maximum number of retries per request - --include-fragments - Enable the checking of fragments in links + [default: 3] - -t, --timeout - Website timeout in seconds from connect to response finished + --min-tls + Minimum accepted TLS Version - [default: 20] + [possible values: TLSv1_0, TLSv1_1, TLSv1_2, TLSv1_3] - -r, --retry-wait-time - Minimum wait time in seconds between retries of failed requests + --mode + Set the output display mode. Determines how results are presented in the terminal - [default: 1] + [default: color] + [possible values: plain, color, emoji, task] - -X, --method - Request method + -n, --no-progress + Do not show progress bar. + This is recommended for non-interactive shells (e.g. for continuous integration) - [default: get] + --no-ignore + Do not skip files that would otherwise be ignored by '.gitignore', '.ignore', or the global ignore file - --base - Deprecated; use `--base-url` instead + -o, --output + Output file of status report - -b, --base-url - Base URL to use when resolving relative URLs in local files. If specified, - relative links in local files are interpreted as being relative to the given - base URL. + --offline + Only check local files and block network requests - For example, given a base URL of `https://example.com/dir/page`, the link `a` - would resolve to `https://example.com/dir/a` and the link `/b` would resolve - to `https://example.com/b`. This behavior is not affected by the filesystem - path of the file containing these links. + -q, --quiet... + Less output per occurrence (e.g. `-q` or `-qq`) - Note that relative URLs without a leading slash become siblings of the base - URL. If, instead, the base URL ended in a slash, the link would become a child - of the base URL. For example, a base URL of `https://example.com/dir/page/` and - a link of `a` would resolve to `https://example.com/dir/page/a`. + -r, --retry-wait-time + Minimum wait time in seconds between retries of failed requests - Basically, the base URL option resolves links as if the local files were hosted - at the given base URL address. + [default: 1] - The provided base URL value must either be a URL (with scheme) or an absolute path. - Note that certain URL schemes cannot be used as a base, e.g., `data` and `mailto`. + --remap + Remap URI matching pattern to different URI + + --require-https + When HTTPS is available, treat HTTP links as errors --root-dir Root directory to use when checking absolute links in local files. This option is @@ -613,68 +637,50 @@ Options: name specified in `--base-url`, followed by the `--root-dir` directory path, followed by the absolute link's own path. - --basic-auth - Basic authentication support. E.g. `http://example.com username:password` - - --github-token - GitHub API token to use when checking github.com links, to avoid rate limiting - - [env: GITHUB_TOKEN] + -s, --scheme + Only test links with the given schemes (e.g. https). Omit to check links with + any other scheme. At the moment, we support http, https, file, and mailto. --skip-missing Skip missing input files (default is to error if they don't exist) - --no-ignore - Do not skip files that would otherwise be ignored by '.gitignore', '.ignore', or the global ignore file - - --hidden - Do not skip hidden directories and files + --suggest + Suggest link replacements for broken links, using a web archive. The web archive can be specified with `--archive` - --include-verbatim - Find links in verbatim sections like `pre`- and `code` blocks + -t, --timeout + Website timeout in seconds from connect to response finished - --glob-ignore-case - Ignore case when expanding filesystem path glob inputs + [default: 20] - -o, --output - Output file of status report + -T, --threads + Number of threads to utilize. Defaults to number of cores available to the system - --mode - Set the output display mode. Determines how results are presented in the terminal + -u, --user-agent + User agent - [default: color] - [possible values: plain, color, emoji, task] + [default: lychee/0.20.1] - -f, --format - Output format of final status report + -v, --verbose... + Set verbosity level; more output per occurrence (e.g. `-v` or `-vv`) - [default: compact] - [possible values: compact, detailed, json, markdown, raw] + -V, --version + Print version - --require-https - When HTTPS is available, treat HTTP links as errors + -X, --method + Request method - --cookie-jar - Tell lychee to read cookies from the given file. Cookies will be stored in the - cookie jar and sent with requests. New cookies will be stored in the cookie jar - and existing cookies will be updated. + [default: get] +``` - --include-wikilinks - Check WikiLinks in Markdown files +### Exit codes - -h, --help - Print help (see a summary with '-h') +0 Success. The operation was completed successfully as instructed. - -V, --version - Print version -``` +1 Missing inputs or any unexpected runtime failures or configuration errors -### Exit codes +2 Link check failures. At least one non-excluded link failed the check. -- `0` for success (all links checked successfully or excluded/skipped as configured) -- `1` for missing inputs and any unexpected runtime failures or config errors -- `2` for link check failures (if any non-excluded link failed the check) -- `3` for errors in the config file +3 Encountered errors in the config file. ### Ignoring links diff --git a/PRE_COMMIT.md b/docs/PRE_COMMIT.md similarity index 100% rename from PRE_COMMIT.md rename to docs/PRE_COMMIT.md diff --git a/fixtures/manpage_examples/README.md b/fixtures/manpage_examples/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/fixtures/manpage_examples/info.txt b/fixtures/manpage_examples/info.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/fixtures/manpage_examples/test.html b/fixtures/manpage_examples/test.html new file mode 100644 index 0000000000..e69de29bb2 diff --git a/fixtures/manpage_examples/test.md b/fixtures/manpage_examples/test.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lychee-bin/Cargo.toml b/lychee-bin/Cargo.toml index e70dda5716..33d717ac02 100644 --- a/lychee-bin/Cargo.toml +++ b/lychee-bin/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lychee" -authors = ["Matthias Endler "] +authors = ["Matthias Endler ", "Thomas Zahner "] description = "A fast, async link checker" documentation = "https://docs.rs/lychee" homepage = "https://github.com/lycheeverse/lychee" @@ -19,7 +19,8 @@ lychee-lib = { path = "../lychee-lib", version = "0.20.1", default-features = fa anyhow = "1.0.99" assert-json-diff = "2.0.2" -clap = { version = "4.5.47", features = ["env", "derive"] } +clap = { version = "4.5.47", features = ["env", "derive", "cargo", "string"] } +clap_mangen = "0.2.29" console = "0.16.1" const_format = "0.2.34" csv = "1.3.1" @@ -58,6 +59,7 @@ tokio = { version = "1.47.1", features = ["full"] } tokio-stream = "0.1.17" toml = "0.9.5" url = "2.5.7" +chrono = { version = "0.4.42", features = ["alloc", "now", "std", "clock"] } [dev-dependencies] diff --git a/lychee-bin/src/commands/check.rs b/lychee-bin/src/commands/check.rs index 0a266c6ffc..8a8e22d8f6 100644 --- a/lychee-bin/src/commands/check.rs +++ b/lychee-bin/src/commands/check.rs @@ -48,7 +48,11 @@ where let client = params.client; let cache = params.cache; - let cache_exclude_status = params.cfg.cache_exclude_status.into_set(); + let cache_exclude_status = params + .cfg + .cache_exclude_status + .unwrap_or_default() + .into_set(); let accept = params.cfg.accept.into(); let pb = if params.cfg.no_progress || params.cfg.verbose.log_level() >= log::Level::Info { diff --git a/lychee-bin/src/commands/generate.rs b/lychee-bin/src/commands/generate.rs new file mode 100644 index 0000000000..dccfb4b16f --- /dev/null +++ b/lychee-bin/src/commands/generate.rs @@ -0,0 +1,242 @@ +//! A module to generate lychee-bin related output for usability purposes. +//! The generated data is not related to the main use-cases of lychee +//! such as link checking but for usability purposes, such as the manual page +//! and shell completions. + +use anyhow::Result; +use clap::{CommandFactory, crate_authors}; +use clap_mangen::{ + Man, + roff::{Roff, roman}, +}; +use serde::Deserialize; +use strum::{Display, EnumIter, EnumString, VariantNames}; + +use crate::LycheeOptions; + +const CONTRIBUTOR_THANK_NOTE: &str = "\n\nA huge thank you to all the wonderful contributors who helped make this project a success."; + +const BUG_SECTION: &str = + "Report any bugs or questions to + +Questions can also be asked on "; + +type Description = &'static str; +type Commands = &'static [&'static str]; +type Example = (Description, Commands); + +/// Used to render the EXAMPLES section in the man page. +/// Note that the `Commands` are executed and tested. +const EXAMPLES: &[Example] = &[ + ( + "Check all links in supported files by specifying a directory", + &["lychee ."], + ), + ( + "Specify files explicitly or use glob patterns", + &[ + "lychee README.md test.html info.txt", + "lychee 'public/**/*.html' '*.md'", + ], + ), + ( + "Check all links on a website", + &["lychee https://example.com"], + ), + ( + "Check links from stdin", + &[ + "cat test.md | lychee -", + "echo 'https://example.com' | lychee -", + ], + ), + ( + "Links can be excluded and included with regular expressions", + &["lychee --exclude '^https?://blog\\.example\\.com' --exclude '\\.(pdf|zip|png|jpg)$' ."], + ), + ( + "Further examples can be found in the online documentation at ", + &[], + ), +]; + +const EXIT_CODE_SECTION: &str = " +0 Success. The operation was completed successfully as instructed. + +1 Missing inputs or any unexpected runtime failures or configuration errors + +2 Link check failures. At least one non-excluded link failed the check. + +3 Encountered errors in the config file. +"; + +/// What to generate when providing the --generate flag +#[derive(Debug, Deserialize, Clone, Display, EnumIter, EnumString, VariantNames, PartialEq)] +#[non_exhaustive] +#[strum(serialize_all = "snake_case")] +#[serde(rename_all = "snake_case")] +pub(crate) enum GenerateMode { + /// Generate roff used for the man page + Man, +} + +/// Generate special output according to the [`GenerateMode`] +pub(crate) fn generate(mode: &GenerateMode) -> Result { + match mode { + GenerateMode::Man => man_page(), + } +} + +/// Generate the lychee man page in roff format using [`clap_mangen`] +fn man_page() -> Result { + let date = chrono::offset::Local::now().format("%Y-%m-%d"); + let authors = crate_authors!("\n\n").to_owned() + CONTRIBUTOR_THANK_NOTE; + + let man = Man::new(LycheeOptions::command().author(authors)).date(format!("{date}")); + let buffer = &mut Vec::default(); + + // Manually customise `Man::render` (see https://github.com/clap-rs/clap/issues/3354) + man.render_title(buffer)?; + man.render_name_section(buffer)?; + man.render_synopsis_section(buffer)?; + man.render_description_section(buffer)?; + man.render_options_section(buffer)?; + render_examples(buffer)?; + render_exit_codes(buffer)?; + render_bug_reporting(buffer)?; + man.render_version_section(buffer)?; + man.render_authors_section(buffer)?; + + Ok(std::str::from_utf8(buffer)?.to_owned()) +} + +fn render_exit_codes(buffer: &mut Vec) -> Result<()> { + render_section("EXIT CODES", EXIT_CODE_SECTION, buffer) +} + +fn render_examples(buffer: &mut Vec) -> Result<()> { + let section = EXAMPLES + .iter() + .map(|(description, examples)| { + let examples = examples + .iter() + .map(|example| format!(" $ {example}")) + .collect::>() + .join("\n"); + format!("{description}\n\n{examples}") + }) + .collect::>() + .join("\n\n"); + render_section("EXAMPLES", §ion, buffer) +} + +fn render_bug_reporting(buffer: &mut Vec) -> Result<()> { + render_section("REPORTING BUGS", BUG_SECTION, buffer) +} + +fn render_section(title: &str, content: &str, buffer: &mut Vec) -> Result<()> { + let mut roff = Roff::default(); + roff.control("SH", [title]); + roff.text([roman(content)]); + roff.to_writer(buffer)?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{EXAMPLES, man_page}; + use crate::generate::{CONTRIBUTOR_THANK_NOTE, EXIT_CODE_SECTION}; + use anyhow::Result; + use assert_cmd::Command; + use test_utils::{fixtures_path, main_command}; + + #[test] + fn test_man_page() -> Result<()> { + let roff = man_page()?; + + // Must contain description + assert!(roff.contains("lychee \\- A fast, async link checker")); + assert!(roff.contains( + "lychee is a fast, asynchronous link checker which detects broken URLs and mail addresses in local files and websites. It supports Markdown and HTML and works well with many plain text file formats." + )); + assert!( + roff.contains("lychee is powered by lychee\\-lib, the Rust library for link checking.") + ); + + // Must contain authors and thank note + assert!(roff.contains("Matthias Endler")); + assert!(roff.contains(CONTRIBUTOR_THANK_NOTE)); + + // Flags should normally occur exactly twice. + // Once in SYNOPSIS and once in OPTIONS. + assert_eq!(roff.matches("\\-\\-version").count(), 2); + Ok(()) + } + + /// Test that the Exit Codes section in `README.md` is up to date with + /// lychee's manual page. + #[test] + #[cfg(unix)] + fn test_readme_exit_codes_up_to_date() -> Result<(), Box> { + use test_utils::load_readme_text; + + const BEGIN: &str = "### Exit codes"; + const END: &str = "# "; + + let readme = load_readme_text!(); + let start = readme.find(BEGIN).ok_or("Beginning not found in README")? + BEGIN.len(); + let end = readme[start..].find(END).ok_or("End not found in README")? - END.len(); + + let section = &readme[start..start + end]; + assert_eq!( + filter_empty_lines(section), + filter_empty_lines(EXIT_CODE_SECTION) + ); + + Ok(()) + } + + #[test] + fn test_examples_work() -> Result<()> { + let results: Vec<_> = EXAMPLES + .iter() + .flat_map(|(_, examples)| examples.iter()) + .map(|example| { + let command = example.replace( + "lychee", + main_command!() + .get_program() + .to_str() + .expect("Unable to convert to string"), + ); + + ( + command.clone(), + Command::new("sh") + .arg("-c") + .arg(command) + .current_dir(fixtures_path!().join("manpage_examples")) + .output(), + ) + }) + .collect(); + + for (command, result) in results { + let result = result?; + let output = str::from_utf8(&result.stderr)?; + assert!( + result.status.success(), + "The command '{command}' failed with: {output}", + ); + } + + Ok(()) + } + + fn filter_empty_lines(s: &str) -> String { + s.lines() + .filter(|line| !line.trim().is_empty()) + .collect::>() + .join("\n") + } +} diff --git a/lychee-bin/src/commands/mod.rs b/lychee-bin/src/commands/mod.rs index 1f00503f02..295f598dea 100644 --- a/lychee-bin/src/commands/mod.rs +++ b/lychee-bin/src/commands/mod.rs @@ -1,6 +1,7 @@ pub(crate) mod check; pub(crate) mod dump; pub(crate) mod dump_inputs; +pub(crate) mod generate; pub(crate) use check::check; pub(crate) use dump::dump; diff --git a/lychee-bin/src/formatters/response/color.rs b/lychee-bin/src/formatters/response/color.rs index fd99c24ba9..baf4ef65d9 100644 --- a/lychee-bin/src/formatters/response/color.rs +++ b/lychee-bin/src/formatters/response/color.rs @@ -72,20 +72,13 @@ mod tests { use http::StatusCode; use lychee_lib::{ErrorKind, Status, Uri}; use pretty_assertions::assert_eq; + use test_utils::mock_response_body; /// Helper function to strip ANSI color codes for tests fn strip_ansi_codes(s: &str) -> String { console::strip_ansi_codes(s).to_string() } - // Helper function to create a ResponseBody with a given status and URI - fn mock_response_body(status: Status, uri: &str) -> ResponseBody { - ResponseBody { - uri: Uri::try_from(uri).unwrap(), - status, - } - } - #[test] fn test_format_status() { let status = Status::Ok(StatusCode::OK); @@ -95,7 +88,7 @@ mod tests { #[test] fn test_format_response_with_ok_status() { let formatter = ColorFormatter; - let body = mock_response_body(Status::Ok(StatusCode::OK), "https://example.com"); + let body = mock_response_body!(Status::Ok(StatusCode::OK), "https://example.com"); let formatted_response = strip_ansi_codes(&formatter.format_response(&body)); assert_eq!(formatted_response, " [200] https://example.com/"); } @@ -103,7 +96,7 @@ mod tests { #[test] fn test_format_response_with_error_status() { let formatter = ColorFormatter; - let body = mock_response_body( + let body = mock_response_body!( Status::Error(ErrorKind::TestError), "https://example.com/404", ); @@ -116,7 +109,7 @@ mod tests { let formatter = ColorFormatter; let long_uri = "https://example.com/some/very/long/path/to/a/resource/that/exceeds/normal/lengths"; - let body = mock_response_body(Status::Ok(StatusCode::OK), long_uri); + let body = mock_response_body!(Status::Ok(StatusCode::OK), long_uri); let formatted_response = strip_ansi_codes(&formatter.format_response(&body)); assert!(formatted_response.contains(long_uri)); } @@ -124,7 +117,7 @@ mod tests { #[test] fn test_detailed_response_output() { let formatter = ColorFormatter; - let body = mock_response_body( + let body = mock_response_body!( Status::Error(ErrorKind::TestError), "https://example.com/404", ); diff --git a/lychee-bin/src/formatters/response/emoji.rs b/lychee-bin/src/formatters/response/emoji.rs index 06179920fd..616670ac08 100644 --- a/lychee-bin/src/formatters/response/emoji.rs +++ b/lychee-bin/src/formatters/response/emoji.rs @@ -41,26 +41,19 @@ mod emoji_tests { use super::*; use http::StatusCode; use lychee_lib::{ErrorKind, Redirects, Status, Uri}; - - // Helper function to create a ResponseBody with a given status and URI - fn mock_response_body(status: Status, uri: &str) -> ResponseBody { - ResponseBody { - uri: Uri::try_from(uri).unwrap(), - status, - } - } + use test_utils::mock_response_body; #[test] fn test_format_response_with_ok_status() { let formatter = EmojiFormatter; - let body = mock_response_body(Status::Ok(StatusCode::OK), "https://example.com"); + let body = mock_response_body!(Status::Ok(StatusCode::OK), "https://example.com"); assert_eq!(formatter.format_response(&body), "✅ https://example.com/"); } #[test] fn test_format_response_with_error_status() { let formatter = EmojiFormatter; - let body = mock_response_body( + let body = mock_response_body!( Status::Error(ErrorKind::TestError), "https://example.com/404", ); @@ -73,7 +66,7 @@ mod emoji_tests { #[test] fn test_format_response_with_excluded_status() { let formatter = EmojiFormatter; - let body = mock_response_body(Status::Excluded, "https://example.com/not-checked"); + let body = mock_response_body!(Status::Excluded, "https://example.com/not-checked"); assert_eq!( formatter.format_response(&body), "🚫 https://example.com/not-checked" @@ -83,7 +76,7 @@ mod emoji_tests { #[test] fn test_format_response_with_redirect_status() { let formatter = EmojiFormatter; - let body = mock_response_body( + let body = mock_response_body!( Status::Redirected(StatusCode::MOVED_PERMANENTLY, Redirects::none()), "https://example.com/redirect", ); @@ -96,7 +89,7 @@ mod emoji_tests { #[test] fn test_format_response_with_unknown_status_code() { let formatter = EmojiFormatter; - let body = mock_response_body( + let body = mock_response_body!( Status::UnknownStatusCode(StatusCode::from_u16(999).unwrap()), "https://example.com/unknown", ); @@ -109,7 +102,7 @@ mod emoji_tests { #[test] fn test_detailed_response_output() { let formatter = EmojiFormatter; - let body = mock_response_body( + let body = mock_response_body!( Status::Error(ErrorKind::TestError), "https://example.com/404", ); diff --git a/lychee-bin/src/formatters/response/plain.rs b/lychee-bin/src/formatters/response/plain.rs index 80eeac3aa6..104c1c0920 100644 --- a/lychee-bin/src/formatters/response/plain.rs +++ b/lychee-bin/src/formatters/response/plain.rs @@ -24,19 +24,12 @@ mod plain_tests { use http::StatusCode; use lychee_lib::Redirects; use lychee_lib::{ErrorKind, Status, Uri}; - - // Helper function to create a ResponseBody with a given status and URI - fn mock_response_body(status: Status, uri: &str) -> ResponseBody { - ResponseBody { - uri: Uri::try_from(uri).unwrap(), - status, - } - } + use test_utils::mock_response_body; #[test] fn test_format_response_with_ok_status() { let formatter = PlainFormatter; - let body = mock_response_body(Status::Ok(StatusCode::OK), "https://example.com"); + let body = mock_response_body!(Status::Ok(StatusCode::OK), "https://example.com"); assert_eq!( formatter.format_response(&body), "[200] https://example.com/" @@ -46,7 +39,7 @@ mod plain_tests { #[test] fn test_format_response_with_error_status() { let formatter = PlainFormatter; - let body = mock_response_body( + let body = mock_response_body!( Status::Error(ErrorKind::TestError), "https://example.com/404", ); @@ -59,7 +52,7 @@ mod plain_tests { #[test] fn test_format_response_with_excluded_status() { let formatter = PlainFormatter; - let body = mock_response_body(Status::Excluded, "https://example.com/not-checked"); + let body = mock_response_body!(Status::Excluded, "https://example.com/not-checked"); assert_eq!( formatter.format_response(&body), "[EXCLUDED] https://example.com/not-checked" @@ -69,7 +62,7 @@ mod plain_tests { #[test] fn test_format_response_with_redirect_status() { let formatter = PlainFormatter; - let body = mock_response_body( + let body = mock_response_body!( Status::Redirected( StatusCode::MOVED_PERMANENTLY, Redirects::from(vec![ @@ -88,7 +81,7 @@ mod plain_tests { #[test] fn test_format_response_with_unknown_status_code() { let formatter = PlainFormatter; - let body = mock_response_body( + let body = mock_response_body!( Status::UnknownStatusCode(StatusCode::from_u16(999).unwrap()), "https://example.com/unknown", ); diff --git a/lychee-bin/src/formatters/response/task.rs b/lychee-bin/src/formatters/response/task.rs index 9cf8e5af65..0775858c62 100644 --- a/lychee-bin/src/formatters/response/task.rs +++ b/lychee-bin/src/formatters/response/task.rs @@ -14,19 +14,12 @@ mod task_tests { use super::*; use http::StatusCode; use lychee_lib::{ErrorKind, Redirects, Status, Uri}; - - // Helper function to create a ResponseBody with a given status and URI - fn mock_response_body(status: Status, uri: &str) -> ResponseBody { - ResponseBody { - uri: Uri::try_from(uri).unwrap(), - status, - } - } + use test_utils::mock_response_body; #[test] fn test_format_response_with_ok_status() { let formatter = TaskFormatter; - let body = mock_response_body(Status::Ok(StatusCode::OK), "https://example.com"); + let body = mock_response_body!(Status::Ok(StatusCode::OK), "https://example.com"); assert_eq!( formatter.format_response(&body), "- [ ] [200] https://example.com/" @@ -36,7 +29,7 @@ mod task_tests { #[test] fn test_format_response_with_error_status() { let formatter = TaskFormatter; - let body = mock_response_body( + let body = mock_response_body!( Status::Error(ErrorKind::TestError), "https://example.com/404", ); @@ -49,7 +42,7 @@ mod task_tests { #[test] fn test_format_response_with_excluded_status() { let formatter = TaskFormatter; - let body = mock_response_body(Status::Excluded, "https://example.com/not-checked"); + let body = mock_response_body!(Status::Excluded, "https://example.com/not-checked"); assert_eq!( formatter.format_response(&body), "- [ ] [EXCLUDED] https://example.com/not-checked" @@ -59,7 +52,7 @@ mod task_tests { #[test] fn test_format_response_with_redirect_status() { let formatter = TaskFormatter; - let body = mock_response_body( + let body = mock_response_body!( Status::Redirected( StatusCode::MOVED_PERMANENTLY, Redirects::from(vec![ @@ -78,7 +71,7 @@ mod task_tests { #[test] fn test_format_response_with_unknown_status_code() { let formatter = TaskFormatter; - let body = mock_response_body( + let body = mock_response_body!( Status::UnknownStatusCode(StatusCode::from_u16(999).unwrap()), "https://example.com/unknown", ); diff --git a/lychee-bin/src/main.rs b/lychee-bin/src/main.rs index b915652ad2..e3264a7268 100644 --- a/lychee-bin/src/main.rs +++ b/lychee-bin/src/main.rs @@ -1,6 +1,6 @@ //! `lychee` is a fast, asynchronous, resource-friendly link checker. //! It is able to find broken hyperlinks and mail addresses inside Markdown, -//! HTML, `reStructuredText`, and any other format. +//! HTML, reStructuredText, and any other format. //! //! The lychee binary is a wrapper around lychee-lib, which provides //! convenience functions for calling lychee from the command-line. @@ -65,7 +65,7 @@ use std::sync::Arc; use anyhow::{Context, Error, Result, bail}; use clap::Parser; -use commands::CommandParams; +use commands::{CommandParams, generate}; use formatters::{get_stats_formatter, log::init_logging}; use http::HeaderMap; use log::{error, info, warn}; @@ -92,10 +92,10 @@ mod stats; mod time; mod verbosity; -use crate::formatters::duration::Duration; use crate::{ cache::{Cache, StoreExt}, - formatters::stats::StatsFormatter, + formatters::{duration::Duration, stats::StatsFormatter}, + generate::generate, options::{Config, LYCHEE_CACHE_FILE, LYCHEE_IGNORE_FILE, LycheeOptions}, }; @@ -248,7 +248,7 @@ fn load_cache(cfg: &Config) -> Option { let cache = Cache::load( LYCHEE_CACHE_FILE, cfg.max_cache_age.as_secs(), - &cfg.cache_exclude_status, + &cfg.cache_exclude_status.clone().unwrap_or_default(), ); match cache { Ok(cache) => Some(cache), @@ -271,6 +271,11 @@ fn run_main() -> Result { } }; + if let Some(mode) = opts.config.generate { + print!("{}", generate(&mode)?); + exit(ExitCode::Success as i32); + } + let runtime = match opts.config.threads { Some(threads) => { // We define our own runtime instead of the `tokio::main` attribute diff --git a/lychee-bin/src/options.rs b/lychee-bin/src/options.rs index 50e1732db8..847a6edffc 100644 --- a/lychee-bin/src/options.rs +++ b/lychee-bin/src/options.rs @@ -1,4 +1,5 @@ use crate::files_from::FilesFrom; +use crate::generate::GenerateMode; use crate::parse::parse_base; use crate::verbosity::Verbosity; use anyhow::{Context, Error, Result, anyhow}; @@ -193,7 +194,6 @@ default_function! { retry_wait_time: usize = DEFAULT_RETRY_WAIT_TIME_SECS; method: String = DEFAULT_METHOD.to_string(); verbosity: Verbosity = Verbosity::default(); - cache_exclude_selector: StatusCodeExcluder = StatusCodeExcluder::new(); accept_selector: StatusCodeSelector = StatusCodeSelector::default(); } @@ -312,12 +312,13 @@ impl HeaderMapExt for HeaderMap { } } -/// A fast, async link checker +/// lychee is a fast, asynchronous link checker which detects broken URLs and mail addresses +/// in local files and websites. It supports Markdown and HTML and works well +/// with many plain text file formats. /// -/// Finds broken URLs and mail addresses inside Markdown, HTML, -/// `reStructuredText`, websites and more! +/// lychee is powered by lychee-lib, the Rust library for link checking. #[derive(Parser, Debug)] -#[command(version, about)] +#[command(version, about, next_display_order = None)] pub(crate) struct LycheeOptions { /// Inputs for link checking (where to get links to check from). #[arg( @@ -462,7 +463,6 @@ specify both extensions explicitly." /// A list of status codes that will be excluded from the cache #[arg( long, - default_value_t, long_help = "A list of status codes that will be ignored from the cache The following exclude range syntax is supported: [start]..[[=]end]|code. Some valid @@ -478,8 +478,7 @@ Use \"lychee --cache-exclude-status '429, 500..502' ...\" to provide a comma-separated list of excluded status codes. This example will not cache results with a status code of 429, 500 and 501." )] - #[serde(default = "cache_exclude_selector")] - pub(crate) cache_exclude_status: StatusCodeExcluder, + pub(crate) cache_exclude_status: Option, /// Don't perform any link checking. /// Instead, dump all the links extracted from inputs that would be checked @@ -829,6 +828,10 @@ followed by the absolute link's own path." #[serde(default)] pub(crate) format: StatsFormat, + /// Generate special output (e.g. the man page) instead of performing link checking + #[arg(long, value_parser = PossibleValuesParser::new(GenerateMode::VARIANTS).map(|s| s.parse::().unwrap()))] + pub(crate) generate: Option, + /// When HTTPS is available, treat HTTP links as errors #[arg(long)] #[serde(default)] @@ -903,7 +906,7 @@ impl Config { base_url: None, basic_auth: None, cache: false, - cache_exclude_status: StatusCodeExcluder::default(), + cache_exclude_status: None, cookie_jar: None, default_extension: None, dump: false, @@ -918,6 +921,7 @@ impl Config { extensions: FileType::default_extensions(), fallback_extensions: Vec::::new(), format: StatsFormat::default(), + generate: None, glob_ignore_case: false, hidden: false, include: Vec::::new(), @@ -985,7 +989,6 @@ mod tests { cli.accept, StatusCodeSelector::from_str("100..=103,200..=299").expect("no error") ); - assert_eq!(cli.cache_exclude_status, StatusCodeExcluder::new()); } #[test] diff --git a/lychee-bin/tests/cli.rs b/lychee-bin/tests/cli.rs index 74a8d4aa12..216f36eb59 100644 --- a/lychee-bin/tests/cli.rs +++ b/lychee-bin/tests/cli.rs @@ -18,11 +18,13 @@ mod cli { error::Error, fs::{self, File}, io::{BufRead, Write}, - path::{Path, PathBuf}, + path::Path, time::Duration, }; use tempfile::{NamedTempFile, tempdir}; - use test_utils::{mock_server, redirecting_mock_server}; + use test_utils::{ + fixtures_path, main_command, mock_server, redirecting_mock_server, root_path, + }; use uuid::Uuid; use wiremock::{ @@ -50,24 +52,6 @@ mod cli { }}; } - /// Gets the "main" binary name (e.g. `lychee`) - fn main_command() -> Command { - Command::cargo_bin(env!("CARGO_PKG_NAME")).expect("Couldn't get cargo package name") - } - - /// Get the root path of the project. - fn root_path() -> PathBuf { - Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .to_path_buf() - } - - /// Get the path to the fixtures directory. - fn fixtures_path() -> PathBuf { - root_path().join("fixtures") - } - /// Convert a relative path to an absolute path string /// starting from a base directory. fn path_str(base: &Path, relative_path: &str) -> String { @@ -97,8 +81,8 @@ mod cli { /// Test the output of the JSON format. macro_rules! test_json_output { ($test_file:expr, $expected:expr $(, $arg:expr)*) => {{ - let mut cmd = main_command(); - let test_path = fixtures_path().join($test_file); + let mut cmd = main_command!(); + let test_path = fixtures_path!().join($test_file); let outfile = format!("{}.json", uuid::Uuid::new_v4()); let result = cmd$(.arg($arg))*.arg("--output").arg(&outfile).arg("--format").arg("json").arg(test_path).assert(); @@ -138,9 +122,9 @@ mod cli { /// sure that the status code only occurs once. #[test] fn test_compact_output_format_contains_status() -> Result<()> { - let test_path = fixtures_path().join("TEST_INVALID_URLS.html"); + let test_path = fixtures_path!().join("TEST_INVALID_URLS.html"); - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--format") .arg("compact") .arg("--mode") @@ -182,7 +166,7 @@ mod cli { async fn test_json_output() -> Result<()> { // Server that returns a bunch of 200 OK responses let mock_server_ok = mock_server!(StatusCode::OK); - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--format") .arg("json") .arg("-vv") @@ -237,9 +221,9 @@ mod cli { /// See https://github.com/lycheeverse/lychee/issues/1355 #[test] fn test_valid_json_output_to_stdout_on_error() -> Result<()> { - let test_path = fixtures_path().join("TEST_GITHUB_404.md"); + let test_path = fixtures_path!().join("TEST_GITHUB_404.md"); - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--format") .arg("json") .arg(test_path) @@ -256,9 +240,9 @@ mod cli { #[test] fn test_detailed_json_output_on_error() -> Result<()> { - let test_path = fixtures_path().join("TEST_DETAILED_JSON_OUTPUT_ERROR.md"); + let test_path = fixtures_path!().join("TEST_DETAILED_JSON_OUTPUT_ERROR.md"); - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--format") .arg("json") .arg(&test_path) @@ -337,8 +321,8 @@ mod cli { #[test] fn test_email_html_with_subject() -> Result<()> { - let mut cmd = main_command(); - let input = fixtures_path().join("TEST_EMAIL_QUERY_PARAMS.html"); + let mut cmd = main_command!(); + let input = fixtures_path!().join("TEST_EMAIL_QUERY_PARAMS.html"); cmd.arg("--dump") .arg(input) @@ -352,8 +336,8 @@ mod cli { #[test] fn test_email_markdown_with_subject() -> Result<()> { - let mut cmd = main_command(); - let input = fixtures_path().join("TEST_EMAIL_QUERY_PARAMS.md"); + let mut cmd = main_command!(); + let input = fixtures_path!().join("TEST_EMAIL_QUERY_PARAMS.md"); cmd.arg("--dump") .arg(input) @@ -392,8 +376,8 @@ mod cli { /// Test unsupported URI schemes #[test] fn test_unsupported_uri_schemes_are_ignored() { - let mut cmd = main_command(); - let test_schemes_path = fixtures_path().join("TEST_SCHEMES.txt"); + let mut cmd = main_command!(); + let test_schemes_path = fixtures_path!().join("TEST_SCHEMES.txt"); // Exclude file link because it doesn't exist on the filesystem. // (File URIs are absolute paths, which we don't have.) @@ -411,8 +395,8 @@ mod cli { #[test] fn test_resolve_paths() { - let mut cmd = main_command(); - let dir = fixtures_path().join("resolve_paths"); + let mut cmd = main_command!(); + let dir = fixtures_path!().join("resolve_paths"); cmd.arg("--offline") .arg("--base-url") @@ -427,8 +411,8 @@ mod cli { #[test] fn test_resolve_paths_from_root_dir() { - let mut cmd = main_command(); - let dir = fixtures_path().join("resolve_paths_from_root_dir"); + let mut cmd = main_command!(); + let dir = fixtures_path!().join("resolve_paths_from_root_dir"); cmd.arg("--offline") .arg("--include-fragments") @@ -445,8 +429,8 @@ mod cli { #[test] fn test_resolve_paths_from_root_dir_and_base_url() { - let mut cmd = main_command(); - let dir = fixtures_path(); + let mut cmd = main_command!(); + let dir = fixtures_path!(); cmd.arg("--offline") .arg("--root-dir") @@ -465,7 +449,7 @@ mod cli { fn test_youtube_quirk() { let url = "https://www.youtube.com/watch?v=NlKuICiT470&list=PLbWDhxwM_45mPVToqaIZNbZeIzFchsKKQ&index=7"; - main_command() + main_command!() .write_stdin(url) .arg("--verbose") .arg("--no-progress") @@ -480,7 +464,7 @@ mod cli { fn test_crates_io_quirk() { let url = "https://crates.io/crates/lychee"; - main_command() + main_command!() .write_stdin(url) .arg("--verbose") .arg("--no-progress") @@ -498,7 +482,7 @@ mod cli { fn test_ignored_hosts() { let url = "https://twitter.com/zarfeblong/status/1339742840142872577"; - main_command() + main_command!() .write_stdin(url) .arg("--verbose") .arg("--no-progress") @@ -517,7 +501,7 @@ mod cli { let mut file = File::create(&file_path)?; writeln!(file, "{}", mock_server.uri())?; - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg(file_path) .write_stdin(mock_server.uri()) .assert() @@ -529,8 +513,8 @@ mod cli { #[test] fn test_schemes() { - let mut cmd = main_command(); - let test_schemes_path = fixtures_path().join("TEST_SCHEMES.md"); + let mut cmd = main_command!(); + let test_schemes_path = fixtures_path!().join("TEST_SCHEMES.md"); cmd.arg(test_schemes_path) .arg("--scheme") @@ -547,9 +531,9 @@ mod cli { #[test] fn test_caching_single_file() { - let mut cmd = main_command(); + let mut cmd = main_command!(); // Repetitions in one file shall all be checked and counted only once. - let test_schemes_path_1 = fixtures_path().join("TEST_REPETITION_1.txt"); + let test_schemes_path_1 = fixtures_path!().join("TEST_REPETITION_1.txt"); cmd.arg(&test_schemes_path_1) .env_clear() @@ -563,7 +547,7 @@ mod cli { // Test that two identical requests don't get executed twice. fn test_caching_across_files() -> Result<()> { // Repetitions across multiple files shall all be checked only once. - let repeated_uris = fixtures_path().join("TEST_REPETITION_*.txt"); + let repeated_uris = fixtures_path!().join("TEST_REPETITION_*.txt"); test_json_output!( repeated_uris, @@ -584,8 +568,8 @@ mod cli { #[test] fn test_failure_github_404_no_token() { - let mut cmd = main_command(); - let test_github_404_path = fixtures_path().join("TEST_GITHUB_404.md"); + let mut cmd = main_command!(); + let test_github_404_path = fixtures_path!().join("TEST_GITHUB_404.md"); cmd.arg(test_github_404_path) .arg("--no-progress") @@ -603,7 +587,7 @@ mod cli { #[tokio::test] async fn test_stdin_input() { - let mut cmd = main_command(); + let mut cmd = main_command!(); let mock_server = mock_server!(StatusCode::OK); cmd.arg("-") @@ -614,7 +598,7 @@ mod cli { #[tokio::test] async fn test_stdin_input_failure() { - let mut cmd = main_command(); + let mut cmd = main_command!(); let mock_server = mock_server!(StatusCode::INTERNAL_SERVER_ERROR); cmd.arg("-") @@ -626,7 +610,7 @@ mod cli { #[tokio::test] async fn test_stdin_input_multiple() { - let mut cmd = main_command(); + let mut cmd = main_command!(); let mock_server_a = mock_server!(StatusCode::OK); let mock_server_b = mock_server!(StatusCode::OK); @@ -642,7 +626,7 @@ mod cli { #[test] fn test_missing_file_ok_if_skip_missing() { - let mut cmd = main_command(); + let mut cmd = main_command!(); let filename = format!("non-existing-file-{}", uuid::Uuid::new_v4()); cmd.arg(&filename).arg("--skip-missing").assert().success(); @@ -650,8 +634,8 @@ mod cli { #[test] fn test_skips_hidden_files_by_default() { - main_command() - .arg(fixtures_path().join("hidden/")) + main_command!() + .arg(fixtures_path!().join("hidden/")) .assert() .success() .stdout(contains("0 Total")); @@ -659,8 +643,8 @@ mod cli { #[test] fn test_include_hidden_file() { - main_command() - .arg(fixtures_path().join("hidden/")) + main_command!() + .arg(fixtures_path!().join("hidden/")) .arg("--hidden") .assert() .success() @@ -669,8 +653,8 @@ mod cli { #[test] fn test_skips_ignored_files_by_default() { - main_command() - .arg(fixtures_path().join("ignore/")) + main_command!() + .arg(fixtures_path!().join("ignore/")) .assert() .success() .stdout(contains("0 Total")); @@ -678,8 +662,8 @@ mod cli { #[test] fn test_include_ignored_file() { - main_command() - .arg(fixtures_path().join("ignore/")) + main_command!() + .arg(fixtures_path!().join("ignore/")) .arg("--no-ignore") .assert() .success() @@ -689,7 +673,7 @@ mod cli { #[tokio::test] async fn test_glob() -> Result<()> { // using Result to be able to use `?` - let mut cmd = main_command(); + let mut cmd = main_command!(); let dir = tempfile::tempdir()?; let mock_server_a = mock_server!(StatusCode::OK); @@ -712,7 +696,7 @@ mod cli { #[cfg(target_os = "linux")] // MacOS and Windows have case-insensitive filesystems #[tokio::test] async fn test_glob_ignore_case() -> Result<()> { - let mut cmd = main_command(); + let mut cmd = main_command!(); let dir = tempfile::tempdir()?; let mock_server_a = mock_server!(StatusCode::OK); @@ -735,7 +719,7 @@ mod cli { #[tokio::test] async fn test_glob_recursive() -> Result<()> { - let mut cmd = main_command(); + let mut cmd = main_command!(); let dir = tempfile::tempdir()?; let subdir_level_1 = tempfile::tempdir_in(&dir)?; @@ -773,8 +757,8 @@ mod cli { /// Test writing output of `--dump` command to file #[test] fn test_dump_to_file() -> Result<()> { - let mut cmd = main_command(); - let test_path = fixtures_path().join("TEST.md"); + let mut cmd = main_command!(); + let test_path = fixtures_path!().join("TEST.md"); let outfile = format!("{}", Uuid::new_v4()); cmd.arg("--output") @@ -799,8 +783,8 @@ mod cli { /// Test excludes #[test] fn test_exclude_wildcard() -> Result<()> { - let mut cmd = main_command(); - let test_path = fixtures_path().join("TEST.md"); + let mut cmd = main_command!(); + let test_path = fixtures_path!().join("TEST.md"); cmd.arg(test_path) .arg("--exclude") @@ -814,8 +798,8 @@ mod cli { #[test] fn test_exclude_multiple_urls() -> Result<()> { - let mut cmd = main_command(); - let test_path = fixtures_path().join("TEST.md"); + let mut cmd = main_command!(); + let test_path = fixtures_path!().join("TEST.md"); cmd.arg(test_path) .arg("--exclude") @@ -832,8 +816,8 @@ mod cli { #[tokio::test] async fn test_empty_config() -> Result<()> { let mock_server = mock_server!(StatusCode::OK); - let config = fixtures_path().join("configs").join("empty.toml"); - let mut cmd = main_command(); + let config = fixtures_path!().join("configs").join("empty.toml"); + let mut cmd = main_command!(); cmd.arg("--config") .arg(config) .arg("-") @@ -849,8 +833,8 @@ mod cli { #[test] fn test_invalid_default_config() -> Result<()> { - let test_path = fixtures_path().join("configs"); - let mut cmd = main_command(); + let test_path = fixtures_path!().join("configs"); + let mut cmd = main_command!(); cmd.current_dir(test_path) .arg(".") .assert() @@ -867,7 +851,7 @@ mod cli { let mut config = NamedTempFile::new()?; writeln!(config, "include_mail = false")?; - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--config") .arg(config.path().to_str().unwrap()) .arg("-") @@ -881,7 +865,7 @@ mod cli { let mut config = NamedTempFile::new()?; writeln!(config, "include_mail = true")?; - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--config") .arg(config.path().to_str().unwrap()) .arg("-") @@ -898,8 +882,8 @@ mod cli { #[tokio::test] async fn test_cache_config() -> Result<()> { let mock_server = mock_server!(StatusCode::OK); - let config = fixtures_path().join("configs").join("cache.toml"); - let mut cmd = main_command(); + let config = fixtures_path!().join("configs").join("cache.toml"); + let mut cmd = main_command!(); cmd.arg("--config") .arg(config) .arg("-") @@ -915,8 +899,8 @@ mod cli { #[tokio::test] async fn test_invalid_config() { - let config = fixtures_path().join("configs").join("invalid.toml"); - let mut cmd = main_command(); + let config = fixtures_path!().join("configs").join("invalid.toml"); + let mut cmd = main_command!(); cmd.arg("--config") .arg(config) .arg("-") @@ -931,7 +915,7 @@ mod cli { #[tokio::test] async fn test_missing_config_error() { let mock_server = mock_server!(StatusCode::OK); - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--config") .arg("config.does.not.exist.toml") .arg("-") @@ -944,8 +928,8 @@ mod cli { #[tokio::test] async fn test_config_example() { let mock_server = mock_server!(StatusCode::OK); - let config = root_path().join("lychee.example.toml"); - let mut cmd = main_command(); + let config = root_path!().join("lychee.example.toml"); + let mut cmd = main_command!(); cmd.arg("--config") .arg(config) .arg("-") @@ -958,8 +942,8 @@ mod cli { #[tokio::test] async fn test_config_smoketest() { let mock_server = mock_server!(StatusCode::OK); - let config = fixtures_path().join("configs").join("smoketest.toml"); - let mut cmd = main_command(); + let config = fixtures_path!().join("configs").join("smoketest.toml"); + let mut cmd = main_command!(); cmd.arg("--config") .arg(config) .arg("-") @@ -972,8 +956,8 @@ mod cli { #[tokio::test] async fn test_config_accept() { let mock_server = mock_server!(StatusCode::OK); - let config = fixtures_path().join("configs").join("accept.toml"); - let mut cmd = main_command(); + let config = fixtures_path!().join("configs").join("accept.toml"); + let mut cmd = main_command!(); cmd.arg("--config") .arg(config) .arg("-") @@ -985,8 +969,8 @@ mod cli { #[test] fn test_lycheeignore_file() -> Result<()> { - let mut cmd = main_command(); - let test_path = fixtures_path().join("lycheeignore"); + let mut cmd = main_command!(); + let test_path = fixtures_path!().join("lycheeignore"); let cmd = cmd .current_dir(test_path) @@ -1006,8 +990,8 @@ mod cli { #[test] fn test_lycheeignore_and_exclude_file() -> Result<()> { - let mut cmd = main_command(); - let test_path = fixtures_path().join("lycheeignore"); + let mut cmd = main_command!(); + let test_path = fixtures_path!().join("lycheeignore"); let excludes_path = test_path.join("normal-exclude-file"); cmd.current_dir(test_path) @@ -1024,7 +1008,7 @@ mod cli { #[tokio::test] async fn test_lycheecache_file() -> Result<()> { - let base_path = fixtures_path().join("cache"); + let base_path = fixtures_path!().join("cache"); let cache_file = base_path.join(LYCHEE_CACHE_FILE); // Ensure clean state @@ -1049,7 +1033,7 @@ mod cli { file.sync_all()?; // Create and run command - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.current_dir(&base_path) .arg(&file_path) .arg("--verbose") @@ -1108,7 +1092,7 @@ mod cli { #[tokio::test] async fn test_lycheecache_exclude_custom_status_codes() -> Result<()> { - let base_path = fixtures_path().join("cache"); + let base_path = fixtures_path!().join("cache"); let cache_file = base_path.join(LYCHEE_CACHE_FILE); // Unconditionally remove cache file if it exists @@ -1125,7 +1109,7 @@ mod cli { writeln!(file, "{}", mock_server_no_content.uri().as_str())?; writeln!(file, "{}", mock_server_too_many_requests.uri().as_str())?; - let mut cmd = main_command(); + let mut cmd = main_command!(); let test_cmd = cmd .current_dir(&base_path) .arg(dir.path().join("c.md")) @@ -1171,7 +1155,7 @@ mod cli { #[tokio::test] async fn test_lycheecache_accept_custom_status_codes() -> Result<()> { - let base_path = fixtures_path().join("cache_accept_custom_status_codes"); + let base_path = fixtures_path!().join("cache_accept_custom_status_codes"); let cache_file = base_path.join(LYCHEE_CACHE_FILE); // Unconditionally remove cache file if it exists @@ -1188,7 +1172,7 @@ mod cli { writeln!(file, "{}", mock_server_teapot.uri().as_str())?; writeln!(file, "{}", mock_server_server_error.uri().as_str())?; - let mut cmd = main_command(); + let mut cmd = main_command!(); let test_cmd = cmd .current_dir(&base_path) .arg(dir.path().join("c.md")) @@ -1248,7 +1232,7 @@ mod cli { async fn test_accept_overrides_defaults_not_additive() -> Result<()> { let mock_server_200 = mock_server!(StatusCode::OK); - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--accept") .arg("404") // ONLY accept 404 - should reject 200 as we overwrite the default .arg("-") @@ -1266,7 +1250,7 @@ mod cli { #[tokio::test] async fn test_skip_cache_unsupported() -> Result<()> { - let base_path = fixtures_path().join("cache"); + let base_path = fixtures_path!().join("cache"); let cache_file = base_path.join(LYCHEE_CACHE_FILE); // Unconditionally remove cache file if it exists @@ -1276,7 +1260,7 @@ mod cli { let excluded_url = "https://example.com/"; // run first without cache to generate the cache file - main_command() + main_command!() .current_dir(&base_path) .write_stdin(format!("{unsupported_url}\n{excluded_url}")) .arg("--cache") @@ -1317,7 +1301,7 @@ mod cli { /// status codes. #[tokio::test] async fn test_skip_cache_unknown_status_code() -> Result<()> { - let base_path = fixtures_path().join("cache"); + let base_path = fixtures_path!().join("cache"); let cache_file = base_path.join(LYCHEE_CACHE_FILE); // Unconditionally remove cache file if it exists @@ -1328,7 +1312,7 @@ mod cli { let unknown_url = "https://www.linkedin.com/company/corrode"; // run first without cache to generate the cache file - main_command() + main_command!() .current_dir(&base_path) .write_stdin(unknown_url.to_string()) .arg("--cache") @@ -1360,8 +1344,8 @@ mod cli { #[test] fn test_verbatim_skipped_by_default() -> Result<()> { - let mut cmd = main_command(); - let input = fixtures_path().join("TEST_CODE_BLOCKS.md"); + let mut cmd = main_command!(); + let input = fixtures_path!().join("TEST_CODE_BLOCKS.md"); cmd.arg(input) .arg("--dump") @@ -1374,8 +1358,8 @@ mod cli { #[test] fn test_include_verbatim() -> Result<()> { - let mut cmd = main_command(); - let input = fixtures_path().join("TEST_CODE_BLOCKS.md"); + let mut cmd = main_command!(); + let input = fixtures_path!().join("TEST_CODE_BLOCKS.md"); cmd.arg("--include-verbatim") .arg(input) @@ -1390,9 +1374,9 @@ mod cli { } #[tokio::test] async fn test_verbatim_skipped_by_default_via_file() -> Result<()> { - let file = fixtures_path().join("TEST_VERBATIM.html"); + let file = fixtures_path!().join("TEST_VERBATIM.html"); - main_command() + main_command!() .arg("--dump") .arg(file) .assert() @@ -1404,8 +1388,8 @@ mod cli { #[tokio::test] async fn test_verbatim_skipped_by_default_via_remote_url() -> Result<()> { - let mut cmd = main_command(); - let file = fixtures_path().join("TEST_VERBATIM.html"); + let mut cmd = main_command!(); + let file = fixtures_path!().join("TEST_VERBATIM.html"); let body = fs::read_to_string(file)?; let mock_server = mock_response!(body); @@ -1420,8 +1404,8 @@ mod cli { #[tokio::test] async fn test_include_verbatim_via_remote_url() -> Result<()> { - let mut cmd = main_command(); - let file = fixtures_path().join("TEST_VERBATIM.html"); + let mut cmd = main_command!(); + let file = fixtures_path!().join("TEST_VERBATIM.html"); let body = fs::read_to_string(file)?; let mock_server = mock_response!(body); @@ -1441,11 +1425,11 @@ mod cli { #[test] fn test_require_https() -> Result<()> { - let mut cmd = main_command(); - let test_path = fixtures_path().join("TEST_HTTP.html"); + let mut cmd = main_command!(); + let test_path = fixtures_path!().join("TEST_HTTP.html"); cmd.arg(&test_path).assert().success(); - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--require-https") .arg(test_path) .assert() @@ -1460,9 +1444,9 @@ mod cli { /// Instead, simply ignore the link. #[test] fn test_ignore_absolute_local_links_without_base() -> Result<()> { - let mut cmd = main_command(); + let mut cmd = main_command!(); - let offline_dir = fixtures_path().join("offline"); + let offline_dir = fixtures_path!().join("offline"); cmd.arg("--offline") .arg(offline_dir.join("index.html")) @@ -1476,8 +1460,8 @@ mod cli { #[test] fn test_inputs_without_scheme() -> Result<()> { - let test_path = fixtures_path().join("TEST_HTTP.html"); - let mut cmd = main_command(); + let test_path = fixtures_path!().join("TEST_HTTP.html"); + let mut cmd = main_command!(); cmd.arg("--dump") .arg("example.com") @@ -1490,8 +1474,8 @@ mod cli { #[test] fn test_print_excluded_links_in_verbose_mode() -> Result<()> { - let test_path = fixtures_path().join("TEST_DUMP_EXCLUDE.txt"); - let mut cmd = main_command(); + let test_path = fixtures_path!().join("TEST_DUMP_EXCLUDE.txt"); + let mut cmd = main_command!(); cmd.arg("--dump") .arg("--verbose") @@ -1518,7 +1502,7 @@ mod cli { #[test] fn test_remap_uri() -> Result<()> { - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--dump") .arg("--remap") @@ -1541,7 +1525,7 @@ mod cli { #[test] #[ignore = "Skipping test until https://github.com/robinst/linkify/pull/58 is merged"] fn test_remap_path() -> Result<()> { - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--dump") .arg("--remap") @@ -1559,7 +1543,7 @@ mod cli { #[test] fn test_remap_capture() -> Result<()> { - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--dump") .arg("--remap") @@ -1577,7 +1561,7 @@ mod cli { #[test] fn test_remap_named_capture() -> Result<()> { - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--dump") .arg("--remap") @@ -1595,10 +1579,10 @@ mod cli { #[test] fn test_excluded_paths_regex() -> Result<()> { - let test_path = fixtures_path().join("exclude-path"); + let test_path = fixtures_path!().join("exclude-path"); let excluded_path_1 = "\\/excluded?\\/"; // exclude paths containing a directory "exclude" and "excluded" let excluded_path_2 = "(\\.mdx|\\.txt)$"; // exclude .mdx and .txt files - let mut cmd = main_command(); + let mut cmd = main_command!(); let result = cmd .arg("--exclude-path") @@ -1624,8 +1608,8 @@ mod cli { #[test] fn test_handle_relative_paths_as_input() -> Result<()> { - let test_path = fixtures_path(); - let mut cmd = main_command(); + let test_path = fixtures_path!(); + let mut cmd = main_command!(); cmd.current_dir(&test_path) .arg("--verbose") @@ -1643,8 +1627,8 @@ mod cli { #[test] fn test_handle_nonexistent_relative_paths_as_input() -> Result<()> { - let test_path = fixtures_path(); - let mut cmd = main_command(); + let test_path = fixtures_path!(); + let mut cmd = main_command!(); cmd.current_dir(&test_path) .arg("--verbose") @@ -1661,7 +1645,7 @@ mod cli { #[test] fn test_prevent_too_many_redirects() -> Result<()> { - let mut cmd = main_command(); + let mut cmd = main_command!(); let url = "https://http.codes/308"; cmd.write_stdin(url) @@ -1681,8 +1665,8 @@ mod cli { for _ in 0..3 { // This can be flaky. Try up to 3 times - let mut cmd = main_command(); - let input = fixtures_path().join("INTERNET_ARCHIVE.md"); + let mut cmd = main_command!(); + let input = fixtures_path!().join("INTERNET_ARCHIVE.md"); cmd.arg("--no-progress").arg("--suggest").arg(input); @@ -1725,7 +1709,7 @@ mod cli { .await; // Configure the command to use the BasicAuthExtractor - main_command() + main_command!() .arg("--verbose") .arg("--basic-auth") .arg(format!("{} {username}:{password}", mock_server.uri())) @@ -1737,7 +1721,7 @@ mod cli { .stdout(contains("1 OK")); // Websites as direct arguments must also use authentication - main_command() + main_command!() .arg(mock_server.uri()) .arg("--verbose") .arg("--basic-auth") @@ -1769,7 +1753,7 @@ mod cli { .await; // Configure the command to use the BasicAuthExtractor - main_command() + main_command!() .arg("--verbose") .arg("--basic-auth") .arg(format!("{} {username1}:{password1}", mock_server1.uri())) @@ -1789,7 +1773,7 @@ mod cli { async fn test_cookie_jar() -> Result<()> { // Create a random cookie jar file let cookie_jar = NamedTempFile::new()?; - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--cookie-jar") .arg(cookie_jar.path().to_str().unwrap()) .arg("-") @@ -1811,9 +1795,9 @@ mod cli { #[test] fn test_dump_inputs_does_not_include_duplicates() -> Result<()> { - let pattern = fixtures_path().join("dump_inputs/markdown.md"); + let pattern = fixtures_path!().join("dump_inputs/markdown.md"); - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--dump-inputs") .arg(&pattern) .arg(&pattern) @@ -1826,10 +1810,10 @@ mod cli { #[test] fn test_dump_inputs_glob_does_not_include_duplicates() -> Result<()> { - let pattern1 = fixtures_path().join("**/markdown.*"); - let pattern2 = fixtures_path().join("**/*.md"); + let pattern1 = fixtures_path!().join("**/markdown.*"); + let pattern2 = fixtures_path!().join("**/*.md"); - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--dump-inputs") .arg(pattern1) .arg(pattern2) @@ -1842,9 +1826,9 @@ mod cli { #[test] fn test_dump_inputs_glob_md() -> Result<()> { - let pattern = fixtures_path().join("**/*.md"); + let pattern = fixtures_path!().join("**/*.md"); - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--dump-inputs") .arg(pattern) .assert() @@ -1857,9 +1841,9 @@ mod cli { #[test] fn test_dump_inputs_glob_all() -> Result<()> { - let pattern = fixtures_path().join("**/*"); + let pattern = fixtures_path!().join("**/*"); - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--dump-inputs") .arg(pattern) .assert() @@ -1875,13 +1859,13 @@ mod cli { #[test] fn test_dump_inputs_glob_exclude_path() -> Result<()> { - let pattern = fixtures_path().join("**/*"); + let pattern = fixtures_path!().join("**/*"); - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--dump-inputs") .arg(pattern) .arg("--exclude-path") - .arg(fixtures_path().join("dump_inputs/subfolder")) + .arg(fixtures_path!().join("dump_inputs/subfolder")) .assert() .success() .stdout(contains("fixtures/dump_inputs/subfolder/test.html").not()) @@ -1893,7 +1877,7 @@ mod cli { #[test] fn test_dump_inputs_url() -> Result<()> { - let mut cmd = main_command(); + let mut cmd = main_command!(); let result = cmd .arg("--dump-inputs") .arg("https://example.com") @@ -1906,14 +1890,14 @@ mod cli { #[test] fn test_dump_inputs_path() -> Result<()> { - let mut cmd = main_command(); + let mut cmd = main_command!(); let result = cmd .arg("--dump-inputs") - .arg(fixtures_path().join("dump_inputs")) + .arg(fixtures_path!().join("dump_inputs")) .assert() .success(); - let base_path = fixtures_path().join("dump_inputs"); + let base_path = fixtures_path!().join("dump_inputs"); let expected_lines = [ "some_file.txt", "subfolder/file2.md", @@ -1932,8 +1916,8 @@ mod cli { // as `stdin` is not a path #[test] fn test_dump_inputs_with_extensions() -> Result<()> { - let mut cmd = main_command(); - let test_dir = fixtures_path().join("dump_inputs"); + let mut cmd = main_command!(); + let test_dir = fixtures_path!().join("dump_inputs"); let output = cmd .arg("--dump-inputs") @@ -1952,7 +1936,7 @@ mod cli { .collect(); actual_lines.sort(); - let base_path = fixtures_path().join("dump_inputs"); + let base_path = fixtures_path!().join("dump_inputs"); let mut expected_lines = vec![ path_str(&base_path, "some_file.txt"), path_str(&base_path, "subfolder/file2.md"), @@ -1975,10 +1959,10 @@ mod cli { #[test] fn test_dump_inputs_skip_hidden() -> Result<()> { - let test_dir = fixtures_path().join("hidden"); + let test_dir = fixtures_path!().join("hidden"); // Test default behavior (skip hidden) - main_command() + main_command!() .arg("--dump-inputs") .arg(&test_dir) .assert() @@ -1986,7 +1970,7 @@ mod cli { .stdout(is_empty()); // Test with --hidden flag - main_command() + main_command!() .arg("--dump-inputs") .arg("--hidden") .arg(test_dir) @@ -1999,8 +1983,8 @@ mod cli { #[test] fn test_dump_inputs_individual_file() -> Result<()> { - let mut cmd = main_command(); - let test_file = fixtures_path().join("TEST.md"); + let mut cmd = main_command!(); + let test_file = fixtures_path!().join("TEST.md"); cmd.arg("--dump-inputs") .arg(&test_file) @@ -2013,7 +1997,7 @@ mod cli { #[test] fn test_dump_inputs_stdin() -> Result<()> { - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--dump-inputs") .arg("-") @@ -2026,8 +2010,8 @@ mod cli { #[test] fn test_fragments_regression() { - let mut cmd = main_command(); - let input = fixtures_path().join("FRAGMENT_REGRESSION.md"); + let mut cmd = main_command!(); + let input = fixtures_path!().join("FRAGMENT_REGRESSION.md"); cmd.arg("--include-fragments") .arg("--verbose") @@ -2038,8 +2022,8 @@ mod cli { #[test] fn test_fragments() { - let mut cmd = main_command(); - let input = fixtures_path().join("fragments"); + let mut cmd = main_command!(); + let input = fixtures_path!().join("fragments"); let mut result = cmd .arg("--include-fragments") @@ -2131,8 +2115,8 @@ mod cli { #[test] fn test_fragments_when_accept_error_status_codes() { - let mut cmd = main_command(); - let input = fixtures_path().join("TEST_FRAGMENT_ERR_CODE.md"); + let mut cmd = main_command!(); + let input = fixtures_path!().join("TEST_FRAGMENT_ERR_CODE.md"); // it's common for user to accept 429, but let's test with 404 since // triggering 429 may annoy the server @@ -2152,8 +2136,8 @@ mod cli { #[test] fn test_fallback_extensions() { - let mut cmd = main_command(); - let input = fixtures_path().join("fallback-extensions"); + let mut cmd = main_command!(); + let input = fixtures_path!().join("fallback-extensions"); cmd.arg("--verbose") .arg("--fallback-extensions=htm,html") @@ -2165,8 +2149,8 @@ mod cli { #[test] fn test_fragments_fallback_extensions() { - let mut cmd = main_command(); - let input = fixtures_path().join("fragments-fallback-extensions"); + let mut cmd = main_command!(); + let input = fixtures_path!().join("fragments-fallback-extensions"); cmd.arg("--include-fragments") .arg("--fallback-extensions=html") @@ -2212,7 +2196,7 @@ mod cli { .mount(&mock_server) .await; - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--verbose") .arg(format!("{}/test/index.html", mock_server.uri())) .assert() @@ -2226,8 +2210,8 @@ mod cli { #[tokio::test] async fn test_json_format_in_config() -> Result<()> { let mock_server = mock_server!(StatusCode::OK); - let config = fixtures_path().join("configs").join("format.toml"); - let mut cmd = main_command(); + let config = fixtures_path!().join("configs").join("format.toml"); + let mut cmd = main_command!(); let output = cmd .arg("--config") .arg(config) @@ -2252,7 +2236,7 @@ mod cli { async fn test_redirect_json() { use serde_json::json; redirecting_mock_server!(async |redirect_url: Url, ok_url| { - let mut cmd = main_command(); + let mut cmd = main_command!(); let output = cmd .arg("-") .arg("--format") @@ -2301,7 +2285,7 @@ mod cli { .mount(&mock_server) .await; - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("-") .write_stdin(mock_server.uri()) .assert() @@ -2312,7 +2296,7 @@ mod cli { #[tokio::test] async fn test_no_header_set_on_input() -> Result<()> { - let mut cmd = main_command(); + let mut cmd = main_command!(); let server = wiremock::MockServer::start().await; server .register( @@ -2338,7 +2322,7 @@ mod cli { #[tokio::test] async fn test_header_set_on_input() -> Result<()> { - let mut cmd = main_command(); + let mut cmd = main_command!(); let server = wiremock::MockServer::start().await; server .register( @@ -2365,7 +2349,7 @@ mod cli { #[tokio::test] async fn test_multi_header_set_on_input() -> Result<()> { - let mut cmd = main_command(); + let mut cmd = main_command!(); let server = wiremock::MockServer::start().await; server .register( @@ -2395,7 +2379,7 @@ mod cli { #[tokio::test] async fn test_header_set_in_config() -> Result<()> { - let mut cmd = main_command(); + let mut cmd = main_command!(); let server = wiremock::MockServer::start().await; server .register( @@ -2409,7 +2393,7 @@ mod cli { ) .await; - let config = fixtures_path().join("configs").join("headers.toml"); + let config = fixtures_path!().join("configs").join("headers.toml"); cmd.arg("--verbose") .arg("--config") .arg(config) @@ -2432,11 +2416,11 @@ mod cli { "https://httpbin.org/status/502", ]; - let cmd = &mut main_command() + let cmd = &mut main_command!() .arg("--format") .arg("compact") - .arg(fixtures_path().join(test_files[1])) - .arg(fixtures_path().join(test_files[0])) + .arg(fixtures_path!().join(test_files[1])) + .arg(fixtures_path!().join(test_files[0])) .assert() .failure() .code(2); @@ -2471,9 +2455,9 @@ mod cli { #[test] fn test_extract_url_ending_with_period_file() { - let test_path = fixtures_path().join("LINK_PERIOD.html"); + let test_path = fixtures_path!().join("LINK_PERIOD.html"); - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--dump") .arg(test_path) .assert() @@ -2483,7 +2467,7 @@ mod cli { #[tokio::test] async fn test_extract_url_ending_with_period_webserver() { - let mut cmd = main_command(); + let mut cmd = main_command!(); let body = r#"link"#; let mock_server = mock_response!(body); @@ -2496,9 +2480,9 @@ mod cli { #[test] fn test_wikilink_extract_when_specified() { - let test_path = fixtures_path().join("TEST_WIKI.md"); + let test_path = fixtures_path!().join("TEST_WIKI.md"); - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--dump") .arg("--include-wikilinks") .arg(test_path) @@ -2509,9 +2493,9 @@ mod cli { #[test] fn test_wikilink_dont_extract_when_not_specified() { - let test_path = fixtures_path().join("TEST_WIKI.md"); + let test_path = fixtures_path!().join("TEST_WIKI.md"); - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--dump") .arg(test_path) .assert() @@ -2521,10 +2505,10 @@ mod cli { #[test] fn test_index_files_default() { - let input = fixtures_path().join("filechecker/dir_links.md"); + let input = fixtures_path!().join("filechecker/dir_links.md"); // the dir links in this file all exist. - main_command() + main_command!() .arg(&input) .arg("--verbose") .assert() @@ -2533,7 +2517,7 @@ mod cli { // ... but checking fragments will find none, because dirs // have no fragments and no index file given. let dir_links_with_fragment = 2; - main_command() + main_command!() .arg(&input) .arg("--include-fragments") .assert() @@ -2544,11 +2528,11 @@ mod cli { #[test] fn test_index_files_specified() { - let input = fixtures_path().join("filechecker/dir_links.md"); + let input = fixtures_path!().join("filechecker/dir_links.md"); // passing `--index-files index.html,index.htm` should reject all links // to /empty_dir because it doesn't have the index file - let result = main_command() + let result = main_command!() .arg(&input) .arg("--index-files") .arg("index.html,index.htm") @@ -2566,7 +2550,7 @@ mod cli { // within the error message, formatting of the index file name list should // omit empty names. - main_command() + main_command!() .arg(&input) .arg("--index-files") .arg(",index.html,,,index.htm,") @@ -2577,11 +2561,11 @@ mod cli { #[test] fn test_index_files_dot_in_list() { - let input = fixtures_path().join("filechecker/dir_links.md"); + let input = fixtures_path!().join("filechecker/dir_links.md"); // passing `.` in the index files list should accept a directory // even if no other index file is found. - main_command() + main_command!() .arg(&input) .arg("--index-files") .arg("index.html,.") @@ -2592,7 +2576,7 @@ mod cli { // checking fragments will accept the index_dir#fragment link, // but reject empty_dir#fragment because empty_dir doesn’t have // index.html. - main_command() + main_command!() .arg(&input) .arg("--index-files") .arg("index.html,.") @@ -2607,11 +2591,11 @@ mod cli { #[test] fn test_index_files_empty_list() { - let input = fixtures_path().join("filechecker/dir_links.md"); + let input = fixtures_path!().join("filechecker/dir_links.md"); // passing an empty list to --index-files should reject /all/ // directory links. - let result = main_command() + let result = main_command!() .arg(&input) .arg("--index-files") .arg("") @@ -2625,7 +2609,7 @@ mod cli { .stdout(contains("0 OK")); // ... as should passing a number of empty index file names - main_command() + main_command!() .arg(&input) .arg("--index-files") .arg(",,,,,") @@ -2638,10 +2622,10 @@ mod cli { #[test] fn test_skip_binary_input() { // A path containing a binary file - let inputs = fixtures_path().join("invalid_utf8"); + let inputs = fixtures_path!().join("invalid_utf8"); // Run the command with the binary input - let mut cmd = main_command(); + let mut cmd = main_command!(); let result = cmd .arg("--verbose") .arg(&inputs) @@ -2668,10 +2652,10 @@ mod cli { #[test] fn test_dump_invalid_utf8_inputs() { // A path containing a binary file - let inputs = fixtures_path().join("invalid_utf8"); + let inputs = fixtures_path!().join("invalid_utf8"); // Run the command with the binary input - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--dump-inputs") .arg(inputs) .assert() @@ -2687,7 +2671,7 @@ mod cli { /// See https://github.com/lycheeverse/lychee-action/issues/305 #[test] fn test_globbed_files_are_always_checked() { - let input = fixtures_path().join("glob_dir/**/*.tsx"); + let input = fixtures_path!().join("glob_dir/**/*.tsx"); // The directory contains: // - example.ts @@ -2695,7 +2679,7 @@ mod cli { // - example.md // - example.html // But the user only specified the .tsx file via the glob pattern. - main_command() + main_command!() .arg("--verbose") // Only check ts, js, and html files by default. // However, all files explicitly specified by the user @@ -2710,11 +2694,11 @@ mod cli { #[test] fn test_extensions_work_on_glob_files_directory() { - let input = fixtures_path().join("glob_dir"); + let input = fixtures_path!().join("glob_dir"); // Make sure all files matching the given extensions are checked // if we specify a directory (and not a glob pattern). - main_command() + main_command!() .arg("--verbose") .arg("--extensions=ts,html") .arg(input) @@ -2735,10 +2719,10 @@ mod cli { /// The extensions should only apply to the directory path, not the glob pattern. #[test] fn test_extensions_apply_to_files_not_globs() { - let glob_input = fixtures_path().join("glob_dir/**/*.tsx"); - let dir_input = fixtures_path().join("example_dir"); + let glob_input = fixtures_path!().join("glob_dir/**/*.tsx"); + let dir_input = fixtures_path!().join("example_dir"); - main_command() + main_command!() .arg("--verbose") .arg("--extensions=html,md") .arg(glob_input) @@ -2764,10 +2748,10 @@ mod cli { /// extension does not match the given extensions. #[test] fn test_file_inputs_always_get_checked_no_matter_their_extension() { - let ts_input_file = fixtures_path().join("glob_dir/example.ts"); - let md_input_file = fixtures_path().join("glob_dir/example.md"); + let ts_input_file = fixtures_path!().join("glob_dir/example.ts"); + let md_input_file = fixtures_path!().join("glob_dir/example.md"); - main_command() + main_command!() .arg("--verbose") .arg("--dump") .arg("--extensions=html,md") @@ -2787,7 +2771,7 @@ mod cli { fn test_url_inputs_always_get_checked_no_matter_their_extension() { let url_input = "https://example.com/sitemap.xml"; - main_command() + main_command!() .arg("--verbose") .arg("--dump") .arg(url_input) @@ -2807,7 +2791,7 @@ mod cli { fs::write(&test_md, "# Test\n[link](https://example.com)")?; fs::write(&files_list_path, test_md.to_string_lossy().as_ref())?; - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--files-from") .arg(&files_list_path) .arg("--dump-inputs") @@ -2826,7 +2810,7 @@ mod cli { // Create test file fs::write(&test_md, "# Test\n[link](https://example.com)")?; - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--files-from") .arg("-") .arg("--dump-inputs") @@ -2854,7 +2838,7 @@ mod cli { ), )?; - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--files-from") .arg(&files_list_path) .arg("--dump-inputs") @@ -2877,7 +2861,7 @@ mod cli { fs::write(&test_md2, "# Test 2")?; fs::write(&files_list_path, test_md1.to_string_lossy().as_ref())?; - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--files-from") .arg(&files_list_path) .arg(&test_md2) // Regular input argument @@ -2892,7 +2876,7 @@ mod cli { #[test] fn test_files_from_nonexistent_file_error() -> Result<()> { - let mut cmd = main_command(); + let mut cmd = main_command!(); cmd.arg("--files-from") .arg("/nonexistent/file.txt") .arg("--dump-inputs") @@ -2913,7 +2897,7 @@ mod cli { writeln!(file_without_ext, "[Local](local.md)")?; // Test with --default-extension md - main_command() + main_command!() .arg("--default-extension") .arg("md") .arg("--dump") @@ -2932,7 +2916,7 @@ mod cli { writeln!(html_file_without_ext, "")?; // Test with --default-extension html - main_command() + main_command!() .arg("--default-extension") .arg("html") .arg("--dump") @@ -2954,7 +2938,7 @@ mod cli { // Unknown extensions should fall back to default behavior (plaintext) // and still extract links from the content - main_command() + main_command!() .arg("--default-extension") .arg("unknown") .arg("--dump") @@ -2969,7 +2953,7 @@ mod cli { fn test_input_matching_nothing_warns() -> Result<()> { let empty_dir = tempdir()?; - main_command() + main_command!() .arg(format!("{}", empty_dir.path().to_string_lossy())) .arg(format!("{}/*", empty_dir.path().to_string_lossy())) .arg("non-existing-path/*") diff --git a/lychee-bin/tests/usage.rs b/lychee-bin/tests/usage.rs index ce078745ce..469ca60efe 100644 --- a/lychee-bin/tests/usage.rs +++ b/lychee-bin/tests/usage.rs @@ -1,25 +1,15 @@ #[cfg(test)] mod readme { - use std::{fs, path::Path}; - use assert_cmd::Command; use pretty_assertions::assert_eq; - - const USAGE_STRING: &str = "Usage: lychee [OPTIONS] [inputs]...\n"; + use regex::Regex; + use test_utils::load_readme_text; fn main_command() -> Command { // this gets the "main" binary name (e.g. `lychee`) Command::cargo_bin(env!("CARGO_PKG_NAME")).expect("Couldn't get cargo package name") } - fn load_readme_text() -> String { - let readme_path = Path::new(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .join("README.md"); - fs::read_to_string(readme_path).unwrap() - } - /// Remove line `[default: lychee/x.y.z]` from the string fn remove_lychee_version_line(string: &str) -> String { string @@ -44,20 +34,15 @@ mod readme { #[test] #[cfg(unix)] fn test_readme_usage_up_to_date() -> Result<(), Box> { + const BEGIN: &str = "```help-message\n"; let mut cmd = main_command(); let help_cmd = cmd.env_clear().arg("--help").assert().success(); - let help_output = std::str::from_utf8(&help_cmd.get_output().stdout)?; - let usage_in_help_start = help_output - .find(USAGE_STRING) - .ok_or("Usage not found in help")?; - let usage_in_help = &help_output[usage_in_help_start..]; + let usage_in_help = std::str::from_utf8(&help_cmd.get_output().stdout)?; let usage_in_help = trim_empty_lines(&remove_lychee_version_line(usage_in_help)); - let readme = load_readme_text(); - let usage_start = readme - .find(USAGE_STRING) - .ok_or("Usage not found in README")?; + let readme = load_readme_text!(); + let usage_start = readme.find(BEGIN).ok_or("Usage not found in README")? + BEGIN.len(); let usage_end = readme[usage_start..] .find("\n```") .ok_or("End of usage not found in README")?; @@ -67,4 +52,51 @@ mod readme { assert_eq!(usage_in_readme, usage_in_help); Ok(()) } + + /// Test that all the arguments yielded by `lychee --help` + /// are ordered alphabetically for better usability. + /// This behaviour aligns with cURL. (see `man curl`) + #[test] + #[cfg(unix)] + fn test_arguments_ordered_alphabetically() -> Result<(), Box> { + let mut cmd = main_command(); + let help_cmd = cmd.env_clear().arg("--help").assert().success(); + let help_text = std::str::from_utf8(&help_cmd.get_output().stdout)?; + + let regex = Regex::new(r"^\s{2,6}(?:-(?[a-zA-Z]),)?\s--(?[a-zA-Z-]+)")?; + + let arguments: Vec<&str> = help_text + .lines() + .filter_map(|line| { + let captures = regex.captures(line)?; + captures + .name("short") + .or_else(|| captures.name("long")) + .map(|m| m.as_str()) + }) + .collect(); + + let mut sorted = arguments.clone(); + sorted.sort_by_key(|arg| arg.to_lowercase()); + + if arguments != sorted { + // Find all positions where order differs + let mismatches: Vec<_> = arguments + .iter() + .zip(&sorted) + .enumerate() + .filter(|(_, (a, b))| a != b) + .map(|(i, (actual, expected))| format!(" [{i}] '{actual}' should be '{expected}'")) + .collect(); + + panic!( + "\nArguments are not sorted alphabetically!\n\nMismatches:\n{}\n\nFull actual order:\n{:?}\n\nFull expected order:\n{:?}", + mismatches.join("\n"), + arguments, + sorted + ); + } + + Ok(()) + } } diff --git a/lychee-lib/Cargo.toml b/lychee-lib/Cargo.toml index 04f1c2ce8e..d5f2235455 100644 --- a/lychee-lib/Cargo.toml +++ b/lychee-lib/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lychee-lib" -authors = ["Matthias Endler "] +authors = ["Matthias Endler ", "Thomas Zahner "] description = "A fast, async link checker" documentation = "https://docs.rs/lychee_lib" edition = "2024" diff --git a/test-utils/src/lib.rs b/test-utils/src/lib.rs index 17a150a243..4e949b3433 100644 --- a/test-utils/src/lib.rs +++ b/test-utils/src/lib.rs @@ -107,11 +107,18 @@ macro_rules! mail { }}; } -/// Returns the path to the `fixtures` directory. -/// -/// # Panic -/// -/// Panics if the fixtures directory could not be determined. +/// Get the root path of the project. +#[macro_export] +macro_rules! root_path { + () => { + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .to_path_buf() + }; +} + +/// Get the path to the `fixtures` directory. #[macro_export] macro_rules! fixtures_path { () => { @@ -149,3 +156,33 @@ macro_rules! fixture_uri { .expect("expected subpath to form a valid URL") }}; } + +#[macro_export] +macro_rules! load_readme_text { + () => {{ + let readme_path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .join("README.md"); + std::fs::read_to_string(readme_path).unwrap() + }}; +} + +/// Helper function to create a ResponseBody with a given status and URI +#[macro_export] +macro_rules! mock_response_body { + ($status:expr, $uri:expr $(,)?) => {{ + ResponseBody { + uri: Uri::try_from($uri).unwrap(), + status: $status, + } + }}; +} + +/// Gets the "main" binary name (e.g. `lychee`) +#[macro_export] +macro_rules! main_command { + () => { + Command::cargo_bin(env!("CARGO_PKG_NAME")).expect("Couldn't get cargo package name") + }; +}