Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
4a3a2a1
add index_files option
katrinafyi Jul 24, 2025
a1fa4b4
fix compilation
katrinafyi Jul 26, 2025
e8d07c3
phrasing
katrinafyi Jul 26, 2025
5296d23
fjdisaofjdisoajfidosa
katrinafyi Jul 26, 2025
73a6a4d
early switch between fallback extensions and index files
katrinafyi Jul 26, 2025
23b8092
docs and it compiles now
katrinafyi Jul 26, 2025
c3fab82
COW
katrinafyi Jul 26, 2025
51afc96
update readme text
katrinafyi Jul 27, 2025
2fc3f29
Merge remote-tracking branch 'origin/master' into katrinafyi-patch-1
katrinafyi Jul 27, 2025
a59ea60
increment errors due to not searching index.html by default
katrinafyi Jul 27, 2025
5173748
tests and fixtures
katrinafyi Jul 27, 2025
118a6f0
fmt and fix test
katrinafyi Jul 27, 2025
c2504dc
lint
katrinafyi Jul 27, 2025
67322d7
add more corner cases to tests
katrinafyi Jul 28, 2025
b55f493
separate test cases for corner cases
katrinafyi Jul 28, 2025
b217503
fix index_files default in ClientBuilder (oops!)
katrinafyi Jul 30, 2025
bb55866
Merge remote-tracking branch 'origin/master' into katrinafyi-patch-1
katrinafyi Jul 30, 2025
4362cdf
tests: now expected to fail `#a-link-inside-index-html-inside-sub-dir`
katrinafyi Jul 31, 2025
8d25acd
revert README.md change where we hardcoded the version
katrinafyi Jul 31, 2025
f6b678e
review: make `index_files` an Option for clearer default behavior
katrinafyi Jul 31, 2025
1d76955
docs: add `--index-files ''` and `--index-files index.html,.`
katrinafyi Aug 1, 2025
ed499f7
tests: add cli tests for `--index-files`
katrinafyi Aug 1, 2025
a4f7497
review: confirm path traversal is allowed in `--index-files`
katrinafyi Aug 2, 2025
0fd19b7
review: typo
katrinafyi Aug 7, 2025
3490256
fix tests bug, where an assertion was weaker than desired
katrinafyi Aug 7, 2025
82b537b
Merge branch 'master' into katrinafyi-patch-1
katrinafyi Aug 8, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 29 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -452,12 +452,38 @@ Options:
Remap URI matching pattern to different URI

--fallback-extensions <FALLBACK_EXTENSIONS>
Test the specified file extensions for URIs when checking files locally.
Multiple extensions can be separated by commas. Extensions will be checked in
order of appearance.
When checking locally, attempts to locate missing files by trying the given
fallback extensions. Multiple extensions can be separated by commas. Extensions
will be checked in order of appearance.

Example: --fallback-extensions html,htm,php,asp,aspx,jsp,cgi

Note: This option only takes effect on `file://` URIs which do not exist.

--index-files <INDEX_FILES>
When checking locally, resolves directory links to a separate index file.
The argument is a comma-separated list of index file names to search for. Index
names are relative to the link's directory and attempted in the order given.

If `--index-files` is specified, then at least one index file must exist in
order for a directory link to be considered valid. Additionally, the special
name `.` can be used in the list to refer to the directory itself.

If unspecified (the default behavior), index files are disabled and directory
links are considered valid as long as the directory exists on disk.

Example 1: `--index-files index.html,readme.md` looks for index.html or readme.md
and requires that at least one exists.

Example 2: `--index-files index.html,.` will use index.html if it exists, but
still accept the directory link regardless.

Example 3: `--index-files ''` will reject all directory links because there are
no valid index files. This will require every link to explicitly name
a file.

Note: This option only takes effect on `file://` URIs which exist and point to a directory.

-H, --header <HEADER:VALUE>
Set custom header for requests

Expand Down
5 changes: 5 additions & 0 deletions fixtures/filechecker/dir_links.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
[a](empty_dir)
[a](empty_dir#fragment)

[a](index_dir)
[a](index_dir#fragment)
Empty file.
Empty file.
1 change: 1 addition & 0 deletions fixtures/filechecker/index_dir/index.html
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
<p id="fragment">boop</p>
Empty file.
Empty file.
Empty file.
4 changes: 2 additions & 2 deletions fixtures/fragments-fallback-extensions/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@
</head>

<body>
<a href="sub_dir#valid-id">1</a>
<a href="sub_dir#invalid-id">2</a>
<a href="sub_dir/index#valid-id">1</a>
<a href="sub_dir/index#invalid-id">2</a>
<a href="empty_dir#invalid-id">3</a>
</body>

Expand Down
1 change: 1 addition & 0 deletions lychee-bin/src/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ pub(crate) fn create(cfg: &Config, cookie_jar: Option<&Arc<CookieStoreMutex>>) -
.min_tls_version(cfg.min_tls.clone().map(Into::into))
.include_fragments(cfg.include_fragments)
.fallback_extensions(cfg.fallback_extensions.clone())
.index_files(cfg.index_files.clone())
.build()
.client()
.context("Failed to create request client")
Expand Down
42 changes: 37 additions & 5 deletions lychee-bin/src/options.rs
Original file line number Diff line number Diff line change
Expand Up @@ -544,19 +544,51 @@ and 501."
#[arg(long)]
pub(crate) remap: Vec<String>,

/// Automatically append file extensions to `file://` URIs as needed
/// Automatically append file extensions to `file://` URIs for non-existing paths
#[serde(default)]
#[arg(
long,
value_delimiter = ',',
long_help = "Test the specified file extensions for URIs when checking files locally.
Multiple extensions can be separated by commas. Extensions will be checked in
order of appearance.
long_help = "When checking locally, attempts to locate missing files by trying the given
fallback extensions. Multiple extensions can be separated by commas. Extensions
will be checked in order of appearance.

Example: --fallback-extensions html,htm,php,asp,aspx,jsp,cgi

Example: --fallback-extensions html,htm,php,asp,aspx,jsp,cgi"
Note: This option only takes effect on `file://` URIs which do not exist."
)]
pub(crate) fallback_extensions: Vec<String>,

/// Resolve local directory links to specified index files within the directory
#[serde(default)]
#[arg(
long,
value_delimiter = ',',
long_help = "When checking locally, resolves directory links to a separate index file.
The argument is a comma-separated list of index file names to search for. Index
names are relative to the link's directory and attempted in the order given.

If `--index-files` is specified, then at least one index file must exist in
order for a directory link to be considered valid. Additionally, the special
name `.` can be used in the list to refer to the directory itself.

If unspecified (the default behavior), index files are disabled and directory
links are considered valid as long as the directory exists on disk.

Example 1: `--index-files index.html,readme.md` looks for index.html or readme.md
and requires that at least one exists.

Example 2: `--index-files index.html,.` will use index.html if it exists, but
still accept the directory link regardless.

Example 3: `--index-files ''` will reject all directory links because there are
no valid index files. This will require every link to explicitly name
a file.

Note: This option only takes effect on `file://` URIs which exist and point to a directory."
)]
pub(crate) index_files: Option<Vec<String>>,

/// Set custom header for requests
#[arg(
short = 'H',
Expand Down
96 changes: 95 additions & 1 deletion lychee-bin/tests/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1908,7 +1908,6 @@ mod cli {
"fixtures/fragments/file.html#top",
"fixtures/fragments/file.html#Upper-%C3%84%C3%96%C3%B6",
"fixtures/fragments/sub_dir",
"fixtures/fragments/sub_dir#a-link-inside-index-html-inside-sub-dir",
"fixtures/fragments/zero.bin",
"fixtures/fragments/zero.bin#",
"fixtures/fragments/zero.bin#fragment",
Expand All @@ -1922,6 +1921,7 @@ mod cli {
let expected_failures = vec![
"fixtures/fragments/sub_dir_non_existing_1",
"fixtures/fragments/sub_dir#non-existing-fragment-2",
"fixtures/fragments/sub_dir#a-link-inside-index-html-inside-sub-dir",
"fixtures/fragments/empty_dir#non-existing-fragment-3",
"fixtures/fragments/file2.md#missing-fragment",
"fixtures/fragments/sub_dir#non-existing-fragment-1",
Expand Down Expand Up @@ -2288,4 +2288,98 @@ mod cli {
.success()
.stdout(contains("https://www.example.com/smth."));
}

#[test]
fn test_index_files_default() {
let input = fixtures_path().join("filechecker/dir_links.md");

// the dir links in this file all exist.
main_command()
.arg(&input)
.arg("--verbose")
.assert()
.success();

// ... but checking fragments will find none, because dirs
// have no fragments and no index file given.
let dir_links_with_fragment = 2;
main_command()
.arg(&input)
.arg("--include-fragments")
.assert()
.failure()
.stdout(contains("Cannot find fragment").count(dir_links_with_fragment))
.stdout(contains("#").count(dir_links_with_fragment));
}

#[test]
fn test_index_files_specified() {
let input = fixtures_path().join("filechecker/dir_links.md");

// passing `--index-files index.html` should reject all links
// to /empty_dir because it doesn't have the index file
let result = main_command()
.arg(input)
.arg("--index-files")
.arg("index.html")
.arg("--verbose")
.assert()
.failure();

let empty_dir_links = 2;
let index_dir_links = 2;
result
.stdout(contains("Cannot find index file").count(empty_dir_links))
.stdout(contains("/empty_dir").count(empty_dir_links))
.stdout(contains(format!("{index_dir_links} OK")));
}

#[test]
fn test_index_files_dot_in_list() {
let input = fixtures_path().join("filechecker/dir_links.md");

// passing `.` in the index files list should accept a directory
// even if no other index file is found.
main_command()
.arg(&input)
.arg("--index-files")
.arg("index.html,.")
.assert()
.success()
.stdout(contains("4 OK"));

// checking fragments will accept the index_dir#fragment link,
// but reject empty_dir#fragment because empty_dir doesn’t have
// index.html.
main_command()
.arg(&input)
.arg("--index-files")
.arg("index.html,.")
.arg("--include-fragments")
.assert()
.failure()
.stdout(contains("Cannot find fragment").count(1))
.stdout(contains("empty_dir#fragment").count(1))
.stdout(contains("index_dir#fragment").count(0))
.stdout(contains("3 OK"));
}

#[test]
fn test_index_files_empty_list() {
let input = fixtures_path().join("filechecker/dir_links.md");

// passing an empty list to --index-files should reject /all/
// directory links.
let result = main_command()
.arg(input)
.arg("--index-files")
.arg("")
.assert()
.failure();

let num_dir_links = 4;
result
.stdout(contains("Cannot find index file").count(num_dir_links))
.stdout(contains("0 OK"));
}
}
Loading