diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ea8da43dacea7..46eebf9e7fb55 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -75,6 +75,14 @@ jobs: sudo mount -t tmpfs -o size=256m tmpfs /tmpfs sudo chown "$(id -u):$(id -g)" /tmpfs + - name: "Create minix filesystem (low hardlink limit)" + run: | + truncate -s 16M /tmp/minix.img + mkfs.minix /tmp/minix.img + sudo mkdir /minix + sudo mount -o loop /tmp/minix.img /minix + sudo chown "$(id -u):$(id -g)" /minix + - name: "Install cargo nextest" uses: taiki-e/install-action@542cebaaed782771e619bd5609d97659d109c492 # v2.66.7 with: @@ -88,6 +96,7 @@ jobs: UV_INTERNAL__TEST_COW_FS: /btrfs UV_INTERNAL__TEST_NOCOW_FS: /tmpfs UV_INTERNAL__TEST_ALT_FS: /tmpfs + UV_INTERNAL__TEST_LOWLINKS_FS: /minix run: | cargo nextest run \ --cargo-profile fast-build \ @@ -190,6 +199,9 @@ jobs: working-directory: ${{ env.UV_WORKSPACE }} run: rustup show + - name: "Create NTFS test directory (low hardlink limit)" + run: New-Item -Path "C:\uv" -ItemType Directory -Force + - name: "Install cargo nextest" uses: taiki-e/install-action@542cebaaed782771e619bd5609d97659d109c492 # v2.66.7 with: @@ -204,6 +216,7 @@ jobs: # See https://github.com/astral-sh/uv/issues/6940 UV_LINK_MODE: copy RUST_BACKTRACE: 1 + UV_INTERNAL__TEST_LOWLINKS_FS: "C:\\uv" shell: bash run: | cargo nextest run \ diff --git a/crates/uv-fs/src/link.rs b/crates/uv-fs/src/link.rs index 8b73c3b1a8b01..1281c8562fda0 100644 --- a/crates/uv-fs/src/link.rs +++ b/crates/uv-fs/src/link.rs @@ -765,9 +765,32 @@ where }) } -/// Try to create a hard link, returning the `io::Error` on failure. +/// Try to create a hard link, handling `TooManyLinks` (EMLINK/`ERROR_TOO_MANY_LINKS`) +/// by copying the source to a fresh inode and retrying. fn try_hardlink_file(src: &Path, dst: &Path) -> io::Result<()> { - fs_err::hard_link(src, dst) + match fs_err::hard_link(src, dst) { + Ok(()) => Ok(()), + Err(err) if err.kind() == io::ErrorKind::TooManyLinks => { + debug!( + "Hit link limit for {}, creating a fresh copy", + src.display() + ); + let mut parent = src.parent().unwrap_or(Path::new(".")); + if parent.as_os_str().is_empty() { + parent = Path::new("."); + } + let temp = tempfile::NamedTempFile::new_in(parent)?; + // This is a benign race. It can effectively lead to the destination being an + // independent copy. + fs_err::copy(src, temp.path())?; + // Linking a copy before renaming avoids the unlikely race where another process could + // exhaust the fresh inode's links between the rename and our link. + fs_err::hard_link(temp.path(), dst)?; + fs_err::rename(temp.path(), src)?; + Ok(()) + } + Err(err) => Err(err), + } } /// Atomically overwrite an existing file with a hard link. @@ -786,7 +809,7 @@ where let tempdir = tempfile::tempdir_in(parent)?; let tempfile = tempdir.path().join(dst.file_name().unwrap()); - if fs_err::hard_link(src, &tempfile).is_ok() { + if try_hardlink_file(src, &tempfile).is_ok() { fs_err::rename(&tempfile, dst)?; Ok(state.mode_working()) } else { diff --git a/crates/uv-static/src/env_vars.rs b/crates/uv-static/src/env_vars.rs index 9492aaabe7e22..6810316429c50 100644 --- a/crates/uv-static/src/env_vars.rs +++ b/crates/uv-static/src/env_vars.rs @@ -582,6 +582,13 @@ impl EnvVars { #[attr_added_in("0.10.5")] pub const UV_INTERNAL__TEST_ALT_FS: &'static str = "UV_INTERNAL__TEST_ALT_FS"; + /// Path to a directory on a filesystem with a low hardlink limit (e.g., minix with ~250). + /// + /// When populated, uv will run additional tests that exercise EMLINK recovery. + #[attr_hidden] + #[attr_added_in("next release")] + pub const UV_INTERNAL__TEST_LOWLINKS_FS: &'static str = "UV_INTERNAL__TEST_LOWLINKS_FS"; + /// Used to force treating an interpreter as "managed" during tests. #[attr_hidden] #[attr_added_in("0.8.0")] diff --git a/crates/uv-test/src/lib.rs b/crates/uv-test/src/lib.rs index 3bee921b9a48a..37e527eda755d 100755 --- a/crates/uv-test/src/lib.rs +++ b/crates/uv-test/src/lib.rs @@ -711,6 +711,17 @@ impl TestContext { self.with_cache_on_fs(&dir, "ALT_FS").map(Some) } + /// Use a cache directory on the filesystem specified by + /// [`EnvVars::UV_INTERNAL__TEST_LOWLINKS_FS`]. + /// + /// Returns `Ok(None)` if the environment variable is not set. + pub fn with_cache_on_lowlinks_fs(self) -> anyhow::Result> { + let Some(dir) = env::var(EnvVars::UV_INTERNAL__TEST_LOWLINKS_FS).ok() else { + return Ok(None); + }; + self.with_cache_on_fs(&dir, "LOWLINKS_FS").map(Some) + } + /// Use a cache directory on the filesystem specified by /// [`EnvVars::UV_INTERNAL__TEST_NOCOW_FS`]. /// diff --git a/crates/uv/tests/it/pip_sync.rs b/crates/uv/tests/it/pip_sync.rs index 20959f383e239..d5889355dd795 100644 --- a/crates/uv/tests/it/pip_sync.rs +++ b/crates/uv/tests/it/pip_sync.rs @@ -214,6 +214,98 @@ fn install_hardlink() -> Result<()> { Ok(()) } +/// Test that EMLINK (too many hardlinks) is handled gracefully. +/// +/// This test exhausts the hardlink limit on a cached file, then verifies that +/// a subsequent install still succeeds by resetting the file's inode. +/// +/// Requires `UV_INTERNAL__TEST_LOWLINKS_FS` pointing to a filesystem with a +/// low hardlink limit (e.g., minix with ~250). +#[test] +fn install_hardlink_after_emlink() -> anyhow::Result<()> { + use walkdir::WalkDir; + + let Some(context) = uv_test::test_context!("3.12").with_cache_on_lowlinks_fs()? else { + return Ok(()); + }; + + let requirements_txt = context.temp_dir.child("requirements.txt"); + requirements_txt.write_str("iniconfig==2.0.0")?; + + // First install to populate the cache. + context + .pip_sync() + .arg("requirements.txt") + .arg("--link-mode") + .arg("hardlink") + .assert() + .success(); + + // Find a cached .py file from the package. + let cached_file = WalkDir::new(context.cache_dir.path()) + .into_iter() + .filter_map(Result::ok) + .find(|e| e.file_type().is_file() && e.path().extension().is_some_and(|ext| ext == "py")) + .expect("should find a cached file") + .into_path(); + + // Create a temp directory to hold hardlinks on the same filesystem but outside the + // cache tree (so the installer doesn't try to install the link files). + let hardlink_dir = tempfile::tempdir_in( + context + .cache_dir + .parent() + .expect("cache dir should have a parent"), + )?; + + // Create hardlinks until we hit EMLINK or reach 66000 (minix limit is 250, ext4 is 65000). + let mut hit_emlink = false; + for i in 0..66000 { + let link_path = hardlink_dir.path().join(format!("link_{i}")); + match fs::hard_link(&cached_file, &link_path) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::TooManyLinks => { + hit_emlink = true; + break; + } + Err(err) => { + return Err(err.into()); + } + } + } + + assert!( + hit_emlink, + "Expected to hit TooManyLinks while creating hardlinks" + ); + + // Now try to install into a new venv on the same filesystem so the + // hardlink stays same-device and actually hits EMLINK (then recovers). + let venv2_dir = tempfile::tempdir_in( + context + .cache_dir + .parent() + .expect("cache dir should have a parent"), + )?; + let venv2 = venv2_dir.path().join("venv2"); + context.venv().arg(&venv2).assert().success(); + + context + .pip_sync() + .arg("requirements.txt") + .arg("--link-mode") + .arg("hardlink") + .env(EnvVars::VIRTUAL_ENV, &venv2) + .assert() + .success(); + + // Verify that another hardlink can be created after recovery. + let extra_link = hardlink_dir.path().join("post_recovery_link"); + fs::hard_link(&cached_file, &extra_link)?; + + Ok(()) +} + /// Install a package into a virtual environment using symlink semantics. #[test] #[cfg(unix)] // Windows does not allow symlinks by default