fix: Tolerate transient input files

crates/turborepo-scm/src/crlf.rs

@@ -286,7 +286,7 @@ fn stream_normalized(reader: &mut impl Read, mut update: impl FnMut(&[u8])) -> s
 
 /// Incrementally computes a git blob oid using the `sha1` crate, which
 /// dispatches to hardware SHA extensions at runtime (SHA-NI on x86, the
-/// crypto extensions on aarch64). Both [`hash_file_maybe_normalized`] and
+/// crypto extensions on aarch64). Both [`hash_file_as_git_blob`] and
 /// [`manual_hash_file_maybe_normalized`] delegate to
 /// [`hash_file_normalized`], which drives this hasher.
 struct BlobHasher(Sha1);
@@ -432,18 +432,28 @@ fn hash_file_normalized(
 /// place a crafted collision pair in the repo can already modify the build
 /// itself. Outputs are bit-identical to git for all non-colliding inputs,
 /// enforced by differential tests against `git hash-object` below.
-pub(crate) fn hash_file_maybe_normalized(
+pub(crate) fn hash_file_as_git_blob(
     path: &AbsoluteSystemPath,
     attr: TextAttr,
-) -> Result<OidHash, std::io::Error> {
+) -> Result<Option<OidHash>, std::io::Error> {
+    // Avoid opening sockets, FIFOs, devices, and directories. Metadata from the
+    // opened handle below remains authoritative if the path changes afterward.
+    if !std::fs::metadata(path)?.is_file() {
+        return Ok(None);
+    }
+
     let mut file = std::fs::File::open(path)?;
     let metadata = file.metadata()?;
-    validate_file_type(path, &metadata)?;
-    Ok(hash_file_normalized(&mut file, metadata.len(), attr)?.0)
+    if !metadata.is_file() {
+        return Ok(None);
+    }
+    Ok(Some(
+        hash_file_normalized(&mut file, metadata.len(), attr)?.0,
+    ))
 }
 
-/// Like [`hash_file_maybe_normalized`], but also reports how the hash was
-/// produced. Used by the repo-index verification pass.
+/// Hash a working-tree file and report how the hash was produced. Used by the
+/// repo-index verification pass.
 pub(crate) fn hash_file_for_verification(
     path: &AbsoluteSystemPath,
     attr: TextAttr,
@@ -721,7 +731,9 @@ mod tests {
             let expected = expected.trim();
 
             let path = root.join_component(name);
-            let actual = hash_file_maybe_normalized(&path, TextAttr::Auto).unwrap();
+            let actual = hash_file_as_git_blob(&path, TextAttr::Auto)
+                .unwrap()
+                .unwrap();
 
             assert_eq!(
                 &*actual, expected,
@@ -804,7 +816,7 @@ mod tests {
             let expected = String::from_utf8(output.stdout).unwrap();
             let expected = expected.trim();
 
-            let fast_result = hash_file_maybe_normalized(&path, *attr).unwrap();
+            let fast_result = hash_file_as_git_blob(&path, *attr).unwrap().unwrap();
             let manual_result = manual_hash_file_maybe_normalized(&path, *attr).unwrap();
 
             assert_eq!(
@@ -843,7 +855,7 @@ mod tests {
         assert!(GitAttrs::load(&root).is_none());
     }
 
-    // -- hash_file_maybe_normalized edge-case tests --
+    // -- hash_file_as_git_blob edge-case tests --
 
     #[test]
     fn test_hash_binary_file_with_auto_is_raw() {
@@ -853,10 +865,14 @@ mod tests {
         std::fs::write(path.as_std_path(), &content).unwrap();
 
         // With Auto, binary should be hashed raw (no normalization)
-        let auto_result = hash_file_maybe_normalized(&path, TextAttr::Auto).unwrap();
+        let auto_result = hash_file_as_git_blob(&path, TextAttr::Auto)
+            .unwrap()
+            .unwrap();
 
         // With Unspecified, should also be raw
-        let raw_result = hash_file_maybe_normalized(&path, TextAttr::Unspecified).unwrap();
+        let raw_result = hash_file_as_git_blob(&path, TextAttr::Unspecified)
+            .unwrap()
+            .unwrap();
 
         // Both should produce the same hash (raw bytes)
         assert_eq!(
@@ -871,14 +887,20 @@ mod tests {
         let path = root.join_component("text.txt");
         std::fs::write(path.as_std_path(), b"a\r\nb\r\n").unwrap();
 
-        let auto_hash = hash_file_maybe_normalized(&path, TextAttr::Auto).unwrap();
-        let set_hash = hash_file_maybe_normalized(&path, TextAttr::Set).unwrap();
+        let auto_hash = hash_file_as_git_blob(&path, TextAttr::Auto)
+            .unwrap()
+            .unwrap();
+        let set_hash = hash_file_as_git_blob(&path, TextAttr::Set)
+            .unwrap()
+            .unwrap();
 
         // Both Auto and Set should normalize CRLF for a text file
         assert_eq!(auto_hash, set_hash);
 
         // Unspecified should hash raw (different from normalized)
-        let raw_hash = hash_file_maybe_normalized(&path, TextAttr::Unspecified).unwrap();
+        let raw_hash = hash_file_as_git_blob(&path, TextAttr::Unspecified)
+            .unwrap()
+            .unwrap();
         assert_ne!(
             auto_hash, raw_hash,
             "normalized hash should differ from raw for CRLF content"
@@ -924,7 +946,9 @@ mod tests {
         let expected = expected.trim();
 
         let path = root.join_component(name);
-        let actual = hash_file_maybe_normalized(&path, TextAttr::Auto).unwrap();
+        let actual = hash_file_as_git_blob(&path, TextAttr::Auto)
+            .unwrap()
+            .unwrap();
 
         assert_eq!(
             &*actual, expected,

crates/turborepo-scm/src/hash_object.rs

@@ -1,5 +1,5 @@
 use rayon::prelude::*;
-use tracing::debug;
+use tracing::{debug, info};
 use turbopath::{AbsoluteSystemPath, AnchoredSystemPathBuf, RelativeUnixPath, RelativeUnixPathBuf};
 
 use crate::{Error, GitHashes, OidHash};
@@ -8,6 +8,12 @@ const MAX_RETRIES: u32 = 10;
 const BASE_DELAY_MS: u64 = 10;
 const MAX_DELAY_MS: u64 = 1000;
 
+#[derive(Clone, Copy)]
+enum MissingFiles {
+    Error,
+    Ignore,
+}
+
 pub(crate) fn with_emfile_retry<T>(
     f: impl Fn() -> Result<T, std::io::Error>,
 ) -> Result<T, std::io::Error> {
@@ -48,6 +54,45 @@ pub(crate) fn hash_objects(
     hashes: &mut GitHashes,
     cached_attrs: Option<&crate::crlf::GitAttrs>,
     slowest_files: Option<&std::sync::Arc<crate::SlowestFiles>>,
+) -> Result<(), Error> {
+    hash_objects_inner(
+        git_root,
+        pkg_path,
+        to_hash,
+        hashes,
+        MissingFiles::Error,
+        cached_attrs,
+        slowest_files,
+    )
+}
+
+pub(crate) fn hash_discovered_objects(
+    git_root: &AbsoluteSystemPath,
+    pkg_path: &AbsoluteSystemPath,
+    to_hash: Vec<RelativeUnixPathBuf>,
+    hashes: &mut GitHashes,
+    cached_attrs: Option<&crate::crlf::GitAttrs>,
+    slowest_files: Option<&std::sync::Arc<crate::SlowestFiles>>,
+) -> Result<(), Error> {
+    hash_objects_inner(
+        git_root,
+        pkg_path,
+        to_hash,
+        hashes,
+        MissingFiles::Ignore,
+        cached_attrs,
+        slowest_files,
+    )
+}
+
+fn hash_objects_inner(
+    git_root: &AbsoluteSystemPath,
+    pkg_path: &AbsoluteSystemPath,
+    to_hash: Vec<RelativeUnixPathBuf>,
+    hashes: &mut GitHashes,
+    missing_files: MissingFiles,
+    cached_attrs: Option<&crate::crlf::GitAttrs>,
+    slowest_files: Option<&std::sync::Arc<crate::SlowestFiles>>,
 ) -> Result<(), Error> {
     let pkg_prefix = git_root.anchor(pkg_path).ok().map(|a| a.to_unix());
 
@@ -72,12 +117,12 @@ pub(crate) fn hash_objects(
 
                 let _guard = slowest_files.map(|sf| sf.start(filename.clone()));
                 let hash_result = with_emfile_retry(|| {
-                    crate::crlf::hash_file_maybe_normalized(&full_file_path, text_attr)
+                    crate::crlf::hash_file_as_git_blob(&full_file_path, text_attr)
                 });
                 drop(_guard);
 
                 match hash_result {
-                    Ok(hash) => {
+                    Ok(Some(hash)) => {
                         let package_relative_path = pkg_prefix
                             .as_ref()
                             .and_then(|prefix| {
@@ -94,19 +139,18 @@ pub(crate) fn hash_objects(
                             });
                         Ok(Some((package_relative_path, hash)))
                     }
-                    Err(e) => {
-                        // Gracefully skip non-regular files (symlinks, sockets,
-                        // FIFOs, device nodes) that can't be read as normal files.
-                        if full_file_path
-                            .symlink_metadata()
-                            .map(|md| !md.is_file())
-                            .unwrap_or(false)
-                        {
-                            Ok(None)
-                        } else {
-                            Err(Error::git_error(format!("{}: {}", full_file_path, e)))
-                        }
+                    Ok(None) => {
+                        info!(path = %full_file_path, "skipping non-regular hash candidate");
+                        Ok(None)
+                    }
+                    Err(error)
+                        if matches!(missing_files, MissingFiles::Ignore)
+                            && error.kind() == std::io::ErrorKind::NotFound =>
+                    {
+                        info!(path = %full_file_path, "discovered hash candidate disappeared");
+                        Ok(None)
                     }
+                    Err(error) => Err(Error::hash_file(full_file_path, error)),
                 }
             },
         )
@@ -124,7 +168,7 @@ pub(crate) fn hash_objects(
 mod test {
     use turbopath::{AbsoluteSystemPathBuf, RelativeUnixPathBuf, RelativeUnixPathBufTestExt};
 
-    use super::hash_objects;
+    use super::{hash_discovered_objects, hash_objects};
     use crate::{GitHashes, OidHash, find_git_root};
 
     #[test]
@@ -180,26 +224,62 @@ mod test {
             hash_objects(&git_root, pkg_path, to_hash, &mut hashes, None, None).unwrap();
             assert_eq!(hashes, expected_hashes);
         }
+    }
 
-        // paths for files here are relative to the package path.
-        let error_tests: Vec<(Vec<&str>, &AbsoluteSystemPathBuf)> = vec![
-            // skipping test for outside of git repo, we now error earlier in the process
-            (vec!["nonexistent.json"], &fixture_path),
-        ];
+    #[test]
+    fn test_vanished_candidate_is_skipped() {
+        let tmp = tempfile::tempdir().unwrap();
+        let git_root = AbsoluteSystemPathBuf::try_from(tmp.path()).unwrap();
+        let candidate = git_root.join_component("transient.lock");
+        candidate.create_with_contents("lock").unwrap();
 
-        for (to_hash, pkg_path) in error_tests {
-            let git_to_pkg_path = git_root.anchor(pkg_path).unwrap();
-            let pkg_prefix = git_to_pkg_path.to_unix();
+        // Model a glob walk finding a transient file that disappears before
+        // the parallel blob-hashing phase starts.
+        let to_hash = vec![RelativeUnixPathBuf::new("transient.lock").unwrap()];
+        candidate.remove().unwrap();
 
-            let to_hash = to_hash
-                .into_iter()
-                .map(|k| pkg_prefix.join(&RelativeUnixPathBuf::new(k).unwrap()))
-                .collect();
+        let mut hashes = GitHashes::new();
+        hash_discovered_objects(&git_root, &git_root, to_hash, &mut hashes, None, None).unwrap();
+        assert!(hashes.is_empty());
+    }
 
-            let mut hashes = GitHashes::new();
-            let result = hash_objects(&git_root, pkg_path, to_hash, &mut hashes, None, None);
-            assert!(result.is_err());
-        }
+    #[test]
+    fn test_missing_explicit_file_errors() {
+        let tmp = tempfile::tempdir().unwrap();
+        let git_root = AbsoluteSystemPathBuf::try_from(tmp.path()).unwrap();
+        let missing = RelativeUnixPathBuf::new("missing.json").unwrap();
+
+        let mut hashes = GitHashes::new();
+        let error =
+            hash_objects(&git_root, &git_root, vec![missing], &mut hashes, None, None).unwrap_err();
+
+        assert!(error.to_string().contains("missing.json"));
+    }
+
+    #[cfg(unix)]
+    #[test]
+    fn test_non_regular_symlink_is_skipped() {
+        use std::os::unix::net::UnixListener;
+
+        let tmp = tempfile::tempdir().unwrap();
+        let git_root = AbsoluteSystemPathBuf::try_from(tmp.path()).unwrap();
+        let socket = git_root.join_component("server.sock");
+        let _listener = UnixListener::bind(socket.as_std_path()).unwrap();
+        let link = git_root.join_component("socket-link");
+        link.symlink_to_file(socket.to_string()).unwrap();
+
+        let mut hashes = GitHashes::new();
+        hash_objects(
+            &git_root,
+            &git_root,
+            vec![RelativeUnixPathBuf::new("socket-link").unwrap()],
+            &mut hashes,
+            None,
+            None,
+        )
+        .unwrap();
+
+        assert!(hashes.is_empty());
     }
 
     /// Verify that our blob hashing produces OIDs identical to `git

crates/turborepo-scm/src/lib.rs

@@ -55,6 +55,14 @@ pub enum Error {
     GitVersion(String),
     #[error("I/O error: {0}")]
     Io(#[from] std::io::Error, #[backtrace] backtrace::Backtrace),
+    #[error("I/O error while hashing {path}: {source}")]
+    HashFile {
+        path: AbsoluteSystemPathBuf,
+        #[source]
+        source: std::io::Error,
+        #[backtrace]
+        backtrace: backtrace::Backtrace,
+    },
     #[error("Path error: {0}")]
     Path(#[from] PathError, #[backtrace] backtrace::Backtrace),
     #[error("Could not find git binary")]
@@ -123,11 +131,21 @@ impl Error {
         Error::Git(s.into(), Backtrace::capture())
     }
 
+    pub(crate) fn hash_file(path: AbsoluteSystemPathBuf, source: std::io::Error) -> Self {
+        tracing::info!(%path, error = %source, "file hashing failed");
+        Error::HashFile {
+            path,
+            source,
+            backtrace: Backtrace::capture(),
+        }
+    }
+
     /// Returns true if this error indicates OS resource exhaustion (e.g. too
     /// many open files) where a fallback to manual hashing would also fail.
     pub fn is_resource_exhaustion(&self) -> bool {
         match self {
             Error::Io(e, _) => is_os_resource_error(e),
+            Error::HashFile { source, .. } => is_os_resource_error(source),
             Error::Walk(e) => walk_error_is_resource_exhaustion(e),
             _ => false,
         }