glob: validate continuation bytes when stepping over directory entry names

src/bun_core/string/immutable.rs

@@ -40,8 +40,8 @@ pub use unicode_draft::{
     copy_latin1_into_ascii, copy_latin1_into_utf8_stop_on_non_ascii, copy_latin1_into_utf16,
     copy_u8_into_u16, copy_u16_into_u8, copy_utf16_into_utf8_impl,
     element_length_cp1252_into_utf16, element_length_utf8_into_utf16, to_utf8_list_with_type_bun,
-    to_utf16_alloc_maybe_buffered, u16_is_lead, u16_is_trail, utf16_codepoint,
-    utf16_codepoint_with_fffd, wtf8_sequence,
+    to_utf16_alloc_maybe_buffered, u16_is_lead, u16_is_trail, utf8_codepoint_with_fffd,
+    utf16_codepoint, utf16_codepoint_with_fffd, wtf8_sequence,
 };
 
 /// `bun.strings.visible` — terminal-visible-width helpers. The implementation
@@ -2774,4 +2774,42 @@ mod tests {
         let out = super::convert_utf8_to_utf16_in_buffer(&mut buf, b"\xC3\xA9\xF0\x9F\x98\x80");
         assert_eq!(out, &[0x00E9, 0xD83D, 0xDE00][..]);
     }
+
+    #[test]
+    fn utf8_codepoint_with_fffd_steps_by_maximal_subpart() {
+        // (input, code_point, len, fail)
+        let cases: &[(&[u8], u32, u8, bool)] = &[
+            (b"a\x80", 0x61, 1, false),
+            (b"\xC3\xA9x", 0xE9, 2, false),
+            (b"\xE4\xB8\xAD", 0x4E2D, 3, false),
+            (b"\xF0\x9F\x98\x80js", 0x1F600, 4, false),
+            (b"\xEF\xBF\xBD", 0xFFFD, 3, false),
+            // stray continuation byte, invalid leads
+            (b"\x80x", 0xFFFD, 1, true),
+            (b"\xC0\xB8", 0xFFFD, 1, true),
+            (b"\xF5\x80\x80\x80", 0xFFFD, 1, true),
+            // lead followed by a non-continuation byte: the lead alone
+            (b"\xC3x", 0xFFFD, 1, true),
+            // second byte outside the narrowed range for E0 / ED / F0 / F4
+            (b"\xE0\x80\x80", 0xFFFD, 1, true),
+            (b"\xED\xA0\x80", 0xFFFD, 1, true),
+            (b"\xF0\x80\x80\x80", 0xFFFD, 1, true),
+            (b"\xF4\x90\x80\x80", 0xFFFD, 1, true),
+            // truncated sequences: the whole valid prefix is one replacement
+            (b"\xE4\xB8.js", 0xFFFD, 2, true),
+            (b"\xE4\xB8", 0xFFFD, 2, true),
+            (b"\xF0\x9F\x98js", 0xFFFD, 3, true),
+            (b"\xF0\x9F", 0xFFFD, 2, true),
+            (b"\xC3", 0xFFFD, 1, true),
+        ];
+        for &(input, code_point, len, fail) in cases {
+            let r = super::utf8_codepoint_with_fffd(input);
+            assert_eq!(
+                (r.code_point, r.len, r.fail),
+                (code_point, len, fail),
+                "input {:x?}",
+                input
+            );
+        }
+    }
 }

src/bun_core/string/immutable/unicode.rs

@@ -409,6 +409,26 @@ pub(super) fn convert_utf8_bytes_into_utf16(bytes: &[u8]) -> UTF16Replacement {
     convert_utf8_bytes_into_utf16_with_length(sequence, sequence_length, bytes.len())
 }
 
+/// Decodes the codepoint at the front of `bytes` (non-empty). `len` is always
+/// in `1..=bytes.len()`. An ill-formed sequence yields U+FFFD with `fail` set
+/// over its maximal subpart, the same units `to_utf16_alloc` produces when the
+/// bytes become a JS string.
+pub fn utf8_codepoint_with_fffd(bytes: &[u8]) -> UTF16Replacement {
+    let lead = bytes[0];
+    if lead < 0x80 {
+        return UTF16Replacement {
+            code_point: u32::from(lead),
+            len: 1,
+            ..Default::default()
+        };
+    }
+    let r = convert_utf8_bytes_into_utf16(bytes);
+    UTF16Replacement {
+        len: r.len.max(1),
+        ..r
+    }
+}
+
 // SWAR body moved down into `crate::strings_impl` (T0) so the canonical
 // `copy_latin1_into_utf8` is the spec-faithful fast path. Re-export here so
 // `pub use unicode_draft::copy_latin1_into_utf8_stop_on_non_ascii` in

src/glob/matcher.rs

@@ -208,9 +208,7 @@ fn glob_match_impl(
                             state.wildcard.glob_index = state.glob_index;
                             state.wildcard.path_index = state.path_index
                                 + if (state.path_index as usize) < path.len() {
-                                    u32::from(strings::wtf8_byte_sequence_length(
-                                        path[state.path_index as usize],
-                                    ))
+                                    u32::from(rune_len_at(path, state.path_index as usize))
                                 } else {
                                     1
                                 };
@@ -265,9 +263,7 @@ fn glob_match_impl(
                                 if !is_separator(path[state.path_index as usize]) {
                                     state.glob_index += 1;
                                     state.path_index +=
-                                        u32::from(strings::wtf8_byte_sequence_length(
-                                            path[state.path_index as usize],
-                                        ));
+                                        u32::from(rune_len_at(path, state.path_index as usize));
                                     continue 'main_loop;
                                 }
                                 break 'fallthrough;
@@ -292,7 +288,7 @@ fn glob_match_impl(
                                 let mut is_match = false;
 
                                 // source unicode char to match against the target + its byte length in `path`
-                                let (c, len) = decode_wtf8_rune_at(path, state.path_index as usize);
+                                let (c, len) = decode_rune_at(path, state.path_index as usize);
 
                                 while (state.glob_index as usize) < glob.len()
                                     && (first || glob[state.glob_index as usize] != b']')
@@ -619,15 +615,20 @@ fn unescape(c: &mut u8, glob: &[u8], glob_index: &mut u32) -> bool {
     true
 }
 
-/// Decodes the WTF-8 codepoint at `bytes[idx]`, returning `(codepoint, byte_len)`.
+/// Byte length of the codepoint at `bytes[idx]`; see [`decode_rune_at`].
 #[inline(always)]
-fn decode_wtf8_rune_at(bytes: &[u8], idx: usize) -> (u32, u8) {
-    let len = strings::wtf8_byte_sequence_length(bytes[idx]);
-    let mut buf = [0u8; 4];
-    let n = (bytes.len() - idx).min(4);
-    buf[..n].copy_from_slice(&bytes[idx..idx + n]);
-    let cp = strings::decode_wtf8_rune_t::<u32>(buf, len, 0xFFFD);
-    (cp, len)
+fn rune_len_at(bytes: &[u8], idx: usize) -> u8 {
+    decode_rune_at(bytes, idx).1
+}
+
+/// Decodes the codepoint at `bytes[idx]`, returning `(codepoint, byte_len)`.
+/// A directory entry need not be valid UTF-8, so this decodes the way the
+/// entry's name is decoded for JS: an ill-formed sequence is one U+FFFD over
+/// its maximal subpart, and `byte_len` never passes the end of `bytes`.
+#[inline(always)]
+fn decode_rune_at(bytes: &[u8], idx: usize) -> (u32, u8) {
+    let r = strings::utf8_codepoint_with_fffd(&bytes[idx..]);
+    (r.code_point, r.len)
 }
 
 /// Unescapes the character if needed
@@ -658,15 +659,15 @@ fn get_unicode(c: &mut u32, clen: &mut u8, glob: &[u8], glob_index: &mut u32) ->
                 b'r' => b'\r' as u32,
                 b't' => b'\t' as u32,
                 _ => 'brk: {
-                    let (cp, len) = decode_wtf8_rune_at(glob, *glob_index as usize);
+                    let (cp, len) = decode_rune_at(glob, *glob_index as usize);
                     *clen = len;
                     break 'brk cp;
                 }
             };
         }
         // multi-byte sequences
         _ => {
-            let (cp, len) = decode_wtf8_rune_at(glob, *glob_index as usize);
+            let (cp, len) = decode_rune_at(glob, *glob_index as usize);
             *clen = len;
             *c = cp;
         }

test/js/bun/glob/scan.test.ts

@@ -24,7 +24,7 @@ import { Glob, GlobScanOptions } from "bun";
 import { afterAll, beforeAll, describe, expect, test } from "bun:test";
 import { execSync } from "child_process";
 import fg from "fast-glob";
-import { bunEnv, bunExe, isWindows, tempDir, tempDirWithFiles, tmpdirSync } from "harness";
+import { bunEnv, bunExe, isLinux, isWindows, tempDir, tempDirWithFiles, tmpdirSync } from "harness";
 import * as fs from "node:fs";
 import * as path from "path";
 import { createTempDirectoryWithBrokenSymlinks, prepareEntries, tempFixturesDir } from "./util";
@@ -989,6 +989,46 @@ describe("explicit dotfile segments match without dot:true", () => {
   });
 });
 
+// Directory entries on Linux are bytes that need not be valid UTF-8. The
+// matcher must step over them in the same units the returned names use (one
+// U+FFFD per maximal ill-formed subpart), not by the lead byte's nominal
+// length, or `?` and `*` step into the next character or past the end.
+describe.skipIf(!isLinux)("scan over entry names that are not valid UTF-8", () => {
+  let dir: string;
+  beforeAll(() => {
+    dir = String(tempDir("glob-scan-invalid-utf8", {}));
+    // Bytes after "t": `\xc3 x`, `\xe4\xb8 .js`, `é \xc3`, `😀 js`, `x \x80`, `\xed\xa0\x80` (a WTF-8 surrogate).
+    for (const hex of ["c378", "e4b82e6a73", "c3a9c3", "f09f98806a73", "7880", "eda080"]) {
+      fs.writeFileSync(Buffer.concat([Buffer.from(dir + "/t"), Buffer.from(hex, "hex")]), "");
+    }
+  });
+  afterAll(() => {
+    fs.rmSync(dir, { recursive: true, force: true });
+  });
+  const scan = (pattern: string) => [...new Glob(pattern).scanSync({ cwd: dir })].sort();
+  const all = ["t\uFFFDx", "t\uFFFD.js", "té\uFFFD", "t😀js", "tx\uFFFD", "t\uFFFD\uFFFD\uFFFD"];
+
+  test.each([
+    ["*", all],
+    ["t*", all],
+    ["t?", []],
+    ["t??", ["t\uFFFDx", "té\uFFFD", "tx\uFFFD"]],
+    ["t???", ["t😀js", "t\uFFFD\uFFFD\uFFFD"]],
+    ["t????", ["t\uFFFD.js"]],
+    ["t*.js", ["t\uFFFD.js"]],
+    ["t?.js", ["t\uFFFD.js"]],
+    ["t*s", ["t\uFFFD.js", "t😀js"]],
+    ["t[!a-z]x", ["t\uFFFDx"]],
+    ["t[\uFFFD]*", ["t\uFFFDx", "t\uFFFD.js", "t\uFFFD\uFFFD\uFFFD"]],
+  ] as [string, string[]][])("%j", (pattern, expected) => {
+    expect(scan(pattern)).toEqual([...expected].sort());
+    // Wildcards and classes see the raw bytes the way match() sees the decoded
+    // name. (A literal U+FFFD in a pattern still compares bytes, since the
+    // decode is lossy.)
+    expect(scan(pattern)).toEqual(scan("*").filter(name => new Glob(pattern).match(name)));
+  });
+});
+
 // `followSymlinks` controls whether wildcard traversal descends through
 // symlinked directories. A segment that names the symlink literally is an
 // explicit path the user wrote; it should resolve regardless, matching