glob: take a pattern byte that is not ASCII as one literal byte

scripts/rust-test.ts

@@ -40,6 +40,7 @@ const crates = [
   "bun_dispatch",
   "bun_errno",
   "bun_exe_format",
+  "bun_glob",
   "bun_hash",
   "bun_http_types",
   "bun_lint_react_compiler",

src/glob/Cargo.toml

@@ -17,3 +17,6 @@ bun_core.workspace = true
 bun_collections.workspace = true
 bun_paths.workspace = true
 bun_sys.workspace = true
+
+[dev-dependencies]
+bun_sema_standalone = { path = "../sema/standalone" }

src/glob/GlobWalker.rs

@@ -776,6 +776,11 @@ impl<'a, A: Accessor, const SENTINEL: bool> Iterator<'a, A, SENTINEL> {
             return None;
         }
 
+        // The matcher takes a byte that isn't UTF-8 as part of a character of a name, which no filter can say.
+        if !strings::is_valid_utf8(slice) {
+            return None;
+        }
+
         let wide = strings::convert_utf8_to_utf16_in_buffer(&mut self.nt_filter_buf, slice);
         Some(wide)
     }

src/glob/scan.rs

@@ -177,12 +177,18 @@ pub fn is_glob(text: &[u8]) -> bool {
                 Some(b'[') => Some(&mut squares),
                 _ => None,
             };
-            // The `\\` and one UTF-16 unit.
-            index += match at(index + 1) {
-                None | Some(0..0xC0) => 2,
-                Some(0xE0..0xF0) => 4,
-                Some(_) => 3,
+            // The `\\` and one UTF-16 unit. Only the continuation bytes that are there count.
+            let more = match at(index + 1) {
+                None | Some(0..0xC0) => 0,
+                Some(0xE0..0xF0) => 2,
+                Some(_) => 1,
             };
+            index += 2;
+            let rest = text.get(index..).unwrap_or_default();
+            let continuations = rest.iter().take(more);
+            index += continuations
+                .take_while(|it| matches!(it, 0x80..0xC0))
+                .count();
             if let Some(found) = closes.and_then(|it| it.index_from(text, index)) {
                 index = found + 1;
             }
@@ -258,3 +264,24 @@ pub fn glob_parent(pattern: &[u8]) -> Vec<u8> {
     }
     out
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn is_glob_reads_what_follows_an_escaped_byte_that_is_not_utf8() {
+        // `\xe9` announces a character of three bytes, `\xc3` one of two and `\xf0` one of four.
+        assert!(is_glob(b"\\\xe9*.js"));
+        assert!(is_glob(b"a\\\xc3*"));
+        assert!(is_glob(b"\\\xf0{a,b}"));
+        assert!(!is_glob(b"\\\xe9.js"));
+        // A whole character is one step, as it is for the package.
+        assert!(is_glob("\\\u{e9}*".as_bytes()));
+        assert!(is_glob("\\\u{9000}!".as_bytes()));
+        // The package steps over one UTF-16 unit, so the `!` isn't right after the escape.
+        assert!(!is_glob("\\\u{1f600}!".as_bytes()));
+        assert!(!is_glob("\\\u{9000}\\*".as_bytes()));
+        assert!(!is_glob("\\\u{1f600}\\*".as_bytes()));
+    }
+}

test/bundler/esbuild/dce.test.ts

@@ -707,6 +707,33 @@ describe("bundler", () => {
       stdout: "component effect\nutility effect\ndone",
     },
   });
+  // `\xe9` is one byte of a package.json in Latin-1: it announces a character that is not there. What follows it in the
+  // pattern is still syntax. The byte is a literal byte, and U+9000 is the bytes e9 80 80.
+  itBundled("dce/PackageJsonSideEffectsGlobBraceAfterByteThatIsNotUTF8", {
+    files: {
+      "/Users/user/project/src/entry.js": /* js */ `
+        import "demo-pkg/xa.js";
+        import "demo-pkg/x1b.js";
+        import "demo-pkg/zb.js";
+        import "demo-pkg/wb.js";
+        import "demo-pkg/y\u9000b.js";
+        console.log("done");
+      `,
+      "/Users/user/project/node_modules/demo-pkg/xa.js": /* js */ `console.log("xa effect");`,
+      "/Users/user/project/node_modules/demo-pkg/x1b.js": /* js */ `console.log("x1b effect");`,
+      "/Users/user/project/node_modules/demo-pkg/zb.js": /* js */ `console.log("zb effect");`,
+      "/Users/user/project/node_modules/demo-pkg/wb.js": /* js */ `console.log("wb effect");`,
+      "/Users/user/project/node_modules/demo-pkg/y\u9000b.js": /* js */ `console.log("y\u9000b effect");`,
+      "/Users/user/project/node_modules/demo-pkg/package.json": Buffer.from(
+        '{ "sideEffects": ["./{x,y\xe9}{a,*b}.js"] }',
+        "latin1",
+      ),
+    },
+    dce: true,
+    run: {
+      stdout: "xa effect\nx1b effect\ny\u9000b effect\ndone",
+    },
+  });
   itBundled("dce/PackageJsonSideEffectsGlobMixedPatterns", {
     files: {
       "/Users/user/project/src/entry.js": /* js */ `

test/cli/install/bun-pack.test.ts

@@ -3808,6 +3808,50 @@ describe.concurrent("files", () => {
     ]);
   });
 
+  // `\xe9` and `\xc3` are single bytes of a file in Latin-1: each announces a character that is not there. What follows
+  // such a byte in a pattern is still syntax. `npm pack` 11.16.0 packs the same files for these three lines.
+  test.each(["{x,y\xe9}{a,*.js}", "{x,y\xc3}{a,*.js}", "{x,\\\xe9}{a,*.js}"])(
+    ".npmignore: %j keeps its braces after a byte that is not UTF-8",
+    async line => {
+      using dir = tempDir("pack-npmignore-byte", {
+        "package.json": JSON.stringify({ name: "pack-npmignore-byte", version: "1.0.0" }),
+        ".npmignore": Buffer.from(line + "\n", "latin1"),
+        "a.js": indexJs,
+        "index.js": indexJs,
+        "x1.js": indexJs,
+        "xa": indexJs,
+      });
+
+      const { err, exitCode } = await runPack(dir);
+      expect(err).toBe("");
+      expect(tarballEntries(join(dir, "pack-npmignore-byte-1.0.0.tgz")).sort()).toEqual([
+        "package/a.js",
+        "package/index.js",
+        "package/package.json",
+      ]);
+      expect(exitCode).toBe(0);
+    },
+  );
+
+  // Unlike npm, which packs `y\u9000.js`: npm reads the byte as U+FFFD, which no name has. Here it is a literal byte, as
+  // it is for git, and U+9000 is the bytes e9 80 80.
+  test(".npmignore: a byte that is not UTF-8 matches the first byte of a character", async () => {
+    using dir = tempDir("pack-npmignore-lead-byte", {
+      "package.json": JSON.stringify({ name: "pack-npmignore-lead-byte", version: "1.0.0" }),
+      ".npmignore": Buffer.from("y\xe9*.js\n", "latin1"),
+      "index.js": indexJs,
+      "y\u9000.js": indexJs,
+    });
+
+    const { err, exitCode } = await runPack(dir);
+    expect(err).toBe("");
+    expect(tarballEntries(join(dir, "pack-npmignore-lead-byte-1.0.0.tgz")).sort()).toEqual([
+      "package/index.js",
+      "package/package.json",
+    ]);
+    expect(exitCode).toBe(0);
+  });
+
   test("'files' field cannot exclude LICENSE", async () => {
     using dir = tempDir("pack-files-license", {
       "package.json": JSON.stringify({ name: "pack-files-license", version: "1.1.1", files: ["lib", "!LICENSE"] }),
@@ -5494,6 +5538,32 @@ test.concurrent.skipIf(!isLinux)("filenames that are not valid UTF-8 are stored
   ]);
 });
 
+// An ignore file in Latin-1 that lists names in Latin-1. git 2.47 ignores the same three names for these lines. Unlike
+// npm, which leaves out the other three too: it reads `caf\xe9` and `caf\xea` as one name.
+test.concurrent.skipIf(!isLinux)(
+  ".npmignore lines that are not valid UTF-8 exclude the names with those bytes",
+  async () => {
+    using dir = tempDir("pack-npmignore-raw-names", {
+      "package.json": JSON.stringify({ name: "pack-npmignore-raw-names", version: "1.0.0" }),
+      ".npmignore": Buffer.from("caf\xe9\n\xe9*.log\ny\xe9.js\n", "latin1"),
+    });
+    const names = ["caf\xe9", "caf\xea", "\xe91.log", "\xea1.log", "y\xe9.js", "y\xea.js"];
+    await Promise.all(
+      names.map(name => writeFile(Buffer.concat([Buffer.from(`${dir}/`), Buffer.from(name, "latin1")]), name)),
+    );
+
+    await pack(dir, bunEnv);
+
+    const members = rawTarballMembers(await file(join(dir, "pack-npmignore-raw-names-1.0.0.tgz")).bytes());
+    expect(members.flatMap(member => [...member.names]).sort()).toEqual([
+      "package/caf\xea",
+      "package/package.json",
+      "package/y\xea.js",
+      "package/\xea1.log",
+    ]);
+  },
+);
+
 test.concurrent("$npm_command is accurate", async () => {
   using dir = tempDir("pack-command", {
     "package.json": JSON.stringify({

test/cli/install/bun-workspaces.test.ts

@@ -505,6 +505,36 @@ test.concurrent("extglob and brace sequence workspace patterns", async () => {
   expect(Object.keys(lockfile.workspaces).sort()).toEqual(["", ...good].sort());
 });
 
+// `\xe9` is one byte of a package.json in Latin-1: it announces a character that is not there. What follows it in the
+// pattern is still syntax. `npm install` 11.16.0 links `pk/x1b` and `pk/xa`. Unlike npm, the third folder is linked too:
+// U+9000 is the bytes e9 80 80, and the byte is a literal byte here. npm reads it as U+FFFD.
+test.concurrent("a workspace pattern keeps its braces after a byte that is not UTF-8", async () => {
+  using ctx = await setupTest();
+  const { packageDir, env } = ctx;
+  const good = ["pk/x1b", "pk/xa", "pk/y\u9000b"];
+  // Installing any of these would fail.
+  const excluded = ["pk/zb", "pk/keepb"];
+  await Promise.all([
+    write(
+      join(packageDir, "package.json"),
+      Buffer.from('{ "name": "root", "workspaces": ["pk/{x,y\xe9}{a,*b}"] }', "latin1"),
+    ),
+    ...good.map((dir, index) => write(join(packageDir, dir, "package.json"), JSON.stringify({ name: `good${index}` }))),
+    ...excluded.map(dir =>
+      write(
+        join(packageDir, dir, "package.json"),
+        JSON.stringify({ name: dir.split("/")[1], dependencies: { "doesnt-exist-oops": "1.2.3" } }),
+      ),
+    ),
+  ]);
+
+  const { exited } = await runBunInstall(env, packageDir);
+  expect(await exited).toBe(0);
+
+  const lockfile = Bun.JSONC.parse(await file(join(packageDir, "bun.lock")).text()) as { workspaces: object };
+  expect(Object.keys(lockfile.workspaces).sort()).toEqual(["", ...good].sort());
+});
+
 test("dependency on same name as workspace and dist-tag", async () => {
   using ctx = await setupTest();
   const { packageDir, env } = ctx;

test/cli/lint/glob.test.ts

@@ -521,6 +521,31 @@ test.concurrent.each(["#*", "!*"])("the command line pattern %s matches no files
   expect(exitCode).toBe(2);
 });
 
+// `\351` is one byte, which `printf` writes: it announces a character of three bytes that is not there. The `*` after
+// it is still a wildcard. U+9000 is the bytes e9 80 80. An argument of `spawn` is a string, which has no such byte.
+test.concurrent.skipIf(isWindows)(
+  "a command line pattern with an escaped byte that is not UTF-8 keeps its wildcard",
+  async () => {
+    using dir = tempDir("bun-lint-glob", {
+      "\u9000a.js": "debugger;\n",
+      "b.js": "debugger;\n",
+      "eslint.config.mjs": 'export default [{ rules: { "no-debugger": "error" } }];\n',
+    });
+    await using proc = spawn({
+      cmd: ["sh", "-c", `exec "$0" lint -f json -- "$(printf '\\\\\\351*.js')"`, bunExe()],
+      env,
+      cwd: String(dir),
+      stdout: "pipe",
+      stderr: "pipe",
+    });
+    const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]);
+    if (!stdout.startsWith("[")) throw new Error(`exit code ${exitCode}\n${stdout}\n${stderr}`);
+    const results: { filePath: string }[] = JSON.parse(stdout);
+    expect(results.map(it => relative(String(dir), it.filePath))).toEqual(["\u9000a.js"]);
+    expect(exitCode).toBe(1);
+  },
+);
+
 // The expected results come from oxlint 1.87, whose glob matcher never matches a pattern with an unclosed `{` or `[`.
 test.concurrent.each(["{a,", "a{,", "*{a,", "[*", "{*", "a\\"])(
   "the oxlint no-restricted-imports pattern %j restricts nothing",

test/js/bun/shell/brace.test.ts

@@ -175,6 +175,39 @@ describe("brace + glob composition", () => {
     // ...and does not split into a spurious `]foo` branch.
     expect(words).not.toContain("x.]foo");
   });
+
+  // `\xe9` is one raw byte, which the builtin `echo -e` writes. A byte like that is not UTF-8 by itself. The glob
+  // matcher takes it as that one byte: it must not take the `}`, the `{` or the `,` after it along with it.
+  describe("a byte that is not UTF-8 in a group, beside a group with a glob", () => {
+    // U+9000 is the bytes e9 80 80, so only an alternative that starts with `y` and the byte e9, or with `y` alone, can name
+    // the last file.
+    const files = ["x1.bak", "z.bak", "keep.bak", "xa", "y\u9000.bak"];
+
+    // Each row is the word and the files of the folder that it names. bash 5.2 names the same ones.
+    test.concurrent.each([
+      ["{x,$(echo -e 'y\\xe9')}{a,*.bak}", ["x1.bak", "xa", "y\u9000.bak"]],
+      ["{x,$(echo -e 'y\\xc3')}{a,*.bak}", ["x1.bak", "xa"]],
+      ["{x,$(echo -e 'y\\xf0')}{a,*.bak}", ["x1.bak", "xa"]],
+      ["{x,$(echo -e '\\xe9y')}{a,*.bak}", ["x1.bak", "xa"]],
+      ["{$(echo -e 'y\\xe9'),x}{a,*.bak}", ["x1.bak", "xa", "y\u9000.bak"]],
+      ["{x,$(echo -e 'y\\xe9')}{*.bak,a}", ["x1.bak", "xa", "y\u9000.bak"]],
+      // The same words with no such byte, with a whole character, and with a byte that announces no character.
+      ["{x,y}{a,*.bak}", ["x1.bak", "xa", "y\u9000.bak"]],
+      ["{x,$(echo -e 'y\\xc3\\xa9')}{a,*.bak}", ["x1.bak", "xa"]],
+      ["{x,$(echo -e 'y\\x80')}{a,*.bak}", ["x1.bak", "xa"]],
+      ["{x,$(echo -e 'y\\xe9')}{a,b}", ["xa"]],
+    ])("echo %s", async (word, named) => {
+      using dir = tempDir("shell-brace-glob-bytes", Object.fromEntries(files.map(it => [it, ""])));
+      const { stdout, stderr, exitCode } = await $`echo ${{ raw: word }}`.cwd(String(dir)).nothrow().quiet();
+      const words = new Set(stdout.toString("latin1").trim().split(" "));
+      const isNamed = (file: string) => words.has(Buffer.from(file, "utf8").toString("latin1"));
+      expect({ named: files.filter(isNamed).sort(), stderr: stderr.toString(), exitCode }).toEqual({
+        named,
+        stderr: "",
+        exitCode: 0,
+      });
+    });
+  });
 });
 
 // $.braces() recursed once per `{` group (parse_atom <-> parse_expansion /