Handle pathspec v1 changes

CHANGES.md

@@ -9,6 +9,28 @@
 
 <!-- Include any especially major or disruptive changes here -->
 
+This release alo bumps `pathspec` to v1.0.0 and fixes inconsistencies with Git's
+`.gitignore` logic (#4958). Now, files will be ignored if a pattern matches them, even
+if the parent directory is directly unignored. For example, Black would previously
+format `exclude/not_this/foo.py` with this `.gitignore`:
+
+```
+exclude/
+!exclude/not_this/
+```
+
+Now, `exclude/not_this/foo.py` will remain ignored. To ensure `exclude/not_this/` and
+all of it's children are included in formatting (and in Git), use this `.gitignore`:
+
+```
+*/exclude/*
+!*/exclude/not_this/
+```
+
+This new behavior matches Git. The leading `*/` are only necessary if you wish to ignore
+matching subdirectories (like the previous behavior did), and not just matching root
+directories.
+
 ### Stable style
 
 <!-- Changes that affect Black's stable style -->

pyproject.toml

@@ -66,7 +66,7 @@ dependencies = [
   "click>=8.0.0",
   "mypy_extensions>=0.4.3",
   "packaging>=22.0",
-  "pathspec>=0.9.0",
+  "pathspec>=1.0.0",
   "platformdirs>=2",
   "pytokens>=0.3.0",
   "tomli>=1.1.0; python_version < '3.11'",

src/black/__init__.py

@@ -23,8 +23,8 @@
 import click
 from click.core import ParameterSource
 from mypy_extensions import mypyc_attr
-from pathspec import PathSpec
-from pathspec.patterns.gitwildmatch import GitWildMatchPatternError
+from pathspec import GitIgnoreSpec
+from pathspec.patterns.gitignore import GitIgnorePatternError
 
 from _black_version import version as __version__
 from black.cache import Cache
@@ -685,7 +685,7 @@ def main(
                 report=report,
                 stdin_filename=stdin_filename,
             )
-        except GitWildMatchPatternError:
+        except GitIgnorePatternError:
             ctx.exit(1)
 
         if not sources:
@@ -749,7 +749,7 @@ def get_sources(
     assert root.is_absolute(), f"INTERNAL ERROR: `root` must be absolute but is {root}"
     using_default_exclude = exclude is None
     exclude = re_compile_maybe_verbose(DEFAULT_EXCLUDES) if exclude is None else exclude
-    gitignore: dict[Path, PathSpec] | None = None
+    gitignore: dict[Path, GitIgnoreSpec] | None = None
     root_gitignore = get_gitignore(root)
 
     for s in src:

src/black/files.py

@@ -10,8 +10,8 @@
 from mypy_extensions import mypyc_attr
 from packaging.specifiers import InvalidSpecifier, Specifier, SpecifierSet
 from packaging.version import InvalidVersion, Version
-from pathspec import PathSpec
-from pathspec.patterns.gitwildmatch import GitWildMatchPatternError
+from pathspec import GitIgnoreSpec
+from pathspec.patterns.gitignore import GitIgnorePatternError
 
 if sys.version_info >= (3, 11):
     try:
@@ -238,16 +238,16 @@ def find_user_pyproject_toml() -> Path:
 
 
 @lru_cache
-def get_gitignore(root: Path) -> PathSpec:
-    """Return a PathSpec matching gitignore content if present."""
+def get_gitignore(root: Path) -> GitIgnoreSpec:
+    """Return a GitIgnoreSpec matching gitignore content if present."""
     gitignore = root / ".gitignore"
     lines: list[str] = []
     if gitignore.is_file():
         with gitignore.open(encoding="utf-8") as gf:
             lines = gf.readlines()
     try:
-        return PathSpec.from_lines("gitwildmatch", lines)
-    except GitWildMatchPatternError as e:
+        return GitIgnoreSpec.from_lines(lines)
+    except GitIgnorePatternError as e:
         err(f"Could not parse {gitignore}: {e}")
         raise
 
@@ -292,7 +292,7 @@ def best_effort_relative_path(path: Path, root: Path) -> Path:
 def _path_is_ignored(
     root_relative_path: str,
     root: Path,
-    gitignore_dict: dict[Path, PathSpec],
+    gitignore_dict: dict[Path, GitIgnoreSpec],
 ) -> bool:
     path = root / root_relative_path
     # Note that this logic is sensitive to the ordering of gitignore_dict. Callers must
@@ -325,7 +325,7 @@ def gen_python_files(
     extend_exclude: Pattern[str] | None,
     force_exclude: Pattern[str] | None,
     report: Report,
-    gitignore_dict: dict[Path, PathSpec] | None,
+    gitignore_dict: dict[Path, GitIgnoreSpec] | None,
     *,
     verbose: bool,
     quiet: bool,

tests/test_black.py

@@ -28,7 +28,7 @@
 from click import unstyle
 from click.testing import CliRunner
 from packaging.version import Version
-from pathspec import PathSpec
+from pathspec import GitIgnoreSpec
 
 import black
 import black.files
@@ -2514,8 +2514,8 @@ def test_gitignore_exclude(self) -> None:
         include = re.compile(r"\.pyi?$")
         exclude = re.compile(r"")
         report = black.Report()
-        gitignore = PathSpec.from_lines(
-            "gitwildmatch", ["exclude/", ".definitely_exclude"]
+        gitignore = GitIgnoreSpec.from_lines(
+            ["exclude/", ".definitely_exclude", "!exclude/still_exclude/"]
         )
         sources: list[Path] = []
         expected = [
@@ -2539,6 +2539,70 @@ def test_gitignore_exclude(self) -> None:
         )
         assert sorted(expected) == sorted(sources)
 
+    def test_gitignore_reinclude(self) -> None:
+        path = THIS_DIR / "data" / "include_exclude_tests"
+        include = re.compile(r"\.pyi?$")
+        exclude = re.compile(r"")
+        report = black.Report()
+        gitignore = GitIgnoreSpec.from_lines(
+            ["*/exclude/*", ".definitely_exclude", "!*/exclude/still_exclude/"]
+        )
+        sources: list[Path] = []
+        expected = [
+            Path(path / "b/dont_exclude/a.py"),
+            Path(path / "b/dont_exclude/a.pyi"),
+            Path(path / "b/exclude/still_exclude/a.py"),
+            Path(path / "b/exclude/still_exclude/a.pyi"),
+        ]
+        this_abs = THIS_DIR.resolve()
+        sources.extend(
+            black.gen_python_files(
+                path.iterdir(),
+                this_abs,
+                include,
+                exclude,
+                None,
+                None,
+                report,
+                {path: gitignore},
+                verbose=False,
+                quiet=False,
+            )
+        )
+        assert sorted(expected) == sorted(sources)
+
+    def test_gitignore_reinclude_root(self) -> None:
+        path = THIS_DIR / "data" / "include_exclude_tests" / "b"
+        include = re.compile(r"\.pyi?$")
+        exclude = re.compile(r"")
+        report = black.Report()
+        gitignore = GitIgnoreSpec.from_lines(
+            ["exclude/*", ".definitely_exclude", "!exclude/still_exclude/"]
+        )
+        sources: list[Path] = []
+        expected = [
+            Path(path / "dont_exclude/a.py"),
+            Path(path / "dont_exclude/a.pyi"),
+            Path(path / "exclude/still_exclude/a.py"),
+            Path(path / "exclude/still_exclude/a.pyi"),
+        ]
+        this_abs = THIS_DIR.resolve()
+        sources.extend(
+            black.gen_python_files(
+                path.iterdir(),
+                this_abs,
+                include,
+                exclude,
+                None,
+                None,
+                report,
+                {path: gitignore},
+                verbose=False,
+                quiet=False,
+            )
+        )
+        assert sorted(expected) == sorted(sources)
+
     def test_nested_gitignore(self) -> None:
         path = Path(THIS_DIR / "data" / "nested_gitignore_tests")
         include = re.compile(r"\.pyi?$")
@@ -2640,6 +2704,9 @@ def test_empty_include(self) -> None:
             Path(path / "b/exclude/a.pie"),
             Path(path / "b/exclude/a.py"),
             Path(path / "b/exclude/a.pyi"),
+            Path(path / "b/exclude/still_exclude/a.pie"),
+            Path(path / "b/exclude/still_exclude/a.py"),
+            Path(path / "b/exclude/still_exclude/a.pyi"),
             Path(path / "b/dont_exclude/a.pie"),
             Path(path / "b/dont_exclude/a.py"),
             Path(path / "b/dont_exclude/a.pyi"),
@@ -2667,6 +2734,7 @@ def test_exclude_absolute_path(self) -> None:
         src = [path]
         expected = [
             Path(path / "b/dont_exclude/a.py"),
+            Path(path / "b/exclude/still_exclude/a.py"),
             Path(path / "b/.definitely_exclude/a.py"),
         ]
         assert_collected_sources(
@@ -2678,6 +2746,7 @@ def test_extend_exclude(self) -> None:
         src = [path]
         expected = [
             Path(path / "b/exclude/a.py"),
+            Path(path / "b/exclude/still_exclude/a.py"),
             Path(path / "b/dont_exclude/a.py"),
         ]
         assert_collected_sources(
@@ -2690,7 +2759,7 @@ def test_symlinks(self) -> None:
         include = re.compile(black.DEFAULT_INCLUDES)
         exclude = re.compile(black.DEFAULT_EXCLUDES)
         report = black.Report()
-        gitignore = PathSpec.from_lines("gitwildmatch", [])
+        gitignore = GitIgnoreSpec.from_lines([])
 
         regular = MagicMock()
         regular.relative_to.return_value = Path("regular.py")