diff --git a/libs/deepagents/deepagents/backends/filesystem.py b/libs/deepagents/deepagents/backends/filesystem.py index e1e6d9cff9..3da4846e45 100644 --- a/libs/deepagents/deepagents/backends/filesystem.py +++ b/libs/deepagents/deepagents/backends/filesystem.py @@ -12,8 +12,6 @@ from datetime import datetime from pathlib import Path -import wcmatch.glob as wcglob - from deepagents._api.deprecation import warn_deprecated from deepagents.backends.protocol import ( DEFAULT_GREP_TIMEOUT, @@ -40,6 +38,7 @@ MAX_VIDEO_INPUT_BYTES, _get_backend_read_file_type, check_empty_content, + compile_grep_include_glob, perform_string_replacement, ) @@ -811,7 +810,7 @@ def _python_search( # noqa: C901, PLR0912, PLR0915 should treat such results as incomplete. """ deadline = time.monotonic() + timeout - glob_matcher = wcglob.compile(include_glob, flags=wcglob.BRACE | wcglob.GLOBSTAR) if include_glob else None + glob_matcher = compile_grep_include_glob(include_glob) if include_glob else None results: dict[str, list[tuple[int, str]]] = {} file_errors: list[str] = [] @@ -856,8 +855,8 @@ def _safe_detail(exc: Exception) -> str: except (PermissionError, OSError, RuntimeError): continue if glob_matcher is not None: - rel_path = str(fp.relative_to(root)) - if not glob_matcher.match(rel_path): + rel_path = fp.relative_to(root).as_posix() + if not glob_matcher(rel_path): continue try: if fp.stat().st_size > self.max_file_size_bytes: diff --git a/libs/deepagents/deepagents/backends/utils.py b/libs/deepagents/deepagents/backends/utils.py index f59da0120b..9c69bd13c5 100644 --- a/libs/deepagents/deepagents/backends/utils.py +++ b/libs/deepagents/deepagents/backends/utils.py @@ -8,9 +8,9 @@ import functools import os import re -from collections.abc import Sequence +from collections.abc import Callable, Sequence from datetime import UTC, datetime -from pathlib import Path, PurePosixPath +from pathlib import PurePosixPath from typing import Any, Final, Literal, overload import wcmatch.glob as wcglob @@ -80,9 +80,54 @@ @functools.lru_cache(maxsize=256) -def _compile_glob(pattern: str) -> wcglob.WcMatcher: - """Compile a glob pattern once and cache it (BRACE flag).""" - return wcglob.compile(pattern, flags=wcglob.BRACE) +def compile_grep_include_glob(pattern: str) -> Callable[[str], bool]: + """Compile a grep include-glob into a matcher with ripgrep-like semantics. + + Provides one shared include-glob behavior for every backend so the same + `grep(..., glob=...)` call closely mirrors ripgrep for common include + patterns, whether or not ripgrep is installed: + + - Patterns without a `/` match the basename at any depth. + + Example: `*.py` matches `src/app/main.py`. + - Patterns containing a `/` match the path relative to the grep search + root, with `**` support. + + Example: `src/**/*.py` matches `src/app/main.py`. + - A leading `/` anchors the pattern to the search root; it narrows the match + rather than widening it. + + Example: `/*.py` matches `top.py` but not `src/app/main.py`. + + Exclusion/negation patterns (a leading `!`) are not supported: the `!` is + treated literally rather than inverting the match, so results for such + patterns can diverge from `rg --glob '!...'`. + + Args: + pattern: Glob include pattern. + + Returns: + Predicate accepting a search-root-relative POSIX path; returns True when + the path is included by `pattern`. + """ + flags = wcglob.BRACE | wcglob.GLOBSTAR + # A leading `/` anchors to the search root: strip it so it matches against + # the (slash-less) relative path, but decide anchoring from the original + # pattern so `/*.py` stays root-anchored instead of collapsing to a + # basename-at-any-depth match. + anchored = "/" in pattern + compiled = wcglob.compile(pattern.lstrip("/"), flags=flags) + + if anchored: + + def matcher(rel_path: str) -> bool: + return bool(compiled.match(rel_path)) + else: + + def matcher(rel_path: str) -> bool: + return bool(compiled.match(PurePosixPath(rel_path).name)) + + return matcher def _normalize_content(file_data: FileData) -> str: @@ -629,6 +674,26 @@ def _filter_files_by_path(files: dict[str, Any], normalized_path: str) -> dict[s return {fp: fd for fp, fd in files.items() if fp.startswith(dir_prefix)} +def _relative_to_root(file_path: str, normalized_path: str) -> str: + """Return `file_path` relative to a normalized grep/glob search root. + + Args: + file_path: Absolute file path (e.g. "/src/app/main.py"). + normalized_path: Normalized search root from `_normalize_path`. + + Returns: + POSIX path relative to the search root (e.g. "src/app/main.py"). + + When `file_path` equals the search root (an exact-file search), + returns just the basename. + """ + if normalized_path == "/": + return file_path[1:] + if file_path == normalized_path: + return file_path.rsplit("/", maxsplit=1)[-1] + return file_path[len(normalized_path) + 1 :] + + def _glob_search_files( files: dict[str, Any], pattern: str, @@ -747,8 +812,8 @@ def grep_matches_from_files( filtered = _filter_files_by_path(files, normalized_path) if glob: - matcher = _compile_glob(glob) - filtered = {fp: fd for fp, fd in filtered.items() if matcher.match(Path(fp).name)} + matcher = compile_grep_include_glob(glob) + filtered = {fp: fd for fp, fd in filtered.items() if matcher(_relative_to_root(fp, normalized_path))} matches: list[GrepMatch] = [] for file_path, file_data in filtered.items(): diff --git a/libs/deepagents/tests/unit_tests/backends/test_file_format.py b/libs/deepagents/tests/unit_tests/backends/test_file_format.py index c1c0e27acd..25d4be14bc 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_file_format.py +++ b/libs/deepagents/tests/unit_tests/backends/test_file_format.py @@ -18,8 +18,8 @@ from deepagents.backends.protocol import ReadResult from deepagents.backends.store import StoreBackend from deepagents.backends.utils import ( - _compile_glob, _to_legacy_file_data, + compile_grep_include_glob, create_file_data, file_data_to_string, grep_matches_from_files, @@ -300,14 +300,14 @@ def test_grep_glob_matches_nothing(): def test_compile_glob_is_cached(): - """`_compile_glob` returns the identical matcher object for a repeated pattern. + """`compile_grep_include_glob` returns the identical matcher for a repeated pattern. This is the optimization the change exists to provide: the compiled matcher is reused across calls rather than recompiled per candidate file. """ - assert _compile_glob("*.py") is _compile_glob("*.py") + assert compile_grep_include_glob("*.py") is compile_grep_include_glob("*.py") # A distinct pattern produces a distinct matcher. - assert _compile_glob("*.py") is not _compile_glob("*.md") + assert compile_grep_include_glob("*.py") is not compile_grep_include_glob("*.md") def test_grep_glob_repeated_pattern_stays_correct(): diff --git a/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py b/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py index 65215602b7..148cc8553a 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py +++ b/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py @@ -1436,6 +1436,51 @@ def test_grep_surfaces_timeout_with_partial_results(self, tmp_path: Path, monkey assert result.matches[0]["path"] == "/file.txt" +class TestGrepPythonFallbackIncludeGlob: + """The Python grep fallback shares ripgrep-like include-glob semantics. + + Stubbing `_ripgrep_search` to `None` forces the Python fallback regardless + of whether ripgrep is installed, so these lock the shared contract: + + - A slashless pattern (`*.py`) matches the basename at any depth. + - A path-containing pattern (`src/**/*.py`) matches relative to the root. + """ + + def _setup(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> FilesystemBackend: + (tmp_path / "src" / "app").mkdir(parents=True) + (tmp_path / "src" / "app" / "main.py").write_text("import os\n") + (tmp_path / "top.py").write_text("import sys\n") + (tmp_path / "README.md").write_text("import note\n") + be = FilesystemBackend(root_dir=str(tmp_path), virtual_mode=True) + monkeypatch.setattr(be, "_ripgrep_search", lambda *_a, **_k: None) + return be + + def _paths(self, be: FilesystemBackend, glob: str, path: str = "/") -> list[str]: + result = be.grep("import", path=path, glob=glob) + assert result.matches is not None + return sorted(m["path"] for m in result.matches) + + def test_directory_glob_matches_nested(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + be = self._setup(tmp_path, monkeypatch) + assert self._paths(be, "src/**/*.py") == ["/src/app/main.py"] + + def test_recursive_glob_matches_all_python(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + be = self._setup(tmp_path, monkeypatch) + assert self._paths(be, "**/*.py") == ["/src/app/main.py", "/top.py"] + + def test_slashless_glob_matches_at_any_depth(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + be = self._setup(tmp_path, monkeypatch) + assert self._paths(be, "*.py") == ["/src/app/main.py", "/top.py"] + + def test_negative_glob(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + be = self._setup(tmp_path, monkeypatch) + assert self._paths(be, "*.md") == ["/README.md"] + + def test_glob_relative_to_search_root(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + be = self._setup(tmp_path, monkeypatch) + assert self._paths(be, "app/*.py", path="/src") == ["/src/app/main.py"] + + class TestEditCrlfNormalization: """Tests for CRLF normalization in edit(). See #2247.""" diff --git a/libs/deepagents/tests/unit_tests/backends/test_utils.py b/libs/deepagents/tests/unit_tests/backends/test_utils.py index 9e7f919841..cf39e19db8 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_utils.py +++ b/libs/deepagents/tests/unit_tests/backends/test_utils.py @@ -12,6 +12,7 @@ _get_backend_read_file_type, _get_file_type, _glob_search_files, + grep_matches_from_files, perform_string_replacement, slice_read_response, to_posix_path, @@ -196,6 +197,57 @@ def test_leading_slash_pattern_with_subdir_path(self) -> None: assert "/foo/b.txt" not in result +class TestGrepIncludeGlob: + """Shared grep include-glob semantics (ripgrep-like) across backends. + + These document the contract implemented by `compile_grep_include_glob` and + consumed by `grep_matches_from_files` (StateBackend/StoreBackend) and the + FilesystemBackend Python fallback: + + - A pattern with no `/` matches the basename at any depth (`*.py` matches + `/src/app/main.py`). + - A pattern containing `/` matches the path relative to the search root, + with `**` support (`src/**/*.py` matches `/src/app/main.py`). + """ + + @pytest.fixture + def sample_files(self) -> dict[str, Any]: + """Files whose every line contains the literal token `import`.""" + return { + "/src/app/main.py": {"content": "import os\n"}, + "/top.py": {"content": "import sys\n"}, + "/README.md": {"content": "import note\n"}, + } + + def _paths(self, files: dict[str, Any], glob: str | None, path: str = "/") -> list[str]: + result = grep_matches_from_files(files, "import", path, glob=glob) + return sorted(m["path"] for m in result.matches) + + def test_directory_glob_matches_nested(self, sample_files: dict[str, Any]) -> None: + assert self._paths(sample_files, "src/**/*.py") == ["/src/app/main.py"] + + def test_recursive_glob_matches_all_python(self, sample_files: dict[str, Any]) -> None: + assert self._paths(sample_files, "**/*.py") == ["/src/app/main.py", "/top.py"] + + def test_slashless_glob_matches_at_any_depth(self, sample_files: dict[str, Any]) -> None: + assert self._paths(sample_files, "*.py") == ["/src/app/main.py", "/top.py"] + + def test_extension_glob_matches_only_that_extension(self, sample_files: dict[str, Any]) -> None: + assert self._paths(sample_files, "*.md") == ["/README.md"] + + def test_glob_relative_to_search_root(self, sample_files: dict[str, Any]) -> None: + """Path-containing patterns resolve relative to the supplied root.""" + assert self._paths(sample_files, "app/*.py", path="/src") == ["/src/app/main.py"] + + def test_leading_slash_anchors_to_root(self, sample_files: dict[str, Any]) -> None: + """A leading `/` anchors to the root; it narrows rather than widens.""" + assert self._paths(sample_files, "/*.py") == ["/top.py"] + + def test_leading_slash_with_globstar(self, sample_files: dict[str, Any]) -> None: + """A leading `/` still supports `**` for anchored recursive matches.""" + assert self._paths(sample_files, "/src/**/*.py") == ["/src/app/main.py"] + + _content_block_adapter = TypeAdapter(ContentBlock)