From 853ae92783dbb2e67375bd15f590a71212a48e82 Mon Sep 17 00:00:00 2001 From: Mason Daugherty <61371264+mdrxy@users.noreply.github.com> Date: Fri, 12 Jun 2026 22:51:02 +0000 Subject: [PATCH 1/2] fix(sdk): unify `grep` include-glob semantics across backends The same `grep(..., glob=...)` call previously matched different files depending on the backend and whether ripgrep was installed. The in-memory path matched the glob against only the basename (so `src/**/*.py` and `**/*.py` matched nothing), while the FilesystemBackend Python fallback matched against the root-relative path (so slashless `*.py` missed nested files). Add `compile_grep_include_glob` implementing one ripgrep-like contract: slashless patterns match the basename at any depth, while path-containing patterns match relative to the search root with `**` support. Use it in `grep_matches_from_files` and the FilesystemBackend Python fallback. Co-authored-by: open-swe[bot] --- .../deepagents/backends/filesystem.py | 9 ++- libs/deepagents/deepagents/backends/utils.py | 60 ++++++++++++++++--- .../backends/test_filesystem_backend.py | 45 ++++++++++++++ .../tests/unit_tests/backends/test_utils.py | 44 ++++++++++++++ 4 files changed, 146 insertions(+), 12 deletions(-) diff --git a/libs/deepagents/deepagents/backends/filesystem.py b/libs/deepagents/deepagents/backends/filesystem.py index 5971996d3c..d13d55ce4d 100644 --- a/libs/deepagents/deepagents/backends/filesystem.py +++ b/libs/deepagents/deepagents/backends/filesystem.py @@ -12,8 +12,6 @@ from datetime import datetime from pathlib import Path -import wcmatch.glob as wcglob - from deepagents._api.deprecation import warn_deprecated from deepagents.backends.protocol import ( DEFAULT_GREP_TIMEOUT, @@ -38,6 +36,7 @@ from deepagents.backends.utils import ( _get_file_type, check_empty_content, + compile_grep_include_glob, perform_string_replacement, ) @@ -767,7 +766,7 @@ def _python_search( # noqa: C901, PLR0912, PLR0915 should treat such results as incomplete. """ deadline = time.monotonic() + timeout - glob_matcher = wcglob.compile(include_glob, flags=wcglob.BRACE | wcglob.GLOBSTAR) if include_glob else None + glob_matcher = compile_grep_include_glob(include_glob) if include_glob else None results: dict[str, list[tuple[int, str]]] = {} file_errors: list[str] = [] @@ -812,8 +811,8 @@ def _safe_detail(exc: Exception) -> str: except (PermissionError, OSError, RuntimeError): continue if glob_matcher is not None: - rel_path = str(fp.relative_to(root)) - if not glob_matcher.match(rel_path): + rel_path = fp.relative_to(root).as_posix() + if not glob_matcher(rel_path): continue try: if fp.stat().st_size > self.max_file_size_bytes: diff --git a/libs/deepagents/deepagents/backends/utils.py b/libs/deepagents/deepagents/backends/utils.py index 09a5d5bd1e..04f595d206 100644 --- a/libs/deepagents/deepagents/backends/utils.py +++ b/libs/deepagents/deepagents/backends/utils.py @@ -8,9 +8,9 @@ import functools import os import re -from collections.abc import Sequence +from collections.abc import Callable, Sequence from datetime import UTC, datetime -from pathlib import Path, PurePosixPath +from pathlib import PurePosixPath from typing import Any, Literal, overload import wcmatch.glob as wcglob @@ -74,9 +74,38 @@ @functools.lru_cache(maxsize=256) -def _compile_glob(pattern: str) -> wcglob.WcMatcher: - """Compile a glob pattern once and cache it (BRACE flag).""" - return wcglob.compile(pattern, flags=wcglob.BRACE) +def compile_grep_include_glob(pattern: str) -> Callable[[str], bool]: + """Compile a grep include-glob into a matcher with ripgrep-like semantics. + + Provides one shared include-glob behavior for every backend so the same + `grep(..., glob=...)` call matches the same files regardless of backend or + whether ripgrep is installed: + + - Patterns without a `/` match the basename at any depth. Example: `*.py` + matches `src/app/main.py`. + - Patterns containing a `/` match the path relative to the grep search + root, with `**` support. Example: `src/**/*.py` matches `src/app/main.py`. + + Args: + pattern: Glob include pattern. + + Returns: + Predicate accepting a search-root-relative POSIX path; returns True when + the path is included by `pattern`. + """ + flags = wcglob.BRACE | wcglob.GLOBSTAR + compiled = wcglob.compile(pattern, flags=flags) + + if "/" in pattern: + + def matcher(rel_path: str) -> bool: + return bool(compiled.match(rel_path)) + else: + + def matcher(rel_path: str) -> bool: + return bool(compiled.match(PurePosixPath(rel_path).name)) + + return matcher def _normalize_content(file_data: FileData) -> str: @@ -589,6 +618,23 @@ def _filter_files_by_path(files: dict[str, Any], normalized_path: str) -> dict[s return {fp: fd for fp, fd in files.items() if fp.startswith(dir_prefix)} +def _relative_to_root(file_path: str, normalized_path: str) -> str: + """Return `file_path` relative to a normalized grep/glob search root. + + Args: + file_path: Absolute file path (e.g. "/src/app/main.py"). + normalized_path: Normalized search root from `_normalize_path`. + + Returns: + POSIX path relative to the search root (e.g. "src/app/main.py"). + """ + if normalized_path == "/": + return file_path[1:] + if file_path == normalized_path: + return file_path.rsplit("/", maxsplit=1)[-1] + return file_path[len(normalized_path) + 1 :] + + def _glob_search_files( files: dict[str, Any], pattern: str, @@ -705,8 +751,8 @@ def grep_matches_from_files( filtered = _filter_files_by_path(files, normalized_path) if glob: - matcher = _compile_glob(glob) - filtered = {fp: fd for fp, fd in filtered.items() if matcher.match(Path(fp).name)} + matcher = compile_grep_include_glob(glob) + filtered = {fp: fd for fp, fd in filtered.items() if matcher(_relative_to_root(fp, normalized_path))} matches: list[GrepMatch] = [] for file_path, file_data in filtered.items(): diff --git a/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py b/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py index 46cb60466e..a8f96fc4f0 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py +++ b/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py @@ -1392,6 +1392,51 @@ def test_grep_surfaces_timeout_with_partial_results(self, tmp_path: Path, monkey assert result.matches[0]["path"] == "/file.txt" +class TestGrepPythonFallbackIncludeGlob: + """The Python grep fallback shares ripgrep-like include-glob semantics. + + Stubbing `_ripgrep_search` to `None` forces the Python fallback regardless + of whether ripgrep is installed, so these lock the shared contract: + + - A slashless pattern (`*.py`) matches the basename at any depth. + - A path-containing pattern (`src/**/*.py`) matches relative to the root. + """ + + def _setup(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> FilesystemBackend: + (tmp_path / "src" / "app").mkdir(parents=True) + (tmp_path / "src" / "app" / "main.py").write_text("import os\n") + (tmp_path / "top.py").write_text("import sys\n") + (tmp_path / "README.md").write_text("import note\n") + be = FilesystemBackend(root_dir=str(tmp_path), virtual_mode=True) + monkeypatch.setattr(be, "_ripgrep_search", lambda *_a, **_k: None) + return be + + def _paths(self, be: FilesystemBackend, glob: str, path: str = "/") -> list[str]: + result = be.grep("import", path=path, glob=glob) + assert result.matches is not None + return sorted(m["path"] for m in result.matches) + + def test_directory_glob_matches_nested(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + be = self._setup(tmp_path, monkeypatch) + assert self._paths(be, "src/**/*.py") == ["/src/app/main.py"] + + def test_recursive_glob_matches_all_python(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + be = self._setup(tmp_path, monkeypatch) + assert self._paths(be, "**/*.py") == ["/src/app/main.py", "/top.py"] + + def test_slashless_glob_matches_at_any_depth(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + be = self._setup(tmp_path, monkeypatch) + assert self._paths(be, "*.py") == ["/src/app/main.py", "/top.py"] + + def test_negative_glob(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + be = self._setup(tmp_path, monkeypatch) + assert self._paths(be, "*.md") == ["/README.md"] + + def test_glob_relative_to_search_root(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + be = self._setup(tmp_path, monkeypatch) + assert self._paths(be, "app/*.py", path="/src") == ["/src/app/main.py"] + + class TestEditCrlfNormalization: """Tests for CRLF normalization in edit(). See #2247.""" diff --git a/libs/deepagents/tests/unit_tests/backends/test_utils.py b/libs/deepagents/tests/unit_tests/backends/test_utils.py index ee14eb130f..7300754b62 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_utils.py +++ b/libs/deepagents/tests/unit_tests/backends/test_utils.py @@ -11,6 +11,7 @@ _EXTENSION_TO_FILE_TYPE, _get_file_type, _glob_search_files, + grep_matches_from_files, perform_string_replacement, slice_read_response, to_posix_path, @@ -195,6 +196,49 @@ def test_leading_slash_pattern_with_subdir_path(self) -> None: assert "/foo/b.txt" not in result +class TestGrepIncludeGlob: + """Shared grep include-glob semantics (ripgrep-like) across backends. + + These document the contract implemented by `compile_grep_include_glob` and + consumed by `grep_matches_from_files` (StateBackend/StoreBackend) and the + FilesystemBackend Python fallback: + + - A pattern with no `/` matches the basename at any depth (`*.py` matches + `/src/app/main.py`). + - A pattern containing `/` matches the path relative to the search root, + with `**` support (`src/**/*.py` matches `/src/app/main.py`). + """ + + @pytest.fixture + def sample_files(self) -> dict[str, Any]: + """Files whose every line contains the literal token `import`.""" + return { + "/src/app/main.py": {"content": "import os\n"}, + "/top.py": {"content": "import sys\n"}, + "/README.md": {"content": "import note\n"}, + } + + def _paths(self, files: dict[str, Any], glob: str | None, path: str = "/") -> list[str]: + result = grep_matches_from_files(files, "import", path, glob=glob) + return sorted(m["path"] for m in result.matches) + + def test_directory_glob_matches_nested(self, sample_files: dict[str, Any]) -> None: + assert self._paths(sample_files, "src/**/*.py") == ["/src/app/main.py"] + + def test_recursive_glob_matches_all_python(self, sample_files: dict[str, Any]) -> None: + assert self._paths(sample_files, "**/*.py") == ["/src/app/main.py", "/top.py"] + + def test_slashless_glob_matches_at_any_depth(self, sample_files: dict[str, Any]) -> None: + assert self._paths(sample_files, "*.py") == ["/src/app/main.py", "/top.py"] + + def test_negative_glob(self, sample_files: dict[str, Any]) -> None: + assert self._paths(sample_files, "*.md") == ["/README.md"] + + def test_glob_relative_to_search_root(self, sample_files: dict[str, Any]) -> None: + """Path-containing patterns resolve relative to the supplied root.""" + assert self._paths(sample_files, "app/*.py", path="/src") == ["/src/app/main.py"] + + _content_block_adapter = TypeAdapter(ContentBlock) From 7dfb10cca13b5ebd77011c77c031d68993beceb6 Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Thu, 2 Jul 2026 02:57:30 -0400 Subject: [PATCH 2/2] cr --- libs/deepagents/deepagents/backends/utils.py | 33 +++++++++++++++---- .../tests/unit_tests/backends/test_utils.py | 10 +++++- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/libs/deepagents/deepagents/backends/utils.py b/libs/deepagents/deepagents/backends/utils.py index 5cc2213269..9c69bd13c5 100644 --- a/libs/deepagents/deepagents/backends/utils.py +++ b/libs/deepagents/deepagents/backends/utils.py @@ -84,13 +84,24 @@ def compile_grep_include_glob(pattern: str) -> Callable[[str], bool]: """Compile a grep include-glob into a matcher with ripgrep-like semantics. Provides one shared include-glob behavior for every backend so the same - `grep(..., glob=...)` call matches the same files regardless of backend or - whether ripgrep is installed: + `grep(..., glob=...)` call closely mirrors ripgrep for common include + patterns, whether or not ripgrep is installed: - - Patterns without a `/` match the basename at any depth. Example: `*.py` - matches `src/app/main.py`. + - Patterns without a `/` match the basename at any depth. + + Example: `*.py` matches `src/app/main.py`. - Patterns containing a `/` match the path relative to the grep search - root, with `**` support. Example: `src/**/*.py` matches `src/app/main.py`. + root, with `**` support. + + Example: `src/**/*.py` matches `src/app/main.py`. + - A leading `/` anchors the pattern to the search root; it narrows the match + rather than widening it. + + Example: `/*.py` matches `top.py` but not `src/app/main.py`. + + Exclusion/negation patterns (a leading `!`) are not supported: the `!` is + treated literally rather than inverting the match, so results for such + patterns can diverge from `rg --glob '!...'`. Args: pattern: Glob include pattern. @@ -100,9 +111,14 @@ def compile_grep_include_glob(pattern: str) -> Callable[[str], bool]: the path is included by `pattern`. """ flags = wcglob.BRACE | wcglob.GLOBSTAR - compiled = wcglob.compile(pattern, flags=flags) + # A leading `/` anchors to the search root: strip it so it matches against + # the (slash-less) relative path, but decide anchoring from the original + # pattern so `/*.py` stays root-anchored instead of collapsing to a + # basename-at-any-depth match. + anchored = "/" in pattern + compiled = wcglob.compile(pattern.lstrip("/"), flags=flags) - if "/" in pattern: + if anchored: def matcher(rel_path: str) -> bool: return bool(compiled.match(rel_path)) @@ -667,6 +683,9 @@ def _relative_to_root(file_path: str, normalized_path: str) -> str: Returns: POSIX path relative to the search root (e.g. "src/app/main.py"). + + When `file_path` equals the search root (an exact-file search), + returns just the basename. """ if normalized_path == "/": return file_path[1:] diff --git a/libs/deepagents/tests/unit_tests/backends/test_utils.py b/libs/deepagents/tests/unit_tests/backends/test_utils.py index db3b9db97a..cf39e19db8 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_utils.py +++ b/libs/deepagents/tests/unit_tests/backends/test_utils.py @@ -232,13 +232,21 @@ def test_recursive_glob_matches_all_python(self, sample_files: dict[str, Any]) - def test_slashless_glob_matches_at_any_depth(self, sample_files: dict[str, Any]) -> None: assert self._paths(sample_files, "*.py") == ["/src/app/main.py", "/top.py"] - def test_negative_glob(self, sample_files: dict[str, Any]) -> None: + def test_extension_glob_matches_only_that_extension(self, sample_files: dict[str, Any]) -> None: assert self._paths(sample_files, "*.md") == ["/README.md"] def test_glob_relative_to_search_root(self, sample_files: dict[str, Any]) -> None: """Path-containing patterns resolve relative to the supplied root.""" assert self._paths(sample_files, "app/*.py", path="/src") == ["/src/app/main.py"] + def test_leading_slash_anchors_to_root(self, sample_files: dict[str, Any]) -> None: + """A leading `/` anchors to the root; it narrows rather than widens.""" + assert self._paths(sample_files, "/*.py") == ["/top.py"] + + def test_leading_slash_with_globstar(self, sample_files: dict[str, Any]) -> None: + """A leading `/` still supports `**` for anchored recursive matches.""" + assert self._paths(sample_files, "/src/**/*.py") == ["/src/app/main.py"] + _content_block_adapter = TypeAdapter(ContentBlock)