Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 4 additions & 5 deletions libs/deepagents/deepagents/backends/filesystem.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,6 @@
from datetime import datetime
from pathlib import Path

import wcmatch.glob as wcglob

from deepagents._api.deprecation import warn_deprecated
from deepagents.backends.protocol import (
DEFAULT_GREP_TIMEOUT,
Expand All @@ -40,6 +38,7 @@
MAX_VIDEO_INPUT_BYTES,
_get_backend_read_file_type,
check_empty_content,
compile_grep_include_glob,
perform_string_replacement,
)

Expand Down Expand Up @@ -811,7 +810,7 @@ def _python_search( # noqa: C901, PLR0912, PLR0915
should treat such results as incomplete.
"""
deadline = time.monotonic() + timeout
glob_matcher = wcglob.compile(include_glob, flags=wcglob.BRACE | wcglob.GLOBSTAR) if include_glob else None
glob_matcher = compile_grep_include_glob(include_glob) if include_glob else None

results: dict[str, list[tuple[int, str]]] = {}
file_errors: list[str] = []
Expand Down Expand Up @@ -856,8 +855,8 @@ def _safe_detail(exc: Exception) -> str:
except (PermissionError, OSError, RuntimeError):
continue
if glob_matcher is not None:
rel_path = str(fp.relative_to(root))
if not glob_matcher.match(rel_path):
rel_path = fp.relative_to(root).as_posix()
if not glob_matcher(rel_path):
continue
try:
if fp.stat().st_size > self.max_file_size_bytes:
Expand Down
79 changes: 72 additions & 7 deletions libs/deepagents/deepagents/backends/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@
import functools
import os
import re
from collections.abc import Sequence
from collections.abc import Callable, Sequence
from datetime import UTC, datetime
from pathlib import Path, PurePosixPath
from pathlib import PurePosixPath
from typing import Any, Final, Literal, overload

import wcmatch.glob as wcglob
Expand Down Expand Up @@ -80,9 +80,54 @@


@functools.lru_cache(maxsize=256)
def _compile_glob(pattern: str) -> wcglob.WcMatcher:
"""Compile a glob pattern once and cache it (BRACE flag)."""
return wcglob.compile(pattern, flags=wcglob.BRACE)
def compile_grep_include_glob(pattern: str) -> Callable[[str], bool]:
Comment thread
open-swe[bot] marked this conversation as resolved.
"""Compile a grep include-glob into a matcher with ripgrep-like semantics.

Provides one shared include-glob behavior for every backend so the same
`grep(..., glob=...)` call closely mirrors ripgrep for common include
patterns, whether or not ripgrep is installed:

- Patterns without a `/` match the basename at any depth.

Example: `*.py` matches `src/app/main.py`.
- Patterns containing a `/` match the path relative to the grep search
root, with `**` support.

Example: `src/**/*.py` matches `src/app/main.py`.
- A leading `/` anchors the pattern to the search root; it narrows the match
rather than widening it.

Example: `/*.py` matches `top.py` but not `src/app/main.py`.

Exclusion/negation patterns (a leading `!`) are not supported: the `!` is
treated literally rather than inverting the match, so results for such
patterns can diverge from `rg --glob '!...'`.

Args:
pattern: Glob include pattern.

Returns:
Predicate accepting a search-root-relative POSIX path; returns True when
the path is included by `pattern`.
"""
flags = wcglob.BRACE | wcglob.GLOBSTAR
# A leading `/` anchors to the search root: strip it so it matches against
# the (slash-less) relative path, but decide anchoring from the original
# pattern so `/*.py` stays root-anchored instead of collapsing to a
# basename-at-any-depth match.
anchored = "/" in pattern
compiled = wcglob.compile(pattern.lstrip("/"), flags=flags)

if anchored:

def matcher(rel_path: str) -> bool:
return bool(compiled.match(rel_path))
else:

def matcher(rel_path: str) -> bool:
return bool(compiled.match(PurePosixPath(rel_path).name))

return matcher


def _normalize_content(file_data: FileData) -> str:
Expand Down Expand Up @@ -629,6 +674,26 @@ def _filter_files_by_path(files: dict[str, Any], normalized_path: str) -> dict[s
return {fp: fd for fp, fd in files.items() if fp.startswith(dir_prefix)}


def _relative_to_root(file_path: str, normalized_path: str) -> str:
"""Return `file_path` relative to a normalized grep/glob search root.

Args:
file_path: Absolute file path (e.g. "/src/app/main.py").
normalized_path: Normalized search root from `_normalize_path`.

Returns:
POSIX path relative to the search root (e.g. "src/app/main.py").

When `file_path` equals the search root (an exact-file search),
returns just the basename.
"""
if normalized_path == "/":
return file_path[1:]
if file_path == normalized_path:
return file_path.rsplit("/", maxsplit=1)[-1]
return file_path[len(normalized_path) + 1 :]


def _glob_search_files(
files: dict[str, Any],
pattern: str,
Expand Down Expand Up @@ -747,8 +812,8 @@ def grep_matches_from_files(
filtered = _filter_files_by_path(files, normalized_path)

if glob:
matcher = _compile_glob(glob)
filtered = {fp: fd for fp, fd in filtered.items() if matcher.match(Path(fp).name)}
matcher = compile_grep_include_glob(glob)
filtered = {fp: fd for fp, fd in filtered.items() if matcher(_relative_to_root(fp, normalized_path))}

matches: list[GrepMatch] = []
for file_path, file_data in filtered.items():
Expand Down
8 changes: 4 additions & 4 deletions libs/deepagents/tests/unit_tests/backends/test_file_format.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@
from deepagents.backends.protocol import ReadResult
from deepagents.backends.store import StoreBackend
from deepagents.backends.utils import (
_compile_glob,
_to_legacy_file_data,
compile_grep_include_glob,
create_file_data,
file_data_to_string,
grep_matches_from_files,
Expand Down Expand Up @@ -300,14 +300,14 @@ def test_grep_glob_matches_nothing():


def test_compile_glob_is_cached():
"""`_compile_glob` returns the identical matcher object for a repeated pattern.
"""`compile_grep_include_glob` returns the identical matcher for a repeated pattern.

This is the optimization the change exists to provide: the compiled matcher
is reused across calls rather than recompiled per candidate file.
"""
assert _compile_glob("*.py") is _compile_glob("*.py")
assert compile_grep_include_glob("*.py") is compile_grep_include_glob("*.py")
# A distinct pattern produces a distinct matcher.
assert _compile_glob("*.py") is not _compile_glob("*.md")
assert compile_grep_include_glob("*.py") is not compile_grep_include_glob("*.md")


def test_grep_glob_repeated_pattern_stays_correct():
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1436,6 +1436,51 @@ def test_grep_surfaces_timeout_with_partial_results(self, tmp_path: Path, monkey
assert result.matches[0]["path"] == "/file.txt"


class TestGrepPythonFallbackIncludeGlob:
"""The Python grep fallback shares ripgrep-like include-glob semantics.

Stubbing `_ripgrep_search` to `None` forces the Python fallback regardless
of whether ripgrep is installed, so these lock the shared contract:

- A slashless pattern (`*.py`) matches the basename at any depth.
- A path-containing pattern (`src/**/*.py`) matches relative to the root.
"""

def _setup(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> FilesystemBackend:
(tmp_path / "src" / "app").mkdir(parents=True)
(tmp_path / "src" / "app" / "main.py").write_text("import os\n")
(tmp_path / "top.py").write_text("import sys\n")
(tmp_path / "README.md").write_text("import note\n")
be = FilesystemBackend(root_dir=str(tmp_path), virtual_mode=True)
monkeypatch.setattr(be, "_ripgrep_search", lambda *_a, **_k: None)
return be

def _paths(self, be: FilesystemBackend, glob: str, path: str = "/") -> list[str]:
result = be.grep("import", path=path, glob=glob)
assert result.matches is not None
return sorted(m["path"] for m in result.matches)

def test_directory_glob_matches_nested(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
be = self._setup(tmp_path, monkeypatch)
assert self._paths(be, "src/**/*.py") == ["/src/app/main.py"]

def test_recursive_glob_matches_all_python(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
be = self._setup(tmp_path, monkeypatch)
assert self._paths(be, "**/*.py") == ["/src/app/main.py", "/top.py"]

def test_slashless_glob_matches_at_any_depth(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
be = self._setup(tmp_path, monkeypatch)
assert self._paths(be, "*.py") == ["/src/app/main.py", "/top.py"]

def test_negative_glob(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
be = self._setup(tmp_path, monkeypatch)
assert self._paths(be, "*.md") == ["/README.md"]

def test_glob_relative_to_search_root(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
be = self._setup(tmp_path, monkeypatch)
assert self._paths(be, "app/*.py", path="/src") == ["/src/app/main.py"]


class TestEditCrlfNormalization:
"""Tests for CRLF normalization in edit(). See #2247."""

Expand Down
52 changes: 52 additions & 0 deletions libs/deepagents/tests/unit_tests/backends/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
_get_backend_read_file_type,
_get_file_type,
_glob_search_files,
grep_matches_from_files,
perform_string_replacement,
slice_read_response,
to_posix_path,
Expand Down Expand Up @@ -196,6 +197,57 @@ def test_leading_slash_pattern_with_subdir_path(self) -> None:
assert "/foo/b.txt" not in result


class TestGrepIncludeGlob:
"""Shared grep include-glob semantics (ripgrep-like) across backends.

These document the contract implemented by `compile_grep_include_glob` and
consumed by `grep_matches_from_files` (StateBackend/StoreBackend) and the
FilesystemBackend Python fallback:

- A pattern with no `/` matches the basename at any depth (`*.py` matches
`/src/app/main.py`).
- A pattern containing `/` matches the path relative to the search root,
with `**` support (`src/**/*.py` matches `/src/app/main.py`).
"""

@pytest.fixture
def sample_files(self) -> dict[str, Any]:
"""Files whose every line contains the literal token `import`."""
return {
"/src/app/main.py": {"content": "import os\n"},
"/top.py": {"content": "import sys\n"},
"/README.md": {"content": "import note\n"},
}

def _paths(self, files: dict[str, Any], glob: str | None, path: str = "/") -> list[str]:
result = grep_matches_from_files(files, "import", path, glob=glob)
return sorted(m["path"] for m in result.matches)

def test_directory_glob_matches_nested(self, sample_files: dict[str, Any]) -> None:
assert self._paths(sample_files, "src/**/*.py") == ["/src/app/main.py"]

def test_recursive_glob_matches_all_python(self, sample_files: dict[str, Any]) -> None:
assert self._paths(sample_files, "**/*.py") == ["/src/app/main.py", "/top.py"]

def test_slashless_glob_matches_at_any_depth(self, sample_files: dict[str, Any]) -> None:
assert self._paths(sample_files, "*.py") == ["/src/app/main.py", "/top.py"]

def test_extension_glob_matches_only_that_extension(self, sample_files: dict[str, Any]) -> None:
assert self._paths(sample_files, "*.md") == ["/README.md"]

def test_glob_relative_to_search_root(self, sample_files: dict[str, Any]) -> None:
"""Path-containing patterns resolve relative to the supplied root."""
assert self._paths(sample_files, "app/*.py", path="/src") == ["/src/app/main.py"]

def test_leading_slash_anchors_to_root(self, sample_files: dict[str, Any]) -> None:
"""A leading `/` anchors to the root; it narrows rather than widens."""
assert self._paths(sample_files, "/*.py") == ["/top.py"]

def test_leading_slash_with_globstar(self, sample_files: dict[str, Any]) -> None:
"""A leading `/` still supports `**` for anchored recursive matches."""
assert self._paths(sample_files, "/src/**/*.py") == ["/src/app/main.py"]


_content_block_adapter = TypeAdapter(ContentBlock)


Expand Down