From aa8e8dc03cf1112066fe22494c71e34af738be2d Mon Sep 17 00:00:00 2001 From: Mason Daugherty <61371264+mdrxy@users.noreply.github.com> Date: Wed, 8 Jul 2026 18:15:52 +0000 Subject: [PATCH 01/14] chore(sdk): init grep max_count branch Co-authored-by: open-swe[bot] From 53bbeca84faef850c6e98abea0614014fd602f56 Mon Sep 17 00:00:00 2001 From: Mason Daugherty <61371264+mdrxy@users.noreply.github.com> Date: Wed, 8 Jul 2026 18:25:51 +0000 Subject: [PATCH 02/14] feat(sdk): stream and cap grep in filesystem backend Add max_count to the grep/agrep protocol and stream the local ripgrep path via Popen, terminating early once the total match cap is reached. Co-authored-by: open-swe[bot] --- .../deepagents/backends/filesystem.py | 272 +++++++++++------- .../deepagents/backends/protocol.py | 21 +- 2 files changed, 193 insertions(+), 100 deletions(-) diff --git a/libs/deepagents/deepagents/backends/filesystem.py b/libs/deepagents/deepagents/backends/filesystem.py index 5eeea5894a8..44d6501d114 100644 --- a/libs/deepagents/deepagents/backends/filesystem.py +++ b/libs/deepagents/deepagents/backends/filesystem.py @@ -8,6 +8,7 @@ import os import shutil import subprocess +import threading import time from datetime import datetime from pathlib import Path @@ -624,6 +625,8 @@ def grep( pattern: str, path: str | None = None, glob: str | None = None, + *, + max_count: int | None = None, ) -> GrepResult: """Search for a literal text pattern in files. @@ -633,6 +636,9 @@ def grep( pattern: Literal string to search for (NOT regex). path: Directory or file path to search in. Defaults to current directory. glob: Optional glob pattern to filter which files to search. + max_count: Optional total cap on returned matches across all files. + `None` returns every match; an int stops the search once the cap + is reached and flags the result with `truncated=True`. Returns: `GrepResult` with matches or error. @@ -654,11 +660,11 @@ def grep( return GrepResult(error=f"Error searching path '{search_path}': {e}", matches=[]) # Try ripgrep first (with -F flag for literal search) - results, truncated = self._ripgrep_search(pattern, base_full, glob) + results, truncated = self._ripgrep_search(pattern, base_full, glob, max_count) partial_error: str | None = None if results is None: # Python fallback does literal substring matching on the raw pattern. - results, truncated, partial_error = self._python_search(pattern, base_full, glob) + results, truncated, partial_error = self._python_search(pattern, base_full, glob, max_count=max_count) matches: list[GrepMatch] = [] for fpath, items in results.items(): @@ -666,29 +672,50 @@ def grep( matches.append({"path": fpath, "line": int(line_num), "text": line_text}) return GrepResult(error=partial_error, matches=matches, truncated=truncated) - def _ripgrep_search(self, pattern: str, base_full: Path, include_glob: str | None) -> tuple[dict[str, list[tuple[int, str]]] | None, bool]: # noqa: C901, PLR0912, PLR0915 # except clauses split per-exception for targeted logging (timeout vs exec-race vs ripgrep hard-error) + def _ripgrep_search( # noqa: C901, PLR0912, PLR0915 # streaming loop + per-exception logging (timeout vs exec-race vs hard-error vs cap) keeps branches explicit + self, + pattern: str, + base_full: Path, + include_glob: str | None, + max_count: int | None = None, + ) -> tuple[dict[str, list[tuple[int, str]]] | None, bool]: """Search using ripgrep with fixed-string (literal) mode. + Streams ripgrep's newline-delimited `--json` output line-by-line via + `subprocess.Popen` instead of buffering all of stdout, so a pathological + pattern on a huge repository cannot spike memory. Once `max_count` total + matches have been collected the process is terminated and the search + stops early. + Args: pattern: Literal string to search for (unescaped). base_full: Resolved base path to search in. include_glob: Optional glob pattern to filter files. + max_count: Optional total cap on collected matches across all files. + `None` disables the cap. Returns: A `(results, truncated)` tuple. `results` maps file paths to a list of `(line_number, line_text)` tuples, or is `None` when ripgrep is unavailable, hard-errored, or timed out before emitting any output — in each case the caller should fall back to the Python - search. `truncated` is `True` when ripgrep timed out but had - already emitted partial output (returned here instead of falling - back). Results whose resolved path lies outside `base_full` are - silently filtered regardless of `virtual_mode`. + search. `truncated` is `True` when ripgrep hit the `max_count` + cap, or timed out but had already emitted partial output + (returned here instead of falling back). Results whose resolved + path lies outside `base_full` are silently filtered regardless + of `virtual_mode`. """ rg_path = _resolve_ripgrep_path() if rg_path is None: return None, False cmd = [rg_path, "--json", "-F"] # -F enables fixed-string (literal) mode + if max_count is not None: + # Secondary, cheap per-file guard. `rg -m` is per file so it does + # not bound the total on its own (a repo with many files each + # contributing one match still overflows) — the total cap below is + # what actually stops the search — but it trims runaway single files. + cmd.extend(["-m", str(max_count)]) if include_glob: cmd.extend(["--glob", include_glob]) # When rg is given an absolute search path, directory-component @@ -696,8 +723,8 @@ def _ripgrep_search(self, pattern: str, base_full: Path, include_glob: str | Non # != search root (#2732). For a directory, set `cwd=base_full` and # use `.` as the search path so `--glob` resolves correctly. For a # single file, leave `cwd` unset and keep the absolute path — - # `subprocess.run` would raise `NotADirectoryError` if passed a file - # path as `cwd`, and globs are irrelevant for single-file searches. + # passing a file path as `cwd` raises `NotADirectoryError`, and globs + # are irrelevant for single-file searches. rg_cwd: str | None = None if base_full.is_dir(): cmd.extend(["--", pattern, "."]) @@ -705,35 +732,14 @@ def _ripgrep_search(self, pattern: str, base_full: Path, include_glob: str | Non else: cmd.extend(["--", pattern, str(base_full)]) - truncated = False try: - proc = subprocess.run( # noqa: S603 + proc = subprocess.Popen( # noqa: S603 cmd, - capture_output=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, - timeout=DEFAULT_GREP_TIMEOUT, - check=False, cwd=rg_cwd, ) - stdout = proc.stdout - except subprocess.TimeoutExpired as exc: - # `subprocess.run` attaches whatever ripgrep wrote before the kill to - # `exc.stdout` on both POSIX (drained during `communicate`) and - # Windows (via a post-`kill` `communicate`), so this path needs no - # per-platform branch. `--json` is newline-delimited so a truncated - # trailing frame just fails to parse and is skipped below; the - # matches that did land are still usable. Only fall back to the - # (slower) Python search when nothing was captured. - # `TimeoutExpired.stdout` is bytes even under `text=True`, so decode - # before the emptiness check or real partial output looks empty. - stdout = exc.stdout or "" - if isinstance(stdout, bytes): - stdout = stdout.decode(errors="replace") - if not stdout: - logger.warning("ripgrep timed out after %ds with no output; using Python grep fallback", DEFAULT_GREP_TIMEOUT) - return None, False - logger.warning("ripgrep timed out after %ds; returning partial results", DEFAULT_GREP_TIMEOUT) - truncated = True except (FileNotFoundError, PermissionError, NotADirectoryError) as e: # `rg` resolved at cache time but failed at exec — treat as a # runtime anomaly (uninstall, permission change, or `which`-vs-exec @@ -744,80 +750,141 @@ def _ripgrep_search(self, pattern: str, base_full: Path, include_glob: str | Non _resolve_ripgrep_path.cache_clear() return None, False - # Ripgrep exits 0 on match, 1 on no-match (both expected), 2+ on a hard - # error (invalid pattern, unreadable directory, malformed glob, etc.). - # Silently parsing stdout on a hard error reports zero matches to the - # agent — exactly the silent failure this resolver is meant to avoid. - # A timeout has no return code yet, so skip this guard for that path. - if not truncated and proc.returncode not in (0, 1): - stderr = proc.stderr.strip()[:500] if proc.stderr else "" - logger.warning("ripgrep exited %d (stderr=%r); using Python grep fallback", proc.returncode, stderr) - return None, False - results: dict[str, list[tuple[int, str]]] = {} base_resolved = base_full.resolve() - for line in stdout.splitlines(): - try: - data = json.loads(line) - except json.JSONDecodeError: - continue - data_type = data.get("type") - if data_type == "error": - # Per-file errors in `--json` mode (e.g., non-UTF-8 file - # ripgrep refused to read). Surface at DEBUG so debugging is - # possible without spamming WARNING for every binary file. - logger.debug("ripgrep per-file error frame: %s", data.get("data")) - continue - if data_type != "match": - continue - pdata = data.get("data", {}) - ftext = pdata.get("path", {}).get("text") - if not ftext: - continue - # When rg ran from cwd=base_full it emits paths relative to that - # cwd; join (don't `.resolve()`) so symlink form is preserved for - # callers. When rg searched a single file it emits the absolute - # path we passed in. - raw = Path(ftext) - p = raw if raw.is_absolute() else (base_full / raw) - # Defensive containment check: resolve both sides only for the - # comparison so symlinks that resolve to paths outside `base_full` - # can't leak results, while `p` itself keeps its original shape. - # OSError guards against unresolvable symlink targets. - try: - p.resolve().relative_to(base_resolved) - except (ValueError, OSError): - logger.warning( - "Skipping ripgrep result outside search root: path=%s root=%s", - p, - base_full, - ) - continue - if self.virtual_mode: + total = 0 + truncated = False + # A watchdog kills ripgrep if it outruns the time budget; a blocking + # `readline` cannot honor a deadline on its own, so the timer is what + # bounds a hang that never reaches the cap. + timed_out = threading.Event() + + def _kill_on_timeout() -> None: + timed_out.set() + proc.kill() + + timer = threading.Timer(DEFAULT_GREP_TIMEOUT, _kill_on_timeout) + timer.start() + try: + # `proc.stdout` is a text stream because `text=True`; iterating it + # yields one `--json` frame per line as ripgrep emits them. + for line in proc.stdout: # type: ignore[union-attr] try: - virt = self._to_virtual_path(p) - except ValueError: - logger.debug("Skipping grep result outside root: %s", p) + data = json.loads(line) + except json.JSONDecodeError: continue - except (OSError, RuntimeError): - logger.warning("Could not resolve grep result path: %s", p, exc_info=True) + data_type = data.get("type") + if data_type == "error": + # Per-file errors in `--json` mode (e.g., non-UTF-8 file + # ripgrep refused to read). Surface at DEBUG so debugging is + # possible without spamming WARNING for every binary file. + logger.debug("ripgrep per-file error frame: %s", data.get("data")) continue - else: - virt = str(p) - ln = pdata.get("line_number") - lt = pdata.get("lines", {}).get("text", "").rstrip("\n") - if ln is None: - continue - results.setdefault(virt, []).append((int(ln), lt)) + if data_type != "match": + continue + pdata = data.get("data", {}) + ftext = pdata.get("path", {}).get("text") + if not ftext: + continue + # When rg ran from cwd=base_full it emits paths relative to that + # cwd; join (don't `.resolve()`) so symlink form is preserved for + # callers. When rg searched a single file it emits the absolute + # path we passed in. + raw = Path(ftext) + p = raw if raw.is_absolute() else (base_full / raw) + # Defensive containment check: resolve both sides only for the + # comparison so symlinks that resolve to paths outside `base_full` + # can't leak results, while `p` itself keeps its original shape. + # OSError guards against unresolvable symlink targets. + try: + p.resolve().relative_to(base_resolved) + except (ValueError, OSError): + logger.warning( + "Skipping ripgrep result outside search root: path=%s root=%s", + p, + base_full, + ) + continue + if self.virtual_mode: + try: + virt = self._to_virtual_path(p) + except ValueError: + logger.debug("Skipping grep result outside root: %s", p) + continue + except (OSError, RuntimeError): + logger.warning("Could not resolve grep result path: %s", p, exc_info=True) + continue + else: + virt = str(p) + ln = pdata.get("line_number") + lt = pdata.get("lines", {}).get("text", "").rstrip("\n") + if ln is None: + continue + results.setdefault(virt, []).append((int(ln), lt)) + total += 1 + if max_count is not None and total >= max_count: + # Stop the process so it cannot keep buffering/emitting + # output once the caller's cap is satisfied. + truncated = True + proc.terminate() + break + finally: + timer.cancel() + stderr = self._drain_and_reap(proc) + + if timed_out.is_set(): + if results: + logger.warning("ripgrep timed out after %ds; returning partial results", DEFAULT_GREP_TIMEOUT) + return results, True + logger.warning("ripgrep timed out after %ds with no output; using Python grep fallback", DEFAULT_GREP_TIMEOUT) + return None, False + + if truncated: + # Hit the total match cap; `results` is intentionally incomplete. + return results, True + + # Ripgrep exits 0 on match, 1 on no-match (both expected), 2+ on a hard + # error (invalid pattern, unreadable directory, malformed glob, etc.). + # Reporting zero matches on a hard error would be the silent failure + # this resolver is meant to avoid, so fall back to the Python search. + if proc.returncode not in (0, 1) and not results: + logger.warning("ripgrep exited %d (stderr=%r); using Python grep fallback", proc.returncode, stderr.strip()[:500]) + return None, False return results, truncated + @staticmethod + def _drain_and_reap(proc: "subprocess.Popen[str]") -> str: + """Read any remaining stderr, close pipes, and reap `proc`. + + Returns the captured stderr so callers can log hard-error diagnostics. + Reaping avoids leaking a zombie/handle after the stdout loop stops + (whether via EOF, the match cap, or the timeout watchdog). + """ + stderr = "" + try: + if proc.stderr is not None: + stderr = proc.stderr.read() or "" + except (OSError, ValueError): + stderr = "" + if proc.stdout is not None: + proc.stdout.close() + if proc.stderr is not None: + proc.stderr.close() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait() + return stderr + def _python_search( # noqa: C901, PLR0912, PLR0915 self, pattern: str, base_full: Path, include_glob: str | None, *, + max_count: int | None = None, timeout: int = DEFAULT_GREP_TIMEOUT, ) -> tuple[dict[str, list[tuple[int, str]]], bool, str | None]: """Fallback search using Python when ripgrep is unavailable. @@ -829,19 +896,23 @@ def _python_search( # noqa: C901, PLR0912, PLR0915 pattern: Literal string to search for (substring match, not regex). base_full: Resolved base path to search in. include_glob: Optional glob pattern to filter files by name. + max_count: Optional total cap on collected matches across all files. + `None` disables the cap; when set, the walk stops once the cap + is reached and the result is flagged `truncated=True`. timeout: Maximum wall-clock seconds before the search is aborted. Returns: A `(results, truncated, error)` tuple. `results` contains every match found before iteration stopped. `truncated` is `True` when - the wall-clock `timeout` elapsed, leaving `results` valid but - incomplete. `error` is `None` on a clean walk, otherwise a - human-readable message when at least one file could not be - opened or fully read, or the walk aborted early (e.g., a - directory entry was removed mid-walk). + the wall-clock `timeout` elapsed or the `max_count` cap was + reached, leaving `results` valid but incomplete. `error` is + `None` on a clean walk, otherwise a human-readable message when + at least one file could not be opened or fully read, or the walk + aborted early (e.g., a directory entry was removed mid-walk). """ deadline = time.monotonic() + timeout glob_matcher = compile_grep_include_glob(include_glob) if include_glob else None + total = 0 results: dict[str, list[tuple[int, str]]] = {} file_errors: list[str] = [] @@ -919,6 +990,11 @@ def _safe_detail(exc: Exception) -> str: continue line = raw_line.rstrip("\n") results.setdefault(virt_path, []).append((line_num, line)) + total += 1 + if max_count is not None and total >= max_count: + # Hit the total match cap; stop scanning and + # report the partial result as truncated. + return results, True, _file_errors_msg() except UnicodeDecodeError as e: # A file that fails to decode before any line is scanned is # treated as binary and skipped silently, mirroring ripgrep's diff --git a/libs/deepagents/deepagents/backends/protocol.py b/libs/deepagents/deepagents/backends/protocol.py index cd9b36abd6c..6de8ae8ee1d 100644 --- a/libs/deepagents/deepagents/backends/protocol.py +++ b/libs/deepagents/deepagents/backends/protocol.py @@ -11,7 +11,7 @@ import logging from collections.abc import Callable from dataclasses import dataclass -from functools import lru_cache +from functools import lru_cache, partial from typing import Any, Final, Literal, NotRequired, TypeAlias from langchain.tools import ToolRuntime @@ -442,6 +442,8 @@ def grep( pattern: str, path: str | None = None, glob: str | None = None, + *, + max_count: int | None = None, ) -> "GrepResult": """Search for a literal text pattern in files. @@ -475,6 +477,15 @@ def grep( - `'src/**/*.js'` - search JS files under src/ - `'test[0-9].txt'` - search `test0.txt`, `test1.txt`, etc. + max_count: Optional total cap on the number of matches returned + across all files. + + `None` (the default) preserves existing backend behavior and + returns every match. When set to an int, the search stops once + that many matches have been collected and the result is flagged + with `GrepResult.truncated=True`. Interpreted as a total cap, not + a per-file cap. + Returns: `GrepResult` with matches or error. @@ -492,6 +503,10 @@ def grep( result = self.grep_raw(pattern, path, glob) if isinstance(result, str): return GrepResult(error=result) + # `grep_raw` predates `max_count`, so enforce the cap post-hoc for + # legacy backends that only implement it. + if max_count is not None and result is not None and len(result) > max_count: + return GrepResult(matches=result[:max_count], truncated=True) return GrepResult(matches=result) raise NotImplementedError @@ -501,6 +516,8 @@ async def agrep( pattern: str, path: str | None = None, glob: str | None = None, + *, + max_count: int | None = None, ) -> "GrepResult": """Async version of `grep`. @@ -510,7 +527,7 @@ async def agrep( """ try: return await asyncio.wait_for( - asyncio.to_thread(self.grep, pattern, path, glob), + asyncio.to_thread(partial(self.grep, pattern, path, glob, max_count=max_count)), timeout=ASYNC_GREP_TIMEOUT, ) except TimeoutError: From 5003f7a59b65273bdf8ab65b7a86016b133859f9 Mon Sep 17 00:00:00 2001 From: Mason Daugherty <61371264+mdrxy@users.noreply.github.com> Date: Wed, 8 Jul 2026 18:29:51 +0000 Subject: [PATCH 03/14] feat(sdk): cap grep matches in state, store, and hub backends Co-authored-by: open-swe[bot] --- libs/deepagents/deepagents/backends/context_hub.py | 10 +++++++++- libs/deepagents/deepagents/backends/state.py | 4 +++- libs/deepagents/deepagents/backends/store.py | 4 +++- libs/deepagents/deepagents/backends/utils.py | 8 +++++++- 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/libs/deepagents/deepagents/backends/context_hub.py b/libs/deepagents/deepagents/backends/context_hub.py index d014be26f06..b4cbb05d7e5 100644 --- a/libs/deepagents/deepagents/backends/context_hub.py +++ b/libs/deepagents/deepagents/backends/context_hub.py @@ -275,8 +275,14 @@ def grep( pattern: str, path: str | None = None, glob: str | None = None, + *, + max_count: int | None = None, ) -> GrepResult: - """Search contents for `pattern` (optional `path` / `glob` filters).""" + """Search contents for `pattern` (optional `path` / `glob` filters). + + When `max_count` is set, the search stops once that many total matches + have been collected and the result is flagged `truncated=True`. + """ try: cache = self._ensure_cache() except LangSmithError as exc: @@ -301,6 +307,8 @@ def grep( for i, line in enumerate(content.splitlines(), start=1): if regex.search(line): matches.append(GrepMatch(path=f"/{file_path}", line=i, text=line)) + if max_count is not None and len(matches) >= max_count: + return GrepResult(matches=matches, truncated=True) return GrepResult(matches=matches) diff --git a/libs/deepagents/deepagents/backends/state.py b/libs/deepagents/deepagents/backends/state.py index 00b7f6aa724..d859a220c55 100644 --- a/libs/deepagents/deepagents/backends/state.py +++ b/libs/deepagents/deepagents/backends/state.py @@ -322,10 +322,12 @@ def grep( pattern: str, path: str | None = None, glob: str | None = None, + *, + max_count: int | None = None, ) -> GrepResult: """Search state files for a literal text pattern.""" files = self._read_files() - return grep_matches_from_files(files, pattern, path if path is not None else "/", glob) + return grep_matches_from_files(files, pattern, path if path is not None else "/", glob, max_count=max_count) def glob(self, pattern: str, path: str | None = None) -> GlobResult: """Get `FileInfo` for files matching glob pattern.""" diff --git a/libs/deepagents/deepagents/backends/store.py b/libs/deepagents/deepagents/backends/store.py index ad37a529e07..27dd7d04e36 100644 --- a/libs/deepagents/deepagents/backends/store.py +++ b/libs/deepagents/deepagents/backends/store.py @@ -768,6 +768,8 @@ def grep( pattern: str, path: str | None = None, glob: str | None = None, + *, + max_count: int | None = None, ) -> GrepResult: """Search store files for a literal text pattern.""" store = self._get_store() @@ -779,7 +781,7 @@ def grep( files[item.key] = self._convert_store_item_to_file_data(item) except ValueError: continue - return grep_matches_from_files(files, pattern, path, glob) + return grep_matches_from_files(files, pattern, path, glob, max_count=max_count) def glob(self, pattern: str, path: str | None = None) -> GlobResult: """Find files matching a glob pattern in the store.""" diff --git a/libs/deepagents/deepagents/backends/utils.py b/libs/deepagents/deepagents/backends/utils.py index 54b2cd64055..94ab19c4f0e 100644 --- a/libs/deepagents/deepagents/backends/utils.py +++ b/libs/deepagents/deepagents/backends/utils.py @@ -826,12 +826,16 @@ def grep_matches_from_files( pattern: str, path: str | None = None, glob: str | None = None, + *, + max_count: int | None = None, ) -> GrepResult: """Return structured grep matches from an in-memory files mapping. Performs literal text search (not regex). - Returns a `GrepResult` with matches on success. + Returns a `GrepResult` with matches on success. When `max_count` is set, the + scan stops once that many total matches have been collected and the result + is flagged `truncated=True`. We deliberately do not raise here to keep backends non-throwing in tool contexts and preserve user-facing error messages. @@ -853,6 +857,8 @@ def grep_matches_from_files( for line_num, line in enumerate(content_str.split("\n"), 1): if pattern in line: # Simple substring search for literal matching matches.append({"path": file_path, "line": int(line_num), "text": line}) + if max_count is not None and len(matches) >= max_count: + return GrepResult(matches=matches, truncated=True) return GrepResult(matches=matches) From a88823bca86af8c368b633c5b688bbe740dcbcc6 Mon Sep 17 00:00:00 2001 From: Mason Daugherty <61371264+mdrxy@users.noreply.github.com> Date: Wed, 8 Jul 2026 18:35:51 +0000 Subject: [PATCH 04/14] feat(sdk): enforce grep max_count in sandbox and composite backends Co-authored-by: open-swe[bot] --- .../deepagents/backends/composite.py | 52 ++++++++++++++++--- .../deepagents/deepagents/backends/sandbox.py | 32 +++++++++--- 2 files changed, 69 insertions(+), 15 deletions(-) diff --git a/libs/deepagents/deepagents/backends/composite.py b/libs/deepagents/deepagents/backends/composite.py index eb5f0d911c8..f534f7c6c8e 100644 --- a/libs/deepagents/deepagents/backends/composite.py +++ b/libs/deepagents/deepagents/backends/composite.py @@ -42,6 +42,18 @@ def _remap_grep_path(m: GrepMatch, route_prefix: str) -> GrepMatch: ) +def _remaining_grep_budget(max_count: int | None, collected: int) -> int | None: + """Return the match budget left for the next routed grep. + + `None` means "no cap" (propagate `max_count=None` downstream). An int is the + number of matches still allowed before the global cap is hit; `0` signals + the caller to short-circuit the remaining routes. + """ + if max_count is None: + return None + return max(max_count - collected, 0) + + def _strip_route_from_pattern(pattern: str, route_prefix: str) -> str: """Strip a route prefix from a glob pattern when the pattern targets that route. @@ -359,6 +371,8 @@ def grep( pattern: str, path: str | None = None, glob: str | None = None, + *, + max_count: int | None = None, ) -> GrepResult: """Search files for literal text pattern. @@ -372,6 +386,10 @@ def grep( glob: Glob pattern to filter files (e.g., `"*.py"`, `"**/*.txt"`). Filters by filename, not content. + max_count: Optional total cap on returned matches across all routed + backends. `None` returns every match; an int enforces the cap + globally (not per backend), short-circuits remaining routes once + the cap is reached, and flags the result `truncated=True`. Returns: `GrepResult` with matches or error. @@ -390,7 +408,7 @@ def grep( path=path, ) if route_prefix is not None: - grep_result = self._coerce_grep_result(backend.grep(pattern, backend_path, glob)) + grep_result = self._coerce_grep_result(backend.grep(pattern, backend_path, glob, max_count=max_count)) if grep_result.error: return grep_result return GrepResult( @@ -403,28 +421,38 @@ def grep( if path is None or path == "/": all_matches: list[GrepMatch] = [] truncated = False - default_result = self._coerce_grep_result(self.default.grep(pattern, path, glob)) + default_result = self._coerce_grep_result(self.default.grep(pattern, path, glob, max_count=max_count)) if default_result.error: return default_result all_matches.extend(default_result.matches or []) truncated = truncated or default_result.truncated for route_prefix, backend in self.routes.items(): - grep_result = self._coerce_grep_result(backend.grep(pattern, "/", glob)) + remaining = _remaining_grep_budget(max_count, len(all_matches)) + if remaining == 0: + # Cap already met by earlier routes; skip the rest. + truncated = True + break + grep_result = self._coerce_grep_result(backend.grep(pattern, "/", glob, max_count=remaining)) if grep_result.error: return grep_result all_matches.extend(_remap_grep_path(m, route_prefix) for m in (grep_result.matches or [])) truncated = truncated or grep_result.truncated + if max_count is not None and len(all_matches) > max_count: + all_matches = all_matches[:max_count] + truncated = True return GrepResult(matches=all_matches, truncated=truncated) # Path specified but doesn't match a route - search only default - return self._coerce_grep_result(self.default.grep(pattern, path, glob)) + return self._coerce_grep_result(self.default.grep(pattern, path, glob, max_count=max_count)) async def agrep( self, pattern: str, path: str | None = None, glob: str | None = None, + *, + max_count: int | None = None, ) -> GrepResult: """Async version of grep. @@ -437,7 +465,7 @@ async def agrep( path=path, ) if route_prefix is not None: - grep_result = self._coerce_grep_result(await backend.agrep(pattern, backend_path, glob)) + grep_result = self._coerce_grep_result(await backend.agrep(pattern, backend_path, glob, max_count=max_count)) if grep_result.error: return grep_result return GrepResult( @@ -450,22 +478,30 @@ async def agrep( if path is None or path == "/": all_matches: list[GrepMatch] = [] truncated = False - default_result = self._coerce_grep_result(await self.default.agrep(pattern, path, glob)) + default_result = self._coerce_grep_result(await self.default.agrep(pattern, path, glob, max_count=max_count)) if default_result.error: return default_result all_matches.extend(default_result.matches or []) truncated = truncated or default_result.truncated for route_prefix, backend in self.routes.items(): - grep_result = self._coerce_grep_result(await backend.agrep(pattern, "/", glob)) + remaining = _remaining_grep_budget(max_count, len(all_matches)) + if remaining == 0: + # Cap already met by earlier routes; skip the rest. + truncated = True + break + grep_result = self._coerce_grep_result(await backend.agrep(pattern, "/", glob, max_count=remaining)) if grep_result.error: return grep_result all_matches.extend(_remap_grep_path(m, route_prefix) for m in (grep_result.matches or [])) truncated = truncated or grep_result.truncated + if max_count is not None and len(all_matches) > max_count: + all_matches = all_matches[:max_count] + truncated = True return GrepResult(matches=all_matches, truncated=truncated) # Path specified but doesn't match a route - search only default - return self._coerce_grep_result(await self.default.agrep(pattern, path, glob)) + return self._coerce_grep_result(await self.default.agrep(pattern, path, glob, max_count=max_count)) def glob(self, pattern: str, path: str | None = None) -> GlobResult: """Find files matching a glob pattern, routing by path prefix. diff --git a/libs/deepagents/deepagents/backends/sandbox.py b/libs/deepagents/deepagents/backends/sandbox.py index a07c3034e26..babc38c31ee 100644 --- a/libs/deepagents/deepagents/backends/sandbox.py +++ b/libs/deepagents/deepagents/backends/sandbox.py @@ -544,7 +544,7 @@ def _check_preflight_result(result: ExecuteResponse, file_path: str) -> WriteRes return None -def _build_grep_cmd(pattern: str, path: str | None, glob: str | None) -> str: +def _build_grep_cmd(pattern: str, path: str | None, glob: str | None, max_count: int | None = None) -> str: search_path = shlex.quote(path or ".") # `-Z` separates the filename from line data with NUL, so filenames may # contain `:` without making the output ambiguous. @@ -567,10 +567,17 @@ def _build_grep_cmd(pattern: str, path: str | None, glob: str | None) -> str: ) glob_pattern = f"--include={shlex.quote(glob)}" if glob else "" - return f"grep {grep_opts} {glob_pattern} -e {pattern_escaped} {search_path} 2>/dev/null || true" + base = f"grep {grep_opts} {glob_pattern} -e {pattern_escaped} {search_path} 2>/dev/null" + if max_count is not None: + # Read one record beyond the cap so the parser can distinguish "exactly + # at the cap" (complete) from "capped early" (truncated). `head` closing + # the pipe delivers SIGPIPE to grep, stopping it early rather than + # letting it keep scanning a huge tree after the cap is met. + return f"{base} | head -n {int(max_count) + 1} || true" + return f"{base} || true" -def _parse_grep_output(result: ExecuteResponse, path: str | None) -> GrepResult: +def _parse_grep_output(result: ExecuteResponse, path: str | None, max_count: int | None = None) -> GrepResult: output = result.output.rstrip("\n") if result.exit_code is not None and result.exit_code != 0: detail = output.strip() if output else f"exit code {result.exit_code}" @@ -589,6 +596,10 @@ def _parse_grep_output(result: ExecuteResponse, path: str | None) -> GrepResult: parse_error = line if parse_error is not None and not matches: return GrepResult(error=f"Path '{path or '.'}': {parse_error}") + if max_count is not None and len(matches) > max_count: + # More matches existed than the caller asked for; return the cap and + # flag the result as incomplete. + return GrepResult(matches=matches[:max_count], truncated=True) return GrepResult(matches=matches) @@ -1263,6 +1274,8 @@ def grep( pattern: str, path: str | None = None, glob: str | None = None, + *, + max_count: int | None = None, ) -> GrepResult: """Search file contents for a literal string using `grep -F`. @@ -1276,23 +1289,28 @@ def grep( `grep --include`; patterns containing a `/` (e.g. `'src/**/*.py'`) match the search-root-relative path via an in-process Python glob. + max_count: Optional total cap on returned matches across all files. + `None` returns every match; an int stops the search once the cap + is reached and flags the result with `truncated=True`. Returns: `GrepResult` with a list of `GrepMatch` dicts, or `error` on failure. """ - result = self.execute(_build_grep_cmd(pattern, path, glob)) - return _parse_grep_output(result, path) + result = self.execute(_build_grep_cmd(pattern, path, glob, max_count)) + return _parse_grep_output(result, path, max_count) async def agrep( self, pattern: str, path: str | None = None, glob: str | None = None, + *, + max_count: int | None = None, ) -> GrepResult: """Async version of `grep`, delegating to `aexecute` with timeout guard.""" try: result = await asyncio.wait_for( - self.aexecute(_build_grep_cmd(pattern, path, glob)), + self.aexecute(_build_grep_cmd(pattern, path, glob, max_count)), timeout=ASYNC_GREP_TIMEOUT, ) except TimeoutError: @@ -1306,7 +1324,7 @@ async def agrep( return GrepResult( error=f"Error: grep timed out after {ASYNC_GREP_TIMEOUT}s. Try a more specific pattern or a narrower path.", ) - return _parse_grep_output(result, path) + return _parse_grep_output(result, path, max_count) def glob(self, pattern: str, path: str | None = None) -> GlobResult: """Structured glob matching returning `GlobResult`.""" From e725ea98db55d9c997e8f286926d676503f061c1 Mon Sep 17 00:00:00 2001 From: Mason Daugherty <61371264+mdrxy@users.noreply.github.com> Date: Wed, 8 Jul 2026 18:37:07 +0000 Subject: [PATCH 05/14] feat(sdk): add grep_max_count to FilesystemMiddleware grep tool Default the grep tool to a 1000-match cap, overridable per call via a new max_count argument, and generalize the truncation note to cover the cap. Co-authored-by: open-swe[bot] --- .../deepagents/middleware/filesystem.py | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index 6ea29998816..c6bdb5e1189 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -536,8 +536,9 @@ def _format_glob_tool_result(paths: list[str], *, truncated: bool) -> str: GLOB_TIMEOUT = 10.0 # seconds LINE_NUMBER_WIDTH = 6 SEARCH_TRUNCATION_NOTE = ( - "Note: the search stopped early because it hit its time limit. The matches above are valid but incomplete. " - "Narrow the search (a more specific pattern or a narrower path) to see the rest." + "Note: the search stopped early (it hit its time limit or the maximum match count). " + "The matches above are valid but incomplete. Narrow the search (a more specific pattern or a " + "narrower path), or raise max_count, to see the rest." ) @@ -747,6 +748,15 @@ class GrepSchema(BaseModel): description=GREP_OUTPUT_MODE_DESCRIPTION, ) + max_count: int | None = Field( + default=None, + description=( + "Optional cap on the total number of matches returned across all files. " + "Leave unset to use the configured default. When the cap is hit, results " + "are truncated and a note says so; narrow the pattern or path to see the rest." + ), + ) + class ExecuteSchema(BaseModel): """Input schema for the `execute` tool.""" @@ -1241,6 +1251,7 @@ def __init__( tool_token_limit_before_evict: int | None = 20000, human_message_token_limit_before_evict: int | None = 50000, max_execute_timeout: int = 3600, + grep_max_count: int | None = 1000, tools: list[FsToolName] | Literal["all"] | None = None, _permissions: list[FilesystemPermission] | None = None, ) -> None: @@ -1259,6 +1270,13 @@ def __init__( Defaults to 3600 seconds (1 hour). Any per-command timeout exceeding this value will be rejected with an error message. + grep_max_count: Default total cap on the number of matches the + `grep` tool returns across all files. + + Defaults to `1000`, which bounds memory use and context size on + very large repositories. The model can override it per call via + the tool's `max_count` argument. Set to `None` to disable the + default cap (return every match unless a per-call cap is given). tools: Allowlist of tool names to expose to the model. ``"all"` indicates all tools. If unset, defaults to `"all"`. Pass a list containing any of `"ls"`, `"read_file"`, @@ -1281,6 +1299,9 @@ def __init__( if max_execute_timeout <= 0: msg = f"max_execute_timeout must be positive, got {max_execute_timeout}" raise ValueError(msg) + if grep_max_count is not None and grep_max_count <= 0: + msg = f"grep_max_count must be positive or None, got {grep_max_count}" + raise ValueError(msg) # Use provided backend or default to StateBackend instance self.backend = backend if backend is not None else StateBackend() if ( @@ -1313,6 +1334,7 @@ def __init__( self._tool_token_limit_before_evict = tool_token_limit_before_evict self._human_message_token_limit_before_evict = human_message_token_limit_before_evict self._max_execute_timeout = max_execute_timeout + self._grep_max_count = grep_max_count if isinstance(tools, list): self._enabled_tools: frozenset[str] | None = frozenset(tools) elif tools == "all": @@ -2131,6 +2153,7 @@ def sync_grep( path: str | None = None, glob: str | None = None, output_mode: Literal["files_with_matches", "content", "count"] = "files_with_matches", + max_count: int | None = None, ) -> ToolMessage: """Synchronous wrapper for grep tool.""" if path is not None: @@ -2151,7 +2174,8 @@ def sync_grep( status="error", ) resolved_backend = self._get_backend(runtime) - grep_result = resolved_backend.grep(pattern, path=path, glob=glob) + effective_max_count = max_count if max_count is not None else self._grep_max_count + grep_result = resolved_backend.grep(pattern, path=path, glob=glob, max_count=effective_max_count) matches = grep_result.matches or [] filtered_matches = _filter_grep_matches_by_permission(self._permissions, matches, operation="read") formatted, status = _format_grep_tool_result( @@ -2175,6 +2199,7 @@ async def async_grep( path: str | None = None, glob: str | None = None, output_mode: Literal["files_with_matches", "content", "count"] = "files_with_matches", + max_count: int | None = None, ) -> ToolMessage: """Asynchronous wrapper for grep tool.""" if path is not None: @@ -2195,7 +2220,8 @@ async def async_grep( status="error", ) resolved_backend = self._get_backend(runtime) - grep_result = await resolved_backend.agrep(pattern, path=path, glob=glob) + effective_max_count = max_count if max_count is not None else self._grep_max_count + grep_result = await resolved_backend.agrep(pattern, path=path, glob=glob, max_count=effective_max_count) matches = grep_result.matches or [] filtered_matches = _filter_grep_matches_by_permission(self._permissions, matches, operation="read") formatted, status = _format_grep_tool_result( From c0ea2872cb2eac9b65cb1a608bf6664b3b51d9c7 Mon Sep 17 00:00:00 2001 From: Mason Daugherty <61371264+mdrxy@users.noreply.github.com> Date: Wed, 8 Jul 2026 18:38:19 +0000 Subject: [PATCH 06/14] test(sdk): cover grep max_count in composite, sandbox, middleware Co-authored-by: open-swe[bot] --- .../backends/test_composite_backend.py | 56 +++++++++++++++ .../backends/test_sandbox_backend.py | 33 +++++++++ .../tests/unit_tests/test_middleware.py | 69 +++++++++++++++++++ 3 files changed, 158 insertions(+) diff --git a/libs/deepagents/tests/unit_tests/backends/test_composite_backend.py b/libs/deepagents/tests/unit_tests/backends/test_composite_backend.py index bf93b9720ac..9ee01718e5a 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_composite_backend.py +++ b/libs/deepagents/tests/unit_tests/backends/test_composite_backend.py @@ -1370,6 +1370,62 @@ def test_grep_path_stripping_matches_get_backend_and_key() -> None: assert matches2 is not None +def test_grep_max_count_enforced_across_routes() -> None: + """`max_count` caps total matches across the default and all routed backends.""" + mem_store = InMemoryStore() + store_be = StoreBackend(store=mem_store, namespace=lambda _rt: ("filesystem",)) + state = StoreBackend(store=mem_store, namespace=lambda _rt: ("default",)) + comp = CompositeBackend(default=state, routes={"/memories/": store_be}) + + comp.write("/root_a.txt", "hit\nhit\n") + comp.write("/memories/mem_a.txt", "hit\nhit\n") + + result = comp.grep("hit", path="/", max_count=3) + + assert result.truncated is True + assert result.matches is not None + assert len(result.matches) == 3 + + +def test_grep_max_count_short_circuits_routes() -> None: + """Once the default backend fills the cap, routed backends are not consulted.""" + mem_store = InMemoryStore() + state = StoreBackend(store=mem_store, namespace=lambda _rt: ("default",)) + + class _RaisingBackend(StoreBackend): + def grep(self, *_args: object, **_kwargs: object) -> GrepResult: + msg = "routed backend should not be queried once the cap is met" + raise AssertionError(msg) + + route = _RaisingBackend(store=mem_store, namespace=lambda _rt: ("filesystem",)) + comp = CompositeBackend(default=state, routes={"/memories/": route}) + + comp.write("/root_a.txt", "hit\nhit\nhit\n") + + result = comp.grep("hit", path="/", max_count=2) + + assert result.truncated is True + assert result.matches is not None + assert len(result.matches) == 2 + + +def test_grep_no_cap_returns_all_across_routes() -> None: + """`max_count=None` preserves prior behavior: every match across routes is returned.""" + mem_store = InMemoryStore() + store_be = StoreBackend(store=mem_store, namespace=lambda _rt: ("filesystem",)) + state = StoreBackend(store=mem_store, namespace=lambda _rt: ("default",)) + comp = CompositeBackend(default=state, routes={"/memories/": store_be}) + + comp.write("/root_a.txt", "hit\nhit\n") + comp.write("/memories/mem_a.txt", "hit\nhit\n") + + result = comp.grep("hit", path="/") + + assert result.truncated is False + assert result.matches is not None + assert len(result.matches) == 4 + + def test_glob_path_stripping_matches_get_backend_and_key() -> None: """Verify glob strips route prefix the same way as _get_backend_and_key.""" mem_store = InMemoryStore() diff --git a/libs/deepagents/tests/unit_tests/backends/test_sandbox_backend.py b/libs/deepagents/tests/unit_tests/backends/test_sandbox_backend.py index 8b4b6892824..3f8b4ff9966 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_sandbox_backend.py +++ b/libs/deepagents/tests/unit_tests/backends/test_sandbox_backend.py @@ -517,6 +517,39 @@ def test_build_grep_cmd_no_glob_uses_grep() -> None: assert "python3" not in cmd +def test_build_grep_cmd_max_count_adds_head_guard() -> None: + """A `max_count` bounds output with `head -n ` so grep stops early via SIGPIPE.""" + cmd = _build_grep_cmd("needle", "/test", None, 10) + # One record beyond the cap so the parser can distinguish complete from capped. + assert "head -n 11" in cmd + + +def test_build_grep_cmd_no_head_guard_without_max_count() -> None: + """Without `max_count`, the command carries no `head` guard (unchanged behavior).""" + cmd = _build_grep_cmd("needle", "/test", None) + assert "head -n" not in cmd + + +def test_parse_grep_output_caps_and_flags_truncation() -> None: + """`_parse_grep_output` caps matches to `max_count` and flags truncation when exceeded.""" + lines = [f"/test/f{i}.py\x001:needle" for i in range(5)] + resp = ExecuteResponse(output="\n".join(lines), exit_code=0) + result = _parse_grep_output(resp, "/test", 3) + assert result.truncated is True + assert result.matches is not None + assert len(result.matches) == 3 + + +def test_parse_grep_output_below_cap_not_truncated() -> None: + """When matches are at or below the cap, the result is not flagged truncated.""" + lines = [f"/test/f{i}.py\x001:needle" for i in range(3)] + resp = ExecuteResponse(output="\n".join(lines), exit_code=0) + result = _parse_grep_output(resp, "/test", 3) + assert result.truncated is False + assert result.matches is not None + assert len(result.matches) == 3 + + def test_grep_slash_glob_returns_matches_from_python_template() -> None: """grep() with a slash-containing glob parses output from the Python template.""" sandbox = MockSandbox() diff --git a/libs/deepagents/tests/unit_tests/test_middleware.py b/libs/deepagents/tests/unit_tests/test_middleware.py index 02fc73f05de..200c659008d 100644 --- a/libs/deepagents/tests/unit_tests/test_middleware.py +++ b/libs/deepagents/tests/unit_tests/test_middleware.py @@ -755,6 +755,75 @@ def test_grep_not_truncated_omits_note(self): assert result.status == "success" assert SEARCH_TRUNCATION_NOTE not in result.content + def test_grep_forwards_default_max_count_to_backend(self): + """The grep tool forwards the middleware's `grep_max_count` default to the backend.""" + backend, _ = _make_backend() + middleware = FilesystemMiddleware(backend=backend, grep_max_count=250) + grep_search_tool = next(tool for tool in middleware.tools if tool.name == "grep") + backend_obj = middleware._get_backend(_runtime()) + + captured: dict[str, object] = {} + + def _grep(_pattern, path=None, glob=None, *, max_count=None): # noqa: ARG001 + captured["max_count"] = max_count + return GrepResult(matches=[]) + + with ( + patch.object(middleware, "_get_backend", return_value=backend_obj), + patch.object(backend_obj, "grep", side_effect=_grep), + ): + grep_search_tool.invoke({"pattern": "import", "runtime": _runtime()}) + + assert captured["max_count"] == 250 + + def test_grep_per_call_max_count_overrides_default(self): + """A per-call `max_count` argument overrides the configured default.""" + backend, _ = _make_backend() + middleware = FilesystemMiddleware(backend=backend, grep_max_count=1000) + grep_search_tool = next(tool for tool in middleware.tools if tool.name == "grep") + backend_obj = middleware._get_backend(_runtime()) + + captured: dict[str, object] = {} + + def _grep(_pattern, path=None, glob=None, *, max_count=None): # noqa: ARG001 + captured["max_count"] = max_count + return GrepResult(matches=[]) + + with ( + patch.object(middleware, "_get_backend", return_value=backend_obj), + patch.object(backend_obj, "grep", side_effect=_grep), + ): + grep_search_tool.invoke({"pattern": "import", "max_count": 5, "runtime": _runtime()}) + + assert captured["max_count"] == 5 + + def test_grep_max_count_none_disables_default_cap(self): + """`grep_max_count=None` forwards no cap to the backend when no per-call value is given.""" + backend, _ = _make_backend() + middleware = FilesystemMiddleware(backend=backend, grep_max_count=None) + grep_search_tool = next(tool for tool in middleware.tools if tool.name == "grep") + backend_obj = middleware._get_backend(_runtime()) + + captured: dict[str, object] = {"max_count": "unset"} + + def _grep(_pattern, path=None, glob=None, *, max_count=None): # noqa: ARG001 + captured["max_count"] = max_count + return GrepResult(matches=[]) + + with ( + patch.object(middleware, "_get_backend", return_value=backend_obj), + patch.object(backend_obj, "grep", side_effect=_grep), + ): + grep_search_tool.invoke({"pattern": "import", "runtime": _runtime()}) + + assert captured["max_count"] is None + + def test_invalid_grep_max_count_raises(self): + """A non-positive `grep_max_count` is rejected at construction.""" + backend, _ = _make_backend() + with pytest.raises(ValueError, match="grep_max_count must be positive"): + FilesystemMiddleware(backend=backend, grep_max_count=0) + def test_glob_not_truncated_omits_note(self): """A complete glob must not carry the truncation note.""" backend, _ = _make_backend() From 7327645c451296f5833d1a8ff25720056ff794c8 Mon Sep 17 00:00:00 2001 From: Mason Daugherty <61371264+mdrxy@users.noreply.github.com> Date: Wed, 8 Jul 2026 18:40:39 +0000 Subject: [PATCH 07/14] test(sdk): cover grep streaming cap in filesystem backend Co-authored-by: open-swe[bot] --- .../backends/test_filesystem_backend.py | 230 ++++++++++++++---- 1 file changed, 187 insertions(+), 43 deletions(-) diff --git a/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py b/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py index e1165d611f9..ba17878c23f 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py +++ b/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py @@ -1,8 +1,10 @@ import base64 +import io import json import logging import shutil import subprocess +import threading import warnings from collections.abc import Iterator from pathlib import Path @@ -798,11 +800,65 @@ def _isolate_rg_cache() -> Iterator[None]: fs_module._resolve_ripgrep_path.cache_clear() -class _FakeProc: - def __init__(self, stdout: str = "", stderr: str = "", returncode: int = 0) -> None: - self.stdout = stdout - self.stderr = stderr +class _BlockingStdout: + """Text stream that yields queued lines then blocks until the proc is killed. + + Mirrors how ripgrep's stdout behaves under `subprocess.Popen(text=True)`: + iterating yields one JSON frame per line, and once ripgrep is terminated the + pipe closes so iteration ends. The `killed` event stands in for that close, + letting the timeout watchdog end the loop without real timing races. + """ + + def __init__(self, lines: list[str], killed: threading.Event) -> None: + self._lines = iter(lines) + self._killed = killed + + def __iter__(self) -> "_BlockingStdout": + return self + + def __next__(self) -> str: + if self._killed.is_set(): + raise StopIteration + try: + return next(self._lines) + except StopIteration: + # No more queued frames: emulate a stream that stays open until the + # process is killed (e.g. by the timeout watchdog). + self._killed.wait() + raise + + def close(self) -> None: + pass + + +class _FakePopen: + """Minimal `subprocess.Popen` stand-in for the streaming ripgrep path.""" + + def __init__(self, stdout_lines: list[str] | None = None, stderr: str = "", returncode: int = 0, *, block: bool = False) -> None: + self._killed = threading.Event() self.returncode = returncode + self.terminated = False + self.killed = False + lines = stdout_lines or [] + if block: + self.stdout: object = _BlockingStdout(lines, self._killed) + else: + self.stdout = io.StringIO("".join(lines)) + self._killed.set() # non-blocking: nothing to wait on + self.stderr = io.StringIO(stderr) + + def terminate(self) -> None: + self.terminated = True + self._killed.set() + + def kill(self) -> None: + self.killed = True + # A real kill yields a negative (signal) return code. + self.returncode = -9 + self._killed.set() + + def wait(self, timeout: float | None = None) -> int: # noqa: ARG002 + return self.returncode @pytest.mark.usefixtures("_isolate_rg_cache") @@ -833,11 +889,11 @@ def test_resolve_ripgrep_uses_resolved_path_in_argv(tmp_path: Path, monkeypatch: captured: dict[str, list[str]] = {} - def fake_run(cmd: list[str], **_kwargs: object) -> _FakeProc: + def fake_popen(cmd: list[str], **_kwargs: object) -> _FakePopen: captured["cmd"] = cmd - return _FakeProc() + return _FakePopen() - monkeypatch.setattr(fs_module.subprocess, "run", fake_run) + monkeypatch.setattr(fs_module.subprocess, "Popen", fake_popen) (tmp_path / "a.txt").write_text("hello\n") be = FilesystemBackend(root_dir=str(tmp_path), virtual_mode=False) @@ -852,13 +908,16 @@ def fake_run(cmd: list[str], **_kwargs: object) -> _FakeProc: @pytest.mark.usefixtures("_isolate_rg_cache") def test_ripgrep_timeout_logs_warning(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture) -> None: - """A `TimeoutExpired` from `subprocess.run` should emit a `WARNING`.""" + """When the streaming watchdog kills ripgrep with no output, we warn and fall back.""" monkeypatch.setattr(fs_module.shutil, "which", lambda _name: "/usr/bin/rg") + # Tiny budget so the watchdog timer fires almost immediately; the fake + # stdout blocks until that kill closes the stream (no timing race). + monkeypatch.setattr(fs_module, "DEFAULT_GREP_TIMEOUT", 0.05) - def timeout_run(cmd: list[str], **_kwargs: object) -> object: - raise subprocess.TimeoutExpired(cmd, timeout=30) + def timeout_popen(_cmd: list[str], **_kwargs: object) -> _FakePopen: + return _FakePopen(stdout_lines=[], block=True) - monkeypatch.setattr(fs_module.subprocess, "run", timeout_run) + monkeypatch.setattr(fs_module.subprocess, "Popen", timeout_popen) (tmp_path / "a.txt").write_text("hello\n") be = FilesystemBackend(root_dir=str(tmp_path), virtual_mode=False) @@ -875,38 +934,18 @@ def timeout_run(cmd: list[str], **_kwargs: object) -> object: @pytest.mark.usefixtures("_isolate_rg_cache") def test_ripgrep_timeout_returns_partial_results_when_output_captured(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """When ripgrep times out after emitting matches, those partial matches are returned flagged as truncated.""" + """When ripgrep is killed after streaming matches, those partial matches are returned flagged as truncated.""" monkeypatch.setattr(fs_module.shutil, "which", lambda _name: "/usr/bin/rg") + monkeypatch.setattr(fs_module, "DEFAULT_GREP_TIMEOUT", 0.05) (tmp_path / "a.txt").write_text("hello\n") frame = json.dumps({"type": "match", "data": {"path": {"text": "a.txt"}, "lines": {"text": "hello\n"}, "line_number": 1}}) - def timeout_run(cmd: list[str], **_kwargs: object) -> object: - # `subprocess.run` populates `TimeoutExpired.stdout` with output drained - # before the kill; ripgrep's newline-delimited JSON parses cleanly. - raise subprocess.TimeoutExpired(cmd, timeout=30, output=frame + "\n") + def timeout_popen(_cmd: list[str], **_kwargs: object) -> _FakePopen: + # One frame streams through, then the stream blocks until the watchdog + # kills the process (mirroring a search that outran its time budget). + return _FakePopen(stdout_lines=[frame + "\n"], block=True) - monkeypatch.setattr(fs_module.subprocess, "run", timeout_run) - be = FilesystemBackend(root_dir=str(tmp_path), virtual_mode=False) - - result = be.grep("hello", path=str(tmp_path)) - - assert result.truncated is True - assert result.error is None - assert result.matches and any(m["path"].endswith("a.txt") and m["text"] == "hello" for m in result.matches) - - -@pytest.mark.usefixtures("_isolate_rg_cache") -def test_ripgrep_timeout_decodes_bytes_output(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """`TimeoutExpired.stdout` is bytes even under `text=True`; partial output must still be parsed.""" - monkeypatch.setattr(fs_module.shutil, "which", lambda _name: "/usr/bin/rg") - (tmp_path / "a.txt").write_text("hello\n") - frame = json.dumps({"type": "match", "data": {"path": {"text": "a.txt"}, "lines": {"text": "hello\n"}, "line_number": 1}}) - - def timeout_run(cmd: list[str], **_kwargs: object) -> object: - # Bytes output mirrors what `subprocess.run` populates on `TimeoutExpired`. - raise subprocess.TimeoutExpired(cmd, timeout=30, output=(frame + "\n").encode()) - - monkeypatch.setattr(fs_module.subprocess, "run", timeout_run) + monkeypatch.setattr(fs_module.subprocess, "Popen", timeout_popen) be = FilesystemBackend(root_dir=str(tmp_path), virtual_mode=False) result = be.grep("hello", path=str(tmp_path)) @@ -1008,10 +1047,10 @@ def counting_which(_name: str) -> str | None: monkeypatch.setattr(fs_module.shutil, "which", counting_which) - def missing_run(cmd: list[str], **_kwargs: object) -> object: + def missing_popen(cmd: list[str], **_kwargs: object) -> object: raise FileNotFoundError(2, "No such file or directory", cmd[0]) - monkeypatch.setattr(fs_module.subprocess, "run", missing_run) + monkeypatch.setattr(fs_module.subprocess, "Popen", missing_popen) (tmp_path / "a.txt").write_text("hello\n") be = FilesystemBackend(root_dir=str(tmp_path), virtual_mode=False) @@ -1040,10 +1079,10 @@ def test_ripgrep_nonzero_returncode_falls_back_with_warning( """ monkeypatch.setattr(fs_module.shutil, "which", lambda _name: "/usr/bin/rg") - def erroring_run(_cmd: list[str], **_kwargs: object) -> _FakeProc: - return _FakeProc(stdout="", stderr="rg: error parsing glob 'docs/[': unclosed character class", returncode=2) + def erroring_popen(_cmd: list[str], **_kwargs: object) -> _FakePopen: + return _FakePopen(stdout_lines=[], stderr="rg: error parsing glob 'docs/[': unclosed character class", returncode=2) - monkeypatch.setattr(fs_module.subprocess, "run", erroring_run) + monkeypatch.setattr(fs_module.subprocess, "Popen", erroring_popen) (tmp_path / "a.txt").write_text("hello\n") be = FilesystemBackend(root_dir=str(tmp_path), virtual_mode=False) @@ -1058,6 +1097,111 @@ def erroring_run(_cmd: list[str], **_kwargs: object) -> _FakeProc: assert result.matches and any(m["path"].endswith("a.txt") for m in result.matches) +def _rg_match_frame(path: str, line_number: int, text: str) -> str: + """Build a ripgrep `--json` match frame line for the streaming fake.""" + return json.dumps({"type": "match", "data": {"path": {"text": path}, "lines": {"text": text}, "line_number": line_number}}) + "\n" + + +@pytest.mark.usefixtures("_isolate_rg_cache") +def test_ripgrep_streaming_caps_total_and_terminates(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The streaming ripgrep path stops at `max_count` total matches and kills the process early.""" + monkeypatch.setattr(fs_module.shutil, "which", lambda _name: "/usr/bin/rg") + for name in ("a.txt", "b.txt"): + (tmp_path / name).write_text("hello\nhello\nhello\n") + + # More frames than the cap, spread across files, so the cap must apply + # across files rather than per file. + frames = [ + _rg_match_frame("a.txt", 1, "hello\n"), + _rg_match_frame("a.txt", 2, "hello\n"), + _rg_match_frame("b.txt", 1, "hello\n"), + _rg_match_frame("b.txt", 2, "hello\n"), + ] + created: dict[str, _FakePopen] = {} + + def fake_popen(cmd: list[str], **_kwargs: object) -> _FakePopen: + # `-m ` is passed to ripgrep as a secondary per-file guard. + assert "-m" in cmd and str(2) in cmd + proc = _FakePopen(stdout_lines=frames) + created["proc"] = proc + return proc + + monkeypatch.setattr(fs_module.subprocess, "Popen", fake_popen) + be = FilesystemBackend(root_dir=str(tmp_path), virtual_mode=False) + + result = be.grep("hello", path=str(tmp_path), max_count=2) + + assert result.truncated is True + assert result.matches is not None + assert len(result.matches) == 2 + # The process was terminated once the cap was reached instead of draining. + assert created["proc"].terminated is True + + +@pytest.mark.usefixtures("_isolate_rg_cache") +def test_ripgrep_streaming_below_cap_not_truncated(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """When fewer matches than `max_count` are emitted, the result completes untruncated.""" + monkeypatch.setattr(fs_module.shutil, "which", lambda _name: "/usr/bin/rg") + (tmp_path / "a.txt").write_text("hello\n") + frames = [_rg_match_frame("a.txt", 1, "hello\n")] + + def fake_popen(_cmd: list[str], **_kwargs: object) -> _FakePopen: + return _FakePopen(stdout_lines=frames) + + monkeypatch.setattr(fs_module.subprocess, "Popen", fake_popen) + be = FilesystemBackend(root_dir=str(tmp_path), virtual_mode=False) + + result = be.grep("hello", path=str(tmp_path), max_count=100) + + assert result.truncated is False + assert result.matches is not None + assert len(result.matches) == 1 + + +def test_python_fallback_caps_total_matches_across_files(tmp_path: Path) -> None: + """The Python fallback caps total matches across files and flags truncation. + + ripgrep is absent in this environment, so `grep` uses `_python_search`. + """ + for name in ("a.txt", "b.txt", "c.txt"): + (tmp_path / name).write_text("needle\nneedle\n") + + be = FilesystemBackend(root_dir=str(tmp_path), virtual_mode=False) + + result = be.grep("needle", path=str(tmp_path), max_count=3) + + assert result.truncated is True + assert result.matches is not None + assert len(result.matches) == 3 + + +def test_python_fallback_no_cap_returns_all_matches(tmp_path: Path) -> None: + """With `max_count=None` the Python fallback returns every match, untruncated.""" + for name in ("a.txt", "b.txt", "c.txt"): + (tmp_path / name).write_text("needle\nneedle\n") + + be = FilesystemBackend(root_dir=str(tmp_path), virtual_mode=False) + + result = be.grep("needle", path=str(tmp_path)) + + assert result.truncated is False + assert result.matches is not None + assert len(result.matches) == 6 + + +def test_python_fallback_below_cap_not_truncated(tmp_path: Path) -> None: + """A cap larger than the number of matches leaves the result untruncated.""" + (tmp_path / "a.txt").write_text("needle\nneedle\n") + + be = FilesystemBackend(root_dir=str(tmp_path), virtual_mode=False) + + result = be.grep("needle", path=str(tmp_path), max_count=100) + + assert result.truncated is False + assert result.matches is not None + assert len(result.matches) == 2 + + def _install_flaky_rglob(monkeypatch: pytest.MonkeyPatch, exc: Exception, after_yields: int = 1) -> None: """Replace `Path.rglob` with a generator that yields N entries then raises.""" real_rglob = Path.rglob From 3c0e33a6d6beea60ad109a72a07bb2f3ec74bde6 Mon Sep 17 00:00:00 2001 From: Mason Daugherty <61371264+mdrxy@users.noreply.github.com> Date: Wed, 8 Jul 2026 18:49:04 +0000 Subject: [PATCH 08/14] fix(sdk): tidy grep max_count lint, docs, and test doubles Move the max_count doc into the grep Args section, satisfy the type checker on the streamed ripgrep stdout, and let composite test doubles accept the new keyword. Co-authored-by: open-swe[bot] --- libs/deepagents/deepagents/backends/filesystem.py | 6 ++++-- libs/deepagents/deepagents/backends/protocol.py | 13 +++++++------ .../unit_tests/backends/test_composite_backend.py | 6 +++--- .../backends/test_composite_backend_async.py | 6 +++--- .../unit_tests/backends/test_filesystem_backend.py | 3 +-- 5 files changed, 18 insertions(+), 16 deletions(-) diff --git a/libs/deepagents/deepagents/backends/filesystem.py b/libs/deepagents/deepagents/backends/filesystem.py index 44d6501d114..884c9d5c036 100644 --- a/libs/deepagents/deepagents/backends/filesystem.py +++ b/libs/deepagents/deepagents/backends/filesystem.py @@ -672,7 +672,7 @@ def grep( matches.append({"path": fpath, "line": int(line_num), "text": line_text}) return GrepResult(error=partial_error, matches=matches, truncated=truncated) - def _ripgrep_search( # noqa: C901, PLR0912, PLR0915 # streaming loop + per-exception logging (timeout vs exec-race vs hard-error vs cap) keeps branches explicit + def _ripgrep_search( # noqa: C901, PLR0911, PLR0912, PLR0915 self, pattern: str, base_full: Path, @@ -768,7 +768,9 @@ def _kill_on_timeout() -> None: try: # `proc.stdout` is a text stream because `text=True`; iterating it # yields one `--json` frame per line as ripgrep emits them. - for line in proc.stdout: # type: ignore[union-attr] + # `stdout=PIPE` guarantees a stream; narrow it for the type checker. + assert proc.stdout is not None # noqa: S101 + for line in proc.stdout: try: data = json.loads(line) except json.JSONDecodeError: diff --git a/libs/deepagents/deepagents/backends/protocol.py b/libs/deepagents/deepagents/backends/protocol.py index 6de8ae8ee1d..e49a67a83e3 100644 --- a/libs/deepagents/deepagents/backends/protocol.py +++ b/libs/deepagents/deepagents/backends/protocol.py @@ -471,12 +471,6 @@ def grep( - `?` matches single character - `[abc]` matches one character from set - Examples: - - `'*.py'` - only search Python files - - `'**/*.txt'` - search all `.txt` files recursively - - `'src/**/*.js'` - search JS files under src/ - - `'test[0-9].txt'` - search `test0.txt`, `test1.txt`, etc. - max_count: Optional total cap on the number of matches returned across all files. @@ -486,6 +480,13 @@ def grep( with `GrepResult.truncated=True`. Interpreted as a total cap, not a per-file cap. + Examples: + - `'*.py'` - only search Python files + - `'**/*.txt'` - search all `.txt` files recursively + - `'src/**/*.js'` - search JS files under src/ + - `'test[0-9].txt'` - search `test0.txt`, `test1.txt`, etc. + + Returns: `GrepResult` with matches or error. diff --git a/libs/deepagents/tests/unit_tests/backends/test_composite_backend.py b/libs/deepagents/tests/unit_tests/backends/test_composite_backend.py index 9ee01718e5a..749c8283223 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_composite_backend.py +++ b/libs/deepagents/tests/unit_tests/backends/test_composite_backend.py @@ -1206,7 +1206,7 @@ def test_composite_grep_error_in_routed_backend() -> None: # Create a mock backend that returns error strings for grep class ErrorBackend(StoreBackend): - def grep(self, pattern: str, path: str | None = None, glob: str | None = None): + def grep(self, pattern: str, path: str | None = None, glob: str | None = None, *, max_count: int | None = None): return "Invalid regex pattern error" error_backend = ErrorBackend() @@ -1225,7 +1225,7 @@ def test_composite_grep_error_in_routed_backend_at_root() -> None: # Create a mock backend that returns error strings for grep class ErrorBackend(StoreBackend): - def grep(self, pattern: str, path: str | None = None, glob: str | None = None): + def grep(self, pattern: str, path: str | None = None, glob: str | None = None, *, max_count: int | None = None): return "Backend error occurred" error_backend = ErrorBackend() @@ -1244,7 +1244,7 @@ def test_composite_grep_error_in_default_backend_at_root() -> None: # Create a mock backend that returns error strings for grep class ErrorDefaultBackend(StoreBackend): - def grep(self, pattern: str, path: str | None = None, glob: str | None = None): + def grep(self, pattern: str, path: str | None = None, glob: str | None = None, *, max_count: int | None = None): return "Default backend error" error_default = ErrorDefaultBackend() diff --git a/libs/deepagents/tests/unit_tests/backends/test_composite_backend_async.py b/libs/deepagents/tests/unit_tests/backends/test_composite_backend_async.py index 064c37e02f9..d72ff2ea814 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_composite_backend_async.py +++ b/libs/deepagents/tests/unit_tests/backends/test_composite_backend_async.py @@ -952,7 +952,7 @@ async def test_composite_agrep_error_in_routed_backend_async() -> None: # Create a mock backend that returns error strings for grep class ErrorBackend(StoreBackend): - async def agrep(self, pattern: str, path: str | None = None, glob: str | None = None): + async def agrep(self, pattern: str, path: str | None = None, glob: str | None = None, *, max_count: int | None = None): return "Invalid regex pattern error" error_backend = ErrorBackend() @@ -971,7 +971,7 @@ async def test_composite_agrep_error_in_routed_backend_at_root_async() -> None: # Create a mock backend that returns error strings for grep class ErrorBackend(StoreBackend): - async def agrep(self, pattern: str, path: str | None = None, glob: str | None = None): + async def agrep(self, pattern: str, path: str | None = None, glob: str | None = None, *, max_count: int | None = None): return "Backend error occurred" error_backend = ErrorBackend() @@ -990,7 +990,7 @@ async def test_composite_agrep_error_in_default_backend_at_root_async() -> None: # Create a mock backend that returns error strings for grep class ErrorDefaultBackend(StoreBackend): - async def agrep(self, pattern: str, path: str | None = None, glob: str | None = None): + async def agrep(self, pattern: str, path: str | None = None, glob: str | None = None, *, max_count: int | None = None): return "Default backend error" error_default = ErrorDefaultBackend() diff --git a/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py b/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py index ba17878c23f..38947ace52c 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py +++ b/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py @@ -3,7 +3,6 @@ import json import logging import shutil -import subprocess import threading import warnings from collections.abc import Iterator @@ -857,7 +856,7 @@ def kill(self) -> None: self.returncode = -9 self._killed.set() - def wait(self, timeout: float | None = None) -> int: # noqa: ARG002 + def wait(self, timeout: float | None = None) -> int: return self.returncode From 3452c819e619c90d948fb4de9e2b6b2cd537b076 Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Mon, 13 Jul 2026 17:14:31 -0400 Subject: [PATCH 09/14] cr --- .../deepagents/backends/protocol.py | 21 ++++++++- .../deepagents/deepagents/backends/sandbox.py | 6 +++ .../deepagents/middleware/filesystem.py | 42 ++++++++++++++++- .../backends/test_sandbox_backend.py | 28 ++++++++++- .../tests/unit_tests/test_middleware.py | 46 +++++++++++++++++++ 5 files changed, 138 insertions(+), 5 deletions(-) diff --git a/libs/deepagents/deepagents/backends/protocol.py b/libs/deepagents/deepagents/backends/protocol.py index e49a67a83e3..559afb77b72 100644 --- a/libs/deepagents/deepagents/backends/protocol.py +++ b/libs/deepagents/deepagents/backends/protocol.py @@ -526,9 +526,12 @@ async def agrep( bounds how long the caller waits; it does not stop the worker thread created by `asyncio.to_thread`. """ + grep_call = partial(self.grep, pattern, path, glob) + if _method_accepts_max_count(type(self), "grep"): + grep_call = partial(self.grep, pattern, path, glob, max_count=max_count) try: return await asyncio.wait_for( - asyncio.to_thread(partial(self.grep, pattern, path, glob, max_count=max_count)), + asyncio.to_thread(grep_call), timeout=ASYNC_GREP_TIMEOUT, ) except TimeoutError: @@ -955,6 +958,22 @@ async def aexecute( return await asyncio.to_thread(self.execute, command) +@lru_cache(maxsize=256) +def _method_accepts_max_count(cls: type[BackendProtocol], method_name: Literal["grep", "agrep"]) -> bool: + """Check whether a backend method accepts the optional `max_count` keyword.""" + try: + sig = inspect.signature(getattr(cls, method_name)) + except (AttributeError, ValueError, TypeError): + logger.warning( + "Could not inspect signature of %s.%s; assuming max_count is not supported.", + cls.__qualname__, + method_name, + exc_info=True, + ) + return False + return "max_count" in sig.parameters or any(parameter.kind is inspect.Parameter.VAR_KEYWORD for parameter in sig.parameters.values()) + + @lru_cache(maxsize=128) def execute_accepts_timeout(cls: type[SandboxBackendProtocol]) -> bool: """Check whether a backend class's `execute` accepts a `timeout` kwarg. diff --git a/libs/deepagents/deepagents/backends/sandbox.py b/libs/deepagents/deepagents/backends/sandbox.py index 5177f7535cc..dd1df034900 100644 --- a/libs/deepagents/deepagents/backends/sandbox.py +++ b/libs/deepagents/deepagents/backends/sandbox.py @@ -97,6 +97,8 @@ search_path = base64.b64decode('{path_b64}').decode('utf-8') glob_pat = base64.b64decode('{glob_b64}').decode('utf-8') pattern = base64.b64decode('{pattern_b64}').decode('utf-8') +max_count = {max_count} +match_count = 0 # When the search path is a directory, chdir to it so glob patterns # resolve relative to it. When it is a single file, search it directly @@ -139,6 +141,9 @@ # one so records never concatenate when a file's last # line lacks a final newline. sys.stdout.write(display_path + chr(0) + str(i) + ':' + line.rstrip(chr(10)) + chr(10)) + match_count += 1 + if max_count is not None and match_count > max_count: + sys.exit(0) except OSError: pass " 2>/dev/null""" @@ -582,6 +587,7 @@ def _build_grep_cmd(pattern: str, path: str | None, glob: str | None, max_count: path_b64=path_b64, glob_b64=glob_b64, pattern_b64=pattern_b64, + max_count=None if max_count is None else int(max_count), ) glob_pattern = f"--include={shlex.quote(glob)}" if glob else "" diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index c6bdb5e1189..046700ee0db 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -56,6 +56,7 @@ ReadResult, SandboxBackendProtocol, WriteResult, + _method_accepts_max_count, _resolve_backend, _supports_delete, execute_accepts_timeout, @@ -457,6 +458,43 @@ def _filter_grep_matches_by_permission( return [m for m in matches if _check_fs_permission(rules, operation, m.get("path", "")) != "deny"] +def _apply_grep_max_count(result: GrepResult, max_count: int | None) -> GrepResult: + """Enforce the tool-level cap when a legacy backend cannot do so while searching.""" + if max_count is None or result.matches is None or len(result.matches) <= max_count: + return result + return GrepResult(error=result.error, matches=result.matches[:max_count], truncated=True) + + +def _grep_backend( + backend: BackendProtocol, + pattern: str, + path: str | None, + glob: str | None, + max_count: int | None, +) -> GrepResult: + """Call `grep` without breaking backends that use the previous signature.""" + if _method_accepts_max_count(type(backend), "grep"): + result = backend.grep(pattern, path=path, glob=glob, max_count=max_count) + else: + result = backend.grep(pattern, path=path, glob=glob) + return _apply_grep_max_count(result, max_count) + + +async def _agrep_backend( + backend: BackendProtocol, + pattern: str, + path: str | None, + glob: str | None, + max_count: int | None, +) -> GrepResult: + """Call `agrep` without breaking backends that use the previous signature.""" + if _method_accepts_max_count(type(backend), "agrep"): + result = await backend.agrep(pattern, path=path, glob=glob, max_count=max_count) + else: + result = await backend.agrep(pattern, path=path, glob=glob) + return _apply_grep_max_count(result, max_count) + + def _format_grep_tool_result( result: GrepResult, output_mode: Literal["files_with_matches", "content", "count"], @@ -2175,7 +2213,7 @@ def sync_grep( ) resolved_backend = self._get_backend(runtime) effective_max_count = max_count if max_count is not None else self._grep_max_count - grep_result = resolved_backend.grep(pattern, path=path, glob=glob, max_count=effective_max_count) + grep_result = _grep_backend(resolved_backend, pattern, path, glob, effective_max_count) matches = grep_result.matches or [] filtered_matches = _filter_grep_matches_by_permission(self._permissions, matches, operation="read") formatted, status = _format_grep_tool_result( @@ -2221,7 +2259,7 @@ async def async_grep( ) resolved_backend = self._get_backend(runtime) effective_max_count = max_count if max_count is not None else self._grep_max_count - grep_result = await resolved_backend.agrep(pattern, path=path, glob=glob, max_count=effective_max_count) + grep_result = await _agrep_backend(resolved_backend, pattern, path, glob, effective_max_count) matches = grep_result.matches or [] filtered_matches = _filter_grep_matches_by_permission(self._permissions, matches, operation="read") formatted, status = _format_grep_tool_result( diff --git a/libs/deepagents/tests/unit_tests/backends/test_sandbox_backend.py b/libs/deepagents/tests/unit_tests/backends/test_sandbox_backend.py index 0887b29f981..6757365e009 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_sandbox_backend.py +++ b/libs/deepagents/tests/unit_tests/backends/test_sandbox_backend.py @@ -1491,7 +1491,12 @@ def test_glob_script_rejects_traversal_pattern(tmp_path: Path) -> None: # the silent-zero-results class this route exists to fix. -def _run_grep_glob_script(path: Path, pattern: str, glob: str) -> subprocess.CompletedProcess[str]: +def _run_grep_glob_script( + path: Path, + pattern: str, + glob: str, + max_count: int | None = None, +) -> subprocess.CompletedProcess[str]: """Execute the formatted `_GREP_PATH_GLOB_TEMPLATE` script directly. Extracts the inline `python3 -c` body from the command `_build_grep_cmd` @@ -1500,7 +1505,7 @@ def _run_grep_glob_script(path: Path, pattern: str, glob: str) -> subprocess.Com assert on both stdout and the exit code — the template's error-surfacing contract depends on a non-zero exit propagating rather than being masked. """ - cmd = _build_grep_cmd(pattern, str(path), glob) + cmd = _build_grep_cmd(pattern, str(path), glob, max_count) _, _, tail = cmd.partition('python3 -c "') script, _, _ = tail.rpartition('"') return subprocess.run( # noqa: S603 # script is the project's own _GREP_PATH_GLOB_TEMPLATE, not user input @@ -1544,6 +1549,25 @@ def test_grep_glob_script_matches_recursively_and_prefixes_path(tmp_path: Path) assert "other.py" not in proc.stdout +def test_grep_glob_script_stops_after_cap_probe(tmp_path: Path) -> None: + """A slash-glob search emits only the cap plus one truncation probe record.""" + (tmp_path / "src").mkdir() + (tmp_path / "src" / "matches.py").write_text("needle\n" * 20) + + proc = _run_grep_glob_script(tmp_path, "needle", "src/*.py", max_count=2) + + assert proc.returncode == 0 + assert len(_parse_script_records(proc.stdout)) == 3 + result = _parse_grep_output( + ExecuteResponse(output=proc.stdout, exit_code=proc.returncode, truncated=False), + str(tmp_path), + max_count=2, + ) + assert result.truncated is True + assert result.matches is not None + assert len(result.matches) == 2 + + def test_grep_glob_script_terminates_records_end_to_end(tmp_path: Path) -> None: """Two matched files whose last line lacks a newline parse as two records. diff --git a/libs/deepagents/tests/unit_tests/test_middleware.py b/libs/deepagents/tests/unit_tests/test_middleware.py index 200c659008d..f1e52940b4d 100644 --- a/libs/deepagents/tests/unit_tests/test_middleware.py +++ b/libs/deepagents/tests/unit_tests/test_middleware.py @@ -818,6 +818,52 @@ def _grep(_pattern, path=None, glob=None, *, max_count=None): # noqa: ARG001 assert captured["max_count"] is None + def test_grep_caps_legacy_backend_without_forwarding_max_count(self): + """The default cap remains compatible with a backend using the previous `grep` signature.""" + + class LegacyBackend(StateBackend): + def grep(self, pattern, path=None, glob=None): # type: ignore[override] + return GrepResult( + matches=[ + {"path": "/one.py", "line": 1, "text": "needle"}, + {"path": "/two.py", "line": 1, "text": "needle"}, + {"path": "/three.py", "line": 1, "text": "needle"}, + ] + ) + + middleware = FilesystemMiddleware(backend=LegacyBackend(), grep_max_count=2) + grep_search_tool = next(tool for tool in middleware.tools if tool.name == "grep") + + result = grep_search_tool.invoke({"pattern": "needle", "output_mode": "content", "runtime": _runtime()}) + + assert result.status == "success" + assert "/one.py" in result.content + assert "/two.py" in result.content + assert "/three.py" not in result.content + assert SEARCH_TRUNCATION_NOTE in result.content + + async def test_async_grep_caps_legacy_backend_without_forwarding_max_count(self): + """The inherited async wrapper also supports the previous `grep` signature.""" + + class LegacyBackend(StateBackend): + def grep(self, pattern, path=None, glob=None): # type: ignore[override] + return GrepResult( + matches=[ + {"path": "/one.py", "line": 1, "text": "needle"}, + {"path": "/two.py", "line": 1, "text": "needle"}, + ] + ) + + middleware = FilesystemMiddleware(backend=LegacyBackend(), grep_max_count=1) + grep_search_tool = next(tool for tool in middleware.tools if tool.name == "grep") + + result = await grep_search_tool.ainvoke({"pattern": "needle", "output_mode": "content", "runtime": _runtime()}) + + assert result.status == "success" + assert "/one.py" in result.content + assert "/two.py" not in result.content + assert SEARCH_TRUNCATION_NOTE in result.content + def test_invalid_grep_max_count_raises(self): """A non-positive `grep_max_count` is rejected at construction.""" backend, _ = _make_backend() From df5c0dae2deae5ea661462ee5248e559aba2c6a3 Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Mon, 13 Jul 2026 17:28:06 -0400 Subject: [PATCH 10/14] cr --- .../deepagents/backends/composite.py | 48 +++++++++++++++---- .../deepagents/backends/filesystem.py | 6 +-- .../deepagents/backends/protocol.py | 10 +++- .../deepagents/middleware/filesystem.py | 9 +--- .../backends/test_composite_backend.py | 30 ++++++++++++ .../backends/test_composite_backend_async.py | 32 +++++++++++++ .../backends/test_filesystem_backend.py | 26 ++++++++++ .../unit_tests/backends/test_protocol.py | 24 ++++++++++ .../tests/unit_tests/test_middleware.py | 8 ++++ 9 files changed, 174 insertions(+), 19 deletions(-) diff --git a/libs/deepagents/deepagents/backends/composite.py b/libs/deepagents/deepagents/backends/composite.py index f534f7c6c8e..557aaab461b 100644 --- a/libs/deepagents/deepagents/backends/composite.py +++ b/libs/deepagents/deepagents/backends/composite.py @@ -24,6 +24,8 @@ ReadResult, SandboxBackendProtocol, WriteResult, + _apply_grep_max_count, + _method_accepts_max_count, execute_accepts_timeout, ) from deepagents.backends.state import StateBackend @@ -366,6 +368,36 @@ def _coerce_grep_result(raw: GrepResult | list[GrepMatch] | str) -> GrepResult: return GrepResult(error=raw) return GrepResult(matches=raw) + def _grep_backend( + self, + backend: BackendProtocol, + pattern: str, + path: str | None, + glob: str | None, + max_count: int | None, + ) -> GrepResult: + """Call `grep` while supporting backends with the previous signature.""" + if _method_accepts_max_count(type(backend), "grep"): + raw = backend.grep(pattern, path, glob, max_count=max_count) + else: + raw = backend.grep(pattern, path, glob) + return _apply_grep_max_count(self._coerce_grep_result(raw), max_count) + + async def _agrep_backend( + self, + backend: BackendProtocol, + pattern: str, + path: str | None, + glob: str | None, + max_count: int | None, + ) -> GrepResult: + """Call `agrep` while supporting backends with the previous signature.""" + if _method_accepts_max_count(type(backend), "agrep"): + raw = await backend.agrep(pattern, path, glob, max_count=max_count) + else: + raw = await backend.agrep(pattern, path, glob) + return _apply_grep_max_count(self._coerce_grep_result(raw), max_count) + def grep( self, pattern: str, @@ -408,7 +440,7 @@ def grep( path=path, ) if route_prefix is not None: - grep_result = self._coerce_grep_result(backend.grep(pattern, backend_path, glob, max_count=max_count)) + grep_result = self._grep_backend(backend, pattern, backend_path, glob, max_count) if grep_result.error: return grep_result return GrepResult( @@ -421,7 +453,7 @@ def grep( if path is None or path == "/": all_matches: list[GrepMatch] = [] truncated = False - default_result = self._coerce_grep_result(self.default.grep(pattern, path, glob, max_count=max_count)) + default_result = self._grep_backend(self.default, pattern, path, glob, max_count) if default_result.error: return default_result all_matches.extend(default_result.matches or []) @@ -433,7 +465,7 @@ def grep( # Cap already met by earlier routes; skip the rest. truncated = True break - grep_result = self._coerce_grep_result(backend.grep(pattern, "/", glob, max_count=remaining)) + grep_result = self._grep_backend(backend, pattern, "/", glob, remaining) if grep_result.error: return grep_result all_matches.extend(_remap_grep_path(m, route_prefix) for m in (grep_result.matches or [])) @@ -444,7 +476,7 @@ def grep( truncated = True return GrepResult(matches=all_matches, truncated=truncated) # Path specified but doesn't match a route - search only default - return self._coerce_grep_result(self.default.grep(pattern, path, glob, max_count=max_count)) + return self._grep_backend(self.default, pattern, path, glob, max_count) async def agrep( self, @@ -465,7 +497,7 @@ async def agrep( path=path, ) if route_prefix is not None: - grep_result = self._coerce_grep_result(await backend.agrep(pattern, backend_path, glob, max_count=max_count)) + grep_result = await self._agrep_backend(backend, pattern, backend_path, glob, max_count) if grep_result.error: return grep_result return GrepResult( @@ -478,7 +510,7 @@ async def agrep( if path is None or path == "/": all_matches: list[GrepMatch] = [] truncated = False - default_result = self._coerce_grep_result(await self.default.agrep(pattern, path, glob, max_count=max_count)) + default_result = await self._agrep_backend(self.default, pattern, path, glob, max_count) if default_result.error: return default_result all_matches.extend(default_result.matches or []) @@ -490,7 +522,7 @@ async def agrep( # Cap already met by earlier routes; skip the rest. truncated = True break - grep_result = self._coerce_grep_result(await backend.agrep(pattern, "/", glob, max_count=remaining)) + grep_result = await self._agrep_backend(backend, pattern, "/", glob, remaining) if grep_result.error: return grep_result all_matches.extend(_remap_grep_path(m, route_prefix) for m in (grep_result.matches or [])) @@ -501,7 +533,7 @@ async def agrep( truncated = True return GrepResult(matches=all_matches, truncated=truncated) # Path specified but doesn't match a route - search only default - return self._coerce_grep_result(await self.default.agrep(pattern, path, glob, max_count=max_count)) + return await self._agrep_backend(self.default, pattern, path, glob, max_count) def glob(self, pattern: str, path: str | None = None) -> GlobResult: """Find files matching a glob pattern, routing by path prefix. diff --git a/libs/deepagents/deepagents/backends/filesystem.py b/libs/deepagents/deepagents/backends/filesystem.py index 884c9d5c036..19f88405153 100644 --- a/libs/deepagents/deepagents/backends/filesystem.py +++ b/libs/deepagents/deepagents/backends/filesystem.py @@ -847,9 +847,9 @@ def _kill_on_timeout() -> None: # Ripgrep exits 0 on match, 1 on no-match (both expected), 2+ on a hard # error (invalid pattern, unreadable directory, malformed glob, etc.). - # Reporting zero matches on a hard error would be the silent failure - # this resolver is meant to avoid, so fall back to the Python search. - if proc.returncode not in (0, 1) and not results: + # Returning matches gathered before a hard error would present an + # incomplete search as complete, so fall back to the Python search. + if proc.returncode not in (0, 1): logger.warning("ripgrep exited %d (stderr=%r); using Python grep fallback", proc.returncode, stderr.strip()[:500]) return None, False diff --git a/libs/deepagents/deepagents/backends/protocol.py b/libs/deepagents/deepagents/backends/protocol.py index 559afb77b72..44a641ddb3f 100644 --- a/libs/deepagents/deepagents/backends/protocol.py +++ b/libs/deepagents/deepagents/backends/protocol.py @@ -334,6 +334,13 @@ class GrepResult: truncated: bool = False +def _apply_grep_max_count(result: GrepResult, max_count: int | None) -> GrepResult: + """Enforce a match cap after a backend search has completed.""" + if max_count is None or result.matches is None or len(result.matches) <= max_count: + return result + return GrepResult(error=result.error, matches=result.matches[:max_count], truncated=True) + + @dataclass class GlobResult: """Result from backend `glob` operations. @@ -530,10 +537,11 @@ async def agrep( if _method_accepts_max_count(type(self), "grep"): grep_call = partial(self.grep, pattern, path, glob, max_count=max_count) try: - return await asyncio.wait_for( + result = await asyncio.wait_for( asyncio.to_thread(grep_call), timeout=ASYNC_GREP_TIMEOUT, ) + return _apply_grep_max_count(result, max_count) except TimeoutError: logger.warning( "agrep timed out after %ds (pattern=%r, path=%r, glob=%r)", diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index 046700ee0db..b3f0f9b47b9 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -56,6 +56,7 @@ ReadResult, SandboxBackendProtocol, WriteResult, + _apply_grep_max_count, _method_accepts_max_count, _resolve_backend, _supports_delete, @@ -458,13 +459,6 @@ def _filter_grep_matches_by_permission( return [m for m in matches if _check_fs_permission(rules, operation, m.get("path", "")) != "deny"] -def _apply_grep_max_count(result: GrepResult, max_count: int | None) -> GrepResult: - """Enforce the tool-level cap when a legacy backend cannot do so while searching.""" - if max_count is None or result.matches is None or len(result.matches) <= max_count: - return result - return GrepResult(error=result.error, matches=result.matches[:max_count], truncated=True) - - def _grep_backend( backend: BackendProtocol, pattern: str, @@ -788,6 +782,7 @@ class GrepSchema(BaseModel): max_count: int | None = Field( default=None, + gt=0, description=( "Optional cap on the total number of matches returned across all files. " "Leave unset to use the configured default. When the cap is hit, results " diff --git a/libs/deepagents/tests/unit_tests/backends/test_composite_backend.py b/libs/deepagents/tests/unit_tests/backends/test_composite_backend.py index 749c8283223..9cc704bc379 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_composite_backend.py +++ b/libs/deepagents/tests/unit_tests/backends/test_composite_backend.py @@ -1426,6 +1426,36 @@ def test_grep_no_cap_returns_all_across_routes() -> None: assert len(result.matches) == 4 +def test_grep_supports_legacy_backends_across_routes() -> None: + """Composite grep preserves old child signatures and caps their results.""" + + class LegacyBackend(BackendProtocol): + def __init__(self, paths: list[str]) -> None: + self.paths = paths + + def grep( # ty: ignore[invalid-method-override] # Intentionally models the old public signature. + self, + pattern: str, + path: str | None = None, + glob: str | None = None, + ) -> GrepResult: + return GrepResult(matches=[{"path": item, "line": 1, "text": pattern} for item in self.paths]) + + comp = CompositeBackend( + default=LegacyBackend(["/default.txt"]), + routes={"/legacy/": LegacyBackend(["/one.txt", "/two.txt", "/three.txt"])}, + ) + + uncapped = comp.grep("needle", path="/") + capped = comp.grep("needle", path="/", max_count=2) + + assert uncapped.matches is not None + assert len(uncapped.matches) == 4 + assert capped.matches is not None + assert len(capped.matches) == 2 + assert capped.truncated is True + + def test_glob_path_stripping_matches_get_backend_and_key() -> None: """Verify glob strips route prefix the same way as _get_backend_and_key.""" mem_store = InMemoryStore() diff --git a/libs/deepagents/tests/unit_tests/backends/test_composite_backend_async.py b/libs/deepagents/tests/unit_tests/backends/test_composite_backend_async.py index d72ff2ea814..15ee3f7cbc8 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_composite_backend_async.py +++ b/libs/deepagents/tests/unit_tests/backends/test_composite_backend_async.py @@ -8,8 +8,10 @@ from deepagents.backends.composite import CompositeBackend from deepagents.backends.filesystem import FilesystemBackend from deepagents.backends.protocol import ( + BackendProtocol, ExecuteResponse, GlobResult, + GrepResult, SandboxBackendProtocol, WriteResult, ) @@ -1003,6 +1005,36 @@ async def agrep(self, pattern: str, path: str | None = None, glob: str | None = assert result.error == "Default backend error" +async def test_composite_agrep_supports_legacy_child_signatures() -> None: + """Composite async grep avoids forwarding caps to old child signatures.""" + + class LegacyBackend(BackendProtocol): + def __init__(self, paths: list[str]) -> None: + self.paths = paths + + async def agrep( # ty: ignore[invalid-method-override] # Intentionally models the old public signature. + self, + pattern: str, + path: str | None = None, + glob: str | None = None, + ) -> GrepResult: + return GrepResult(matches=[{"path": item, "line": 1, "text": pattern} for item in self.paths]) + + comp = CompositeBackend( + default=LegacyBackend(["/default.txt"]), + routes={"/legacy/": LegacyBackend(["/one.txt", "/two.txt", "/three.txt"])}, + ) + + uncapped = await comp.agrep("needle", path="/") + capped = await comp.agrep("needle", path="/", max_count=2) + + assert uncapped.matches is not None + assert len(uncapped.matches) == 4 + assert capped.matches is not None + assert len(capped.matches) == 2 + assert capped.truncated is True + + async def test_composite_aglob_default_error_short_circuits_routes_async() -> None: """A root glob default error should return before consulting routed backends.""" diff --git a/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py b/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py index 38947ace52c..c8fd224587d 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py +++ b/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py @@ -1096,6 +1096,32 @@ def erroring_popen(_cmd: list[str], **_kwargs: object) -> _FakePopen: assert result.matches and any(m["path"].endswith("a.txt") for m in result.matches) +@pytest.mark.usefixtures("_isolate_rg_cache") +def test_ripgrep_nonzero_returncode_discards_partial_output( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + """A hard ripgrep error after a match reruns the complete Python search.""" + monkeypatch.setattr(fs_module.shutil, "which", lambda _name: "/usr/bin/rg") + frame = _rg_match_frame("a.txt", 1, "hello\n") + + def erroring_popen(_cmd: list[str], **_kwargs: object) -> _FakePopen: + return _FakePopen(stdout_lines=[frame], stderr="rg: unreadable directory", returncode=2) + + monkeypatch.setattr(fs_module.subprocess, "Popen", erroring_popen) + (tmp_path / "a.txt").write_text("hello\nhello\n") + be = FilesystemBackend(root_dir=str(tmp_path), virtual_mode=False) + + with caplog.at_level(logging.WARNING, logger=fs_module.logger.name): + result = be.grep("hello", path=str(tmp_path)) + + assert any("ripgrep exited 2" in record.getMessage() for record in caplog.records) + assert result.error is None + assert result.matches is not None + assert len(result.matches) == 2 + + def _rg_match_frame(path: str, line_number: int, text: str) -> str: """Build a ripgrep `--json` match frame line for the streaming fake.""" return json.dumps({"type": "match", "data": {"path": {"text": path}, "lines": {"text": text}, "line_number": line_number}}) + "\n" diff --git a/libs/deepagents/tests/unit_tests/backends/test_protocol.py b/libs/deepagents/tests/unit_tests/backends/test_protocol.py index 53ec02741b0..2956c091fb6 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_protocol.py +++ b/libs/deepagents/tests/unit_tests/backends/test_protocol.py @@ -258,6 +258,30 @@ async def test_agrep_propagates_not_implemented(self, backend: BareBackend) -> N with pytest.raises(NotImplementedError): await backend.agrep("pattern") + async def test_agrep_caps_legacy_grep_result(self) -> None: + """The inherited async wrapper caps results from an old `grep` signature.""" + + class LegacyBackend(BackendProtocol): + def grep( # ty: ignore[invalid-method-override] # Intentionally models the old public signature. + self, + pattern: str, + path: str | None = None, + glob: str | None = None, + ) -> GrepResult: + return GrepResult( + matches=[ + {"path": "/one.txt", "line": 1, "text": pattern}, + {"path": "/two.txt", "line": 1, "text": pattern}, + {"path": "/three.txt", "line": 1, "text": pattern}, + ] + ) + + result = await LegacyBackend().agrep("needle", max_count=2) + + assert result.matches is not None + assert len(result.matches) == 2 + assert result.truncated is True + def _runtime_error_from_eloop_context() -> RuntimeError: """Create the Python <=3.12 `Path.resolve()` symlink-loop shape via `__context__`.""" diff --git a/libs/deepagents/tests/unit_tests/test_middleware.py b/libs/deepagents/tests/unit_tests/test_middleware.py index f1e52940b4d..69fa129c20c 100644 --- a/libs/deepagents/tests/unit_tests/test_middleware.py +++ b/libs/deepagents/tests/unit_tests/test_middleware.py @@ -17,6 +17,7 @@ from langgraph.graph.message import REMOVE_ALL_MESSAGES from langgraph.store.memory import InMemoryStore from langgraph.types import Command +from pydantic import ValidationError import deepagents.middleware.filesystem as filesystem_middleware from deepagents.backends import CompositeBackend, StateBackend, StoreBackend @@ -51,6 +52,7 @@ FilesystemMiddleware, FilesystemPermission, FilesystemState, + GrepSchema, supports_execution, ) from deepagents.middleware.patch_tool_calls import PatchToolCallsMiddleware @@ -870,6 +872,12 @@ def test_invalid_grep_max_count_raises(self): with pytest.raises(ValueError, match="grep_max_count must be positive"): FilesystemMiddleware(backend=backend, grep_max_count=0) + @pytest.mark.parametrize("max_count", [0, -1]) + def test_non_positive_per_call_max_count_is_rejected(self, max_count: int) -> None: + """The grep tool schema accepts only positive per-call caps.""" + with pytest.raises(ValidationError, match="greater than 0"): + GrepSchema(pattern="needle", max_count=max_count) + def test_glob_not_truncated_omits_note(self): """A complete glob must not carry the truncation note.""" backend, _ = _make_backend() From 8234890b7894f3c8aafeadba792229ac64fb2af4 Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Mon, 13 Jul 2026 17:50:07 -0400 Subject: [PATCH 11/14] cr --- .../deepagents/backends/composite.py | 6 + .../deepagents/backends/context_hub.py | 11 +- .../deepagents/backends/filesystem.py | 171 +++++++++++------- .../deepagents/backends/protocol.py | 24 ++- .../deepagents/deepagents/backends/sandbox.py | 4 + libs/deepagents/deepagents/backends/utils.py | 12 +- .../backends/test_context_hub_backend.py | 35 ++++ .../backends/test_filesystem_backend.py | 43 ++++- .../tests/unit_tests/backends/test_utils.py | 34 ++++ .../custom_system_message_tools.json | 13 ++ .../system_prompt_with_execute_tools.json | 13 ++ ...m_prompt_with_memory_and_skills_tools.json | 13 ++ ...stem_prompt_with_routed_backend_tools.json | 13 ++ ...t_with_sync_and_async_subagents_tools.json | 13 ++ .../system_prompt_without_execute_tools.json | 13 ++ 15 files changed, 330 insertions(+), 88 deletions(-) diff --git a/libs/deepagents/deepagents/backends/composite.py b/libs/deepagents/deepagents/backends/composite.py index 557aaab461b..c7417c68d89 100644 --- a/libs/deepagents/deepagents/backends/composite.py +++ b/libs/deepagents/deepagents/backends/composite.py @@ -471,6 +471,9 @@ def grep( all_matches.extend(_remap_grep_path(m, route_prefix) for m in (grep_result.matches or [])) truncated = truncated or grep_result.truncated + # Defense-in-depth: the budget split above already keeps the total at + # or under `max_count`, so this only trims a non-compliant backend + # that returned more than its allotted budget. if max_count is not None and len(all_matches) > max_count: all_matches = all_matches[:max_count] truncated = True @@ -528,6 +531,9 @@ async def agrep( all_matches.extend(_remap_grep_path(m, route_prefix) for m in (grep_result.matches or [])) truncated = truncated or grep_result.truncated + # Defense-in-depth: the budget split above already keeps the total at + # or under `max_count`, so this only trims a non-compliant backend + # that returned more than its allotted budget. if max_count is not None and len(all_matches) > max_count: all_matches = all_matches[:max_count] truncated = True diff --git a/libs/deepagents/deepagents/backends/context_hub.py b/libs/deepagents/deepagents/backends/context_hub.py index b4cbb05d7e5..f2caf3c2191 100644 --- a/libs/deepagents/deepagents/backends/context_hub.py +++ b/libs/deepagents/deepagents/backends/context_hub.py @@ -280,8 +280,10 @@ def grep( ) -> GrepResult: """Search contents for `pattern` (optional `path` / `glob` filters). - When `max_count` is set, the search stops once that many total matches - have been collected and the result is flagged `truncated=True`. + When `max_count` is set, at most that many matches are returned; if more + exist the search stops and the result is flagged `truncated=True`. + Exactly `max_count` matches with none dropped is reported complete + (`truncated=False`). """ try: cache = self._ensure_cache() @@ -306,9 +308,12 @@ def grep( continue for i, line in enumerate(content.splitlines(), start=1): if regex.search(line): - matches.append(GrepMatch(path=f"/{file_path}", line=i, text=line)) if max_count is not None and len(matches) >= max_count: + # A further match beyond `max_count` proves more exist; + # stop and flag truncation. Checked before appending so + # exactly `max_count` matches is reported complete. return GrepResult(matches=matches, truncated=True) + matches.append(GrepMatch(path=f"/{file_path}", line=i, text=line)) return GrepResult(matches=matches) diff --git a/libs/deepagents/deepagents/backends/filesystem.py b/libs/deepagents/deepagents/backends/filesystem.py index 19f88405153..91cfb6f346d 100644 --- a/libs/deepagents/deepagents/backends/filesystem.py +++ b/libs/deepagents/deepagents/backends/filesystem.py @@ -683,9 +683,9 @@ def _ripgrep_search( # noqa: C901, PLR0911, PLR0912, PLR0915 Streams ripgrep's newline-delimited `--json` output line-by-line via `subprocess.Popen` instead of buffering all of stdout, so a pathological - pattern on a huge repository cannot spike memory. Once `max_count` total - matches have been collected the process is terminated and the search - stops early. + pattern on a huge repository cannot spike memory. Once more than + `max_count` matches are found the process is terminated and the search + stops early, returning exactly `max_count` matches flagged truncated. Args: pattern: Literal string to search for (unescaped). @@ -711,11 +711,13 @@ def _ripgrep_search( # noqa: C901, PLR0911, PLR0912, PLR0915 cmd = [rg_path, "--json", "-F"] # -F enables fixed-string (literal) mode if max_count is not None: - # Secondary, cheap per-file guard. `rg -m` is per file so it does - # not bound the total on its own (a repo with many files each - # contributing one match still overflows) — the total cap below is - # what actually stops the search — but it trims runaway single files. - cmd.extend(["-m", str(max_count)]) + # Per-file guard set to `max_count + 1`, not `max_count`. `rg -m` is + # per file, so it can't bound the total on its own — the streaming + # total cap below is what actually stops the search. The `+ 1` lets a + # single file emit one match past the cap, which is the signal the + # loop needs to distinguish "exactly at the cap" (complete) from + # "more exist" (truncated) without scanning the whole file. + cmd.extend(["-m", str(max_count + 1)]) if include_glob: cmd.extend(["--glob", include_glob]) # When rg is given an absolute search path, directory-component @@ -755,8 +757,8 @@ def _ripgrep_search( # noqa: C901, PLR0911, PLR0912, PLR0915 total = 0 truncated = False # A watchdog kills ripgrep if it outruns the time budget; a blocking - # `readline` cannot honor a deadline on its own, so the timer is what - # bounds a hang that never reaches the cap. + # read on `proc.stdout` cannot honor a deadline on its own, so the timer + # is what bounds a hang that never reaches the cap. timed_out = threading.Event() def _kill_on_timeout() -> None: @@ -771,65 +773,20 @@ def _kill_on_timeout() -> None: # `stdout=PIPE` guarantees a stream; narrow it for the type checker. assert proc.stdout is not None # noqa: S101 for line in proc.stdout: - try: - data = json.loads(line) - except json.JSONDecodeError: - continue - data_type = data.get("type") - if data_type == "error": - # Per-file errors in `--json` mode (e.g., non-UTF-8 file - # ripgrep refused to read). Surface at DEBUG so debugging is - # possible without spamming WARNING for every binary file. - logger.debug("ripgrep per-file error frame: %s", data.get("data")) - continue - if data_type != "match": - continue - pdata = data.get("data", {}) - ftext = pdata.get("path", {}).get("text") - if not ftext: - continue - # When rg ran from cwd=base_full it emits paths relative to that - # cwd; join (don't `.resolve()`) so symlink form is preserved for - # callers. When rg searched a single file it emits the absolute - # path we passed in. - raw = Path(ftext) - p = raw if raw.is_absolute() else (base_full / raw) - # Defensive containment check: resolve both sides only for the - # comparison so symlinks that resolve to paths outside `base_full` - # can't leak results, while `p` itself keeps its original shape. - # OSError guards against unresolvable symlink targets. - try: - p.resolve().relative_to(base_resolved) - except (ValueError, OSError): - logger.warning( - "Skipping ripgrep result outside search root: path=%s root=%s", - p, - base_full, - ) - continue - if self.virtual_mode: - try: - virt = self._to_virtual_path(p) - except ValueError: - logger.debug("Skipping grep result outside root: %s", p) - continue - except (OSError, RuntimeError): - logger.warning("Could not resolve grep result path: %s", p, exc_info=True) - continue - else: - virt = str(p) - ln = pdata.get("line_number") - lt = pdata.get("lines", {}).get("text", "").rstrip("\n") - if ln is None: + parsed = self._parse_rg_match(line, base_full, base_resolved) + if parsed is None: continue - results.setdefault(virt, []).append((int(ln), lt)) - total += 1 + virt, ln, lt = parsed if max_count is not None and total >= max_count: - # Stop the process so it cannot keep buffering/emitting - # output once the caller's cap is satisfied. + # We already hold `max_count` matches and ripgrep emitted + # another (we asked for one past the cap via `-m`), proving + # more exist than requested. Stop without keeping the extra + # so the result is exactly `max_count`, flagged truncated. truncated = True proc.terminate() break + results.setdefault(virt, []).append((ln, lt)) + total += 1 finally: timer.cancel() stderr = self._drain_and_reap(proc) @@ -855,6 +812,71 @@ def _kill_on_timeout() -> None: return results, truncated + def _parse_rg_match( # noqa: PLR0911 + self, + line: str, + base_full: Path, + base_resolved: Path, + ) -> tuple[str, int, str] | None: + """Parse one ripgrep `--json` line into `(virtual_path, line_no, text)`. + + Returns `None` for non-match frames, unparseable lines, matches missing a + path or line number, and matches whose resolved path escapes `base_full` + (each logged and skipped). Extracted from the streaming loop so the + cap/watchdog bookkeeping there stays readable. + """ + try: + data = json.loads(line) + except json.JSONDecodeError: + return None + data_type = data.get("type") + if data_type == "error": + # Per-file errors in `--json` mode (e.g., non-UTF-8 file ripgrep + # refused to read). Surface at DEBUG so debugging is possible + # without spamming WARNING for every binary file. + logger.debug("ripgrep per-file error frame: %s", data.get("data")) + return None + if data_type != "match": + return None + pdata = data.get("data", {}) + ftext = pdata.get("path", {}).get("text") + if not ftext: + return None + # When rg ran from cwd=base_full it emits paths relative to that cwd; + # join (don't `.resolve()`) so symlink form is preserved for callers. + # When rg searched a single file it emits the absolute path we passed in. + raw = Path(ftext) + p = raw if raw.is_absolute() else (base_full / raw) + # Defensive containment check: resolve both sides only for the comparison + # so symlinks that resolve to paths outside `base_full` can't leak + # results, while `p` itself keeps its original shape. OSError guards + # against unresolvable symlink targets. + try: + p.resolve().relative_to(base_resolved) + except (ValueError, OSError): + logger.warning( + "Skipping ripgrep result outside search root: path=%s root=%s", + p, + base_full, + ) + return None + if self.virtual_mode: + try: + virt = self._to_virtual_path(p) + except ValueError: + logger.debug("Skipping grep result outside root: %s", p) + return None + except (OSError, RuntimeError): + logger.warning("Could not resolve grep result path: %s", p, exc_info=True) + return None + else: + virt = str(p) + ln = pdata.get("line_number") + if ln is None: + return None + lt = pdata.get("lines", {}).get("text", "").rstrip("\n") + return virt, int(ln), lt + @staticmethod def _drain_and_reap(proc: "subprocess.Popen[str]") -> str: """Read any remaining stderr, close pipes, and reap `proc`. @@ -863,14 +885,20 @@ def _drain_and_reap(proc: "subprocess.Popen[str]") -> str: Reaping avoids leaking a zombie/handle after the stdout loop stops (whether via EOF, the match cap, or the timeout watchdog). """ + # Close stdout first: on the cap path we stopped reading it, so a child + # still writing gets EPIPE and can exit. Draining stderr before that + # could otherwise block on a child wedged on a full stdout pipe. + if proc.stdout is not None: + proc.stdout.close() stderr = "" try: if proc.stderr is not None: stderr = proc.stderr.read() or "" except (OSError, ValueError): + # Losing the stderr text only costs diagnostics, but log why so a + # missing hard-error message in the caller isn't a dead end. + logger.debug("Failed to read ripgrep stderr during cleanup", exc_info=True) stderr = "" - if proc.stdout is not None: - proc.stdout.close() if proc.stderr is not None: proc.stderr.close() try: @@ -990,13 +1018,16 @@ def _safe_detail(exc: Exception) -> str: return results, True, None if pattern not in raw_line: continue + if max_count is not None and total >= max_count: + # Already collected `max_count` and found another + # match, so more exist than requested: stop and + # report the partial result as truncated. Checked + # before appending so exactly `max_count` matches + # is reported complete, not truncated. + return results, True, _file_errors_msg() line = raw_line.rstrip("\n") results.setdefault(virt_path, []).append((line_num, line)) total += 1 - if max_count is not None and total >= max_count: - # Hit the total match cap; stop scanning and - # report the partial result as truncated. - return results, True, _file_errors_msg() except UnicodeDecodeError as e: # A file that fails to decode before any line is scanned is # treated as binary and skipped silently, mirroring ripgrep's diff --git a/libs/deepagents/deepagents/backends/protocol.py b/libs/deepagents/deepagents/backends/protocol.py index 44a641ddb3f..9da50005a0c 100644 --- a/libs/deepagents/deepagents/backends/protocol.py +++ b/libs/deepagents/deepagents/backends/protocol.py @@ -482,10 +482,12 @@ def grep( across all files. `None` (the default) preserves existing backend behavior and - returns every match. When set to an int, the search stops once - that many matches have been collected and the result is flagged - with `GrepResult.truncated=True`. Interpreted as a total cap, not - a per-file cap. + returns every match. When set to an int, at most that many + matches are returned; if more exist the search stops and the + result is flagged with `GrepResult.truncated=True`. Exactly + `max_count` matches with none dropped is reported complete + (`truncated=False`). Interpreted as a total cap, not a per-file + cap. Examples: - `'*.py'` - only search Python files @@ -493,7 +495,6 @@ def grep( - `'src/**/*.js'` - search JS files under src/ - `'test[0-9].txt'` - search `test0.txt`, `test1.txt`, etc. - Returns: `GrepResult` with matches or error. @@ -532,10 +533,13 @@ async def agrep( Wraps the sync call with an async timeout as a safety net. The timeout bounds how long the caller waits; it does not stop the worker thread created by `asyncio.to_thread`. + + `max_count` is forwarded when the concrete `grep` accepts it; otherwise + the cap is applied post-hoc via `_apply_grep_max_count`, so callers get + the same guarantee regardless of which path runs. """ - grep_call = partial(self.grep, pattern, path, glob) - if _method_accepts_max_count(type(self), "grep"): - grep_call = partial(self.grep, pattern, path, glob, max_count=max_count) + grep_kwargs = {"max_count": max_count} if _method_accepts_max_count(type(self), "grep") else {} + grep_call = partial(self.grep, pattern, path, glob, **grep_kwargs) try: result = await asyncio.wait_for( asyncio.to_thread(grep_call), @@ -973,7 +977,9 @@ def _method_accepts_max_count(cls: type[BackendProtocol], method_name: Literal[" sig = inspect.signature(getattr(cls, method_name)) except (AttributeError, ValueError, TypeError): logger.warning( - "Could not inspect signature of %s.%s; assuming max_count is not supported.", + "Could not inspect signature of %s.%s; assuming max_count is not supported. " + "The cap will be enforced after the search instead of bounding it, so a huge " + "result set is fully materialized before being trimmed.", cls.__qualname__, method_name, exc_info=True, diff --git a/libs/deepagents/deepagents/backends/sandbox.py b/libs/deepagents/deepagents/backends/sandbox.py index dd1df034900..a344f832741 100644 --- a/libs/deepagents/deepagents/backends/sandbox.py +++ b/libs/deepagents/deepagents/backends/sandbox.py @@ -142,6 +142,10 @@ # line lacks a final newline. sys.stdout.write(display_path + chr(0) + str(i) + ':' + line.rstrip(chr(10)) + chr(10)) match_count += 1 + # Emit one record past the cap (match_count > max_count, not + # >=) so the parser can tell "exactly at the cap" (complete) + # from "capped early" (truncated). Mirrors the `head -n + # max_count+1` route in `_build_grep_cmd`. if max_count is not None and match_count > max_count: sys.exit(0) except OSError: diff --git a/libs/deepagents/deepagents/backends/utils.py b/libs/deepagents/deepagents/backends/utils.py index 94ab19c4f0e..95d952e9d0f 100644 --- a/libs/deepagents/deepagents/backends/utils.py +++ b/libs/deepagents/deepagents/backends/utils.py @@ -833,9 +833,10 @@ def grep_matches_from_files( Performs literal text search (not regex). - Returns a `GrepResult` with matches on success. When `max_count` is set, the - scan stops once that many total matches have been collected and the result - is flagged `truncated=True`. + Returns a `GrepResult` with matches on success. When `max_count` is set, at + most that many matches are returned; if more exist the scan stops and the + result is flagged `truncated=True`. Exactly `max_count` matches with none + dropped is reported complete (`truncated=False`). We deliberately do not raise here to keep backends non-throwing in tool contexts and preserve user-facing error messages. @@ -856,9 +857,12 @@ def grep_matches_from_files( content_str = _normalize_content(file_data) for line_num, line in enumerate(content_str.split("\n"), 1): if pattern in line: # Simple substring search for literal matching - matches.append({"path": file_path, "line": int(line_num), "text": line}) if max_count is not None and len(matches) >= max_count: + # A further match beyond `max_count` proves more exist; stop + # and flag truncation. Checked before appending so exactly + # `max_count` matches is reported complete, not truncated. return GrepResult(matches=matches, truncated=True) + matches.append({"path": file_path, "line": int(line_num), "text": line}) return GrepResult(matches=matches) diff --git a/libs/deepagents/tests/unit_tests/backends/test_context_hub_backend.py b/libs/deepagents/tests/unit_tests/backends/test_context_hub_backend.py index 9f5459f29d8..c68185ea88a 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_context_hub_backend.py +++ b/libs/deepagents/tests/unit_tests/backends/test_context_hub_backend.py @@ -365,6 +365,41 @@ def test_grep_invalid_regex() -> None: assert "Invalid regex" in result.error +def _grep_cap_backend() -> ContextHubBackend: + """Backend with three total matches across two files for cap tests.""" + backend, _ = _make_backend( + **{ + "a.md": FileEntry(type="file", content="hit\nhit\n"), + "b.md": FileEntry(type="file", content="hit\n"), + } + ) + return backend + + +def test_grep_max_count_over_cap_truncates() -> None: + """More matches than `max_count` returns the cap flagged truncated.""" + result = _grep_cap_backend().grep("hit", max_count=2) + assert result.matches is not None + assert len(result.matches) == 2 + assert result.truncated is True + + +def test_grep_max_count_exact_cap_not_truncated() -> None: + """Exactly `max_count` matches with none dropped is reported complete.""" + result = _grep_cap_backend().grep("hit", max_count=3) + assert result.matches is not None + assert len(result.matches) == 3 + assert result.truncated is False + + +def test_grep_max_count_none_returns_all() -> None: + """`max_count=None` returns every match, untruncated.""" + result = _grep_cap_backend().grep("hit") + assert result.matches is not None + assert len(result.matches) == 3 + assert result.truncated is False + + def test_glob_matches_pattern() -> None: backend, _ = _make_backend( **{ diff --git a/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py b/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py index c8fd224587d..f02bb47707f 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py +++ b/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py @@ -1145,8 +1145,11 @@ def test_ripgrep_streaming_caps_total_and_terminates(tmp_path: Path, monkeypatch created: dict[str, _FakePopen] = {} def fake_popen(cmd: list[str], **_kwargs: object) -> _FakePopen: - # `-m ` is passed to ripgrep as a secondary per-file guard. - assert "-m" in cmd and str(2) in cmd + # `-m ` is passed to ripgrep as a secondary per-file guard; the + # `+ 1` lets a single file emit one match past the cap so truncation is + # detectable. + assert "-m" in cmd + assert cmd[cmd.index("-m") + 1] == "3" proc = _FakePopen(stdout_lines=frames) created["proc"] = proc return proc @@ -1183,6 +1186,29 @@ def fake_popen(_cmd: list[str], **_kwargs: object) -> _FakePopen: assert len(result.matches) == 1 +@pytest.mark.usefixtures("_isolate_rg_cache") +def test_ripgrep_streaming_exact_cap_not_truncated(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Exactly `max_count` matches with none dropped is reported complete.""" + monkeypatch.setattr(fs_module.shutil, "which", lambda _name: "/usr/bin/rg") + (tmp_path / "a.txt").write_text("hello\nhello\n") + # Exactly `max_count` frames and no more: ripgrep (`-m cap + 1`) would have + # emitted a third if one existed, so the stream ending at the cap proves the + # result is complete. + frames = [_rg_match_frame("a.txt", 1, "hello\n"), _rg_match_frame("a.txt", 2, "hello\n")] + + def fake_popen(_cmd: list[str], **_kwargs: object) -> _FakePopen: + return _FakePopen(stdout_lines=frames) + + monkeypatch.setattr(fs_module.subprocess, "Popen", fake_popen) + be = FilesystemBackend(root_dir=str(tmp_path), virtual_mode=False) + + result = be.grep("hello", path=str(tmp_path), max_count=2) + + assert result.truncated is False + assert result.matches is not None + assert len(result.matches) == 2 + + def test_python_fallback_caps_total_matches_across_files(tmp_path: Path) -> None: """The Python fallback caps total matches across files and flags truncation. @@ -1227,6 +1253,19 @@ def test_python_fallback_below_cap_not_truncated(tmp_path: Path) -> None: assert len(result.matches) == 2 +def test_python_fallback_exact_cap_not_truncated(tmp_path: Path) -> None: + """The Python fallback reports exactly `max_count` matches as complete.""" + (tmp_path / "a.txt").write_text("needle\nneedle\n") + + be = FilesystemBackend(root_dir=str(tmp_path), virtual_mode=False) + + result = be.grep("needle", path=str(tmp_path), max_count=2) + + assert result.truncated is False + assert result.matches is not None + assert len(result.matches) == 2 + + def _install_flaky_rglob(monkeypatch: pytest.MonkeyPatch, exc: Exception, after_yields: int = 1) -> None: """Replace `Path.rglob` with a generator that yields N entries then raises.""" real_rglob = Path.rglob diff --git a/libs/deepagents/tests/unit_tests/backends/test_utils.py b/libs/deepagents/tests/unit_tests/backends/test_utils.py index 3a1f96e5ad7..375e117ba97 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_utils.py +++ b/libs/deepagents/tests/unit_tests/backends/test_utils.py @@ -483,3 +483,37 @@ def test_offset_beyond_file_returns_error_result(self) -> None: assert isinstance(result, ReadResult) assert result.error is not None assert "exceeds file length" in result.error + + +class TestGrepMaxCount: + """`max_count` total-cap semantics for `grep_matches_from_files`. + + Backs `StateBackend`/`StoreBackend`, which delegate their `grep` here. + """ + + @staticmethod + def _files() -> dict[str, Any]: + # Two files, three matching lines total. + return { + "/a.txt": {"content": "hit\nhit\n"}, + "/b.txt": {"content": "hit\n"}, + } + + def test_over_cap_truncates(self) -> None: + result = grep_matches_from_files(self._files(), "hit", "/", max_count=2) + assert result.matches is not None + assert len(result.matches) == 2 + assert result.truncated is True + + def test_exact_cap_not_truncated(self) -> None: + """Exactly `max_count` matches with none dropped is reported complete.""" + result = grep_matches_from_files(self._files(), "hit", "/", max_count=3) + assert result.matches is not None + assert len(result.matches) == 3 + assert result.truncated is False + + def test_no_cap_returns_all(self) -> None: + result = grep_matches_from_files(self._files(), "hit", "/") + assert result.matches is not None + assert len(result.matches) == 3 + assert result.truncated is False diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json index 6dcceb029b6..90575cb2138 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json @@ -212,6 +212,19 @@ "default": null, "description": "Glob pattern (NOT regex) limiting which files are searched (e.g. '*.py', '*.ts'). A pattern without '/' matches the file name at any depth; a pattern containing '/' matches the search-root-relative path (e.g. 'src/**/*.py'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." }, + "max_count": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional cap on the total number of matches returned across all files. Leave unset to use the configured default. When the cap is hit, results are truncated and a note says so; narrow the pattern or path to see the rest." + }, "output_mode": { "default": "files_with_matches", "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file under a ':' header, each line indented and formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json index 365603baac6..881f2480f82 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json @@ -212,6 +212,19 @@ "default": null, "description": "Glob pattern (NOT regex) limiting which files are searched (e.g. '*.py', '*.ts'). A pattern without '/' matches the file name at any depth; a pattern containing '/' matches the search-root-relative path (e.g. 'src/**/*.py'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." }, + "max_count": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional cap on the total number of matches returned across all files. Leave unset to use the configured default. When the cap is hit, results are truncated and a note says so; narrow the pattern or path to see the rest." + }, "output_mode": { "default": "files_with_matches", "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file under a ':' header, each line indented and formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json index 6dcceb029b6..90575cb2138 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json @@ -212,6 +212,19 @@ "default": null, "description": "Glob pattern (NOT regex) limiting which files are searched (e.g. '*.py', '*.ts'). A pattern without '/' matches the file name at any depth; a pattern containing '/' matches the search-root-relative path (e.g. 'src/**/*.py'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." }, + "max_count": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional cap on the total number of matches returned across all files. Leave unset to use the configured default. When the cap is hit, results are truncated and a note says so; narrow the pattern or path to see the rest." + }, "output_mode": { "default": "files_with_matches", "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file under a ':' header, each line indented and formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json index 365603baac6..881f2480f82 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json @@ -212,6 +212,19 @@ "default": null, "description": "Glob pattern (NOT regex) limiting which files are searched (e.g. '*.py', '*.ts'). A pattern without '/' matches the file name at any depth; a pattern containing '/' matches the search-root-relative path (e.g. 'src/**/*.py'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." }, + "max_count": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional cap on the total number of matches returned across all files. Leave unset to use the configured default. When the cap is hit, results are truncated and a note says so; narrow the pattern or path to see the rest." + }, "output_mode": { "default": "files_with_matches", "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file under a ':' header, each line indented and formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json index d83b47335e1..9b9de775582 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json @@ -212,6 +212,19 @@ "default": null, "description": "Glob pattern (NOT regex) limiting which files are searched (e.g. '*.py', '*.ts'). A pattern without '/' matches the file name at any depth; a pattern containing '/' matches the search-root-relative path (e.g. 'src/**/*.py'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." }, + "max_count": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional cap on the total number of matches returned across all files. Leave unset to use the configured default. When the cap is hit, results are truncated and a note says so; narrow the pattern or path to see the rest." + }, "output_mode": { "default": "files_with_matches", "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file under a ':' header, each line indented and formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json index 6dcceb029b6..90575cb2138 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json @@ -212,6 +212,19 @@ "default": null, "description": "Glob pattern (NOT regex) limiting which files are searched (e.g. '*.py', '*.ts'). A pattern without '/' matches the file name at any depth; a pattern containing '/' matches the search-root-relative path (e.g. 'src/**/*.py'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." }, + "max_count": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional cap on the total number of matches returned across all files. Leave unset to use the configured default. When the cap is hit, results are truncated and a note says so; narrow the pattern or path to see the rest." + }, "output_mode": { "default": "files_with_matches", "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file under a ':' header, each line indented and formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", From 90bc6ee92828ff0804219e91431230ef30cb9db9 Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Mon, 13 Jul 2026 22:59:59 -0400 Subject: [PATCH 12/14] cr --- .../deepagents/backends/filesystem.py | 56 +++++++++++-------- .../backends/test_filesystem_backend.py | 39 +++++++++++++ 2 files changed, 73 insertions(+), 22 deletions(-) diff --git a/libs/deepagents/deepagents/backends/filesystem.py b/libs/deepagents/deepagents/backends/filesystem.py index 91cfb6f346d..8e351a3919b 100644 --- a/libs/deepagents/deepagents/backends/filesystem.py +++ b/libs/deepagents/deepagents/backends/filesystem.py @@ -57,6 +57,12 @@ guarded by `test_glob_backend_budget_below_middleware_deadline`. """ +_RIPGREP_STDERR_CAPTURE_LIMIT = 500 +"""Maximum stderr characters retained for ripgrep error diagnostics.""" + +_RIPGREP_STDERR_READ_SIZE = 8192 +"""Number of stderr characters read per chunk while draining ripgrep.""" + @functools.cache def _resolve_ripgrep_path() -> str | None: @@ -752,6 +758,14 @@ def _ripgrep_search( # noqa: C901, PLR0911, PLR0912, PLR0915 _resolve_ripgrep_path.cache_clear() return None, False + stderr_chunks: list[str] = [] + stderr_thread = threading.Thread( + target=self._drain_ripgrep_stderr, + args=(proc, stderr_chunks), + daemon=True, + ) + stderr_thread.start() + results: dict[str, list[tuple[int, str]]] = {} base_resolved = base_full.resolve() total = 0 @@ -789,7 +803,11 @@ def _kill_on_timeout() -> None: total += 1 finally: timer.cancel() - stderr = self._drain_and_reap(proc) + self._reap_ripgrep(proc) + stderr_thread.join() + if proc.stderr is not None: + proc.stderr.close() + stderr = "".join(stderr_chunks) if timed_out.is_set(): if results: @@ -878,35 +896,29 @@ def _parse_rg_match( # noqa: PLR0911 return virt, int(ln), lt @staticmethod - def _drain_and_reap(proc: "subprocess.Popen[str]") -> str: - """Read any remaining stderr, close pipes, and reap `proc`. + def _drain_ripgrep_stderr(proc: "subprocess.Popen[str]", chunks: list[str]) -> None: + """Drain ripgrep stderr while retaining bounded error diagnostics.""" + remaining = _RIPGREP_STDERR_CAPTURE_LIMIT + try: + assert proc.stderr is not None # noqa: S101 # `stderr=PIPE` guarantees a stream + while chunk := proc.stderr.read(_RIPGREP_STDERR_READ_SIZE): + if remaining > 0: + captured = chunk[:remaining] + chunks.append(captured) + remaining -= len(captured) + except (OSError, ValueError): + logger.debug("Failed to read ripgrep stderr", exc_info=True) - Returns the captured stderr so callers can log hard-error diagnostics. - Reaping avoids leaking a zombie/handle after the stdout loop stops - (whether via EOF, the match cap, or the timeout watchdog). - """ - # Close stdout first: on the cap path we stopped reading it, so a child - # still writing gets EPIPE and can exit. Draining stderr before that - # could otherwise block on a child wedged on a full stdout pipe. + @staticmethod + def _reap_ripgrep(proc: "subprocess.Popen[str]") -> None: + """Close stdout and reap ripgrep after EOF, termination, or timeout.""" if proc.stdout is not None: proc.stdout.close() - stderr = "" - try: - if proc.stderr is not None: - stderr = proc.stderr.read() or "" - except (OSError, ValueError): - # Losing the stderr text only costs diagnostics, but log why so a - # missing hard-error message in the caller isn't a dead end. - logger.debug("Failed to read ripgrep stderr during cleanup", exc_info=True) - stderr = "" - if proc.stderr is not None: - proc.stderr.close() try: proc.wait(timeout=5) except subprocess.TimeoutExpired: proc.kill() proc.wait() - return stderr def _python_search( # noqa: C901, PLR0912, PLR0915 self, diff --git a/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py b/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py index f02bb47707f..f419fb7ab9e 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py +++ b/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py @@ -3,6 +3,8 @@ import json import logging import shutil +import subprocess +import sys import threading import warnings from collections.abc import Iterator @@ -1122,6 +1124,43 @@ def erroring_popen(_cmd: list[str], **_kwargs: object) -> _FakePopen: assert len(result.matches) == 2 +@pytest.mark.usefixtures("_isolate_rg_cache") +def test_ripgrep_drains_stderr_while_streaming_stdout(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Large stderr output cannot block ripgrep before it emits a match.""" + monkeypatch.setattr(fs_module.shutil, "which", lambda _name: "/usr/bin/rg") + monkeypatch.setattr(fs_module, "DEFAULT_GREP_TIMEOUT", 1) + frame = _rg_match_frame("a.txt", 1, "hello\n") + child_code = f"import sys\nsys.stderr.write('x' * 1_000_000)\nsys.stderr.flush()\nsys.stdout.write({frame!r})\nsys.stdout.flush()\n" + real_popen = subprocess.Popen + + def noisy_popen( + _cmd: list[str], + *, + stdout: int, + stderr: int, + text: bool, + cwd: str | None, + ) -> subprocess.Popen[str]: + assert text + return real_popen( + [sys.executable, "-c", child_code], + stdout=stdout, + stderr=stderr, + text=True, + cwd=cwd, + ) + + monkeypatch.setattr(fs_module.subprocess, "Popen", noisy_popen) + # The fallback cannot manufacture the synthetic ripgrep match. + (tmp_path / "a.txt").write_text("different text\n") + backend = FilesystemBackend(root_dir=str(tmp_path), virtual_mode=False) + + result = backend.grep("hello", path=str(tmp_path)) + + assert result.truncated is False + assert result.matches == [{"path": str(tmp_path / "a.txt"), "line": 1, "text": "hello"}] + + def _rg_match_frame(path: str, line_number: int, text: str) -> str: """Build a ripgrep `--json` match frame line for the streaming fake.""" return json.dumps({"type": "match", "data": {"path": {"text": path}, "lines": {"text": text}, "line_number": line_number}}) + "\n" From b19fe7b0e32979c7d06df2c2a972a0ca3d1ade7a Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Mon, 13 Jul 2026 23:20:20 -0400 Subject: [PATCH 13/14] cr --- .../deepagents/backends/composite.py | 23 +++++--- .../deepagents/backends/filesystem.py | 34 ++++++++---- .../deepagents/backends/protocol.py | 8 +-- .../deepagents/deepagents/backends/sandbox.py | 9 ++++ .../deepagents/middleware/filesystem.py | 14 +++-- .../unit_tests/backends/test_protocol.py | 52 +++++++++++++++++++ .../tests/unit_tests/test_middleware.py | 49 ++++++++++++----- 7 files changed, 152 insertions(+), 37 deletions(-) diff --git a/libs/deepagents/deepagents/backends/composite.py b/libs/deepagents/deepagents/backends/composite.py index c7417c68d89..60b075b89e2 100644 --- a/libs/deepagents/deepagents/backends/composite.py +++ b/libs/deepagents/deepagents/backends/composite.py @@ -423,6 +423,13 @@ def grep( globally (not per backend), short-circuits remaining routes once the cap is reached, and flags the result `truncated=True`. + Unlike a single backend, composite does not guarantee the + "exactly `max_count` matches means complete" boundary: when an + earlier route fills the budget exactly, the remaining routes are + short-circuited and the result is flagged `truncated=True` even + if those routes would have contributed nothing. The flag is thus + conservative — it may over-report truncation, never under-report. + Returns: `GrepResult` with matches or error. @@ -471,9 +478,11 @@ def grep( all_matches.extend(_remap_grep_path(m, route_prefix) for m in (grep_result.matches or [])) truncated = truncated or grep_result.truncated - # Defense-in-depth: the budget split above already keeps the total at - # or under `max_count`, so this only trims a non-compliant backend - # that returned more than its allotted budget. + # Unreachable safety net: each routed result is already capped to its + # allotted budget by the `_grep_backend`/`_agrep_backend` helpers + # (via `_apply_grep_max_count`), so the running total can never exceed + # `max_count`. Kept as belt-and-suspenders against a future refactor + # that bypasses that per-route capping. if max_count is not None and len(all_matches) > max_count: all_matches = all_matches[:max_count] truncated = True @@ -531,9 +540,11 @@ async def agrep( all_matches.extend(_remap_grep_path(m, route_prefix) for m in (grep_result.matches or [])) truncated = truncated or grep_result.truncated - # Defense-in-depth: the budget split above already keeps the total at - # or under `max_count`, so this only trims a non-compliant backend - # that returned more than its allotted budget. + # Unreachable safety net: each routed result is already capped to its + # allotted budget by the `_grep_backend`/`_agrep_backend` helpers + # (via `_apply_grep_max_count`), so the running total can never exceed + # `max_count`. Kept as belt-and-suspenders against a future refactor + # that bypasses that per-route capping. if max_count is not None and len(all_matches) > max_count: all_matches = all_matches[:max_count] truncated = True diff --git a/libs/deepagents/deepagents/backends/filesystem.py b/libs/deepagents/deepagents/backends/filesystem.py index 8e351a3919b..9050e16af38 100644 --- a/libs/deepagents/deepagents/backends/filesystem.py +++ b/libs/deepagents/deepagents/backends/filesystem.py @@ -678,7 +678,7 @@ def grep( matches.append({"path": fpath, "line": int(line_num), "text": line_text}) return GrepResult(error=partial_error, matches=matches, truncated=truncated) - def _ripgrep_search( # noqa: C901, PLR0911, PLR0912, PLR0915 + def _ripgrep_search( # noqa: C901, PLR0911, PLR0912, PLR0915 # single streaming loop + watchdog + per-branch fallback logging; splitting it would scatter the cap/timeout bookkeeping self, pattern: str, base_full: Path, @@ -688,10 +688,12 @@ def _ripgrep_search( # noqa: C901, PLR0911, PLR0912, PLR0915 """Search using ripgrep with fixed-string (literal) mode. Streams ripgrep's newline-delimited `--json` output line-by-line via - `subprocess.Popen` instead of buffering all of stdout, so a pathological - pattern on a huge repository cannot spike memory. Once more than - `max_count` matches are found the process is terminated and the search - stops early, returning exactly `max_count` matches flagged truncated. + `subprocess.Popen` instead of buffering all of stdout, so it holds only + parsed matches rather than the full JSON output. When `max_count` is set, + memory is additionally bounded: once more than `max_count` matches are + found the process is terminated and the search stops early, returning + exactly `max_count` matches flagged truncated. With no cap, `results` + still grows with the total match count. Args: pattern: Literal string to search for (unescaped). @@ -830,7 +832,7 @@ def _kill_on_timeout() -> None: return results, truncated - def _parse_rg_match( # noqa: PLR0911 + def _parse_rg_match( # noqa: PLR0911 # one early return per skip reason reads clearer than nesting the checks self, line: str, base_full: Path, @@ -838,10 +840,11 @@ def _parse_rg_match( # noqa: PLR0911 ) -> tuple[str, int, str] | None: """Parse one ripgrep `--json` line into `(virtual_path, line_no, text)`. - Returns `None` for non-match frames, unparseable lines, matches missing a - path or line number, and matches whose resolved path escapes `base_full` - (each logged and skipped). Extracted from the streaming loop so the - cap/watchdog bookkeeping there stays readable. + Returns `None` for non-match frames, unparseable lines, and matches + missing a path or line number (all skipped silently), as well as for + matches whose resolved path escapes `base_full` and per-file `error` + frames (both logged, then skipped). Extracted from the streaming loop so + the cap/watchdog bookkeeping there stays readable. """ try: data = json.loads(line) @@ -918,7 +921,16 @@ def _reap_ripgrep(proc: "subprocess.Popen[str]") -> None: proc.wait(timeout=5) except subprocess.TimeoutExpired: proc.kill() - proc.wait() + try: + # Bound the post-SIGKILL wait too. A process wedged in + # uninterruptible I/O (e.g. a dead NFS mount) can ignore even + # SIGKILL until the I/O returns; an unbounded wait here would + # hang the grep call past its deadline — the exact hang the + # watchdog exists to prevent. Abandon the handle after a grace + # period rather than block the caller forever. + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + logger.warning("ripgrep did not exit after SIGKILL; abandoning process handle") def _python_search( # noqa: C901, PLR0912, PLR0915 self, diff --git a/libs/deepagents/deepagents/backends/protocol.py b/libs/deepagents/deepagents/backends/protocol.py index 9da50005a0c..58df3b33cca 100644 --- a/libs/deepagents/deepagents/backends/protocol.py +++ b/libs/deepagents/deepagents/backends/protocol.py @@ -534,9 +534,11 @@ async def agrep( bounds how long the caller waits; it does not stop the worker thread created by `asyncio.to_thread`. - `max_count` is forwarded when the concrete `grep` accepts it; otherwise - the cap is applied post-hoc via `_apply_grep_max_count`, so callers get - the same guarantee regardless of which path runs. + `max_count` is forwarded when the concrete `grep` accepts it (so the + search can bound itself); backends that don't accept it run uncapped and + are trimmed afterward. Either way the return value is always passed + through `_apply_grep_max_count` (a no-op when already within the cap), so + callers get the same guarantee regardless of which path runs. """ grep_kwargs = {"max_count": max_count} if _method_accepts_max_count(type(self), "grep") else {} grep_call = partial(self.grep, pattern, path, glob, **grep_kwargs) diff --git a/libs/deepagents/deepagents/backends/sandbox.py b/libs/deepagents/deepagents/backends/sandbox.py index a344f832741..62e58d32c04 100644 --- a/libs/deepagents/deepagents/backends/sandbox.py +++ b/libs/deepagents/deepagents/backends/sandbox.py @@ -595,6 +595,15 @@ def _build_grep_cmd(pattern: str, path: str | None, glob: str | None, max_count: ) glob_pattern = f"--include={shlex.quote(glob)}" if glob else "" + # Known limitation (pre-existing): `2>/dev/null` + `|| true` means a genuine + # grep failure (exit 2 — unreadable root, bad option) is swallowed and parses + # as an empty "no matches" result, indistinguishable from a real zero-match. + # Surfacing exit 2 while still tolerating no-match (exit 1) AND the SIGPIPE + # (exit 141) that `head` sends grep on the cap path requires `set -o pipefail` + # (bash/zsh only, not POSIX sh/dash/busybox); buffering to a temp file instead + # would defeat the `head` early-stop below. A portable fix belongs in its own + # sandbox-tested change. The in-process `_GREP_PATH_GLOB_TEMPLATE` route does + # surface its errors (see its docstring); only this GNU-grep route swallows. base = f"grep {grep_opts} {glob_pattern} -e {pattern_escaped} {search_path} 2>/dev/null" if max_count is not None: # Read one record beyond the cap so the parser can distinguish "exactly diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index b3f0f9b47b9..0c1c721659e 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -499,7 +499,7 @@ def _format_grep_tool_result( """Format a backend grep result for the tool boundary. Size-truncation is applied to the match body here, before any note is - appended, so a trailing `SEARCH_TRUNCATION_NOTE` survives instead of being + appended, so a trailing `GREP_TRUNCATION_NOTE` survives instead of being sliced off by an outer `truncate_if_too_long` at the call site. Callers should use the returned content as-is rather than re-truncating it. @@ -522,7 +522,7 @@ def _format_grep_tool_result( return f"{error}\n\nPartial matches:\n{formatted}", "error" notes: list[str] = [] if result.truncated: - notes.append(SEARCH_TRUNCATION_NOTE) + notes.append(GREP_TRUNCATION_NOTE) if not result.truncated and not matches and not backend_had_matches and (hint := regex_literal_hint(pattern)): notes.append(hint) if notes: @@ -560,18 +560,24 @@ def _format_glob_tool_result(paths: list[str], *, truncated: bool) -> str: """Render glob paths for the tool boundary, appending the truncation note when partial.""" content = _format_file_paths(paths) if truncated: - return f"{content}\n\n{SEARCH_TRUNCATION_NOTE}" + return f"{content}\n\n{GLOB_TRUNCATION_NOTE}" return content EMPTY_CONTENT_WARNING = "System reminder: File exists but has empty contents" GLOB_TIMEOUT = 10.0 # seconds LINE_NUMBER_WIDTH = 6 -SEARCH_TRUNCATION_NOTE = ( +GREP_TRUNCATION_NOTE = ( "Note: the search stopped early (it hit its time limit or the maximum match count). " "The matches above are valid but incomplete. Narrow the search (a more specific pattern or a " "narrower path), or raise max_count, to see the rest." ) +# Glob has no match-count cap and no `max_count` argument, so its note names only +# the time/size limit and omits the (inapplicable) "raise max_count" remedy. +GLOB_TRUNCATION_NOTE = ( + "Note: the search stopped early because it hit its time limit. The paths above are valid but " + "incomplete. Narrow the search (a more specific pattern or a narrower path) to see the rest." +) def _glob_timeout_message() -> str: diff --git a/libs/deepagents/tests/unit_tests/backends/test_protocol.py b/libs/deepagents/tests/unit_tests/backends/test_protocol.py index 2956c091fb6..9b1d562f420 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_protocol.py +++ b/libs/deepagents/tests/unit_tests/backends/test_protocol.py @@ -21,6 +21,7 @@ GrepResult, LsResult, SandboxBackendProtocol, + _method_accepts_max_count, _supports_delete, ) @@ -205,6 +206,30 @@ def grep_raw(self, pattern: str, path: str | None = None, glob: str | None = Non assert LegacyBackend().grep("x") == GrepResult(matches=[{"path": "/f", "line": 1, "text": "x"}]) assert any("grep_raw" in str(x.message) for x in w) + def test_grep_raw_override_respects_max_count(self) -> None: + """`grep` caps a legacy `grep_raw` override post-hoc, honoring the boundary.""" + + class LegacyBackend(BackendProtocol): + def grep_raw(self, pattern: str, path: str | None = None, glob: str | None = None) -> list[dict[str, str | int]] | str: + return [ + {"path": "/one", "line": 1, "text": pattern}, + {"path": "/two", "line": 2, "text": pattern}, + {"path": "/three", "line": 3, "text": pattern}, + ] + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + capped = LegacyBackend().grep("x", max_count=2) + exact = LegacyBackend().grep("x", max_count=3) + + # More matches than the cap: trimmed to `max_count` and flagged truncated. + assert capped.matches == [{"path": "/one", "line": 1, "text": "x"}, {"path": "/two", "line": 2, "text": "x"}] + assert capped.truncated is True + # Exactly `max_count` matches with none dropped is reported complete. + assert exact.matches is not None + assert len(exact.matches) == 3 + assert exact.truncated is False + def test_glob_routes_to_glob_info_override(self) -> None: class LegacyBackend(BackendProtocol): def glob_info(self, pattern: str, path: str = "/") -> list[dict[str, str]]: @@ -329,3 +354,30 @@ def test_value_error_maps_to_invalid_path(self) -> None: assert _map_exception_to_standard_error(ValueError("unexpected encoding")) == "invalid_path" assert _map_exception_to_standard_error(ValueError("invalid literal for int()")) == "invalid_path" assert _map_exception_to_standard_error(ValueError("Path traversal not allowed")) == "invalid_path" + + +class TestMethodAcceptsMaxCount: + """`_method_accepts_max_count` decides whether the cap is forwarded or applied post-hoc.""" + + def test_explicit_keyword_param_detected(self) -> None: + class Backend(BackendProtocol): + def grep(self, pattern: str, path: str | None = None, glob: str | None = None, *, max_count: int | None = None) -> GrepResult: + return GrepResult(matches=[]) + + assert _method_accepts_max_count(Backend, "grep") is True + + def test_var_keyword_param_detected(self) -> None: + """A `**kwargs` grep is treated as accepting the cap (forwarded, not post-hoc).""" + + class Backend(BackendProtocol): + def grep(self, pattern: str, path: str | None = None, glob: str | None = None, **kwargs: object) -> GrepResult: + return GrepResult(matches=[]) + + assert _method_accepts_max_count(Backend, "grep") is True + + def test_missing_param_not_detected(self) -> None: + class Backend(BackendProtocol): + def grep(self, pattern: str, path: str | None = None, glob: str | None = None) -> GrepResult: # ty: ignore[invalid-method-override] + return GrepResult(matches=[]) + + assert _method_accepts_max_count(Backend, "grep") is False diff --git a/libs/deepagents/tests/unit_tests/test_middleware.py b/libs/deepagents/tests/unit_tests/test_middleware.py index 69fa129c20c..a8c32dec76a 100644 --- a/libs/deepagents/tests/unit_tests/test_middleware.py +++ b/libs/deepagents/tests/unit_tests/test_middleware.py @@ -46,8 +46,9 @@ ) from deepagents.middleware.filesystem import ( EMPTY_CONTENT_WARNING, + GLOB_TRUNCATION_NOTE, + GREP_TRUNCATION_NOTE, NUM_CHARS_PER_TOKEN, - SEARCH_TRUNCATION_NOTE, FileData, FilesystemMiddleware, FilesystemPermission, @@ -688,7 +689,7 @@ def test_grep_truncated_renders_as_success_with_note(self): assert result.status == "success" assert "1: import os" in result.content - assert SEARCH_TRUNCATION_NOTE in result.content + assert GREP_TRUNCATION_NOTE in result.content def test_grep_truncated_regex_pattern_no_matches_keeps_note(self): """A regex-looking miss still reports that the backend search was incomplete.""" @@ -711,7 +712,7 @@ def test_grep_truncated_regex_pattern_no_matches_keeps_note(self): assert result.status == "success" assert result.content.startswith("No matches found") - assert SEARCH_TRUNCATION_NOTE in result.content + assert GREP_TRUNCATION_NOTE in result.content assert "literal text, not regex" not in result.content def test_glob_truncated_renders_as_success_with_note(self): @@ -738,7 +739,7 @@ def test_glob_truncated_renders_as_success_with_note(self): assert result.status == "success" assert "/test.py" in result.content - assert SEARCH_TRUNCATION_NOTE in result.content + assert GLOB_TRUNCATION_NOTE in result.content def test_grep_not_truncated_omits_note(self): """A complete grep must not carry the truncation note.""" @@ -755,7 +756,7 @@ def test_grep_not_truncated_omits_note(self): result = grep_search_tool.invoke({"pattern": "import", "output_mode": "content", "runtime": _runtime()}) assert result.status == "success" - assert SEARCH_TRUNCATION_NOTE not in result.content + assert GREP_TRUNCATION_NOTE not in result.content def test_grep_forwards_default_max_count_to_backend(self): """The grep tool forwards the middleware's `grep_max_count` default to the backend.""" @@ -842,7 +843,7 @@ def grep(self, pattern, path=None, glob=None): # type: ignore[override] assert "/one.py" in result.content assert "/two.py" in result.content assert "/three.py" not in result.content - assert SEARCH_TRUNCATION_NOTE in result.content + assert GREP_TRUNCATION_NOTE in result.content async def test_async_grep_caps_legacy_backend_without_forwarding_max_count(self): """The inherited async wrapper also supports the previous `grep` signature.""" @@ -864,13 +865,35 @@ def grep(self, pattern, path=None, glob=None): # type: ignore[override] assert result.status == "success" assert "/one.py" in result.content assert "/two.py" not in result.content - assert SEARCH_TRUNCATION_NOTE in result.content + assert GREP_TRUNCATION_NOTE in result.content - def test_invalid_grep_max_count_raises(self): + @pytest.mark.parametrize("grep_max_count", [0, -1]) + def test_invalid_grep_max_count_raises(self, grep_max_count: int): """A non-positive `grep_max_count` is rejected at construction.""" backend, _ = _make_backend() with pytest.raises(ValueError, match="grep_max_count must be positive"): - FilesystemMiddleware(backend=backend, grep_max_count=0) + FilesystemMiddleware(backend=backend, grep_max_count=grep_max_count) + + def test_default_grep_max_count_is_1000(self): + """The documented default cap (1000) is forwarded when no override is given.""" + backend, _ = _make_backend() + middleware = FilesystemMiddleware(backend=backend) + grep_search_tool = next(tool for tool in middleware.tools if tool.name == "grep") + backend_obj = middleware._get_backend(_runtime()) + + captured: dict[str, object] = {} + + def _grep(_pattern, path=None, glob=None, *, max_count=None): # noqa: ARG001 + captured["max_count"] = max_count + return GrepResult(matches=[]) + + with ( + patch.object(middleware, "_get_backend", return_value=backend_obj), + patch.object(backend_obj, "grep", side_effect=_grep), + ): + grep_search_tool.invoke({"pattern": "import", "runtime": _runtime()}) + + assert captured["max_count"] == 1000 @pytest.mark.parametrize("max_count", [0, -1]) def test_non_positive_per_call_max_count_is_rejected(self, max_count: int) -> None: @@ -893,7 +916,7 @@ def test_glob_not_truncated_omits_note(self): result = glob_search_tool.invoke({"pattern": "*.py", "runtime": _runtime()}) assert result.status == "success" - assert SEARCH_TRUNCATION_NOTE not in result.content + assert GLOB_TRUNCATION_NOTE not in result.content def test_grep_truncation_note_survives_size_truncation(self): """A grep that is both time-truncated and size-overflowing keeps the truncation note (it isn't tail-cut).""" @@ -914,7 +937,7 @@ def test_grep_truncation_note_survives_size_truncation(self): assert result.status == "success" # Size truncation engaged (body was cut) yet the time-limit note survived at the tail. assert TRUNCATION_GUIDANCE in result.content - assert SEARCH_TRUNCATION_NOTE in result.content + assert GREP_TRUNCATION_NOTE in result.content async def test_async_grep_truncated_renders_as_success_with_note(self): """The async grep handler renders a truncated result as success with the note (parity with sync).""" @@ -932,7 +955,7 @@ async def test_async_grep_truncated_renders_as_success_with_note(self): assert result.status == "success" assert "1: import os" in result.content - assert SEARCH_TRUNCATION_NOTE in result.content + assert GREP_TRUNCATION_NOTE in result.content async def test_async_glob_truncated_renders_as_success_with_note(self): """The async glob handler renders a truncated result as success with the note (parity with sync).""" @@ -950,7 +973,7 @@ async def test_async_glob_truncated_renders_as_success_with_note(self): assert result.status == "success" assert "/test.py" in result.content - assert SEARCH_TRUNCATION_NOTE in result.content + assert GLOB_TRUNCATION_NOTE in result.content def test_grep_search_shortterm_content_mode(self): files = { From ef9d054b020c36247003608290c1d5c6d7727d8d Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Tue, 14 Jul 2026 00:08:41 -0400 Subject: [PATCH 14/14] cr --- .../system_prompt_with_media_extra_tools.json | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_media_extra_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_media_extra_tools.json index 258a939925d..59f5eaacc20 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_media_extra_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_media_extra_tools.json @@ -212,6 +212,19 @@ "default": null, "description": "Glob pattern (NOT regex) limiting which files are searched (e.g. '*.py', '*.ts'). A pattern without '/' matches the file name at any depth; a pattern containing '/' matches the search-root-relative path (e.g. 'src/**/*.py'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." }, + "max_count": { + "anyOf": [ + { + "exclusiveMinimum": 0, + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Optional cap on the total number of matches returned across all files. Leave unset to use the configured default. When the cap is hit, results are truncated and a note says so; narrow the pattern or path to see the rest." + }, "output_mode": { "default": "files_with_matches", "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file under a ':' header, each line indented and formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.",