From 7f2800dd173dec036e9c5f2c2b710086efbf6b15 Mon Sep 17 00:00:00 2001 From: Mason Daugherty <61371264+mdrxy@users.noreply.github.com> Date: Tue, 23 Jun 2026 13:47:14 +0000 Subject: [PATCH 01/13] feat(sdk): warn when literal grep pattern looks like regex The grep tool matches literal text (ripgrep -F / substring), but models routinely pass regex (alternation, .*, escapes) and get a bare "No matches found", masking that the content exists. Sharpen the tool description to forbid regex and, on an empty result whose pattern carries regex signals, append a hint pointing at separate literal searches or execute+rg. Co-authored-by: open-swe[bot] --- libs/deepagents/deepagents/backends/utils.py | 32 ++++++++++ .../deepagents/middleware/filesystem.py | 21 +++++-- .../tests/unit_tests/backends/test_utils.py | 44 +++++++++++++ .../tests/unit_tests/test_middleware.py | 63 +++++++++++++++++++ 4 files changed, 156 insertions(+), 4 deletions(-) diff --git a/libs/deepagents/deepagents/backends/utils.py b/libs/deepagents/deepagents/backends/utils.py index aeb87d7753f..78e5526a8d0 100644 --- a/libs/deepagents/deepagents/backends/utils.py +++ b/libs/deepagents/deepagents/backends/utils.py @@ -737,3 +737,35 @@ def format_grep_matches( if not matches: return "No matches found" return _format_grep_results(build_grep_results_dict(matches), output_mode) + + +# Strong signals that a pattern was written as a regex rather than literal text. +# Deliberately conservative: bare `.`, `(`, `)`, `[`, `]`, `?`, `^`, `$` are +# omitted because they appear routinely in literal code searches (e.g. +# `self.tools`, `def __init__(self):`, `arr[0]`), which would cause false hints. +_REGEX_SIGNAL_RE = re.compile( + r"\|" # alternation + r"|\.\*" # `.*` wildcard + r"|\.\+" # `.+` wildcard + r"|\\[.wWdDsSbB(){}\[\]|+*?^$]" # escaped regex metacharacters / classes +) + + +def looks_like_regex(pattern: str) -> bool: + """Heuristically detect regex syntax in a pattern meant for literal grep.""" + return bool(_REGEX_SIGNAL_RE.search(pattern)) + + +def regex_literal_hint(pattern: str) -> str | None: + """Return a hint when a no-match pattern looks like an (unsupported) regex. + + `grep` matches literal text, so regex metacharacters are searched verbatim + and silently miss. Returns `None` when the pattern has no regex signals. + """ + if not looks_like_regex(pattern): + return None + return ( + "Note: grep matches literal text, not regex, so characters like " + "`|`, `.*`, and `\\.` are searched verbatim. Run a separate literal " + "search for each alternative, or use the execute tool with `rg` for regex." + ) diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index 05ba7e9ed28..a5e33aa7ff0 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -59,6 +59,7 @@ check_empty_content, format_content_with_line_numbers, format_grep_matches, + regex_literal_hint, sanitize_tool_call_id as sanitize_tool_call_id, truncate_if_too_long, validate_path, @@ -204,6 +205,7 @@ def _filter_grep_matches_by_permission( def _format_grep_tool_result( result: GrepResult, output_mode: Literal["files_with_matches", "content", "count"], + pattern: str = "", ) -> tuple[str, Literal["success", "error"]]: """Format a backend grep result for the tool boundary.""" matches = result.matches or [] @@ -213,6 +215,8 @@ def _format_grep_tool_result( formatted = format_grep_matches(matches, output_mode) if result.error: return f"{result.error}\n\nPartial matches:\n{formatted}", "error" + if not matches and (hint := regex_literal_hint(pattern)): + return f"{formatted}\n\n{hint}", "success" return formatted, "success" @@ -470,16 +474,23 @@ class ExecuteSchema(BaseModel): - `*.txt` - Find all text files in the backend's default root - `/subdir/**/*.md` - Find all markdown files under /subdir""" -GREP_TOOL_DESCRIPTION = """Search for a text pattern across files. +GREP_TOOL_DESCRIPTION = """Search for a LITERAL text pattern across files (NOT regex). -Searches for literal text (not regex) and returns matching files or content based on output_mode. -Special characters like parentheses, brackets, pipes, etc. are treated as literal characters, not regex operators. +Returns matching files or content based on output_mode. The pattern is matched +verbatim: regex metacharacters such as `|`, `.*`, `.+`, `\\.`, `\\w`, `[...]`, +`^`, and `$` are treated as ordinary characters, NOT operators. + +Do NOT pass a regex. In particular: +- To match any of several strings, run a SEPARATE grep for each one. There is no + `|` alternation: `grep(pattern="foo|bar")` looks for the literal text "foo|bar". +- Do not use wildcards (`.*`) or escapes (`\\.`); they match those characters literally. +- If you genuinely need regex, use the execute tool with `rg ''` instead. Examples: - Search all files: `grep(pattern="TODO")` - Search Python files only: `grep(pattern="import", glob="*.py")` - Show matching lines: `grep(pattern="error", output_mode="content")` -- Search for code with special chars: `grep(pattern="def __init__(self):")`""" +- Literal special chars are fine: `grep(pattern="def __init__(self):")`""" EXECUTE_TOOL_DESCRIPTION = """Executes a shell command in an isolated sandbox environment. @@ -1611,6 +1622,7 @@ def sync_grep( formatted, status = _format_grep_tool_result( GrepResult(error=grep_result.error, matches=filtered_matches), output_mode, + pattern, ) return ToolMessage( content=truncate_if_too_long(formatted), @@ -1654,6 +1666,7 @@ async def async_grep( formatted, status = _format_grep_tool_result( GrepResult(error=grep_result.error, matches=filtered_matches), output_mode, + pattern, ) return ToolMessage( content=truncate_if_too_long(formatted), diff --git a/libs/deepagents/tests/unit_tests/backends/test_utils.py b/libs/deepagents/tests/unit_tests/backends/test_utils.py index ee14eb130fd..acb3b923590 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_utils.py +++ b/libs/deepagents/tests/unit_tests/backends/test_utils.py @@ -11,13 +11,57 @@ _EXTENSION_TO_FILE_TYPE, _get_file_type, _glob_search_files, + looks_like_regex, perform_string_replacement, + regex_literal_hint, slice_read_response, to_posix_path, validate_path, ) +class TestLooksLikeRegex: + """`looks_like_regex` flags regex syntax in patterns meant for literal grep.""" + + @pytest.mark.parametrize( + "pattern", + [ + "foo|bar", + "def .*model", + "self\\.tools", + "a.+b", + "\\bword\\b", + "\\d+", + "\\w", + ], + ) + def test_detects_regex(self, pattern: str) -> None: + assert looks_like_regex(pattern) is True + + @pytest.mark.parametrize( + "pattern", + [ + "def __init__(self):", + "self.tools", + "arr[0]", + "plain text", + "TODO", + "a == b", + "", + ], + ) + def test_ignores_literal(self, pattern: str) -> None: + assert looks_like_regex(pattern) is False + + def test_hint_present_for_regex(self) -> None: + hint = regex_literal_hint("foo|bar") + assert hint is not None + assert "literal text, not regex" in hint + + def test_hint_absent_for_literal(self) -> None: + assert regex_literal_hint("plain text") is None + + class TestToPosixPath: """`to_posix_path` is the load-bearing primitive for Windows path handling.""" diff --git a/libs/deepagents/tests/unit_tests/test_middleware.py b/libs/deepagents/tests/unit_tests/test_middleware.py index 0199a65eae6..1e989821e2b 100644 --- a/libs/deepagents/tests/unit_tests/test_middleware.py +++ b/libs/deepagents/tests/unit_tests/test_middleware.py @@ -782,6 +782,69 @@ def test_grep_search_shortterm_invalid_regex(self): ) assert "No matches found" in result.content + def test_grep_regex_pattern_no_matches_adds_hint(self): + """A no-match pattern that looks like regex gets a literal-search hint.""" + files = { + "/test.py": FileData( + content=["def hello():"], + modified_at="2021-01-01", + created_at="2021-01-01", + ), + } + backend, _ = _make_backend(files) + middleware = FilesystemMiddleware(backend=backend) + grep_search_tool = next(tool for tool in middleware.tools if tool.name == "grep") + result = grep_search_tool.invoke( + { + "pattern": "def hello|def world", + "runtime": _runtime(), + } + ) + assert result.content.startswith("No matches found") + assert "literal text, not regex" in result.content + + def test_grep_literal_no_matches_omits_hint(self): + """A plain literal no-match pattern does not get the regex hint.""" + files = { + "/test.py": FileData( + content=["print('hello')"], + modified_at="2021-01-01", + created_at="2021-01-01", + ), + } + backend, _ = _make_backend(files) + middleware = FilesystemMiddleware(backend=backend) + grep_search_tool = next(tool for tool in middleware.tools if tool.name == "grep") + result = grep_search_tool.invoke( + { + "pattern": "import", + "runtime": _runtime(), + } + ) + assert result.content == "No matches found" + + def test_grep_regex_pattern_with_matches_omits_hint(self): + """A regex-looking pattern that still matches literally shows no hint.""" + files = { + "/test.py": FileData( + content=["a = b|c"], + modified_at="2021-01-01", + created_at="2021-01-01", + ), + } + backend, _ = _make_backend(files) + middleware = FilesystemMiddleware(backend=backend) + grep_search_tool = next(tool for tool in middleware.tools if tool.name == "grep") + result = grep_search_tool.invoke( + { + "pattern": "b|c", + "output_mode": "content", + "runtime": _runtime(), + } + ) + assert "a = b|c" in result.content + assert "literal text, not regex" not in result.content + def test_search_store_paginated_empty(self): """Test pagination with no items.""" store = InMemoryStore() From e10dc432c2fe2b4c8b3ec0ea50fe98d819e2cc9f Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Tue, 23 Jun 2026 17:19:06 -0400 Subject: [PATCH 02/13] cr --- libs/deepagents/deepagents/backends/utils.py | 14 +++++++++----- .../deepagents/deepagents/middleware/filesystem.py | 3 +-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/libs/deepagents/deepagents/backends/utils.py b/libs/deepagents/deepagents/backends/utils.py index 78e5526a8d0..86baac143bf 100644 --- a/libs/deepagents/deepagents/backends/utils.py +++ b/libs/deepagents/deepagents/backends/utils.py @@ -739,16 +739,18 @@ def format_grep_matches( return _format_grep_results(build_grep_results_dict(matches), output_mode) -# Strong signals that a pattern was written as a regex rather than literal text. -# Deliberately conservative: bare `.`, `(`, `)`, `[`, `]`, `?`, `^`, `$` are -# omitted because they appear routinely in literal code searches (e.g. -# `self.tools`, `def __init__(self):`, `arr[0]`), which would cause false hints. _REGEX_SIGNAL_RE = re.compile( r"\|" # alternation r"|\.\*" # `.*` wildcard r"|\.\+" # `.+` wildcard r"|\\[.wWdDsSbB(){}\[\]|+*?^$]" # escaped regex metacharacters / classes ) +"""Strong signals that a pattern was written as a regex rather than literal text. + +Deliberately conservative: bare `.`, `(`, `)`, `[`, `]`, `?`, `^`, `$` are +omitted because they appear routinely in literal code searches (e.g. +`self.tools`, `def __init__(self):`, `arr[0]`), which would cause false hints. +""" def looks_like_regex(pattern: str) -> bool: @@ -760,7 +762,9 @@ def regex_literal_hint(pattern: str) -> str | None: """Return a hint when a no-match pattern looks like an (unsupported) regex. `grep` matches literal text, so regex metacharacters are searched verbatim - and silently miss. Returns `None` when the pattern has no regex signals. + and silently miss. + + Returns `None` when the pattern has no regex signals. """ if not looks_like_regex(pattern): return None diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index a5e33aa7ff0..136ad8b9c20 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -477,8 +477,7 @@ class ExecuteSchema(BaseModel): GREP_TOOL_DESCRIPTION = """Search for a LITERAL text pattern across files (NOT regex). Returns matching files or content based on output_mode. The pattern is matched -verbatim: regex metacharacters such as `|`, `.*`, `.+`, `\\.`, `\\w`, `[...]`, -`^`, and `$` are treated as ordinary characters, NOT operators. +verbatim: regex metacharacters are treated as ordinary characters, NOT operators. Do NOT pass a regex. In particular: - To match any of several strings, run a SEPARATE grep for each one. There is no From ae32a80248568af4fa09a22da838589ed28656f4 Mon Sep 17 00:00:00 2001 From: Mason Daugherty <61371264+mdrxy@users.noreply.github.com> Date: Tue, 23 Jun 2026 21:55:05 +0000 Subject: [PATCH 03/13] test(evals): smoke evals for grep regex-literal recovery; refresh prompt snapshots Add hillclimb retrieval evals covering the three regex signals the new literal-grep hint detects (`|` alternation, `.*` wildcard, escaped `\.`), asserting the agent still finds and reports the right files. Refresh the stale system-prompt tool snapshots for the updated GREP_TOOL_DESCRIPTION and regenerate EVAL_CATALOG.md. Co-authored-by: open-swe[bot] --- .../custom_system_message_tools.json | 2 +- .../system_prompt_with_execute_tools.json | 2 +- ...m_prompt_with_memory_and_skills_tools.json | 2 +- ...stem_prompt_with_routed_backend_tools.json | 2 +- ...t_with_sync_and_async_subagents_tools.json | 2 +- .../system_prompt_without_execute_tools.json | 2 +- libs/evals/EVAL_CATALOG.md | 19 ++-- .../evals/tests/evals/test_file_operations.py | 96 +++++++++++++++++++ 8 files changed, 113 insertions(+), 14 deletions(-) diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json index c36fbf93a52..330629bb070 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json @@ -177,7 +177,7 @@ }, { "function": { - "description": "Search for a text pattern across files.\n\nSearches for literal text (not regex) and returns matching files or content based on output_mode.\nSpecial characters like parentheses, brackets, pipes, etc. are treated as literal characters, not regex operators.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Search for code with special chars: `grep(pattern=\"def __init__(self):\")`", + "description": "Search for a LITERAL text pattern across files (NOT regex).\n\nReturns matching files or content based on output_mode. The pattern is matched\nverbatim: regex metacharacters are treated as ordinary characters, NOT operators.\n\nDo NOT pass a regex. In particular:\n- To match any of several strings, run a SEPARATE grep for each one. There is no\n `|` alternation: `grep(pattern=\"foo|bar\")` looks for the literal text \"foo|bar\".\n- Do not use wildcards (`.*`) or escapes (`\\.`); they match those characters literally.\n- If you genuinely need regex, use the execute tool with `rg ''` instead.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Literal special chars are fine: `grep(pattern=\"def __init__(self):\")`", "name": "grep", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json index daf4db3b7d5..bd7b57f8f6b 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json @@ -177,7 +177,7 @@ }, { "function": { - "description": "Search for a text pattern across files.\n\nSearches for literal text (not regex) and returns matching files or content based on output_mode.\nSpecial characters like parentheses, brackets, pipes, etc. are treated as literal characters, not regex operators.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Search for code with special chars: `grep(pattern=\"def __init__(self):\")`", + "description": "Search for a LITERAL text pattern across files (NOT regex).\n\nReturns matching files or content based on output_mode. The pattern is matched\nverbatim: regex metacharacters are treated as ordinary characters, NOT operators.\n\nDo NOT pass a regex. In particular:\n- To match any of several strings, run a SEPARATE grep for each one. There is no\n `|` alternation: `grep(pattern=\"foo|bar\")` looks for the literal text \"foo|bar\".\n- Do not use wildcards (`.*`) or escapes (`\\.`); they match those characters literally.\n- If you genuinely need regex, use the execute tool with `rg ''` instead.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Literal special chars are fine: `grep(pattern=\"def __init__(self):\")`", "name": "grep", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json index c36fbf93a52..330629bb070 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json @@ -177,7 +177,7 @@ }, { "function": { - "description": "Search for a text pattern across files.\n\nSearches for literal text (not regex) and returns matching files or content based on output_mode.\nSpecial characters like parentheses, brackets, pipes, etc. are treated as literal characters, not regex operators.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Search for code with special chars: `grep(pattern=\"def __init__(self):\")`", + "description": "Search for a LITERAL text pattern across files (NOT regex).\n\nReturns matching files or content based on output_mode. The pattern is matched\nverbatim: regex metacharacters are treated as ordinary characters, NOT operators.\n\nDo NOT pass a regex. In particular:\n- To match any of several strings, run a SEPARATE grep for each one. There is no\n `|` alternation: `grep(pattern=\"foo|bar\")` looks for the literal text \"foo|bar\".\n- Do not use wildcards (`.*`) or escapes (`\\.`); they match those characters literally.\n- If you genuinely need regex, use the execute tool with `rg ''` instead.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Literal special chars are fine: `grep(pattern=\"def __init__(self):\")`", "name": "grep", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json index daf4db3b7d5..bd7b57f8f6b 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json @@ -177,7 +177,7 @@ }, { "function": { - "description": "Search for a text pattern across files.\n\nSearches for literal text (not regex) and returns matching files or content based on output_mode.\nSpecial characters like parentheses, brackets, pipes, etc. are treated as literal characters, not regex operators.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Search for code with special chars: `grep(pattern=\"def __init__(self):\")`", + "description": "Search for a LITERAL text pattern across files (NOT regex).\n\nReturns matching files or content based on output_mode. The pattern is matched\nverbatim: regex metacharacters are treated as ordinary characters, NOT operators.\n\nDo NOT pass a regex. In particular:\n- To match any of several strings, run a SEPARATE grep for each one. There is no\n `|` alternation: `grep(pattern=\"foo|bar\")` looks for the literal text \"foo|bar\".\n- Do not use wildcards (`.*`) or escapes (`\\.`); they match those characters literally.\n- If you genuinely need regex, use the execute tool with `rg ''` instead.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Literal special chars are fine: `grep(pattern=\"def __init__(self):\")`", "name": "grep", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json index af3990c1cd4..b1bcbe8ae71 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json @@ -177,7 +177,7 @@ }, { "function": { - "description": "Search for a text pattern across files.\n\nSearches for literal text (not regex) and returns matching files or content based on output_mode.\nSpecial characters like parentheses, brackets, pipes, etc. are treated as literal characters, not regex operators.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Search for code with special chars: `grep(pattern=\"def __init__(self):\")`", + "description": "Search for a LITERAL text pattern across files (NOT regex).\n\nReturns matching files or content based on output_mode. The pattern is matched\nverbatim: regex metacharacters are treated as ordinary characters, NOT operators.\n\nDo NOT pass a regex. In particular:\n- To match any of several strings, run a SEPARATE grep for each one. There is no\n `|` alternation: `grep(pattern=\"foo|bar\")` looks for the literal text \"foo|bar\".\n- Do not use wildcards (`.*`) or escapes (`\\.`); they match those characters literally.\n- If you genuinely need regex, use the execute tool with `rg ''` instead.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Literal special chars are fine: `grep(pattern=\"def __init__(self):\")`", "name": "grep", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json index c36fbf93a52..330629bb070 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json @@ -177,7 +177,7 @@ }, { "function": { - "description": "Search for a text pattern across files.\n\nSearches for literal text (not regex) and returns matching files or content based on output_mode.\nSpecial characters like parentheses, brackets, pipes, etc. are treated as literal characters, not regex operators.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Search for code with special chars: `grep(pattern=\"def __init__(self):\")`", + "description": "Search for a LITERAL text pattern across files (NOT regex).\n\nReturns matching files or content based on output_mode. The pattern is matched\nverbatim: regex metacharacters are treated as ordinary characters, NOT operators.\n\nDo NOT pass a regex. In particular:\n- To match any of several strings, run a SEPARATE grep for each one. There is no\n `|` alternation: `grep(pattern=\"foo|bar\")` looks for the literal text \"foo|bar\".\n- Do not use wildcards (`.*`) or escapes (`\\.`); they match those characters literally.\n- If you genuinely need regex, use the execute tool with `rg ''` instead.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Literal special chars are fine: `grep(pattern=\"def __init__(self):\")`", "name": "grep", "parameters": { "properties": { diff --git a/libs/evals/EVAL_CATALOG.md b/libs/evals/EVAL_CATALOG.md index dd9b9efaa83..2b8ab978efe 100644 --- a/libs/evals/EVAL_CATALOG.md +++ b/libs/evals/EVAL_CATALOG.md @@ -10,7 +10,7 @@ Categories (for `--eval-category` filtering): file_operations,retrieval,tool_use,memory,conversation,summarization,unit_test,langchain/middleware ``` -**118 evals** across **8 categories** +**121 evals** across **8 categories** ## File Ops (`file_operations`) (13 evals) @@ -25,17 +25,20 @@ file_operations,retrieval,tool_use,memory,conversation,summarization,unit_test,l - [`test_read_then_write_derived_output`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L248) — `tests/evals/test_file_operations.py:248` - [`test_avoid_unnecessary_tool_calls`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L273) — `tests/evals/test_file_operations.py:273` - [`test_read_files_in_parallel`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L291) — `tests/evals/test_file_operations.py:291` -- [`test_read_file_truncation_recovery_with_pagination`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L561) — `tests/evals/test_file_operations.py:561` -- [`test_read_file_empty_file_reports_empty`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L599) — `tests/evals/test_file_operations.py:599` +- [`test_read_file_truncation_recovery_with_pagination`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L657) — `tests/evals/test_file_operations.py:657` +- [`test_read_file_empty_file_reports_empty`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L695) — `tests/evals/test_file_operations.py:695` -## Retrieval (`retrieval`) (6 evals) +## Retrieval (`retrieval`) (9 evals) - [`test_frames`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_external_benchmarks.py#L67) — `tests/evals/test_external_benchmarks.py:67` - [`test_grep_finds_matching_paths`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L321) — `tests/evals/test_file_operations.py:321` -- [`test_glob_lists_markdown_files`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L349) — `tests/evals/test_file_operations.py:349` -- [`test_find_magic_phrase_deep_nesting`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L377) — `tests/evals/test_file_operations.py:377` -- [`test_identify_quote_author_from_directory_parallel_reads`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L413) — `tests/evals/test_file_operations.py:413` -- [`test_identify_quote_author_from_directory_unprompted_efficiency`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L488) — `tests/evals/test_file_operations.py:488` +- [`test_grep_alternation_regex_recovers`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L349) — `tests/evals/test_file_operations.py:349` +- [`test_grep_wildcard_regex_recovers`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L383) — `tests/evals/test_file_operations.py:383` +- [`test_grep_escaped_metachar_regex_recovers`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L414) — `tests/evals/test_file_operations.py:414` +- [`test_glob_lists_markdown_files`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L445) — `tests/evals/test_file_operations.py:445` +- [`test_find_magic_phrase_deep_nesting`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L473) — `tests/evals/test_file_operations.py:473` +- [`test_identify_quote_author_from_directory_parallel_reads`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L509) — `tests/evals/test_file_operations.py:509` +- [`test_identify_quote_author_from_directory_unprompted_efficiency`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L584) — `tests/evals/test_file_operations.py:584` ## Tool Use (`tool_use`) (53 evals) diff --git a/libs/evals/tests/evals/test_file_operations.py b/libs/evals/tests/evals/test_file_operations.py index c61632ec289..17ea85fcbf1 100644 --- a/libs/evals/tests/evals/test_file_operations.py +++ b/libs/evals/tests/evals/test_file_operations.py @@ -343,6 +343,102 @@ def test_grep_finds_matching_paths(model: BaseChatModel) -> None: ) +@pytest.mark.eval_tier("hillclimb") +@pytest.mark.eval_category("retrieval") +@pytest.mark.langsmith +def test_grep_alternation_regex_recovers(model: BaseChatModel) -> None: + """Recovers when an `|` alternation grep misses, finding both literal terms. + + `grep` matches literal text, so `cat|dog` is searched verbatim and misses. + A model that reaches for regex should be steered by the no-match hint into + running a separate literal search per term instead of burning calls on + regex variants. Either way it must end up reporting both matching files. + """ + agent = create_deep_agent(model=model) + run_agent( + agent, + model=model, + initial_files={ + "/pets/a.txt": "the cat sat\n", + "/pets/b.txt": "the dog ran\n", + "/pets/c.txt": "the bird flew\n", + }, + query="Using grep, find which files mention either 'cat' or 'dog'. Answer with the matching file paths only.", + # 1st step (ideal): two literal greps, one per term. + # 2nd step: answer with the matching paths. + # A single `cat|dog` regex attempt misses and surfaces the literal hint. + scorer=TrajectoryScorer() + .expect(agent_steps=2, tool_call_requests=2) + .success( + final_text_contains("/pets/a.txt"), + final_text_contains("/pets/b.txt"), + final_text_excludes("/pets/c.txt"), + ), + ) + + +@pytest.mark.eval_tier("hillclimb") +@pytest.mark.eval_category("retrieval") +@pytest.mark.langsmith +def test_grep_wildcard_regex_recovers(model: BaseChatModel) -> None: + """Recovers when a `.*` wildcard grep misses, locating the literal line. + + A model may search `TODO.*cache`; literal grep treats `.*` verbatim and + misses. The no-match hint should steer it back to a plain literal search + (e.g. `TODO`) so it still finds and reports the file. + """ + agent = create_deep_agent(model=model) + run_agent( + agent, + model=model, + initial_files={ + "/src/app.py": "x = 1\n# TODO: add a cache layer here\n", + "/src/util.py": "y = 2\n", + }, + query="Using grep, find which file has a TODO comment about a cache. Answer with the matching file path only.", + # 1st step (ideal): literal grep for 'TODO'. + # 2nd step: answer with the matching path. + # A `TODO.*cache` regex attempt misses and surfaces the literal hint. + scorer=TrajectoryScorer() + .expect(agent_steps=2, tool_call_requests=1) + .success( + final_text_contains("/src/app.py"), + final_text_excludes("/src/util.py"), + ), + ) + + +@pytest.mark.eval_tier("hillclimb") +@pytest.mark.eval_category("retrieval") +@pytest.mark.langsmith +def test_grep_escaped_metachar_regex_recovers(model: BaseChatModel) -> None: + """Recovers when an escaped-regex grep misses a literal filename. + + A model may search `config\\.yaml`; literal grep matches the backslash + verbatim and misses `config.yaml`. The no-match hint should steer it back + to a plain literal search so it still finds and reports the file. + """ + agent = create_deep_agent(model=model) + run_agent( + agent, + model=model, + initial_files={ + "/svc/readme.md": "Edit config.yaml to set up the service.\n", + "/svc/notes.txt": "nothing relevant here\n", + }, + query="Using grep, find which file mentions the config.yaml file. Answer with the matching file path only.", + # 1st step (ideal): literal grep for 'config.yaml'. + # 2nd step: answer with the matching path. + # A `config\\.yaml` escaped-regex attempt misses and surfaces the literal hint. + scorer=TrajectoryScorer() + .expect(agent_steps=2, tool_call_requests=1) + .success( + final_text_contains("/svc/readme.md"), + final_text_excludes("/svc/notes.txt"), + ), + ) + + @pytest.mark.eval_tier("baseline") @pytest.mark.eval_category("retrieval") @pytest.mark.langsmith From 96be360652da452937bba9cf689a3f637ffca892 Mon Sep 17 00:00:00 2001 From: Mason Daugherty <61371264+mdrxy@users.noreply.github.com> Date: Tue, 23 Jun 2026 22:01:26 +0000 Subject: [PATCH 04/13] feat(sdk): clarify grep glob filter and output_mode descriptions The grep `glob` param is a filename glob (NOT regex) and an in-tool filter, not a call to the separate glob tool; say so explicitly. Replace the vague `output_mode` description (which implied 'content' included surrounding context) with the exact returned shapes per mode. Share the wording across the schema and both tool signatures and refresh prompt snapshots. Co-authored-by: open-swe[bot] --- .../deepagents/middleware/filesystem.py | 26 ++++++++++++++----- .../custom_system_message_tools.json | 4 +-- .../system_prompt_with_execute_tools.json | 4 +-- ...m_prompt_with_memory_and_skills_tools.json | 4 +-- ...stem_prompt_with_routed_backend_tools.json | 4 +-- ...t_with_sync_and_async_subagents_tools.json | 4 +-- .../system_prompt_without_execute_tools.json | 4 +-- 7 files changed, 32 insertions(+), 18 deletions(-) diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index 136ad8b9c20..b6a76eff1f6 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -340,6 +340,20 @@ class FilesystemState(AgentState): """Files in the filesystem. Uses DeltaChannel with snapshots every ~50 pregel steps to bound read depth.""" +GREP_GLOB_DESCRIPTION = ( + "Glob pattern (NOT regex) limiting which files are searched, matched against " + "the file name (e.g. '*.py', '*.{ts,tsx}'). This is an in-tool file filter, " + "not a call to the separate glob tool." +) + +GREP_OUTPUT_MODE_DESCRIPTION = ( + "Shape of the returned text. 'files_with_matches' (default): newline-separated " + "matching file paths. 'content': matching lines grouped by file, each formatted " + "': ' (only the matched line, no surrounding context). " + "'count': one ': ' line per file." +) + + class LsSchema(BaseModel): """Input schema for the `ls` tool.""" @@ -400,11 +414,11 @@ class GrepSchema(BaseModel): path: str | None = Field(default=None, description="Directory to search in. Defaults to current working directory.") - glob: str | None = Field(default=None, description="Glob pattern to filter which files to search (e.g., '*.py').") + glob: str | None = Field(default=None, description=GREP_GLOB_DESCRIPTION) output_mode: Literal["files_with_matches", "content", "count"] = Field( default="files_with_matches", - description="Output format: 'files_with_matches' (file paths only, default), 'content' (matching lines with context), 'count' (match counts per file).", + description=GREP_OUTPUT_MODE_DESCRIPTION, ) @@ -1590,10 +1604,10 @@ def sync_grep( pattern: Annotated[str, "Text pattern to search for (literal string, not regex)."], runtime: ToolRuntime[None, FilesystemState], path: Annotated[str | None, "Directory to search in. Defaults to current working directory."] = None, - glob: Annotated[str | None, "Glob pattern to filter which files to search (e.g., '*.py')."] = None, + glob: Annotated[str | None, GREP_GLOB_DESCRIPTION] = None, output_mode: Annotated[ Literal["files_with_matches", "content", "count"], - "Output format: 'files_with_matches' (file paths only, default), 'content' (matching lines with context), 'count' (match counts per file).", + GREP_OUTPUT_MODE_DESCRIPTION, ] = "files_with_matches", ) -> ToolMessage: """Synchronous wrapper for grep tool.""" @@ -1634,10 +1648,10 @@ async def async_grep( pattern: Annotated[str, "Text pattern to search for (literal string, not regex)."], runtime: ToolRuntime[None, FilesystemState], path: Annotated[str | None, "Directory to search in. Defaults to current working directory."] = None, - glob: Annotated[str | None, "Glob pattern to filter which files to search (e.g., '*.py')."] = None, + glob: Annotated[str | None, GREP_GLOB_DESCRIPTION] = None, output_mode: Annotated[ Literal["files_with_matches", "content", "count"], - "Output format: 'files_with_matches' (file paths only, default), 'content' (matching lines with context), 'count' (match counts per file).", + GREP_OUTPUT_MODE_DESCRIPTION, ] = "files_with_matches", ) -> ToolMessage: """Asynchronous wrapper for grep tool.""" diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json index 330629bb070..3b38fe9a112 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json @@ -191,11 +191,11 @@ } ], "default": null, - "description": "Glob pattern to filter which files to search (e.g., '*.py')." + "description": "Glob pattern (NOT regex) limiting which files are searched, matched against the file name (e.g. '*.py', '*.{ts,tsx}'). This is an in-tool file filter, not a call to the separate glob tool." }, "output_mode": { "default": "files_with_matches", - "description": "Output format: 'files_with_matches' (file paths only, default), 'content' (matching lines with context), 'count' (match counts per file).", + "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file, each formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", "enum": [ "files_with_matches", "content", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json index bd7b57f8f6b..869d22b2708 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json @@ -191,11 +191,11 @@ } ], "default": null, - "description": "Glob pattern to filter which files to search (e.g., '*.py')." + "description": "Glob pattern (NOT regex) limiting which files are searched, matched against the file name (e.g. '*.py', '*.{ts,tsx}'). This is an in-tool file filter, not a call to the separate glob tool." }, "output_mode": { "default": "files_with_matches", - "description": "Output format: 'files_with_matches' (file paths only, default), 'content' (matching lines with context), 'count' (match counts per file).", + "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file, each formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", "enum": [ "files_with_matches", "content", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json index 330629bb070..3b38fe9a112 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json @@ -191,11 +191,11 @@ } ], "default": null, - "description": "Glob pattern to filter which files to search (e.g., '*.py')." + "description": "Glob pattern (NOT regex) limiting which files are searched, matched against the file name (e.g. '*.py', '*.{ts,tsx}'). This is an in-tool file filter, not a call to the separate glob tool." }, "output_mode": { "default": "files_with_matches", - "description": "Output format: 'files_with_matches' (file paths only, default), 'content' (matching lines with context), 'count' (match counts per file).", + "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file, each formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", "enum": [ "files_with_matches", "content", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json index bd7b57f8f6b..869d22b2708 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json @@ -191,11 +191,11 @@ } ], "default": null, - "description": "Glob pattern to filter which files to search (e.g., '*.py')." + "description": "Glob pattern (NOT regex) limiting which files are searched, matched against the file name (e.g. '*.py', '*.{ts,tsx}'). This is an in-tool file filter, not a call to the separate glob tool." }, "output_mode": { "default": "files_with_matches", - "description": "Output format: 'files_with_matches' (file paths only, default), 'content' (matching lines with context), 'count' (match counts per file).", + "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file, each formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", "enum": [ "files_with_matches", "content", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json index b1bcbe8ae71..225983be995 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json @@ -191,11 +191,11 @@ } ], "default": null, - "description": "Glob pattern to filter which files to search (e.g., '*.py')." + "description": "Glob pattern (NOT regex) limiting which files are searched, matched against the file name (e.g. '*.py', '*.{ts,tsx}'). This is an in-tool file filter, not a call to the separate glob tool." }, "output_mode": { "default": "files_with_matches", - "description": "Output format: 'files_with_matches' (file paths only, default), 'content' (matching lines with context), 'count' (match counts per file).", + "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file, each formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", "enum": [ "files_with_matches", "content", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json index 330629bb070..3b38fe9a112 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json @@ -191,11 +191,11 @@ } ], "default": null, - "description": "Glob pattern to filter which files to search (e.g., '*.py')." + "description": "Glob pattern (NOT regex) limiting which files are searched, matched against the file name (e.g. '*.py', '*.{ts,tsx}'). This is an in-tool file filter, not a call to the separate glob tool." }, "output_mode": { "default": "files_with_matches", - "description": "Output format: 'files_with_matches' (file paths only, default), 'content' (matching lines with context), 'count' (match counts per file).", + "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file, each formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", "enum": [ "files_with_matches", "content", From bf4bfcb6703f077d4a21971b4d79c748d0cb7f5e Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Thu, 2 Jul 2026 15:20:28 -0400 Subject: [PATCH 05/13] fix(sdk): correct grep glob description: brace expansion varies across backends The glob parameter description claimed brace alternation is "not supported across backends." In reality, the wcmatch-backed backends (FilesystemBackend, SandboxBackend, and the in-memory helpers) enable BRACE expansion via / ; only the Context Hub backend (which uses fnmatch) treats braces literally. Reword to "not supported on all backends" with "run a separate search per extension for reliable results" so the guidance is accurate without overstating the limitation. --- libs/deepagents/deepagents/backends/utils.py | 2 +- .../deepagents/middleware/filesystem.py | 15 ++++++++---- .../tests/unit_tests/backends/test_utils.py | 1 + .../custom_system_message_tools.json | 2 +- .../system_prompt_with_execute_tools.json | 2 +- ...m_prompt_with_memory_and_skills_tools.json | 2 +- ...stem_prompt_with_routed_backend_tools.json | 2 +- ...t_with_sync_and_async_subagents_tools.json | 2 +- .../system_prompt_without_execute_tools.json | 2 +- .../tests/unit_tests/test_middleware.py | 24 +++++++++++++++++++ 10 files changed, 42 insertions(+), 12 deletions(-) diff --git a/libs/deepagents/deepagents/backends/utils.py b/libs/deepagents/deepagents/backends/utils.py index 8e4967f01e1..0e61a562087 100644 --- a/libs/deepagents/deepagents/backends/utils.py +++ b/libs/deepagents/deepagents/backends/utils.py @@ -906,5 +906,5 @@ def regex_literal_hint(pattern: str) -> str | None: return ( "Note: grep matches literal text, not regex, so characters like " "`|`, `.*`, and `\\.` are searched verbatim. Run a separate literal " - "search for each alternative, or use the execute tool with `rg` for regex." + "search for each alternative." ) diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index 58dd51b39b8..ba3c247c4a5 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -481,10 +481,14 @@ def _format_grep_tool_result( # "Partial matches:" section past the token limit and cut it off. error = truncate_if_too_long(result.error) return f"{error}\n\nPartial matches:\n{formatted}", "error" - if not matches and (hint := regex_literal_hint(pattern)): - return f"{formatted}\n\n{hint}", "success" + notes = [] if result.truncated: - return f"{formatted}\n\n{SEARCH_TRUNCATION_NOTE}", "success" + notes.append(SEARCH_TRUNCATION_NOTE) + if not matches and (hint := regex_literal_hint(pattern)): + notes.append(hint) + if notes: + formatted_notes = "\n\n".join(notes) + return f"{formatted}\n\n{formatted_notes}", "success" return formatted, "success" @@ -630,8 +634,9 @@ class FilesystemState(AgentState): GREP_GLOB_DESCRIPTION = ( "Glob pattern (NOT regex) limiting which files are searched, matched against " "the file name (e.g. '*.py', '*.ts'). This is an in-tool file filter, " - "not a call to the separate glob tool. Brace alternation like '*.{ts,tsx}' is " - "not supported across backends; run a separate search per extension instead." + "not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') " + "is not supported on all backends; run a separate search per extension for " + "reliable results." ) GREP_OUTPUT_MODE_DESCRIPTION = ( diff --git a/libs/deepagents/tests/unit_tests/backends/test_utils.py b/libs/deepagents/tests/unit_tests/backends/test_utils.py index fb967891269..15281c465e0 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_utils.py +++ b/libs/deepagents/tests/unit_tests/backends/test_utils.py @@ -59,6 +59,7 @@ def test_hint_present_for_regex(self) -> None: hint = regex_literal_hint("foo|bar") assert hint is not None assert "literal text, not regex" in hint + assert "execute" not in hint def test_hint_absent_for_literal(self) -> None: assert regex_literal_hint("plain text") is None diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json index 7e6898ed42c..95634b38f30 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json @@ -210,7 +210,7 @@ } ], "default": null, - "description": "Glob pattern (NOT regex) limiting which files are searched, matched against the file name (e.g. '*.py', '*.ts'). This is an in-tool file filter, not a call to the separate glob tool. Brace alternation like '*.{ts,tsx}' is not supported across backends; run a separate search per extension instead." + "description": "Glob pattern (NOT regex) limiting which files are searched, matched against the file name (e.g. '*.py', '*.ts'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." }, "output_mode": { "default": "files_with_matches", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json index 1e6c9c7cc6e..3c4d6ddd3b0 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json @@ -210,7 +210,7 @@ } ], "default": null, - "description": "Glob pattern (NOT regex) limiting which files are searched, matched against the file name (e.g. '*.py', '*.ts'). This is an in-tool file filter, not a call to the separate glob tool. Brace alternation like '*.{ts,tsx}' is not supported across backends; run a separate search per extension instead." + "description": "Glob pattern (NOT regex) limiting which files are searched, matched against the file name (e.g. '*.py', '*.ts'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." }, "output_mode": { "default": "files_with_matches", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json index 7e6898ed42c..95634b38f30 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json @@ -210,7 +210,7 @@ } ], "default": null, - "description": "Glob pattern (NOT regex) limiting which files are searched, matched against the file name (e.g. '*.py', '*.ts'). This is an in-tool file filter, not a call to the separate glob tool. Brace alternation like '*.{ts,tsx}' is not supported across backends; run a separate search per extension instead." + "description": "Glob pattern (NOT regex) limiting which files are searched, matched against the file name (e.g. '*.py', '*.ts'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." }, "output_mode": { "default": "files_with_matches", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json index 1e6c9c7cc6e..3c4d6ddd3b0 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json @@ -210,7 +210,7 @@ } ], "default": null, - "description": "Glob pattern (NOT regex) limiting which files are searched, matched against the file name (e.g. '*.py', '*.ts'). This is an in-tool file filter, not a call to the separate glob tool. Brace alternation like '*.{ts,tsx}' is not supported across backends; run a separate search per extension instead." + "description": "Glob pattern (NOT regex) limiting which files are searched, matched against the file name (e.g. '*.py', '*.ts'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." }, "output_mode": { "default": "files_with_matches", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json index 3462756ac87..22e2c25444c 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json @@ -210,7 +210,7 @@ } ], "default": null, - "description": "Glob pattern (NOT regex) limiting which files are searched, matched against the file name (e.g. '*.py', '*.ts'). This is an in-tool file filter, not a call to the separate glob tool. Brace alternation like '*.{ts,tsx}' is not supported across backends; run a separate search per extension instead." + "description": "Glob pattern (NOT regex) limiting which files are searched, matched against the file name (e.g. '*.py', '*.ts'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." }, "output_mode": { "default": "files_with_matches", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json index 7e6898ed42c..95634b38f30 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json @@ -210,7 +210,7 @@ } ], "default": null, - "description": "Glob pattern (NOT regex) limiting which files are searched, matched against the file name (e.g. '*.py', '*.ts'). This is an in-tool file filter, not a call to the separate glob tool. Brace alternation like '*.{ts,tsx}' is not supported across backends; run a separate search per extension instead." + "description": "Glob pattern (NOT regex) limiting which files are searched, matched against the file name (e.g. '*.py', '*.ts'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." }, "output_mode": { "default": "files_with_matches", diff --git a/libs/deepagents/tests/unit_tests/test_middleware.py b/libs/deepagents/tests/unit_tests/test_middleware.py index 617b7e9bc1b..f99895baddd 100644 --- a/libs/deepagents/tests/unit_tests/test_middleware.py +++ b/libs/deepagents/tests/unit_tests/test_middleware.py @@ -687,6 +687,30 @@ def test_grep_truncated_renders_as_success_with_note(self): assert "1: import os" in result.content assert SEARCH_TRUNCATION_NOTE in result.content + def test_grep_truncated_regex_pattern_no_matches_keeps_note(self): + """A regex-looking miss still reports that the backend search was incomplete.""" + backend, _ = _make_backend() + middleware = FilesystemMiddleware(backend=backend) + grep_search_tool = next(tool for tool in middleware.tools if tool.name == "grep") + backend_obj = middleware._get_backend(_runtime()) + + truncated_result = GrepResult(matches=[], truncated=True) + with ( + patch.object(middleware, "_get_backend", return_value=backend_obj), + patch.object(backend_obj, "grep", return_value=truncated_result), + ): + result = grep_search_tool.invoke( + { + "pattern": "def hello|def world", + "runtime": _runtime(), + } + ) + + assert result.status == "success" + assert result.content.startswith("No matches found") + assert SEARCH_TRUNCATION_NOTE in result.content + assert "literal text, not regex" in result.content + def test_glob_truncated_renders_as_success_with_note(self): """A truncated glob returns its partial paths as a success plus the narrow-your-search note.""" backend, _ = _make_backend() From 5a7cd3713c6cfe27b27d0473d671f34e4fce9090 Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Thu, 2 Jul 2026 15:33:44 -0400 Subject: [PATCH 06/13] cr --- libs/deepagents/deepagents/backends/utils.py | 5 +++-- .../deepagents/middleware/filesystem.py | 7 ++++--- .../tests/unit_tests/backends/test_utils.py | 8 +++++++ .../custom_system_message_tools.json | 2 +- .../system_prompt_with_execute_tools.json | 2 +- ...m_prompt_with_memory_and_skills_tools.json | 2 +- ...stem_prompt_with_routed_backend_tools.json | 2 +- ...t_with_sync_and_async_subagents_tools.json | 2 +- .../system_prompt_without_execute_tools.json | 2 +- .../tests/unit_tests/test_middleware_async.py | 21 +++++++++++++++++++ 10 files changed, 42 insertions(+), 11 deletions(-) diff --git a/libs/deepagents/deepagents/backends/utils.py b/libs/deepagents/deepagents/backends/utils.py index 0e61a562087..8b435f46752 100644 --- a/libs/deepagents/deepagents/backends/utils.py +++ b/libs/deepagents/deepagents/backends/utils.py @@ -894,10 +894,11 @@ def _looks_like_regex(pattern: str) -> bool: def regex_literal_hint(pattern: str) -> str | None: - """Return a hint when a no-match pattern looks like an (unsupported) regex. + """Return a hint when a pattern looks like an (unsupported) regex. `grep` matches literal text, so regex metacharacters are searched verbatim - and silently miss. + and silently miss. Callers gate this on a no-match result; the function + itself only inspects the pattern. Returns `None` when the pattern has no regex signals. """ diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index ba3c247c4a5..30df4fa0ae8 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -641,9 +641,10 @@ class FilesystemState(AgentState): GREP_OUTPUT_MODE_DESCRIPTION = ( "Shape of the returned text. 'files_with_matches' (default): newline-separated " - "matching file paths. 'content': matching lines grouped by file, each formatted " - "': ' (only the matched line, no surrounding context). " - "'count': one ': ' line per file." + "matching file paths. 'content': matching lines grouped by file under a " + "':' header, each line indented and formatted ': ' " + "(only the matched line, no surrounding context). 'count': one " + "': ' line per file." ) diff --git a/libs/deepagents/tests/unit_tests/backends/test_utils.py b/libs/deepagents/tests/unit_tests/backends/test_utils.py index 15281c465e0..a9a465f1402 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_utils.py +++ b/libs/deepagents/tests/unit_tests/backends/test_utils.py @@ -50,6 +50,14 @@ def test_detects_regex(self, pattern: str) -> None: "TODO", "a == b", "", + # Metacharacters deliberately treated as literal (see `_REGEX_SIGNAL_RE` + # docstring): bare `^`, `$`, `?`, `*`, `+` are common in literal code + # searches and must not trip a false hint. + "^main", + "cost$5", + "value?", + "a*b", + "c++", ], ) def test_ignores_literal(self, pattern: str) -> None: diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json index 95634b38f30..1e3c16ea111 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json @@ -214,7 +214,7 @@ }, "output_mode": { "default": "files_with_matches", - "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file, each formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", + "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file under a ':' header, each line indented and formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", "enum": [ "files_with_matches", "content", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json index 3c4d6ddd3b0..92a41fe960c 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json @@ -214,7 +214,7 @@ }, "output_mode": { "default": "files_with_matches", - "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file, each formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", + "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file under a ':' header, each line indented and formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", "enum": [ "files_with_matches", "content", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json index 95634b38f30..1e3c16ea111 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json @@ -214,7 +214,7 @@ }, "output_mode": { "default": "files_with_matches", - "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file, each formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", + "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file under a ':' header, each line indented and formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", "enum": [ "files_with_matches", "content", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json index 3c4d6ddd3b0..92a41fe960c 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json @@ -214,7 +214,7 @@ }, "output_mode": { "default": "files_with_matches", - "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file, each formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", + "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file under a ':' header, each line indented and formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", "enum": [ "files_with_matches", "content", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json index 22e2c25444c..3bbcbdfe135 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json @@ -214,7 +214,7 @@ }, "output_mode": { "default": "files_with_matches", - "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file, each formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", + "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file under a ':' header, each line indented and formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", "enum": [ "files_with_matches", "content", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json index 95634b38f30..1e3c16ea111 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json @@ -214,7 +214,7 @@ }, "output_mode": { "default": "files_with_matches", - "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file, each formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", + "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file under a ':' header, each line indented and formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", "enum": [ "files_with_matches", "content", diff --git a/libs/deepagents/tests/unit_tests/test_middleware_async.py b/libs/deepagents/tests/unit_tests/test_middleware_async.py index 12b3a943d4a..5969a5f1ee2 100644 --- a/libs/deepagents/tests/unit_tests/test_middleware_async.py +++ b/libs/deepagents/tests/unit_tests/test_middleware_async.py @@ -600,6 +600,27 @@ async def test_agrep_search_shortterm_no_matches(self): ) assert result.content == "No matches found" + async def test_agrep_regex_pattern_no_matches_adds_hint(self): + """A no-match pattern that looks like regex gets a literal-search hint (async path).""" + files = { + "/test.py": FileData( + content=["def hello():"], + modified_at="2021-01-01", + created_at="2021-01-01", + ), + } + backend, _ = _make_backend(files) + middleware = FilesystemMiddleware(backend=backend) + grep_search_tool = next(tool for tool in middleware.tools if tool.name == "grep") + result = await grep_search_tool.ainvoke( + { + "pattern": "def hello|def world", + "runtime": _runtime(), + } + ) + assert result.content.startswith("No matches found") + assert "literal text, not regex" in result.content + async def test_agrep_search_shortterm_invalid_regex(self): """Test async grep with special characters (literal search, not regex).""" files = { From ff10cde0cd3c16632834c8689235bccb7350f7f7 Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Thu, 2 Jul 2026 15:42:29 -0400 Subject: [PATCH 07/13] cr --- .../deepagents/middleware/filesystem.py | 137 ++++++++++++++---- .../system_prompt_without_execute_tools.json | 2 +- .../tests/unit_tests/test_middleware.py | 81 +++++++++++ 3 files changed, 189 insertions(+), 31 deletions(-) diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index 30df4fa0ae8..22b406473f1 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -837,7 +837,9 @@ class ExecuteSchema(BaseModel): - `*.txt` - Find all text files in the backend's default root - `/subdir/**/*.md` - Find all markdown files under /subdir""" -GREP_TOOL_DESCRIPTION = """Search for a LITERAL text pattern across files (NOT regex). +_GREP_REGEX_EXECUTE_FALLBACK = "- If you genuinely need regex, use the execute tool with `rg ''` instead." + +_GREP_TOOL_DESCRIPTION_TEMPLATE = """Search for a LITERAL text pattern across files (NOT regex). Returns matching files or content based on output_mode. The pattern is matched verbatim: regex metacharacters are treated as ordinary characters, NOT operators. @@ -846,7 +848,7 @@ class ExecuteSchema(BaseModel): - To match any of several strings, run a SEPARATE grep for each one. There is no `|` alternation: `grep(pattern="foo|bar")` looks for the literal text "foo|bar". - Do not use wildcards (`.*`) or escapes (`\\.`); they match those characters literally. -- If you genuinely need regex, use the execute tool with `rg ''` instead. +{execute_fallback} Examples: - Search all files: `grep(pattern="TODO")` @@ -854,6 +856,12 @@ class ExecuteSchema(BaseModel): - Show matching lines: `grep(pattern="error", output_mode="content")` - Literal special chars are fine: `grep(pattern="def __init__(self):")`""" +GREP_TOOL_DESCRIPTION = _GREP_TOOL_DESCRIPTION_TEMPLATE.format(execute_fallback=_GREP_REGEX_EXECUTE_FALLBACK) +_GREP_TOOL_DESCRIPTION_WITHOUT_EXECUTE = _GREP_TOOL_DESCRIPTION_TEMPLATE.format(execute_fallback="").replace( + "\n\nExamples:", + "\nExamples:", +) + EXECUTE_TOOL_DESCRIPTION = """Executes a shell command in an isolated sandbox environment. Usage: @@ -2104,7 +2112,7 @@ async def async_glob( def _create_grep_tool(self) -> BaseTool: """Create the grep tool.""" - tool_description = self._custom_tool_descriptions.get("grep") or GREP_TOOL_DESCRIPTION + tool_description = self._grep_tool_description(include_execution=True) def sync_grep( pattern: str, @@ -2201,6 +2209,94 @@ async def async_grep( args_schema=GrepSchema, ) + def _grep_tool_description(self, *, include_execution: bool) -> str: + """Return the grep description for the current execution visibility.""" + return self._custom_tool_descriptions.get("grep") or (GREP_TOOL_DESCRIPTION if include_execution else _GREP_TOOL_DESCRIPTION_WITHOUT_EXECUTE) + + def _with_filtered_grep_description( + self, + tools: list[BaseTool | dict[str, Any]], + *, + include_execution: bool, + ) -> list[BaseTool | dict[str, Any]]: + """Copy default grep tools when their execution-specific guidance changes.""" + if self._custom_tool_descriptions.get("grep"): + return tools + + target_description = self._grep_tool_description(include_execution=include_execution) + default_descriptions = {GREP_TOOL_DESCRIPTION, _GREP_TOOL_DESCRIPTION_WITHOUT_EXECUTE} + rewritten: list[BaseTool | dict[str, Any]] = [] + changed = False + + for tool in tools: + tool_name = self._tool_name(tool) + if tool_name != "grep": + rewritten.append(tool) + continue + + if isinstance(tool, BaseTool): + if tool.description in default_descriptions and tool.description != target_description: + rewritten.append(tool.model_copy(update={"description": target_description})) + changed = True + else: + rewritten.append(tool) + continue + + if not isinstance(tool, dict): + rewritten.append(cast("BaseTool | dict[str, Any]", tool)) + continue + + if tool.get("description") in default_descriptions and tool.get("description") != target_description: + copied_tool = tool.copy() + copied_tool["description"] = target_description + rewritten.append(copied_tool) + changed = True + else: + rewritten.append(tool) + + return rewritten if changed else tools + + @staticmethod + def _tool_name(tool: object) -> str | None: + """Extract a request tool name from `BaseTool`, dict, or test doubles.""" + if isinstance(tool, BaseTool): + return tool.name + if isinstance(tool, dict): + return cast("str | None", cast("dict[str, Any]", tool).get("name")) + if hasattr(tool, "name"): + return cast("str | None", tool.name) + get = getattr(tool, "get", None) + if callable(get): + return cast("str | None", get("name")) + return None + + def _unsupported_tools_and_execution_state( + self, + tool_names: set[str | None], + runtime: Runtime[ContextT], + ) -> tuple[set[str | None], bool, BackendProtocol | None]: + """Return unsupported filesystem tools and whether execute remains active.""" + unsupported: set[str | None] = ( + {name for name in tool_names if name in _ALL_FS_TOOL_NAMES and name not in self._enabled_tools} + if self._enabled_tools is not None + else set() + ) + execution_active = False + backend = None + has_execute_tool = "execute" in tool_names + has_delete_tool = "delete" in tool_names + if not has_delete_tool and not has_execute_tool: + return unsupported, execution_active, backend + + backend = self._get_backend(runtime) # ty: ignore[invalid-argument-type] + if has_execute_tool and "execute" not in unsupported: + execution_active = supports_execution(backend) + if not execution_active: + unsupported.add("execute") + if has_delete_tool and "delete" not in unsupported and not _supports_delete(backend): + unsupported.add("delete") + return unsupported, execution_active, backend + def _resolve_capture(self, resolved_backend: BackendProtocol, tool_call_id: str | None) -> tuple[BaseSandbox, str] | None: """Resolve the executing sandbox and offload path for capture-at-source. @@ -2469,34 +2565,15 @@ def _filter_unsupported_tools_and_apply_prompt(self, request: ModelRequest[Conte Returns the request with unsupported tools removed and the filesystem system prompt appended. """ - tool_names: set[str | None] = cast( - "set[str | None]", - {(tool.name if hasattr(tool, "name") else tool.get("name")) for tool in request.tools}, - ) - has_execute_tool = "execute" in tool_names - has_delete_tool = "delete" in tool_names - - # Tools excluded by the tools allowlist (empty when no allowlist is set). - # Only filesystem tools are subject to the allowlist — user-provided tools always pass through. - # `execute` and `delete` are also filtered when the backend can't serve them. - unsupported: set[str | None] = ( - {name for name in tool_names if name in _ALL_FS_TOOL_NAMES and name not in self._enabled_tools} - if self._enabled_tools is not None - else set() - ) - execution_active = False # tracks whether execute should get system-prompt instructions - backend = None - if has_delete_tool or has_execute_tool: - backend = self._get_backend(request.runtime) # ty: ignore[invalid-argument-type] - if has_execute_tool and "execute" not in unsupported: - execution_active = supports_execution(backend) - if not execution_active: - unsupported.add("execute") - if has_delete_tool and "delete" not in unsupported and not _supports_delete(backend): - unsupported.add("delete") + tool_names: set[str | None] = {self._tool_name(tool) for tool in request.tools} + unsupported, execution_active, backend = self._unsupported_tools_and_execution_state(tool_names, request.runtime) + visible_tools = [tool for tool in request.tools if self._tool_name(tool) not in unsupported] if unsupported: - filtered_tools = [tool for tool in request.tools if (tool.name if hasattr(tool, "name") else tool.get("name")) not in unsupported] - request = request.override(tools=filtered_tools) + request = request.override(tools=visible_tools) + + described_tools = self._with_filtered_grep_description(visible_tools, include_execution=execution_active) + if described_tools is not visible_tools: + request = request.override(tools=described_tools) # Use custom system prompt if provided, otherwise generate dynamically if self._custom_system_prompt is not None: diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json index 1e3c16ea111..7ec32cdbca3 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json @@ -196,7 +196,7 @@ }, { "function": { - "description": "Search for a LITERAL text pattern across files (NOT regex).\n\nReturns matching files or content based on output_mode. The pattern is matched\nverbatim: regex metacharacters are treated as ordinary characters, NOT operators.\n\nDo NOT pass a regex. In particular:\n- To match any of several strings, run a SEPARATE grep for each one. There is no\n `|` alternation: `grep(pattern=\"foo|bar\")` looks for the literal text \"foo|bar\".\n- Do not use wildcards (`.*`) or escapes (`\\.`); they match those characters literally.\n- If you genuinely need regex, use the execute tool with `rg ''` instead.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Literal special chars are fine: `grep(pattern=\"def __init__(self):\")`", + "description": "Search for a LITERAL text pattern across files (NOT regex).\n\nReturns matching files or content based on output_mode. The pattern is matched\nverbatim: regex metacharacters are treated as ordinary characters, NOT operators.\n\nDo NOT pass a regex. In particular:\n- To match any of several strings, run a SEPARATE grep for each one. There is no\n `|` alternation: `grep(pattern=\"foo|bar\")` looks for the literal text \"foo|bar\".\n- Do not use wildcards (`.*`) or escapes (`\\.`); they match those characters literally.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Literal special chars are fine: `grep(pattern=\"def __init__(self):\")`", "name": "grep", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/test_middleware.py b/libs/deepagents/tests/unit_tests/test_middleware.py index f99895baddd..30d245a652a 100644 --- a/libs/deepagents/tests/unit_tests/test_middleware.py +++ b/libs/deepagents/tests/unit_tests/test_middleware.py @@ -2223,6 +2223,87 @@ def test_enabled_tools_execute_listed_but_backend_unsupported_is_noop(self): assert "execute" not in filtered_names assert "ls" in filtered_names + def test_grep_description_omits_execute_fallback_when_backend_lacks_execution(self): + """Default grep docs must not advertise hidden execute tooling.""" + middleware = FilesystemMiddleware(backend=StateBackend(), system_prompt="") + request = MagicMock() + request.tools = middleware.tools + request.runtime = _runtime() + request.override.return_value = request + + middleware._filter_unsupported_tools_and_apply_prompt(request) + + tools_override = [c.kwargs["tools"] for c in request.override.call_args_list if "tools" in c.kwargs][-1] + grep_tool = next(tool for tool in tools_override if tool.name == "grep") + assert "execute tool" not in grep_tool.description + assert "rg ''" not in grep_tool.description + assert next(tool for tool in middleware.tools if tool.name == "grep").description != grep_tool.description + + def test_grep_description_omits_execute_fallback_when_execute_unlisted(self): + """Default grep docs follow the tools allowlist even if the backend can execute.""" + + class _SandboxBackend(SandboxBackendProtocol, StateBackend): + def execute(self, command: str, *, timeout: int | None = None) -> ExecuteResponse: + return ExecuteResponse(output="", exit_code=0) + + middleware = FilesystemMiddleware( + backend=_SandboxBackend(), + system_prompt="", + tools=["read_file", "ls", "grep"], + ) + request = MagicMock() + request.tools = middleware.tools + request.runtime = _runtime() + request.override.return_value = request + + middleware._filter_unsupported_tools_and_apply_prompt(request) + + tools_override = [c.kwargs["tools"] for c in request.override.call_args_list if "tools" in c.kwargs][-1] + grep_tool = next(tool for tool in tools_override if tool.name == "grep") + assert "execute" not in {tool.name for tool in tools_override} + assert "execute tool" not in grep_tool.description + assert "rg ''" not in grep_tool.description + + def test_grep_description_keeps_execute_fallback_when_execution_visible(self): + """Default grep docs can recommend execute only when execute is available.""" + + class _SandboxBackend(SandboxBackendProtocol, StateBackend): + def execute(self, command: str, *, timeout: int | None = None) -> ExecuteResponse: + return ExecuteResponse(output="", exit_code=0) + + middleware = FilesystemMiddleware(backend=_SandboxBackend(), system_prompt="") + request = MagicMock() + request.tools = middleware.tools + request.runtime = _runtime() + request.override.return_value = request + + middleware._filter_unsupported_tools_and_apply_prompt(request) + + tools_overrides = [c for c in request.override.call_args_list if "tools" in c.kwargs] + grep_tool = next(tool for tool in middleware.tools if tool.name == "grep") + assert tools_overrides == [] + assert "execute tool" in grep_tool.description + assert "rg ''" in grep_tool.description + + def test_custom_grep_description_is_not_rewritten_when_execute_hidden(self): + """User-provided grep docs remain authoritative.""" + custom_description = "Custom literal search guidance." + middleware = FilesystemMiddleware( + backend=StateBackend(), + system_prompt="", + custom_tool_descriptions={"grep": custom_description}, + ) + request = MagicMock() + request.tools = middleware.tools + request.runtime = _runtime() + request.override.return_value = request + + middleware._filter_unsupported_tools_and_apply_prompt(request) + + tools_override = [c.kwargs["tools"] for c in request.override.call_args_list if "tools" in c.kwargs][-1] + grep_tool = next(tool for tool in tools_override if tool.name == "grep") + assert grep_tool.description == custom_description + def test_enabled_tools_system_prompt_lists_only_enabled_tools(self): """Dynamic system prompt only mentions the tools that survived filtering.""" middleware = FilesystemMiddleware( From 4515ec86cfa59f2ca9321c35bc0e0466dcc47208 Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Thu, 2 Jul 2026 15:54:12 -0400 Subject: [PATCH 08/13] cr --- libs/deepagents/deepagents/backends/utils.py | 5 +- .../deepagents/middleware/filesystem.py | 24 ++++++--- .../tests/unit_tests/backends/test_utils.py | 2 + .../custom_system_message_tools.json | 4 +- .../system_prompt_with_execute_tools.json | 2 +- ...m_prompt_with_memory_and_skills_tools.json | 4 +- ...stem_prompt_with_routed_backend_tools.json | 2 +- ...t_with_sync_and_async_subagents_tools.json | 4 +- .../system_prompt_without_execute_tools.json | 2 +- .../tests/unit_tests/test_middleware.py | 52 +++++++++++++++++++ 10 files changed, 83 insertions(+), 18 deletions(-) diff --git a/libs/deepagents/deepagents/backends/utils.py b/libs/deepagents/deepagents/backends/utils.py index 8b435f46752..a0f72312e41 100644 --- a/libs/deepagents/deepagents/backends/utils.py +++ b/libs/deepagents/deepagents/backends/utils.py @@ -906,6 +906,7 @@ def regex_literal_hint(pattern: str) -> str | None: return None return ( "Note: grep matches literal text, not regex, so characters like " - "`|`, `.*`, and `\\.` are searched verbatim. Run a separate literal " - "search for each alternative." + "`|`, `.*`, and `\\.` are searched verbatim. Search for the literal " + "text you need instead; for `|` alternation, run a separate search " + "per alternative." ) diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index 22b406473f1..b75bb88d3f3 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -461,6 +461,8 @@ def _format_grep_tool_result( result: GrepResult, output_mode: Literal["files_with_matches", "content", "count"], pattern: str, + *, + backend_had_matches: bool, ) -> tuple[str, Literal["success", "error"]]: """Format a backend grep result for the tool boundary. @@ -468,6 +470,11 @@ def _format_grep_tool_result( appended, so a trailing `SEARCH_TRUNCATION_NOTE` survives instead of being sliced off by an outer `truncate_if_too_long` at the call site. Callers should use the returned content as-is rather than re-truncating it. + + `backend_had_matches` reports whether the backend found anything *before* + permission filtering, so the regex hint fires only on a genuine no-match — + not when matches existed but were all redacted by read permissions (a + redaction miss has nothing to do with regex syntax). """ matches = result.matches or [] if result.error and not matches: @@ -481,10 +488,10 @@ def _format_grep_tool_result( # "Partial matches:" section past the token limit and cut it off. error = truncate_if_too_long(result.error) return f"{error}\n\nPartial matches:\n{formatted}", "error" - notes = [] + notes: list[str] = [] if result.truncated: notes.append(SEARCH_TRUNCATION_NOTE) - if not matches and (hint := regex_literal_hint(pattern)): + if not matches and not backend_had_matches and (hint := regex_literal_hint(pattern)): notes.append(hint) if notes: formatted_notes = "\n\n".join(notes) @@ -632,11 +639,12 @@ class FilesystemState(AgentState): GREP_GLOB_DESCRIPTION = ( - "Glob pattern (NOT regex) limiting which files are searched, matched against " - "the file name (e.g. '*.py', '*.ts'). This is an in-tool file filter, " - "not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') " - "is not supported on all backends; run a separate search per extension for " - "reliable results." + "Glob pattern (NOT regex) limiting which files are searched (e.g. '*.py', " + "'*.ts'). A pattern without '/' matches the file name at any depth; a pattern " + "containing '/' matches the search-root-relative path (e.g. 'src/**/*.py'). " + "This is an in-tool file filter, not a call to the separate glob tool. Brace " + "expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a " + "separate search per extension for reliable results." ) GREP_OUTPUT_MODE_DESCRIPTION = ( @@ -2147,6 +2155,7 @@ def sync_grep( GrepResult(error=grep_result.error, matches=filtered_matches, truncated=grep_result.truncated), output_mode, pattern, + backend_had_matches=bool(matches), ) return ToolMessage( # `formatted` is already size-truncated inside @@ -2190,6 +2199,7 @@ async def async_grep( GrepResult(error=grep_result.error, matches=filtered_matches, truncated=grep_result.truncated), output_mode, pattern, + backend_had_matches=bool(matches), ) return ToolMessage( # `formatted` is already size-truncated inside diff --git a/libs/deepagents/tests/unit_tests/backends/test_utils.py b/libs/deepagents/tests/unit_tests/backends/test_utils.py index a9a465f1402..3a1f96e5ad7 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_utils.py +++ b/libs/deepagents/tests/unit_tests/backends/test_utils.py @@ -35,6 +35,8 @@ class TestLooksLikeRegex: "\\bword\\b", "\\d+", "\\w", + "\\s+", + "foo\\(bar\\)", ], ) def test_detects_regex(self, pattern: str) -> None: diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json index 1e3c16ea111..6dcceb029b6 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json @@ -196,7 +196,7 @@ }, { "function": { - "description": "Search for a LITERAL text pattern across files (NOT regex).\n\nReturns matching files or content based on output_mode. The pattern is matched\nverbatim: regex metacharacters are treated as ordinary characters, NOT operators.\n\nDo NOT pass a regex. In particular:\n- To match any of several strings, run a SEPARATE grep for each one. There is no\n `|` alternation: `grep(pattern=\"foo|bar\")` looks for the literal text \"foo|bar\".\n- Do not use wildcards (`.*`) or escapes (`\\.`); they match those characters literally.\n- If you genuinely need regex, use the execute tool with `rg ''` instead.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Literal special chars are fine: `grep(pattern=\"def __init__(self):\")`", + "description": "Search for a LITERAL text pattern across files (NOT regex).\n\nReturns matching files or content based on output_mode. The pattern is matched\nverbatim: regex metacharacters are treated as ordinary characters, NOT operators.\n\nDo NOT pass a regex. In particular:\n- To match any of several strings, run a SEPARATE grep for each one. There is no\n `|` alternation: `grep(pattern=\"foo|bar\")` looks for the literal text \"foo|bar\".\n- Do not use wildcards (`.*`) or escapes (`\\.`); they match those characters literally.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Literal special chars are fine: `grep(pattern=\"def __init__(self):\")`", "name": "grep", "parameters": { "properties": { @@ -210,7 +210,7 @@ } ], "default": null, - "description": "Glob pattern (NOT regex) limiting which files are searched, matched against the file name (e.g. '*.py', '*.ts'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." + "description": "Glob pattern (NOT regex) limiting which files are searched (e.g. '*.py', '*.ts'). A pattern without '/' matches the file name at any depth; a pattern containing '/' matches the search-root-relative path (e.g. 'src/**/*.py'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." }, "output_mode": { "default": "files_with_matches", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json index 92a41fe960c..365603baac6 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json @@ -210,7 +210,7 @@ } ], "default": null, - "description": "Glob pattern (NOT regex) limiting which files are searched, matched against the file name (e.g. '*.py', '*.ts'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." + "description": "Glob pattern (NOT regex) limiting which files are searched (e.g. '*.py', '*.ts'). A pattern without '/' matches the file name at any depth; a pattern containing '/' matches the search-root-relative path (e.g. 'src/**/*.py'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." }, "output_mode": { "default": "files_with_matches", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json index 1e3c16ea111..6dcceb029b6 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json @@ -196,7 +196,7 @@ }, { "function": { - "description": "Search for a LITERAL text pattern across files (NOT regex).\n\nReturns matching files or content based on output_mode. The pattern is matched\nverbatim: regex metacharacters are treated as ordinary characters, NOT operators.\n\nDo NOT pass a regex. In particular:\n- To match any of several strings, run a SEPARATE grep for each one. There is no\n `|` alternation: `grep(pattern=\"foo|bar\")` looks for the literal text \"foo|bar\".\n- Do not use wildcards (`.*`) or escapes (`\\.`); they match those characters literally.\n- If you genuinely need regex, use the execute tool with `rg ''` instead.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Literal special chars are fine: `grep(pattern=\"def __init__(self):\")`", + "description": "Search for a LITERAL text pattern across files (NOT regex).\n\nReturns matching files or content based on output_mode. The pattern is matched\nverbatim: regex metacharacters are treated as ordinary characters, NOT operators.\n\nDo NOT pass a regex. In particular:\n- To match any of several strings, run a SEPARATE grep for each one. There is no\n `|` alternation: `grep(pattern=\"foo|bar\")` looks for the literal text \"foo|bar\".\n- Do not use wildcards (`.*`) or escapes (`\\.`); they match those characters literally.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Literal special chars are fine: `grep(pattern=\"def __init__(self):\")`", "name": "grep", "parameters": { "properties": { @@ -210,7 +210,7 @@ } ], "default": null, - "description": "Glob pattern (NOT regex) limiting which files are searched, matched against the file name (e.g. '*.py', '*.ts'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." + "description": "Glob pattern (NOT regex) limiting which files are searched (e.g. '*.py', '*.ts'). A pattern without '/' matches the file name at any depth; a pattern containing '/' matches the search-root-relative path (e.g. 'src/**/*.py'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." }, "output_mode": { "default": "files_with_matches", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json index 92a41fe960c..365603baac6 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json @@ -210,7 +210,7 @@ } ], "default": null, - "description": "Glob pattern (NOT regex) limiting which files are searched, matched against the file name (e.g. '*.py', '*.ts'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." + "description": "Glob pattern (NOT regex) limiting which files are searched (e.g. '*.py', '*.ts'). A pattern without '/' matches the file name at any depth; a pattern containing '/' matches the search-root-relative path (e.g. 'src/**/*.py'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." }, "output_mode": { "default": "files_with_matches", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json index 3bbcbdfe135..d83b47335e1 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json @@ -196,7 +196,7 @@ }, { "function": { - "description": "Search for a LITERAL text pattern across files (NOT regex).\n\nReturns matching files or content based on output_mode. The pattern is matched\nverbatim: regex metacharacters are treated as ordinary characters, NOT operators.\n\nDo NOT pass a regex. In particular:\n- To match any of several strings, run a SEPARATE grep for each one. There is no\n `|` alternation: `grep(pattern=\"foo|bar\")` looks for the literal text \"foo|bar\".\n- Do not use wildcards (`.*`) or escapes (`\\.`); they match those characters literally.\n- If you genuinely need regex, use the execute tool with `rg ''` instead.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Literal special chars are fine: `grep(pattern=\"def __init__(self):\")`", + "description": "Search for a LITERAL text pattern across files (NOT regex).\n\nReturns matching files or content based on output_mode. The pattern is matched\nverbatim: regex metacharacters are treated as ordinary characters, NOT operators.\n\nDo NOT pass a regex. In particular:\n- To match any of several strings, run a SEPARATE grep for each one. There is no\n `|` alternation: `grep(pattern=\"foo|bar\")` looks for the literal text \"foo|bar\".\n- Do not use wildcards (`.*`) or escapes (`\\.`); they match those characters literally.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Literal special chars are fine: `grep(pattern=\"def __init__(self):\")`", "name": "grep", "parameters": { "properties": { @@ -210,7 +210,7 @@ } ], "default": null, - "description": "Glob pattern (NOT regex) limiting which files are searched, matched against the file name (e.g. '*.py', '*.ts'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." + "description": "Glob pattern (NOT regex) limiting which files are searched (e.g. '*.py', '*.ts'). A pattern without '/' matches the file name at any depth; a pattern containing '/' matches the search-root-relative path (e.g. 'src/**/*.py'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." }, "output_mode": { "default": "files_with_matches", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json index 7ec32cdbca3..6dcceb029b6 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json @@ -210,7 +210,7 @@ } ], "default": null, - "description": "Glob pattern (NOT regex) limiting which files are searched, matched against the file name (e.g. '*.py', '*.ts'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." + "description": "Glob pattern (NOT regex) limiting which files are searched (e.g. '*.py', '*.ts'). A pattern without '/' matches the file name at any depth; a pattern containing '/' matches the search-root-relative path (e.g. 'src/**/*.py'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." }, "output_mode": { "default": "files_with_matches", diff --git a/libs/deepagents/tests/unit_tests/test_middleware.py b/libs/deepagents/tests/unit_tests/test_middleware.py index 30d245a652a..1803aef7c9d 100644 --- a/libs/deepagents/tests/unit_tests/test_middleware.py +++ b/libs/deepagents/tests/unit_tests/test_middleware.py @@ -49,6 +49,7 @@ SEARCH_TRUNCATION_NOTE, FileData, FilesystemMiddleware, + FilesystemPermission, FilesystemState, supports_execution, ) @@ -1055,6 +1056,57 @@ def test_grep_regex_pattern_with_matches_omits_hint(self): assert "a = b|c" in result.content assert "literal text, not regex" not in result.content + def test_grep_regex_pattern_all_matches_permission_filtered_omits_hint(self): + """A miss caused by permission redaction, not regex syntax, shows no hint. + + The backend matches the literal `b|c` in `/secret.py`, but a deny rule + strips that match. The empty result then reads "No matches found" for a + redaction reason, so the regex hint would misattribute the cause. + """ + files = { + "/secret.py": FileData( + content=["a = b|c"], + modified_at="2021-01-01", + created_at="2021-01-01", + ), + } + backend, _ = _make_backend(files) + middleware = FilesystemMiddleware( + backend=backend, + _permissions=[FilesystemPermission(operations=["read"], paths=["/secret.py"], mode="deny")], + ) + grep_search_tool = next(tool for tool in middleware.tools if tool.name == "grep") + result = grep_search_tool.invoke( + { + "pattern": "b|c", + "runtime": _runtime(), + } + ) + assert result.content == "No matches found" + assert "literal text, not regex" not in result.content + + def test_grep_error_regex_pattern_omits_hint(self): + """A backend error result never gets the regex hint appended.""" + backend, _ = _make_backend() + middleware = FilesystemMiddleware(backend=backend) + grep_search_tool = next(tool for tool in middleware.tools if tool.name == "grep") + backend_obj = middleware._get_backend(_runtime()) + + error_result = GrepResult(error="boom: backend exploded", matches=[]) + with ( + patch.object(middleware, "_get_backend", return_value=backend_obj), + patch.object(backend_obj, "grep", return_value=error_result), + ): + result = grep_search_tool.invoke( + { + "pattern": "def hello|def world", + "runtime": _runtime(), + } + ) + + assert result.status == "error" + assert "literal text, not regex" not in result.content + def test_search_store_paginated_empty(self): """Test pagination with no items.""" store = InMemoryStore() From 80f938017cb536a6c4426d8c277eb350276434be Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Thu, 2 Jul 2026 16:17:23 -0400 Subject: [PATCH 09/13] cr --- libs/deepagents/deepagents/middleware/filesystem.py | 2 +- libs/deepagents/tests/unit_tests/test_middleware.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index b75bb88d3f3..44e069b74bc 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -491,7 +491,7 @@ def _format_grep_tool_result( notes: list[str] = [] if result.truncated: notes.append(SEARCH_TRUNCATION_NOTE) - if not matches and not backend_had_matches and (hint := regex_literal_hint(pattern)): + if not result.truncated and not matches and not backend_had_matches and (hint := regex_literal_hint(pattern)): notes.append(hint) if notes: formatted_notes = "\n\n".join(notes) diff --git a/libs/deepagents/tests/unit_tests/test_middleware.py b/libs/deepagents/tests/unit_tests/test_middleware.py index 1803aef7c9d..ff382da18d4 100644 --- a/libs/deepagents/tests/unit_tests/test_middleware.py +++ b/libs/deepagents/tests/unit_tests/test_middleware.py @@ -710,7 +710,7 @@ def test_grep_truncated_regex_pattern_no_matches_keeps_note(self): assert result.status == "success" assert result.content.startswith("No matches found") assert SEARCH_TRUNCATION_NOTE in result.content - assert "literal text, not regex" in result.content + assert "literal text, not regex" not in result.content def test_glob_truncated_renders_as_success_with_note(self): """A truncated glob returns its partial paths as a success plus the narrow-your-search note.""" From 220007ae50065dcac77a8515f8a297acb6517fb9 Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Thu, 2 Jul 2026 16:26:45 -0400 Subject: [PATCH 10/13] cr --- libs/deepagents/deepagents/backends/utils.py | 7 ++++++- .../deepagents/middleware/filesystem.py | 17 ++++++++++------- .../tests/unit_tests/test_middleware.py | 17 +++++++++++++++++ 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/libs/deepagents/deepagents/backends/utils.py b/libs/deepagents/deepagents/backends/utils.py index a0f72312e41..54b2cd64055 100644 --- a/libs/deepagents/deepagents/backends/utils.py +++ b/libs/deepagents/deepagents/backends/utils.py @@ -900,7 +900,12 @@ def regex_literal_hint(pattern: str) -> str | None: and silently miss. Callers gate this on a no-match result; the function itself only inspects the pattern. - Returns `None` when the pattern has no regex signals. + Args: + pattern: The literal grep pattern to inspect for regex signals. + + Returns: + A one-line hint steering the caller toward literal search, or `None` + when the pattern has no regex signals. """ if not _looks_like_regex(pattern): return None diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index 44e069b74bc..7a4b89df0fc 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -845,7 +845,9 @@ class ExecuteSchema(BaseModel): - `*.txt` - Find all text files in the backend's default root - `/subdir/**/*.md` - Find all markdown files under /subdir""" -_GREP_REGEX_EXECUTE_FALLBACK = "- If you genuinely need regex, use the execute tool with `rg ''` instead." +# Carries its own leading newline so the empty-string substitution below drops +# the whole line cleanly, with no blank line left behind. +_GREP_REGEX_EXECUTE_FALLBACK = "\n- If you genuinely need regex, use the execute tool with `rg ''` instead." _GREP_TOOL_DESCRIPTION_TEMPLATE = """Search for a LITERAL text pattern across files (NOT regex). @@ -855,8 +857,7 @@ class ExecuteSchema(BaseModel): Do NOT pass a regex. In particular: - To match any of several strings, run a SEPARATE grep for each one. There is no `|` alternation: `grep(pattern="foo|bar")` looks for the literal text "foo|bar". -- Do not use wildcards (`.*`) or escapes (`\\.`); they match those characters literally. -{execute_fallback} +- Do not use wildcards (`.*`) or escapes (`\\.`); they match those characters literally.{execute_fallback} Examples: - Search all files: `grep(pattern="TODO")` @@ -865,10 +866,7 @@ class ExecuteSchema(BaseModel): - Literal special chars are fine: `grep(pattern="def __init__(self):")`""" GREP_TOOL_DESCRIPTION = _GREP_TOOL_DESCRIPTION_TEMPLATE.format(execute_fallback=_GREP_REGEX_EXECUTE_FALLBACK) -_GREP_TOOL_DESCRIPTION_WITHOUT_EXECUTE = _GREP_TOOL_DESCRIPTION_TEMPLATE.format(execute_fallback="").replace( - "\n\nExamples:", - "\nExamples:", -) +_GREP_TOOL_DESCRIPTION_WITHOUT_EXECUTE = _GREP_TOOL_DESCRIPTION_TEMPLATE.format(execute_fallback="") EXECUTE_TOOL_DESCRIPTION = """Executes a shell command in an isolated sandbox environment. @@ -2120,6 +2118,11 @@ async def async_glob( def _create_grep_tool(self) -> BaseTool: """Create the grep tool.""" + # Provisional default: assume execute is available so the description can + # point at `rg` for genuine regex. `_filter_unsupported_tools_and_apply_prompt` + # reconciles this to the backend's actual execute capability at request time, + # swapping in the without-execute variant when execute isn't active. The static + # description on `self.tools` is therefore only a placeholder until a request runs. tool_description = self._grep_tool_description(include_execution=True) def sync_grep( diff --git a/libs/deepagents/tests/unit_tests/test_middleware.py b/libs/deepagents/tests/unit_tests/test_middleware.py index ff382da18d4..02fc73f05de 100644 --- a/libs/deepagents/tests/unit_tests/test_middleware.py +++ b/libs/deepagents/tests/unit_tests/test_middleware.py @@ -2356,6 +2356,23 @@ def test_custom_grep_description_is_not_rewritten_when_execute_hidden(self): grep_tool = next(tool for tool in tools_override if tool.name == "grep") assert grep_tool.description == custom_description + def test_grep_description_swap_copies_dict_tool_specs(self): + """Dict-shaped grep specs are swapped via a copy, leaving the input untouched.""" + middleware = FilesystemMiddleware(backend=StateBackend(), system_prompt="") + default_description = next(tool for tool in middleware.tools if tool.name == "grep").description + original = {"name": "grep", "description": default_description} + + rewritten = middleware._with_filtered_grep_description([original], include_execution=False) + + rewritten_grep = next(tool for tool in rewritten if tool["name"] == "grep") + # Swapped via a copy, never mutated in place. + assert rewritten_grep is not original + assert original["description"] == default_description + # The copy carries the without-execute variant. + assert "execute tool" not in rewritten_grep["description"] + assert "rg ''" not in rewritten_grep["description"] + assert "LITERAL text pattern" in rewritten_grep["description"] + def test_enabled_tools_system_prompt_lists_only_enabled_tools(self): """Dynamic system prompt only mentions the tools that survived filtering.""" middleware = FilesystemMiddleware( From 8fd16b4df7b3411688ffa48d5ed4d6ad064b2a10 Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Thu, 2 Jul 2026 16:30:40 -0400 Subject: [PATCH 11/13] perf(sdk): halve filesystem tool timeouts DEFAULT_GREP_TIMEOUT: 30s -> 15s (per sync phase) ASYNC_GREP_TIMEOUT: 65s -> 35s (wraps 2 sync phases + headroom) _DEFAULT_GLOB_TIMEOUT: 10s -> 5s (backend walk budget) GLOB_TIMEOUT: 20s -> 10s (middleware outer deadline) The layering invariants still hold: - _DEFAULT_GLOB_TIMEOUT (5) < GLOB_TIMEOUT (10) - ASYNC_GREP_TIMEOUT (35) > 2 * DEFAULT_GREP_TIMEOUT (30) --- libs/deepagents/deepagents/backends/filesystem.py | 4 ++-- libs/deepagents/deepagents/backends/protocol.py | 2 +- libs/deepagents/deepagents/middleware/filesystem.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libs/deepagents/deepagents/backends/filesystem.py b/libs/deepagents/deepagents/backends/filesystem.py index b28e833ac3c..5eeea5894a8 100644 --- a/libs/deepagents/deepagents/backends/filesystem.py +++ b/libs/deepagents/deepagents/backends/filesystem.py @@ -45,13 +45,13 @@ logger = logging.getLogger(__name__) -_DEFAULT_GLOB_TIMEOUT = 10 +_DEFAULT_GLOB_TIMEOUT = 5 """Wall-clock budget in seconds for a single local `glob` walk. A fixed bound keeps `glob` from hanging on huge or slow trees; when it elapses the walk returns whatever it found so far with `GlobResult.truncated=True` rather than erroring. Kept below the middleware's `GLOB_TIMEOUT` -(`deepagents.middleware.filesystem.GLOB_TIMEOUT`, currently 20s) so the backend +(`deepagents.middleware.filesystem.GLOB_TIMEOUT`, currently 10s) so the backend returns partial results before that outer net abandons the call; the ordering is guarded by `test_glob_backend_budget_below_middleware_deadline`. """ diff --git a/libs/deepagents/deepagents/backends/protocol.py b/libs/deepagents/deepagents/backends/protocol.py index f0fc7886277..cd9b36abd6c 100644 --- a/libs/deepagents/deepagents/backends/protocol.py +++ b/libs/deepagents/deepagents/backends/protocol.py @@ -31,7 +31,7 @@ logger = logging.getLogger(__name__) -DEFAULT_GREP_TIMEOUT: Final = 30 +DEFAULT_GREP_TIMEOUT: Final = 15 """Default timeout in seconds for one sync grep phase.""" ASYNC_GREP_TIMEOUT: Final = (2 * DEFAULT_GREP_TIMEOUT) + 5 diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index 7a4b89df0fc..6ea29998816 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -533,7 +533,7 @@ def _format_glob_tool_result(paths: list[str], *, truncated: bool) -> str: EMPTY_CONTENT_WARNING = "System reminder: File exists but has empty contents" -GLOB_TIMEOUT = 20.0 # seconds +GLOB_TIMEOUT = 10.0 # seconds LINE_NUMBER_WIDTH = 6 SEARCH_TRUNCATION_NOTE = ( "Note: the search stopped early because it hit its time limit. The matches above are valid but incomplete. " From fc5a9c950cd1cfbb3e9c6355bc9cc9bcd777df0f Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Thu, 2 Jul 2026 17:42:54 -0400 Subject: [PATCH 12/13] cr --- .../deepagents/deepagents/backends/sandbox.py | 88 +++++- .../backends/test_sandbox_backend.py | 260 ++++++++++++++++++ 2 files changed, 345 insertions(+), 3 deletions(-) diff --git a/libs/deepagents/deepagents/backends/sandbox.py b/libs/deepagents/deepagents/backends/sandbox.py index 57e15b5a79c..a07c3034e26 100644 --- a/libs/deepagents/deepagents/backends/sandbox.py +++ b/libs/deepagents/deepagents/backends/sandbox.py @@ -82,6 +82,69 @@ Uses base64-encoded parameters to avoid shell escaping issues. """ + +_GREP_PATH_GLOB_TEMPLATE = """python3 -c " +import glob, os, base64, sys + +search_path = base64.b64decode('{path_b64}').decode('utf-8') +glob_pat = base64.b64decode('{glob_b64}').decode('utf-8') +pattern = base64.b64decode('{pattern_b64}').decode('utf-8') + +# When the search path is a directory, chdir to it so glob patterns +# resolve relative to it. When it is a single file, search it directly +# (glob filtering is irrelevant for a single-file search). +if os.path.isdir(search_path): + os.chdir(search_path) + # A leading `/` would make `glob.glob` treat the pattern as an + # absolute filesystem path, searching outside the search root (e.g. + # `/*.py` after `chdir('/workspace')` would match `/top.py` on + # the host, not `/workspace/top.py`). Strip it so anchored globs + # stay relative to the search root, matching the `FilesystemBackend` + # semantics where `/` anchors to the root, not the filesystem. + rel_glob = glob_pat.lstrip('/') + rel_files = sorted(glob.glob(rel_glob, recursive=True)) + # Open the glob-relative path (cwd is the search root) but report the + # path prefixed with the search root, so GrepResult.path matches the + # `/` form that `grep -r` emits on the --include route. + targets = [(rel, os.path.join(search_path, rel)) for rel in rel_files] +else: + targets = [(search_path, search_path)] + +for open_path, display_path in targets: + try: + with open(open_path, 'r', encoding='utf-8', errors='ignore') as fh: + for i, line in enumerate(fh, 1): + if pattern in line: + # GNU grep -HnFZ always terminates each record with a + # newline, even when the matched line has none. Strip + # the line's own trailing newline and add an explicit + # one so records never concatenate when a file's last + # line lacks a final newline. + sys.stdout.write(display_path + chr(0) + str(i) + ':' + line.rstrip(chr(10)) + chr(10)) + except OSError: + pass +" 2>/dev/null""" +"""Search file contents for a literal string, filtered by a path-relative glob. + +Used when the glob pattern contains a `/` (e.g. `src/**/*.py`), because +GNU `grep --include` only matches basenames and would silently return zero +results for such patterns. All three parameters are base64-encoded to avoid +shell escaping issues. + +Emits the same `path\0line_num:text` record structure that `grep -HnFZ` +produces — each match path is prefixed with the search root to mirror +grep's output — so `_parse_grep_output` consumes it unchanged. Unlike the +`grep -r` route, results are sorted, hidden files and directories are +skipped (Python `glob` semantics), and file contents are decoded as UTF-8 +with `errors='ignore'` rather than matched byte-for-byte. + +`stderr` is discarded, but `|| true` is deliberately omitted: the script +exits 0 on a legitimate no-match, so a non-zero exit signals a genuine +failure (bad base64, an inaccessible search root) that `_parse_grep_output` +surfaces as an error instead of a silent empty result. +""" + + _WRITE_CHECK_TEMPLATE = """python3 -c " import os, base64 @@ -486,8 +549,24 @@ def _build_grep_cmd(pattern: str, path: str | None, glob: str | None) -> str: # `-Z` separates the filename from line data with NUL, so filenames may # contain `:` without making the output ambiguous. grep_opts = "-rHnFZ" - glob_pattern = f"--include={shlex.quote(glob)}" if glob else "" pattern_escaped = shlex.quote(pattern) + + # GNU `grep --include` only matches basenames, so a slash-containing glob + # like `src/**/*.py` would silently match zero files. Route those to the + # in-process Python template that resolves the glob relative to the search + # root. Basename-only globs (no `/`) work correctly with `--include` and + # are faster to run through GNU grep. + if glob and "/" in glob: + path_b64 = base64.b64encode((path or ".").encode("utf-8")).decode("ascii") + glob_b64 = base64.b64encode(glob.encode("utf-8")).decode("ascii") + pattern_b64 = base64.b64encode(pattern.encode("utf-8")).decode("ascii") + return _GREP_PATH_GLOB_TEMPLATE.format( + path_b64=path_b64, + glob_b64=glob_b64, + pattern_b64=pattern_b64, + ) + + glob_pattern = f"--include={shlex.quote(glob)}" if glob else "" return f"grep {grep_opts} {glob_pattern} -e {pattern_escaped} {search_path} 2>/dev/null || true" @@ -1192,8 +1271,11 @@ def grep( path: Directory or file to search in. Defaults to `"."`. - glob: Optional file-name glob to restrict the search - (e.g. `'*.py'`). + glob: Optional glob to restrict the search. Patterns without a + `/` (e.g. `'*.py'`) match basenames at any depth via + `grep --include`; patterns containing a `/` (e.g. + `'src/**/*.py'`) match the search-root-relative path via an + in-process Python glob. Returns: `GrepResult` with a list of `GrepMatch` dicts, or `error` on failure. diff --git a/libs/deepagents/tests/unit_tests/backends/test_sandbox_backend.py b/libs/deepagents/tests/unit_tests/backends/test_sandbox_backend.py index 5f38bd71266..8b4b6892824 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_sandbox_backend.py +++ b/libs/deepagents/tests/unit_tests/backends/test_sandbox_backend.py @@ -27,9 +27,11 @@ _EDIT_INLINE_MAX_BYTES, _EDIT_TMPFILE_TEMPLATE, _GLOB_COMMAND_TEMPLATE, + _GREP_PATH_GLOB_TEMPLATE, _READ_COMMAND_TEMPLATE, _WRITE_CHECK_TEMPLATE, BaseSandbox, + _build_grep_cmd, _build_read_cmd, _check_preflight_result, _map_edit_error, @@ -464,6 +466,143 @@ def test_grep_passes_glob_include() -> None: assert "--include='*.py'" in sandbox.last_command +def test_build_grep_cmd_uses_grep_include_for_basename_glob() -> None: + """Basename-only globs (no slash) use GNU grep --include.""" + cmd = _build_grep_cmd("needle", "/test", "*.py") + assert "--include='*.py'" in cmd + assert "python3" not in cmd + + +def test_build_grep_cmd_routes_slash_glob_to_python_template() -> None: + """Slash-containing globs use the Python template, not grep --include.""" + cmd = _build_grep_cmd("needle", "/test", "src/**/*.py") + assert "python3" in cmd + assert "--include=" not in cmd + # Raw parameters must not appear in the command... + assert "src/**/*.py" not in cmd + assert "needle" not in cmd + # ...because each is base64-encoded. Assert the encodings are actually + # present (a bare absence check passes vacuously if a param is dropped). + assert base64.b64encode(b"src/**/*.py").decode("ascii") in cmd + assert base64.b64encode(b"needle").decode("ascii") in cmd + assert base64.b64encode(b"/test").decode("ascii") in cmd + + +def test_build_grep_cmd_routes_simple_slash_glob_to_python_template() -> None: + """A simple slash glob (no **) also uses the Python template.""" + cmd = _build_grep_cmd("needle", None, "src/*.py") + assert "python3" in cmd + assert "--include=" not in cmd + # path=None falls back to ".", which must be base64-encoded like the rest. + assert base64.b64encode(b"src/*.py").decode("ascii") in cmd + assert base64.b64encode(b".").decode("ascii") in cmd + + +def test_build_grep_cmd_slash_glob_does_not_mask_errors() -> None: + """The Python-template command must not force exit 0 with `|| true`. + + Unlike grep (which exits 1 on no-match), the template exits 0 on a + legitimate no-match, so `|| true` would only mask genuine crashes and + reintroduce the silent-zero-results failure this route exists to fix. + """ + cmd = _build_grep_cmd("needle", "/test", "src/**/*.py") + assert "|| true" not in cmd + + +def test_build_grep_cmd_no_glob_uses_grep() -> None: + """No glob falls through to the plain grep command.""" + cmd = _build_grep_cmd("needle", "/test", None) + assert "grep" in cmd + assert "--include=" not in cmd + assert "python3" not in cmd + + +def test_grep_slash_glob_returns_matches_from_python_template() -> None: + """grep() with a slash-containing glob parses output from the Python template.""" + sandbox = MockSandbox() + # Record structure: path\0line_num:text. The template prefixes each match + # with the search root, mirroring grep -HnFZ's `/` output. + sandbox._next_output = "/test/src/pkg/a.py\0001:needle" + + result = sandbox.grep("needle", "/test", "src/**/*.py") + + assert result.error is None + assert result.matches == [ + {"path": "/test/src/pkg/a.py", "line": 1, "text": "needle"}, + ] + assert sandbox.last_command is not None + assert "python3" in sandbox.last_command + + +def test_grep_slash_glob_empty_results() -> None: + """grep() with a slash glob and no matches returns empty list.""" + sandbox = MockSandbox() + sandbox._next_output = "" + + result = sandbox.grep("needle", "/test", "src/**/*.py") + + assert result.error is None + assert result.matches == [] + + +def test_grep_path_glob_template_no_shell_injection() -> None: + """The Python path-glob template base64-encodes all parameters.""" + malicious_glob = "src/x' ; echo injected ; #" + cmd = _build_grep_cmd("needle", "/test", malicious_glob) + # The raw glob must not appear in the command — only its base64 encoding. + assert malicious_glob not in cmd + assert "echo injected" not in cmd + assert "python3" in cmd + # The glob is neutralized because it is base64-encoded, not because it was + # silently dropped: assert the encoding is present. + assert base64.b64encode(malicious_glob.encode("utf-8")).decode("ascii") in cmd + + +def test_grep_path_glob_is_routed_for_slash_in_glob() -> None: + """grep() routes slash-containing globs to the Python template.""" + sandbox = MockSandbox() + sandbox._next_output = "" + + sandbox.grep("needle", "/test", "src/**/*.py") + + assert sandbox.last_command is not None + assert "python3" in sandbox.last_command + assert "--include=" not in sandbox.last_command + + +def test_grep_path_glob_template_strips_leading_slash() -> None: + """Anchored globs (leading /) stay relative to the search root, not the filesystem root.""" + assert "lstrip" in _GREP_PATH_GLOB_TEMPLATE + assert "rel_glob" in _GREP_PATH_GLOB_TEMPLATE + # The raw glob_pat must not be passed directly to glob.glob — only rel_glob. + # Verify the template uses rel_glob in the glob() call, not glob_pat. + assert "glob.glob(rel_glob" in _GREP_PATH_GLOB_TEMPLATE + assert "glob.glob(glob_pat" not in _GREP_PATH_GLOB_TEMPLATE + + +def test_grep_path_glob_template_terminates_each_record() -> None: + """Each match record is explicitly newline-terminated to prevent concatenation.""" + # The template must strip the line's trailing newline and add an explicit one + # so a file whose last line lacks a final newline doesn't merge with the next. + assert "rstrip" in _GREP_PATH_GLOB_TEMPLATE + assert "line.rstrip" in _GREP_PATH_GLOB_TEMPLATE + + +def test_grep_path_glob_parses_multiple_matches_no_trailing_newline() -> None: + """Two matches where the first line has no trailing newline parse correctly.""" + # Simulate the fixed template output: each record explicitly newline-terminated. + sandbox = MockSandbox() + sandbox._next_output = "file1.py\x001:needle\nfile2.py\x002:needle\n" + + result = sandbox.grep("needle", "/test", "src/**/*.py") + + assert result.error is None + assert result.matches == [ + {"path": "file1.py", "line": 1, "text": "needle"}, + {"path": "file2.py", "line": 2, "text": "needle"}, + ] + + def test_grep_returns_empty_matches_for_successful_empty_output() -> None: """grep() returns no matches when grep succeeds with no output.""" sandbox = MockSandbox() @@ -1281,6 +1420,127 @@ def test_glob_script_permission_denied(tmp_path: Path) -> None: locked.chmod(stat.S_IRWXU) +# -- grep path-glob template runtime behavior --------------------------------- +# Direct execution of the formatted _GREP_PATH_GLOB_TEMPLATE script. Mock-based +# tests stub execute() output and so cannot catch a real defect in the template +# body (a dropped `recursive=True`, a wrong delimiter, a broken chdir) — exactly +# the silent-zero-results class this route exists to fix. + + +def _run_grep_glob_script(path: Path, pattern: str, glob: str) -> subprocess.CompletedProcess[str]: + """Execute the formatted `_GREP_PATH_GLOB_TEMPLATE` script directly. + + Extracts the inline `python3 -c` body from the command `_build_grep_cmd` + produces (dropping the shell wrapper) and runs it via the interpreter, + mirroring `_run_glob_script`. Returns the `CompletedProcess` so callers can + assert on both stdout and the exit code — the template's error-surfacing + contract depends on a non-zero exit propagating rather than being masked. + """ + cmd = _build_grep_cmd(pattern, str(path), glob) + _, _, tail = cmd.partition('python3 -c "') + script, _, _ = tail.rpartition('"') + return subprocess.run( # noqa: S603 # script is the project's own _GREP_PATH_GLOB_TEMPLATE, not user input + [sys.executable, "-c", script], + capture_output=True, + text=True, + check=False, + ) + + +def _parse_script_records(stdout: str) -> list[tuple[str, int, str]]: + r"""Parse the template's `path\0line_num:text\n` records for assertions.""" + records: list[tuple[str, int, str]] = [] + for line in stdout.split("\n"): + if not line: + continue + file_path, rest = line.split("\0", 1) + num, text = rest.split(":", 1) + records.append((file_path, int(num), text)) + return records + + +def test_grep_glob_script_matches_recursively_and_prefixes_path(tmp_path: Path) -> None: + """`src/**/*.py` matches .py files at any depth under src, path-prefixed and sorted.""" + (tmp_path / "src" / "pkg").mkdir(parents=True) + (tmp_path / "src" / "a.py").write_text("alpha needle here\n") + (tmp_path / "src" / "pkg" / "b.py").write_text("needle in b\n") + (tmp_path / "src" / "pkg" / "c.txt").write_text("needle in c\n") # excluded: not .py + (tmp_path / "other.py").write_text("needle in other\n") # excluded: not under src + + proc = _run_grep_glob_script(tmp_path, "needle", "src/**/*.py") + + assert proc.returncode == 0 + records = _parse_script_records(proc.stdout) + # Paths are prefixed with the search root and sorted, and only src/*.py match. + assert [r[0] for r in records] == [ + str(tmp_path / "src" / "a.py"), + str(tmp_path / "src" / "pkg" / "b.py"), + ] + assert "c.txt" not in proc.stdout + assert "other.py" not in proc.stdout + + +def test_grep_glob_script_terminates_records_end_to_end(tmp_path: Path) -> None: + """Two matched files whose last line lacks a newline parse as two records. + + Drives the real template (not a hand-written mock string) through + `_parse_grep_output` to prove the newline-termination fix prevents record + concatenation across files. + """ + (tmp_path / "src").mkdir() + (tmp_path / "src" / "a.py").write_bytes(b"needle") # no trailing newline + (tmp_path / "src" / "b.py").write_bytes(b"needle") # no trailing newline + + proc = _run_grep_glob_script(tmp_path, "needle", "src/*.py") + + assert proc.returncode == 0 + resp = ExecuteResponse(output=proc.stdout, exit_code=proc.returncode, truncated=False) + result = _parse_grep_output(resp, str(tmp_path)) + assert result.error is None + assert result.matches is not None + assert len(result.matches) == 2 + assert {m["path"] for m in result.matches} == { + str(tmp_path / "src" / "a.py"), + str(tmp_path / "src" / "b.py"), + } + + +def test_grep_glob_script_strips_leading_slash(tmp_path: Path) -> None: + """A leading `/` in the glob stays relative to the search root, not the host root.""" + (tmp_path / "sub").mkdir() + (tmp_path / "top.py").write_text("needle\n") + (tmp_path / "sub" / "deep.py").write_text("needle\n") + + proc = _run_grep_glob_script(tmp_path, "needle", "/*.py") + + assert proc.returncode == 0 + records = _parse_script_records(proc.stdout) + # `/*.py` -> `*.py`: matches only the top-level file under the search root. + assert [r[0] for r in records] == [str(tmp_path / "top.py")] + + +@_PERMISSION_DENIED_SKIP +def test_grep_glob_script_surfaces_error_on_unreadable_root(tmp_path: Path) -> None: + """An inaccessible search root yields a non-zero exit, not a silent empty result. + + Confirms `|| true` was removed: a `chdir` failure propagates so + `_parse_grep_output` reports an error instead of `matches=[]`. + """ + locked = tmp_path / "locked_dir" + locked.mkdir() + locked.chmod(0o000) + try: + proc = _run_grep_glob_script(locked, "needle", "src/*.py") + finally: + locked.chmod(stat.S_IRWXU) + + assert proc.returncode != 0 + resp = ExecuteResponse(output=proc.stdout, exit_code=proc.returncode, truncated=False) + result = _parse_grep_output(resp, str(locked)) + assert result.error is not None + assert result.matches is None + + # -- glob host-side error surfacing ------------------------------------------- From f5e9de354ee4d4dcf1311f345dfcadc8c77885f3 Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Thu, 2 Jul 2026 18:35:55 -0400 Subject: [PATCH 13/13] test(evals): move grep regex-recovery evals from hillclimb to baseline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These evals pass deterministically across all tested models (100% correctness, 4 models × 3 trials × 3 cases = 36/36), with perfect step_ratio and tool_call_ratio. They test a concrete SDK feature (the literal-grep regex hint) and behave as regression gates, not progress signals. --- libs/evals/tests/evals/test_file_operations.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/evals/tests/evals/test_file_operations.py b/libs/evals/tests/evals/test_file_operations.py index 3047248b637..c7de0f4c479 100644 --- a/libs/evals/tests/evals/test_file_operations.py +++ b/libs/evals/tests/evals/test_file_operations.py @@ -428,7 +428,7 @@ def test_grep_finds_matching_paths(model: BaseChatModel) -> None: ) -@pytest.mark.eval_tier("hillclimb") +@pytest.mark.eval_tier("baseline") @pytest.mark.eval_category("retrieval") @pytest.mark.langsmith def test_grep_alternation_regex_recovers(model: BaseChatModel) -> None: @@ -462,7 +462,7 @@ def test_grep_alternation_regex_recovers(model: BaseChatModel) -> None: ) -@pytest.mark.eval_tier("hillclimb") +@pytest.mark.eval_tier("baseline") @pytest.mark.eval_category("retrieval") @pytest.mark.langsmith def test_grep_wildcard_regex_recovers(model: BaseChatModel) -> None: @@ -493,7 +493,7 @@ def test_grep_wildcard_regex_recovers(model: BaseChatModel) -> None: ) -@pytest.mark.eval_tier("hillclimb") +@pytest.mark.eval_tier("baseline") @pytest.mark.eval_category("retrieval") @pytest.mark.langsmith def test_grep_escaped_metachar_regex_recovers(model: BaseChatModel) -> None: