diff --git a/libs/deepagents/deepagents/backends/filesystem.py b/libs/deepagents/deepagents/backends/filesystem.py index b28e833ac3c..5eeea5894a8 100644 --- a/libs/deepagents/deepagents/backends/filesystem.py +++ b/libs/deepagents/deepagents/backends/filesystem.py @@ -45,13 +45,13 @@ logger = logging.getLogger(__name__) -_DEFAULT_GLOB_TIMEOUT = 10 +_DEFAULT_GLOB_TIMEOUT = 5 """Wall-clock budget in seconds for a single local `glob` walk. A fixed bound keeps `glob` from hanging on huge or slow trees; when it elapses the walk returns whatever it found so far with `GlobResult.truncated=True` rather than erroring. Kept below the middleware's `GLOB_TIMEOUT` -(`deepagents.middleware.filesystem.GLOB_TIMEOUT`, currently 20s) so the backend +(`deepagents.middleware.filesystem.GLOB_TIMEOUT`, currently 10s) so the backend returns partial results before that outer net abandons the call; the ordering is guarded by `test_glob_backend_budget_below_middleware_deadline`. """ diff --git a/libs/deepagents/deepagents/backends/protocol.py b/libs/deepagents/deepagents/backends/protocol.py index f0fc7886277..cd9b36abd6c 100644 --- a/libs/deepagents/deepagents/backends/protocol.py +++ b/libs/deepagents/deepagents/backends/protocol.py @@ -31,7 +31,7 @@ logger = logging.getLogger(__name__) -DEFAULT_GREP_TIMEOUT: Final = 30 +DEFAULT_GREP_TIMEOUT: Final = 15 """Default timeout in seconds for one sync grep phase.""" ASYNC_GREP_TIMEOUT: Final = (2 * DEFAULT_GREP_TIMEOUT) + 5 diff --git a/libs/deepagents/deepagents/backends/sandbox.py b/libs/deepagents/deepagents/backends/sandbox.py index 57e15b5a79c..a07c3034e26 100644 --- a/libs/deepagents/deepagents/backends/sandbox.py +++ b/libs/deepagents/deepagents/backends/sandbox.py @@ -82,6 +82,69 @@ Uses base64-encoded parameters to avoid shell escaping issues. """ + +_GREP_PATH_GLOB_TEMPLATE = """python3 -c " +import glob, os, base64, sys + +search_path = base64.b64decode('{path_b64}').decode('utf-8') +glob_pat = base64.b64decode('{glob_b64}').decode('utf-8') +pattern = base64.b64decode('{pattern_b64}').decode('utf-8') + +# When the search path is a directory, chdir to it so glob patterns +# resolve relative to it. When it is a single file, search it directly +# (glob filtering is irrelevant for a single-file search). +if os.path.isdir(search_path): + os.chdir(search_path) + # A leading `/` would make `glob.glob` treat the pattern as an + # absolute filesystem path, searching outside the search root (e.g. + # `/*.py` after `chdir('/workspace')` would match `/top.py` on + # the host, not `/workspace/top.py`). Strip it so anchored globs + # stay relative to the search root, matching the `FilesystemBackend` + # semantics where `/` anchors to the root, not the filesystem. + rel_glob = glob_pat.lstrip('/') + rel_files = sorted(glob.glob(rel_glob, recursive=True)) + # Open the glob-relative path (cwd is the search root) but report the + # path prefixed with the search root, so GrepResult.path matches the + # `/` form that `grep -r` emits on the --include route. + targets = [(rel, os.path.join(search_path, rel)) for rel in rel_files] +else: + targets = [(search_path, search_path)] + +for open_path, display_path in targets: + try: + with open(open_path, 'r', encoding='utf-8', errors='ignore') as fh: + for i, line in enumerate(fh, 1): + if pattern in line: + # GNU grep -HnFZ always terminates each record with a + # newline, even when the matched line has none. Strip + # the line's own trailing newline and add an explicit + # one so records never concatenate when a file's last + # line lacks a final newline. + sys.stdout.write(display_path + chr(0) + str(i) + ':' + line.rstrip(chr(10)) + chr(10)) + except OSError: + pass +" 2>/dev/null""" +"""Search file contents for a literal string, filtered by a path-relative glob. + +Used when the glob pattern contains a `/` (e.g. `src/**/*.py`), because +GNU `grep --include` only matches basenames and would silently return zero +results for such patterns. All three parameters are base64-encoded to avoid +shell escaping issues. + +Emits the same `path\0line_num:text` record structure that `grep -HnFZ` +produces — each match path is prefixed with the search root to mirror +grep's output — so `_parse_grep_output` consumes it unchanged. Unlike the +`grep -r` route, results are sorted, hidden files and directories are +skipped (Python `glob` semantics), and file contents are decoded as UTF-8 +with `errors='ignore'` rather than matched byte-for-byte. + +`stderr` is discarded, but `|| true` is deliberately omitted: the script +exits 0 on a legitimate no-match, so a non-zero exit signals a genuine +failure (bad base64, an inaccessible search root) that `_parse_grep_output` +surfaces as an error instead of a silent empty result. +""" + + _WRITE_CHECK_TEMPLATE = """python3 -c " import os, base64 @@ -486,8 +549,24 @@ def _build_grep_cmd(pattern: str, path: str | None, glob: str | None) -> str: # `-Z` separates the filename from line data with NUL, so filenames may # contain `:` without making the output ambiguous. grep_opts = "-rHnFZ" - glob_pattern = f"--include={shlex.quote(glob)}" if glob else "" pattern_escaped = shlex.quote(pattern) + + # GNU `grep --include` only matches basenames, so a slash-containing glob + # like `src/**/*.py` would silently match zero files. Route those to the + # in-process Python template that resolves the glob relative to the search + # root. Basename-only globs (no `/`) work correctly with `--include` and + # are faster to run through GNU grep. + if glob and "/" in glob: + path_b64 = base64.b64encode((path or ".").encode("utf-8")).decode("ascii") + glob_b64 = base64.b64encode(glob.encode("utf-8")).decode("ascii") + pattern_b64 = base64.b64encode(pattern.encode("utf-8")).decode("ascii") + return _GREP_PATH_GLOB_TEMPLATE.format( + path_b64=path_b64, + glob_b64=glob_b64, + pattern_b64=pattern_b64, + ) + + glob_pattern = f"--include={shlex.quote(glob)}" if glob else "" return f"grep {grep_opts} {glob_pattern} -e {pattern_escaped} {search_path} 2>/dev/null || true" @@ -1192,8 +1271,11 @@ def grep( path: Directory or file to search in. Defaults to `"."`. - glob: Optional file-name glob to restrict the search - (e.g. `'*.py'`). + glob: Optional glob to restrict the search. Patterns without a + `/` (e.g. `'*.py'`) match basenames at any depth via + `grep --include`; patterns containing a `/` (e.g. + `'src/**/*.py'`) match the search-root-relative path via an + in-process Python glob. Returns: `GrepResult` with a list of `GrepMatch` dicts, or `error` on failure. diff --git a/libs/deepagents/deepagents/backends/utils.py b/libs/deepagents/deepagents/backends/utils.py index 31bde7f60af..54b2cd64055 100644 --- a/libs/deepagents/deepagents/backends/utils.py +++ b/libs/deepagents/deepagents/backends/utils.py @@ -872,3 +872,46 @@ def format_grep_matches( if not matches: return "No matches found" return _format_grep_results(build_grep_results_dict(matches), output_mode) + + +_REGEX_SIGNAL_RE = re.compile( + r"\|" # alternation + r"|\.\*" # `.*` wildcard + r"|\.\+" # `.+` wildcard + r"|\\[.wWdDsSbB(){}\[\]|+*?^$]" # escaped regex metacharacters / classes +) +"""Strong signals that a pattern was written as a regex rather than literal text. + +Deliberately conservative: bare `.`, `(`, `)`, `[`, `]`, `?`, `^`, `$` are +omitted because they appear routinely in literal code searches (e.g. +`self.tools`, `def __init__(self):`, `arr[0]`), which would cause false hints. +""" + + +def _looks_like_regex(pattern: str) -> bool: + """Heuristically detect regex syntax in a pattern meant for literal grep.""" + return bool(_REGEX_SIGNAL_RE.search(pattern)) + + +def regex_literal_hint(pattern: str) -> str | None: + """Return a hint when a pattern looks like an (unsupported) regex. + + `grep` matches literal text, so regex metacharacters are searched verbatim + and silently miss. Callers gate this on a no-match result; the function + itself only inspects the pattern. + + Args: + pattern: The literal grep pattern to inspect for regex signals. + + Returns: + A one-line hint steering the caller toward literal search, or `None` + when the pattern has no regex signals. + """ + if not _looks_like_regex(pattern): + return None + return ( + "Note: grep matches literal text, not regex, so characters like " + "`|`, `.*`, and `\\.` are searched verbatim. Search for the literal " + "text you need instead; for `|` alternation, run a separate search " + "per alternative." + ) diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index 331edb657bc..6ea29998816 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -72,6 +72,7 @@ check_empty_content, format_content_with_line_numbers, format_grep_matches, + regex_literal_hint, sanitize_tool_call_id as sanitize_tool_call_id, truncate_if_too_long, validate_path, @@ -459,6 +460,9 @@ def _filter_grep_matches_by_permission( def _format_grep_tool_result( result: GrepResult, output_mode: Literal["files_with_matches", "content", "count"], + pattern: str, + *, + backend_had_matches: bool, ) -> tuple[str, Literal["success", "error"]]: """Format a backend grep result for the tool boundary. @@ -466,6 +470,11 @@ def _format_grep_tool_result( appended, so a trailing `SEARCH_TRUNCATION_NOTE` survives instead of being sliced off by an outer `truncate_if_too_long` at the call site. Callers should use the returned content as-is rather than re-truncating it. + + `backend_had_matches` reports whether the backend found anything *before* + permission filtering, so the regex hint fires only on a genuine no-match — + not when matches existed but were all redacted by read permissions (a + redaction miss has nothing to do with regex syntax). """ matches = result.matches or [] if result.error and not matches: @@ -479,8 +488,14 @@ def _format_grep_tool_result( # "Partial matches:" section past the token limit and cut it off. error = truncate_if_too_long(result.error) return f"{error}\n\nPartial matches:\n{formatted}", "error" + notes: list[str] = [] if result.truncated: - return f"{formatted}\n\n{SEARCH_TRUNCATION_NOTE}", "success" + notes.append(SEARCH_TRUNCATION_NOTE) + if not result.truncated and not matches and not backend_had_matches and (hint := regex_literal_hint(pattern)): + notes.append(hint) + if notes: + formatted_notes = "\n\n".join(notes) + return f"{formatted}\n\n{formatted_notes}", "success" return formatted, "success" @@ -518,7 +533,7 @@ def _format_glob_tool_result(paths: list[str], *, truncated: bool) -> str: EMPTY_CONTENT_WARNING = "System reminder: File exists but has empty contents" -GLOB_TIMEOUT = 20.0 # seconds +GLOB_TIMEOUT = 10.0 # seconds LINE_NUMBER_WIDTH = 6 SEARCH_TRUNCATION_NOTE = ( "Note: the search stopped early because it hit its time limit. The matches above are valid but incomplete. " @@ -623,6 +638,24 @@ class FilesystemState(AgentState): """Files in the filesystem. Uses DeltaChannel with snapshots every ~50 pregel steps to bound read depth.""" +GREP_GLOB_DESCRIPTION = ( + "Glob pattern (NOT regex) limiting which files are searched (e.g. '*.py', " + "'*.ts'). A pattern without '/' matches the file name at any depth; a pattern " + "containing '/' matches the search-root-relative path (e.g. 'src/**/*.py'). " + "This is an in-tool file filter, not a call to the separate glob tool. Brace " + "expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a " + "separate search per extension for reliable results." +) + +GREP_OUTPUT_MODE_DESCRIPTION = ( + "Shape of the returned text. 'files_with_matches' (default): newline-separated " + "matching file paths. 'content': matching lines grouped by file under a " + "':' header, each line indented and formatted ': ' " + "(only the matched line, no surrounding context). 'count': one " + "': ' line per file." +) + + class LsSchema(BaseModel): """Input schema for the `ls` tool.""" @@ -707,11 +740,11 @@ class GrepSchema(BaseModel): path: str | None = Field(default=None, description="Directory to search in. Defaults to current working directory.") - glob: str | None = Field(default=None, description="Glob pattern to filter which files to search (e.g., '*.py').") + glob: str | None = Field(default=None, description=GREP_GLOB_DESCRIPTION) output_mode: Literal["files_with_matches", "content", "count"] = Field( default="files_with_matches", - description="Output format: 'files_with_matches' (file paths only, default), 'content' (matching lines with context), 'count' (match counts per file).", + description=GREP_OUTPUT_MODE_DESCRIPTION, ) @@ -812,16 +845,28 @@ class ExecuteSchema(BaseModel): - `*.txt` - Find all text files in the backend's default root - `/subdir/**/*.md` - Find all markdown files under /subdir""" -GREP_TOOL_DESCRIPTION = """Search for a text pattern across files. +# Carries its own leading newline so the empty-string substitution below drops +# the whole line cleanly, with no blank line left behind. +_GREP_REGEX_EXECUTE_FALLBACK = "\n- If you genuinely need regex, use the execute tool with `rg ''` instead." + +_GREP_TOOL_DESCRIPTION_TEMPLATE = """Search for a LITERAL text pattern across files (NOT regex). + +Returns matching files or content based on output_mode. The pattern is matched +verbatim: regex metacharacters are treated as ordinary characters, NOT operators. -Searches for literal text (not regex) and returns matching files or content based on output_mode. -Special characters like parentheses, brackets, pipes, etc. are treated as literal characters, not regex operators. +Do NOT pass a regex. In particular: +- To match any of several strings, run a SEPARATE grep for each one. There is no + `|` alternation: `grep(pattern="foo|bar")` looks for the literal text "foo|bar". +- Do not use wildcards (`.*`) or escapes (`\\.`); they match those characters literally.{execute_fallback} Examples: - Search all files: `grep(pattern="TODO")` - Search Python files only: `grep(pattern="import", glob="*.py")` - Show matching lines: `grep(pattern="error", output_mode="content")` -- Search for code with special chars: `grep(pattern="def __init__(self):")`""" +- Literal special chars are fine: `grep(pattern="def __init__(self):")`""" + +GREP_TOOL_DESCRIPTION = _GREP_TOOL_DESCRIPTION_TEMPLATE.format(execute_fallback=_GREP_REGEX_EXECUTE_FALLBACK) +_GREP_TOOL_DESCRIPTION_WITHOUT_EXECUTE = _GREP_TOOL_DESCRIPTION_TEMPLATE.format(execute_fallback="") EXECUTE_TOOL_DESCRIPTION = """Executes a shell command in an isolated sandbox environment. @@ -2073,7 +2118,12 @@ async def async_glob( def _create_grep_tool(self) -> BaseTool: """Create the grep tool.""" - tool_description = self._custom_tool_descriptions.get("grep") or GREP_TOOL_DESCRIPTION + # Provisional default: assume execute is available so the description can + # point at `rg` for genuine regex. `_filter_unsupported_tools_and_apply_prompt` + # reconciles this to the backend's actual execute capability at request time, + # swapping in the without-execute variant when execute isn't active. The static + # description on `self.tools` is therefore only a placeholder until a request runs. + tool_description = self._grep_tool_description(include_execution=True) def sync_grep( pattern: str, @@ -2107,6 +2157,8 @@ def sync_grep( formatted, status = _format_grep_tool_result( GrepResult(error=grep_result.error, matches=filtered_matches, truncated=grep_result.truncated), output_mode, + pattern, + backend_had_matches=bool(matches), ) return ToolMessage( # `formatted` is already size-truncated inside @@ -2149,6 +2201,8 @@ async def async_grep( formatted, status = _format_grep_tool_result( GrepResult(error=grep_result.error, matches=filtered_matches, truncated=grep_result.truncated), output_mode, + pattern, + backend_had_matches=bool(matches), ) return ToolMessage( # `formatted` is already size-truncated inside @@ -2168,6 +2222,94 @@ async def async_grep( args_schema=GrepSchema, ) + def _grep_tool_description(self, *, include_execution: bool) -> str: + """Return the grep description for the current execution visibility.""" + return self._custom_tool_descriptions.get("grep") or (GREP_TOOL_DESCRIPTION if include_execution else _GREP_TOOL_DESCRIPTION_WITHOUT_EXECUTE) + + def _with_filtered_grep_description( + self, + tools: list[BaseTool | dict[str, Any]], + *, + include_execution: bool, + ) -> list[BaseTool | dict[str, Any]]: + """Copy default grep tools when their execution-specific guidance changes.""" + if self._custom_tool_descriptions.get("grep"): + return tools + + target_description = self._grep_tool_description(include_execution=include_execution) + default_descriptions = {GREP_TOOL_DESCRIPTION, _GREP_TOOL_DESCRIPTION_WITHOUT_EXECUTE} + rewritten: list[BaseTool | dict[str, Any]] = [] + changed = False + + for tool in tools: + tool_name = self._tool_name(tool) + if tool_name != "grep": + rewritten.append(tool) + continue + + if isinstance(tool, BaseTool): + if tool.description in default_descriptions and tool.description != target_description: + rewritten.append(tool.model_copy(update={"description": target_description})) + changed = True + else: + rewritten.append(tool) + continue + + if not isinstance(tool, dict): + rewritten.append(cast("BaseTool | dict[str, Any]", tool)) + continue + + if tool.get("description") in default_descriptions and tool.get("description") != target_description: + copied_tool = tool.copy() + copied_tool["description"] = target_description + rewritten.append(copied_tool) + changed = True + else: + rewritten.append(tool) + + return rewritten if changed else tools + + @staticmethod + def _tool_name(tool: object) -> str | None: + """Extract a request tool name from `BaseTool`, dict, or test doubles.""" + if isinstance(tool, BaseTool): + return tool.name + if isinstance(tool, dict): + return cast("str | None", cast("dict[str, Any]", tool).get("name")) + if hasattr(tool, "name"): + return cast("str | None", tool.name) + get = getattr(tool, "get", None) + if callable(get): + return cast("str | None", get("name")) + return None + + def _unsupported_tools_and_execution_state( + self, + tool_names: set[str | None], + runtime: Runtime[ContextT], + ) -> tuple[set[str | None], bool, BackendProtocol | None]: + """Return unsupported filesystem tools and whether execute remains active.""" + unsupported: set[str | None] = ( + {name for name in tool_names if name in _ALL_FS_TOOL_NAMES and name not in self._enabled_tools} + if self._enabled_tools is not None + else set() + ) + execution_active = False + backend = None + has_execute_tool = "execute" in tool_names + has_delete_tool = "delete" in tool_names + if not has_delete_tool and not has_execute_tool: + return unsupported, execution_active, backend + + backend = self._get_backend(runtime) # ty: ignore[invalid-argument-type] + if has_execute_tool and "execute" not in unsupported: + execution_active = supports_execution(backend) + if not execution_active: + unsupported.add("execute") + if has_delete_tool and "delete" not in unsupported and not _supports_delete(backend): + unsupported.add("delete") + return unsupported, execution_active, backend + def _resolve_capture(self, resolved_backend: BackendProtocol, tool_call_id: str | None) -> tuple[BaseSandbox, str] | None: """Resolve the executing sandbox and offload path for capture-at-source. @@ -2436,34 +2578,15 @@ def _filter_unsupported_tools_and_apply_prompt(self, request: ModelRequest[Conte Returns the request with unsupported tools removed and the filesystem system prompt appended. """ - tool_names: set[str | None] = cast( - "set[str | None]", - {(tool.name if hasattr(tool, "name") else tool.get("name")) for tool in request.tools}, - ) - has_execute_tool = "execute" in tool_names - has_delete_tool = "delete" in tool_names - - # Tools excluded by the tools allowlist (empty when no allowlist is set). - # Only filesystem tools are subject to the allowlist — user-provided tools always pass through. - # `execute` and `delete` are also filtered when the backend can't serve them. - unsupported: set[str | None] = ( - {name for name in tool_names if name in _ALL_FS_TOOL_NAMES and name not in self._enabled_tools} - if self._enabled_tools is not None - else set() - ) - execution_active = False # tracks whether execute should get system-prompt instructions - backend = None - if has_delete_tool or has_execute_tool: - backend = self._get_backend(request.runtime) # ty: ignore[invalid-argument-type] - if has_execute_tool and "execute" not in unsupported: - execution_active = supports_execution(backend) - if not execution_active: - unsupported.add("execute") - if has_delete_tool and "delete" not in unsupported and not _supports_delete(backend): - unsupported.add("delete") + tool_names: set[str | None] = {self._tool_name(tool) for tool in request.tools} + unsupported, execution_active, backend = self._unsupported_tools_and_execution_state(tool_names, request.runtime) + visible_tools = [tool for tool in request.tools if self._tool_name(tool) not in unsupported] if unsupported: - filtered_tools = [tool for tool in request.tools if (tool.name if hasattr(tool, "name") else tool.get("name")) not in unsupported] - request = request.override(tools=filtered_tools) + request = request.override(tools=visible_tools) + + described_tools = self._with_filtered_grep_description(visible_tools, include_execution=execution_active) + if described_tools is not visible_tools: + request = request.override(tools=described_tools) # Use custom system prompt if provided, otherwise generate dynamically if self._custom_system_prompt is not None: diff --git a/libs/deepagents/tests/unit_tests/backends/test_sandbox_backend.py b/libs/deepagents/tests/unit_tests/backends/test_sandbox_backend.py index 5f38bd71266..8b4b6892824 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_sandbox_backend.py +++ b/libs/deepagents/tests/unit_tests/backends/test_sandbox_backend.py @@ -27,9 +27,11 @@ _EDIT_INLINE_MAX_BYTES, _EDIT_TMPFILE_TEMPLATE, _GLOB_COMMAND_TEMPLATE, + _GREP_PATH_GLOB_TEMPLATE, _READ_COMMAND_TEMPLATE, _WRITE_CHECK_TEMPLATE, BaseSandbox, + _build_grep_cmd, _build_read_cmd, _check_preflight_result, _map_edit_error, @@ -464,6 +466,143 @@ def test_grep_passes_glob_include() -> None: assert "--include='*.py'" in sandbox.last_command +def test_build_grep_cmd_uses_grep_include_for_basename_glob() -> None: + """Basename-only globs (no slash) use GNU grep --include.""" + cmd = _build_grep_cmd("needle", "/test", "*.py") + assert "--include='*.py'" in cmd + assert "python3" not in cmd + + +def test_build_grep_cmd_routes_slash_glob_to_python_template() -> None: + """Slash-containing globs use the Python template, not grep --include.""" + cmd = _build_grep_cmd("needle", "/test", "src/**/*.py") + assert "python3" in cmd + assert "--include=" not in cmd + # Raw parameters must not appear in the command... + assert "src/**/*.py" not in cmd + assert "needle" not in cmd + # ...because each is base64-encoded. Assert the encodings are actually + # present (a bare absence check passes vacuously if a param is dropped). + assert base64.b64encode(b"src/**/*.py").decode("ascii") in cmd + assert base64.b64encode(b"needle").decode("ascii") in cmd + assert base64.b64encode(b"/test").decode("ascii") in cmd + + +def test_build_grep_cmd_routes_simple_slash_glob_to_python_template() -> None: + """A simple slash glob (no **) also uses the Python template.""" + cmd = _build_grep_cmd("needle", None, "src/*.py") + assert "python3" in cmd + assert "--include=" not in cmd + # path=None falls back to ".", which must be base64-encoded like the rest. + assert base64.b64encode(b"src/*.py").decode("ascii") in cmd + assert base64.b64encode(b".").decode("ascii") in cmd + + +def test_build_grep_cmd_slash_glob_does_not_mask_errors() -> None: + """The Python-template command must not force exit 0 with `|| true`. + + Unlike grep (which exits 1 on no-match), the template exits 0 on a + legitimate no-match, so `|| true` would only mask genuine crashes and + reintroduce the silent-zero-results failure this route exists to fix. + """ + cmd = _build_grep_cmd("needle", "/test", "src/**/*.py") + assert "|| true" not in cmd + + +def test_build_grep_cmd_no_glob_uses_grep() -> None: + """No glob falls through to the plain grep command.""" + cmd = _build_grep_cmd("needle", "/test", None) + assert "grep" in cmd + assert "--include=" not in cmd + assert "python3" not in cmd + + +def test_grep_slash_glob_returns_matches_from_python_template() -> None: + """grep() with a slash-containing glob parses output from the Python template.""" + sandbox = MockSandbox() + # Record structure: path\0line_num:text. The template prefixes each match + # with the search root, mirroring grep -HnFZ's `/` output. + sandbox._next_output = "/test/src/pkg/a.py\0001:needle" + + result = sandbox.grep("needle", "/test", "src/**/*.py") + + assert result.error is None + assert result.matches == [ + {"path": "/test/src/pkg/a.py", "line": 1, "text": "needle"}, + ] + assert sandbox.last_command is not None + assert "python3" in sandbox.last_command + + +def test_grep_slash_glob_empty_results() -> None: + """grep() with a slash glob and no matches returns empty list.""" + sandbox = MockSandbox() + sandbox._next_output = "" + + result = sandbox.grep("needle", "/test", "src/**/*.py") + + assert result.error is None + assert result.matches == [] + + +def test_grep_path_glob_template_no_shell_injection() -> None: + """The Python path-glob template base64-encodes all parameters.""" + malicious_glob = "src/x' ; echo injected ; #" + cmd = _build_grep_cmd("needle", "/test", malicious_glob) + # The raw glob must not appear in the command — only its base64 encoding. + assert malicious_glob not in cmd + assert "echo injected" not in cmd + assert "python3" in cmd + # The glob is neutralized because it is base64-encoded, not because it was + # silently dropped: assert the encoding is present. + assert base64.b64encode(malicious_glob.encode("utf-8")).decode("ascii") in cmd + + +def test_grep_path_glob_is_routed_for_slash_in_glob() -> None: + """grep() routes slash-containing globs to the Python template.""" + sandbox = MockSandbox() + sandbox._next_output = "" + + sandbox.grep("needle", "/test", "src/**/*.py") + + assert sandbox.last_command is not None + assert "python3" in sandbox.last_command + assert "--include=" not in sandbox.last_command + + +def test_grep_path_glob_template_strips_leading_slash() -> None: + """Anchored globs (leading /) stay relative to the search root, not the filesystem root.""" + assert "lstrip" in _GREP_PATH_GLOB_TEMPLATE + assert "rel_glob" in _GREP_PATH_GLOB_TEMPLATE + # The raw glob_pat must not be passed directly to glob.glob — only rel_glob. + # Verify the template uses rel_glob in the glob() call, not glob_pat. + assert "glob.glob(rel_glob" in _GREP_PATH_GLOB_TEMPLATE + assert "glob.glob(glob_pat" not in _GREP_PATH_GLOB_TEMPLATE + + +def test_grep_path_glob_template_terminates_each_record() -> None: + """Each match record is explicitly newline-terminated to prevent concatenation.""" + # The template must strip the line's trailing newline and add an explicit one + # so a file whose last line lacks a final newline doesn't merge with the next. + assert "rstrip" in _GREP_PATH_GLOB_TEMPLATE + assert "line.rstrip" in _GREP_PATH_GLOB_TEMPLATE + + +def test_grep_path_glob_parses_multiple_matches_no_trailing_newline() -> None: + """Two matches where the first line has no trailing newline parse correctly.""" + # Simulate the fixed template output: each record explicitly newline-terminated. + sandbox = MockSandbox() + sandbox._next_output = "file1.py\x001:needle\nfile2.py\x002:needle\n" + + result = sandbox.grep("needle", "/test", "src/**/*.py") + + assert result.error is None + assert result.matches == [ + {"path": "file1.py", "line": 1, "text": "needle"}, + {"path": "file2.py", "line": 2, "text": "needle"}, + ] + + def test_grep_returns_empty_matches_for_successful_empty_output() -> None: """grep() returns no matches when grep succeeds with no output.""" sandbox = MockSandbox() @@ -1281,6 +1420,127 @@ def test_glob_script_permission_denied(tmp_path: Path) -> None: locked.chmod(stat.S_IRWXU) +# -- grep path-glob template runtime behavior --------------------------------- +# Direct execution of the formatted _GREP_PATH_GLOB_TEMPLATE script. Mock-based +# tests stub execute() output and so cannot catch a real defect in the template +# body (a dropped `recursive=True`, a wrong delimiter, a broken chdir) — exactly +# the silent-zero-results class this route exists to fix. + + +def _run_grep_glob_script(path: Path, pattern: str, glob: str) -> subprocess.CompletedProcess[str]: + """Execute the formatted `_GREP_PATH_GLOB_TEMPLATE` script directly. + + Extracts the inline `python3 -c` body from the command `_build_grep_cmd` + produces (dropping the shell wrapper) and runs it via the interpreter, + mirroring `_run_glob_script`. Returns the `CompletedProcess` so callers can + assert on both stdout and the exit code — the template's error-surfacing + contract depends on a non-zero exit propagating rather than being masked. + """ + cmd = _build_grep_cmd(pattern, str(path), glob) + _, _, tail = cmd.partition('python3 -c "') + script, _, _ = tail.rpartition('"') + return subprocess.run( # noqa: S603 # script is the project's own _GREP_PATH_GLOB_TEMPLATE, not user input + [sys.executable, "-c", script], + capture_output=True, + text=True, + check=False, + ) + + +def _parse_script_records(stdout: str) -> list[tuple[str, int, str]]: + r"""Parse the template's `path\0line_num:text\n` records for assertions.""" + records: list[tuple[str, int, str]] = [] + for line in stdout.split("\n"): + if not line: + continue + file_path, rest = line.split("\0", 1) + num, text = rest.split(":", 1) + records.append((file_path, int(num), text)) + return records + + +def test_grep_glob_script_matches_recursively_and_prefixes_path(tmp_path: Path) -> None: + """`src/**/*.py` matches .py files at any depth under src, path-prefixed and sorted.""" + (tmp_path / "src" / "pkg").mkdir(parents=True) + (tmp_path / "src" / "a.py").write_text("alpha needle here\n") + (tmp_path / "src" / "pkg" / "b.py").write_text("needle in b\n") + (tmp_path / "src" / "pkg" / "c.txt").write_text("needle in c\n") # excluded: not .py + (tmp_path / "other.py").write_text("needle in other\n") # excluded: not under src + + proc = _run_grep_glob_script(tmp_path, "needle", "src/**/*.py") + + assert proc.returncode == 0 + records = _parse_script_records(proc.stdout) + # Paths are prefixed with the search root and sorted, and only src/*.py match. + assert [r[0] for r in records] == [ + str(tmp_path / "src" / "a.py"), + str(tmp_path / "src" / "pkg" / "b.py"), + ] + assert "c.txt" not in proc.stdout + assert "other.py" not in proc.stdout + + +def test_grep_glob_script_terminates_records_end_to_end(tmp_path: Path) -> None: + """Two matched files whose last line lacks a newline parse as two records. + + Drives the real template (not a hand-written mock string) through + `_parse_grep_output` to prove the newline-termination fix prevents record + concatenation across files. + """ + (tmp_path / "src").mkdir() + (tmp_path / "src" / "a.py").write_bytes(b"needle") # no trailing newline + (tmp_path / "src" / "b.py").write_bytes(b"needle") # no trailing newline + + proc = _run_grep_glob_script(tmp_path, "needle", "src/*.py") + + assert proc.returncode == 0 + resp = ExecuteResponse(output=proc.stdout, exit_code=proc.returncode, truncated=False) + result = _parse_grep_output(resp, str(tmp_path)) + assert result.error is None + assert result.matches is not None + assert len(result.matches) == 2 + assert {m["path"] for m in result.matches} == { + str(tmp_path / "src" / "a.py"), + str(tmp_path / "src" / "b.py"), + } + + +def test_grep_glob_script_strips_leading_slash(tmp_path: Path) -> None: + """A leading `/` in the glob stays relative to the search root, not the host root.""" + (tmp_path / "sub").mkdir() + (tmp_path / "top.py").write_text("needle\n") + (tmp_path / "sub" / "deep.py").write_text("needle\n") + + proc = _run_grep_glob_script(tmp_path, "needle", "/*.py") + + assert proc.returncode == 0 + records = _parse_script_records(proc.stdout) + # `/*.py` -> `*.py`: matches only the top-level file under the search root. + assert [r[0] for r in records] == [str(tmp_path / "top.py")] + + +@_PERMISSION_DENIED_SKIP +def test_grep_glob_script_surfaces_error_on_unreadable_root(tmp_path: Path) -> None: + """An inaccessible search root yields a non-zero exit, not a silent empty result. + + Confirms `|| true` was removed: a `chdir` failure propagates so + `_parse_grep_output` reports an error instead of `matches=[]`. + """ + locked = tmp_path / "locked_dir" + locked.mkdir() + locked.chmod(0o000) + try: + proc = _run_grep_glob_script(locked, "needle", "src/*.py") + finally: + locked.chmod(stat.S_IRWXU) + + assert proc.returncode != 0 + resp = ExecuteResponse(output=proc.stdout, exit_code=proc.returncode, truncated=False) + result = _parse_grep_output(resp, str(locked)) + assert result.error is not None + assert result.matches is None + + # -- glob host-side error surfacing ------------------------------------------- diff --git a/libs/deepagents/tests/unit_tests/backends/test_utils.py b/libs/deepagents/tests/unit_tests/backends/test_utils.py index cf39e19db80..3a1f96e5ad7 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_utils.py +++ b/libs/deepagents/tests/unit_tests/backends/test_utils.py @@ -12,14 +12,69 @@ _get_backend_read_file_type, _get_file_type, _glob_search_files, + _looks_like_regex, grep_matches_from_files, perform_string_replacement, + regex_literal_hint, slice_read_response, to_posix_path, validate_path, ) +class TestLooksLikeRegex: + """`_looks_like_regex` flags regex syntax in patterns meant for literal grep.""" + + @pytest.mark.parametrize( + "pattern", + [ + "foo|bar", + "def .*model", + "self\\.tools", + "a.+b", + "\\bword\\b", + "\\d+", + "\\w", + "\\s+", + "foo\\(bar\\)", + ], + ) + def test_detects_regex(self, pattern: str) -> None: + assert _looks_like_regex(pattern) is True + + @pytest.mark.parametrize( + "pattern", + [ + "def __init__(self):", + "self.tools", + "arr[0]", + "plain text", + "TODO", + "a == b", + "", + # Metacharacters deliberately treated as literal (see `_REGEX_SIGNAL_RE` + # docstring): bare `^`, `$`, `?`, `*`, `+` are common in literal code + # searches and must not trip a false hint. + "^main", + "cost$5", + "value?", + "a*b", + "c++", + ], + ) + def test_ignores_literal(self, pattern: str) -> None: + assert _looks_like_regex(pattern) is False + + def test_hint_present_for_regex(self) -> None: + hint = regex_literal_hint("foo|bar") + assert hint is not None + assert "literal text, not regex" in hint + assert "execute" not in hint + + def test_hint_absent_for_literal(self) -> None: + assert regex_literal_hint("plain text") is None + + class TestToPosixPath: """`to_posix_path` is the load-bearing primitive for Windows path handling.""" diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json index f3329be6f63..6dcceb029b6 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json @@ -196,7 +196,7 @@ }, { "function": { - "description": "Search for a text pattern across files.\n\nSearches for literal text (not regex) and returns matching files or content based on output_mode.\nSpecial characters like parentheses, brackets, pipes, etc. are treated as literal characters, not regex operators.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Search for code with special chars: `grep(pattern=\"def __init__(self):\")`", + "description": "Search for a LITERAL text pattern across files (NOT regex).\n\nReturns matching files or content based on output_mode. The pattern is matched\nverbatim: regex metacharacters are treated as ordinary characters, NOT operators.\n\nDo NOT pass a regex. In particular:\n- To match any of several strings, run a SEPARATE grep for each one. There is no\n `|` alternation: `grep(pattern=\"foo|bar\")` looks for the literal text \"foo|bar\".\n- Do not use wildcards (`.*`) or escapes (`\\.`); they match those characters literally.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Literal special chars are fine: `grep(pattern=\"def __init__(self):\")`", "name": "grep", "parameters": { "properties": { @@ -210,11 +210,11 @@ } ], "default": null, - "description": "Glob pattern to filter which files to search (e.g., '*.py')." + "description": "Glob pattern (NOT regex) limiting which files are searched (e.g. '*.py', '*.ts'). A pattern without '/' matches the file name at any depth; a pattern containing '/' matches the search-root-relative path (e.g. 'src/**/*.py'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." }, "output_mode": { "default": "files_with_matches", - "description": "Output format: 'files_with_matches' (file paths only, default), 'content' (matching lines with context), 'count' (match counts per file).", + "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file under a ':' header, each line indented and formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", "enum": [ "files_with_matches", "content", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json index 31d4c1e3558..365603baac6 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json @@ -196,7 +196,7 @@ }, { "function": { - "description": "Search for a text pattern across files.\n\nSearches for literal text (not regex) and returns matching files or content based on output_mode.\nSpecial characters like parentheses, brackets, pipes, etc. are treated as literal characters, not regex operators.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Search for code with special chars: `grep(pattern=\"def __init__(self):\")`", + "description": "Search for a LITERAL text pattern across files (NOT regex).\n\nReturns matching files or content based on output_mode. The pattern is matched\nverbatim: regex metacharacters are treated as ordinary characters, NOT operators.\n\nDo NOT pass a regex. In particular:\n- To match any of several strings, run a SEPARATE grep for each one. There is no\n `|` alternation: `grep(pattern=\"foo|bar\")` looks for the literal text \"foo|bar\".\n- Do not use wildcards (`.*`) or escapes (`\\.`); they match those characters literally.\n- If you genuinely need regex, use the execute tool with `rg ''` instead.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Literal special chars are fine: `grep(pattern=\"def __init__(self):\")`", "name": "grep", "parameters": { "properties": { @@ -210,11 +210,11 @@ } ], "default": null, - "description": "Glob pattern to filter which files to search (e.g., '*.py')." + "description": "Glob pattern (NOT regex) limiting which files are searched (e.g. '*.py', '*.ts'). A pattern without '/' matches the file name at any depth; a pattern containing '/' matches the search-root-relative path (e.g. 'src/**/*.py'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." }, "output_mode": { "default": "files_with_matches", - "description": "Output format: 'files_with_matches' (file paths only, default), 'content' (matching lines with context), 'count' (match counts per file).", + "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file under a ':' header, each line indented and formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", "enum": [ "files_with_matches", "content", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json index f3329be6f63..6dcceb029b6 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json @@ -196,7 +196,7 @@ }, { "function": { - "description": "Search for a text pattern across files.\n\nSearches for literal text (not regex) and returns matching files or content based on output_mode.\nSpecial characters like parentheses, brackets, pipes, etc. are treated as literal characters, not regex operators.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Search for code with special chars: `grep(pattern=\"def __init__(self):\")`", + "description": "Search for a LITERAL text pattern across files (NOT regex).\n\nReturns matching files or content based on output_mode. The pattern is matched\nverbatim: regex metacharacters are treated as ordinary characters, NOT operators.\n\nDo NOT pass a regex. In particular:\n- To match any of several strings, run a SEPARATE grep for each one. There is no\n `|` alternation: `grep(pattern=\"foo|bar\")` looks for the literal text \"foo|bar\".\n- Do not use wildcards (`.*`) or escapes (`\\.`); they match those characters literally.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Literal special chars are fine: `grep(pattern=\"def __init__(self):\")`", "name": "grep", "parameters": { "properties": { @@ -210,11 +210,11 @@ } ], "default": null, - "description": "Glob pattern to filter which files to search (e.g., '*.py')." + "description": "Glob pattern (NOT regex) limiting which files are searched (e.g. '*.py', '*.ts'). A pattern without '/' matches the file name at any depth; a pattern containing '/' matches the search-root-relative path (e.g. 'src/**/*.py'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." }, "output_mode": { "default": "files_with_matches", - "description": "Output format: 'files_with_matches' (file paths only, default), 'content' (matching lines with context), 'count' (match counts per file).", + "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file under a ':' header, each line indented and formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", "enum": [ "files_with_matches", "content", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json index 31d4c1e3558..365603baac6 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json @@ -196,7 +196,7 @@ }, { "function": { - "description": "Search for a text pattern across files.\n\nSearches for literal text (not regex) and returns matching files or content based on output_mode.\nSpecial characters like parentheses, brackets, pipes, etc. are treated as literal characters, not regex operators.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Search for code with special chars: `grep(pattern=\"def __init__(self):\")`", + "description": "Search for a LITERAL text pattern across files (NOT regex).\n\nReturns matching files or content based on output_mode. The pattern is matched\nverbatim: regex metacharacters are treated as ordinary characters, NOT operators.\n\nDo NOT pass a regex. In particular:\n- To match any of several strings, run a SEPARATE grep for each one. There is no\n `|` alternation: `grep(pattern=\"foo|bar\")` looks for the literal text \"foo|bar\".\n- Do not use wildcards (`.*`) or escapes (`\\.`); they match those characters literally.\n- If you genuinely need regex, use the execute tool with `rg ''` instead.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Literal special chars are fine: `grep(pattern=\"def __init__(self):\")`", "name": "grep", "parameters": { "properties": { @@ -210,11 +210,11 @@ } ], "default": null, - "description": "Glob pattern to filter which files to search (e.g., '*.py')." + "description": "Glob pattern (NOT regex) limiting which files are searched (e.g. '*.py', '*.ts'). A pattern without '/' matches the file name at any depth; a pattern containing '/' matches the search-root-relative path (e.g. 'src/**/*.py'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." }, "output_mode": { "default": "files_with_matches", - "description": "Output format: 'files_with_matches' (file paths only, default), 'content' (matching lines with context), 'count' (match counts per file).", + "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file under a ':' header, each line indented and formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", "enum": [ "files_with_matches", "content", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json index da35899b741..d83b47335e1 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json @@ -196,7 +196,7 @@ }, { "function": { - "description": "Search for a text pattern across files.\n\nSearches for literal text (not regex) and returns matching files or content based on output_mode.\nSpecial characters like parentheses, brackets, pipes, etc. are treated as literal characters, not regex operators.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Search for code with special chars: `grep(pattern=\"def __init__(self):\")`", + "description": "Search for a LITERAL text pattern across files (NOT regex).\n\nReturns matching files or content based on output_mode. The pattern is matched\nverbatim: regex metacharacters are treated as ordinary characters, NOT operators.\n\nDo NOT pass a regex. In particular:\n- To match any of several strings, run a SEPARATE grep for each one. There is no\n `|` alternation: `grep(pattern=\"foo|bar\")` looks for the literal text \"foo|bar\".\n- Do not use wildcards (`.*`) or escapes (`\\.`); they match those characters literally.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Literal special chars are fine: `grep(pattern=\"def __init__(self):\")`", "name": "grep", "parameters": { "properties": { @@ -210,11 +210,11 @@ } ], "default": null, - "description": "Glob pattern to filter which files to search (e.g., '*.py')." + "description": "Glob pattern (NOT regex) limiting which files are searched (e.g. '*.py', '*.ts'). A pattern without '/' matches the file name at any depth; a pattern containing '/' matches the search-root-relative path (e.g. 'src/**/*.py'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." }, "output_mode": { "default": "files_with_matches", - "description": "Output format: 'files_with_matches' (file paths only, default), 'content' (matching lines with context), 'count' (match counts per file).", + "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file under a ':' header, each line indented and formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", "enum": [ "files_with_matches", "content", diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json index f3329be6f63..6dcceb029b6 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json @@ -196,7 +196,7 @@ }, { "function": { - "description": "Search for a text pattern across files.\n\nSearches for literal text (not regex) and returns matching files or content based on output_mode.\nSpecial characters like parentheses, brackets, pipes, etc. are treated as literal characters, not regex operators.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Search for code with special chars: `grep(pattern=\"def __init__(self):\")`", + "description": "Search for a LITERAL text pattern across files (NOT regex).\n\nReturns matching files or content based on output_mode. The pattern is matched\nverbatim: regex metacharacters are treated as ordinary characters, NOT operators.\n\nDo NOT pass a regex. In particular:\n- To match any of several strings, run a SEPARATE grep for each one. There is no\n `|` alternation: `grep(pattern=\"foo|bar\")` looks for the literal text \"foo|bar\".\n- Do not use wildcards (`.*`) or escapes (`\\.`); they match those characters literally.\n\nExamples:\n- Search all files: `grep(pattern=\"TODO\")`\n- Search Python files only: `grep(pattern=\"import\", glob=\"*.py\")`\n- Show matching lines: `grep(pattern=\"error\", output_mode=\"content\")`\n- Literal special chars are fine: `grep(pattern=\"def __init__(self):\")`", "name": "grep", "parameters": { "properties": { @@ -210,11 +210,11 @@ } ], "default": null, - "description": "Glob pattern to filter which files to search (e.g., '*.py')." + "description": "Glob pattern (NOT regex) limiting which files are searched (e.g. '*.py', '*.ts'). A pattern without '/' matches the file name at any depth; a pattern containing '/' matches the search-root-relative path (e.g. 'src/**/*.py'). This is an in-tool file filter, not a call to the separate glob tool. Brace expansion (e.g. '*.{ts,tsx}') is not supported on all backends; run a separate search per extension for reliable results." }, "output_mode": { "default": "files_with_matches", - "description": "Output format: 'files_with_matches' (file paths only, default), 'content' (matching lines with context), 'count' (match counts per file).", + "description": "Shape of the returned text. 'files_with_matches' (default): newline-separated matching file paths. 'content': matching lines grouped by file under a ':' header, each line indented and formatted ': ' (only the matched line, no surrounding context). 'count': one ': ' line per file.", "enum": [ "files_with_matches", "content", diff --git a/libs/deepagents/tests/unit_tests/test_middleware.py b/libs/deepagents/tests/unit_tests/test_middleware.py index e0eef459dac..02fc73f05de 100644 --- a/libs/deepagents/tests/unit_tests/test_middleware.py +++ b/libs/deepagents/tests/unit_tests/test_middleware.py @@ -49,6 +49,7 @@ SEARCH_TRUNCATION_NOTE, FileData, FilesystemMiddleware, + FilesystemPermission, FilesystemState, supports_execution, ) @@ -687,6 +688,30 @@ def test_grep_truncated_renders_as_success_with_note(self): assert "1: import os" in result.content assert SEARCH_TRUNCATION_NOTE in result.content + def test_grep_truncated_regex_pattern_no_matches_keeps_note(self): + """A regex-looking miss still reports that the backend search was incomplete.""" + backend, _ = _make_backend() + middleware = FilesystemMiddleware(backend=backend) + grep_search_tool = next(tool for tool in middleware.tools if tool.name == "grep") + backend_obj = middleware._get_backend(_runtime()) + + truncated_result = GrepResult(matches=[], truncated=True) + with ( + patch.object(middleware, "_get_backend", return_value=backend_obj), + patch.object(backend_obj, "grep", return_value=truncated_result), + ): + result = grep_search_tool.invoke( + { + "pattern": "def hello|def world", + "runtime": _runtime(), + } + ) + + assert result.status == "success" + assert result.content.startswith("No matches found") + assert SEARCH_TRUNCATION_NOTE in result.content + assert "literal text, not regex" not in result.content + def test_glob_truncated_renders_as_success_with_note(self): """A truncated glob returns its partial paths as a success plus the narrow-your-search note.""" backend, _ = _make_backend() @@ -968,6 +993,120 @@ def test_grep_search_shortterm_invalid_regex(self): ) assert "No matches found" in result.content + def test_grep_regex_pattern_no_matches_adds_hint(self): + """A no-match pattern that looks like regex gets a literal-search hint.""" + files = { + "/test.py": FileData( + content=["def hello():"], + modified_at="2021-01-01", + created_at="2021-01-01", + ), + } + backend, _ = _make_backend(files) + middleware = FilesystemMiddleware(backend=backend) + grep_search_tool = next(tool for tool in middleware.tools if tool.name == "grep") + result = grep_search_tool.invoke( + { + "pattern": "def hello|def world", + "runtime": _runtime(), + } + ) + assert result.content.startswith("No matches found") + assert "literal text, not regex" in result.content + + def test_grep_literal_no_matches_omits_hint(self): + """A plain literal no-match pattern does not get the regex hint.""" + files = { + "/test.py": FileData( + content=["print('hello')"], + modified_at="2021-01-01", + created_at="2021-01-01", + ), + } + backend, _ = _make_backend(files) + middleware = FilesystemMiddleware(backend=backend) + grep_search_tool = next(tool for tool in middleware.tools if tool.name == "grep") + result = grep_search_tool.invoke( + { + "pattern": "import", + "runtime": _runtime(), + } + ) + assert result.content == "No matches found" + + def test_grep_regex_pattern_with_matches_omits_hint(self): + """A regex-looking pattern that still matches literally shows no hint.""" + files = { + "/test.py": FileData( + content=["a = b|c"], + modified_at="2021-01-01", + created_at="2021-01-01", + ), + } + backend, _ = _make_backend(files) + middleware = FilesystemMiddleware(backend=backend) + grep_search_tool = next(tool for tool in middleware.tools if tool.name == "grep") + result = grep_search_tool.invoke( + { + "pattern": "b|c", + "output_mode": "content", + "runtime": _runtime(), + } + ) + assert "a = b|c" in result.content + assert "literal text, not regex" not in result.content + + def test_grep_regex_pattern_all_matches_permission_filtered_omits_hint(self): + """A miss caused by permission redaction, not regex syntax, shows no hint. + + The backend matches the literal `b|c` in `/secret.py`, but a deny rule + strips that match. The empty result then reads "No matches found" for a + redaction reason, so the regex hint would misattribute the cause. + """ + files = { + "/secret.py": FileData( + content=["a = b|c"], + modified_at="2021-01-01", + created_at="2021-01-01", + ), + } + backend, _ = _make_backend(files) + middleware = FilesystemMiddleware( + backend=backend, + _permissions=[FilesystemPermission(operations=["read"], paths=["/secret.py"], mode="deny")], + ) + grep_search_tool = next(tool for tool in middleware.tools if tool.name == "grep") + result = grep_search_tool.invoke( + { + "pattern": "b|c", + "runtime": _runtime(), + } + ) + assert result.content == "No matches found" + assert "literal text, not regex" not in result.content + + def test_grep_error_regex_pattern_omits_hint(self): + """A backend error result never gets the regex hint appended.""" + backend, _ = _make_backend() + middleware = FilesystemMiddleware(backend=backend) + grep_search_tool = next(tool for tool in middleware.tools if tool.name == "grep") + backend_obj = middleware._get_backend(_runtime()) + + error_result = GrepResult(error="boom: backend exploded", matches=[]) + with ( + patch.object(middleware, "_get_backend", return_value=backend_obj), + patch.object(backend_obj, "grep", return_value=error_result), + ): + result = grep_search_tool.invoke( + { + "pattern": "def hello|def world", + "runtime": _runtime(), + } + ) + + assert result.status == "error" + assert "literal text, not regex" not in result.content + def test_search_store_paginated_empty(self): """Test pagination with no items.""" store = InMemoryStore() @@ -2136,6 +2275,104 @@ def test_enabled_tools_execute_listed_but_backend_unsupported_is_noop(self): assert "execute" not in filtered_names assert "ls" in filtered_names + def test_grep_description_omits_execute_fallback_when_backend_lacks_execution(self): + """Default grep docs must not advertise hidden execute tooling.""" + middleware = FilesystemMiddleware(backend=StateBackend(), system_prompt="") + request = MagicMock() + request.tools = middleware.tools + request.runtime = _runtime() + request.override.return_value = request + + middleware._filter_unsupported_tools_and_apply_prompt(request) + + tools_override = [c.kwargs["tools"] for c in request.override.call_args_list if "tools" in c.kwargs][-1] + grep_tool = next(tool for tool in tools_override if tool.name == "grep") + assert "execute tool" not in grep_tool.description + assert "rg ''" not in grep_tool.description + assert next(tool for tool in middleware.tools if tool.name == "grep").description != grep_tool.description + + def test_grep_description_omits_execute_fallback_when_execute_unlisted(self): + """Default grep docs follow the tools allowlist even if the backend can execute.""" + + class _SandboxBackend(SandboxBackendProtocol, StateBackend): + def execute(self, command: str, *, timeout: int | None = None) -> ExecuteResponse: + return ExecuteResponse(output="", exit_code=0) + + middleware = FilesystemMiddleware( + backend=_SandboxBackend(), + system_prompt="", + tools=["read_file", "ls", "grep"], + ) + request = MagicMock() + request.tools = middleware.tools + request.runtime = _runtime() + request.override.return_value = request + + middleware._filter_unsupported_tools_and_apply_prompt(request) + + tools_override = [c.kwargs["tools"] for c in request.override.call_args_list if "tools" in c.kwargs][-1] + grep_tool = next(tool for tool in tools_override if tool.name == "grep") + assert "execute" not in {tool.name for tool in tools_override} + assert "execute tool" not in grep_tool.description + assert "rg ''" not in grep_tool.description + + def test_grep_description_keeps_execute_fallback_when_execution_visible(self): + """Default grep docs can recommend execute only when execute is available.""" + + class _SandboxBackend(SandboxBackendProtocol, StateBackend): + def execute(self, command: str, *, timeout: int | None = None) -> ExecuteResponse: + return ExecuteResponse(output="", exit_code=0) + + middleware = FilesystemMiddleware(backend=_SandboxBackend(), system_prompt="") + request = MagicMock() + request.tools = middleware.tools + request.runtime = _runtime() + request.override.return_value = request + + middleware._filter_unsupported_tools_and_apply_prompt(request) + + tools_overrides = [c for c in request.override.call_args_list if "tools" in c.kwargs] + grep_tool = next(tool for tool in middleware.tools if tool.name == "grep") + assert tools_overrides == [] + assert "execute tool" in grep_tool.description + assert "rg ''" in grep_tool.description + + def test_custom_grep_description_is_not_rewritten_when_execute_hidden(self): + """User-provided grep docs remain authoritative.""" + custom_description = "Custom literal search guidance." + middleware = FilesystemMiddleware( + backend=StateBackend(), + system_prompt="", + custom_tool_descriptions={"grep": custom_description}, + ) + request = MagicMock() + request.tools = middleware.tools + request.runtime = _runtime() + request.override.return_value = request + + middleware._filter_unsupported_tools_and_apply_prompt(request) + + tools_override = [c.kwargs["tools"] for c in request.override.call_args_list if "tools" in c.kwargs][-1] + grep_tool = next(tool for tool in tools_override if tool.name == "grep") + assert grep_tool.description == custom_description + + def test_grep_description_swap_copies_dict_tool_specs(self): + """Dict-shaped grep specs are swapped via a copy, leaving the input untouched.""" + middleware = FilesystemMiddleware(backend=StateBackend(), system_prompt="") + default_description = next(tool for tool in middleware.tools if tool.name == "grep").description + original = {"name": "grep", "description": default_description} + + rewritten = middleware._with_filtered_grep_description([original], include_execution=False) + + rewritten_grep = next(tool for tool in rewritten if tool["name"] == "grep") + # Swapped via a copy, never mutated in place. + assert rewritten_grep is not original + assert original["description"] == default_description + # The copy carries the without-execute variant. + assert "execute tool" not in rewritten_grep["description"] + assert "rg ''" not in rewritten_grep["description"] + assert "LITERAL text pattern" in rewritten_grep["description"] + def test_enabled_tools_system_prompt_lists_only_enabled_tools(self): """Dynamic system prompt only mentions the tools that survived filtering.""" middleware = FilesystemMiddleware( diff --git a/libs/deepagents/tests/unit_tests/test_middleware_async.py b/libs/deepagents/tests/unit_tests/test_middleware_async.py index 12b3a943d4a..5969a5f1ee2 100644 --- a/libs/deepagents/tests/unit_tests/test_middleware_async.py +++ b/libs/deepagents/tests/unit_tests/test_middleware_async.py @@ -600,6 +600,27 @@ async def test_agrep_search_shortterm_no_matches(self): ) assert result.content == "No matches found" + async def test_agrep_regex_pattern_no_matches_adds_hint(self): + """A no-match pattern that looks like regex gets a literal-search hint (async path).""" + files = { + "/test.py": FileData( + content=["def hello():"], + modified_at="2021-01-01", + created_at="2021-01-01", + ), + } + backend, _ = _make_backend(files) + middleware = FilesystemMiddleware(backend=backend) + grep_search_tool = next(tool for tool in middleware.tools if tool.name == "grep") + result = await grep_search_tool.ainvoke( + { + "pattern": "def hello|def world", + "runtime": _runtime(), + } + ) + assert result.content.startswith("No matches found") + assert "literal text, not regex" in result.content + async def test_agrep_search_shortterm_invalid_regex(self): """Test async grep with special characters (literal search, not regex).""" files = { diff --git a/libs/evals/EVAL_CATALOG.md b/libs/evals/EVAL_CATALOG.md index 5122bc1c1ef..b806ee0e92a 100644 --- a/libs/evals/EVAL_CATALOG.md +++ b/libs/evals/EVAL_CATALOG.md @@ -10,7 +10,7 @@ Categories (for `--eval-category` filtering): file_operations,retrieval,tool_use,memory,conversation,summarization,unit_test,langchain/middleware ``` -**126 evals** across **8 categories** +**129 evals** across **8 categories** ## File Ops (`file_operations`) (21 evals) @@ -28,22 +28,25 @@ file_operations,retrieval,tool_use,memory,conversation,summarization,unit_test,l - [`test_read_then_write_derived_output`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L333) — `tests/evals/test_file_operations.py:333` - [`test_avoid_unnecessary_tool_calls`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L358) — `tests/evals/test_file_operations.py:358` - [`test_read_files_in_parallel`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L376) — `tests/evals/test_file_operations.py:376` -- [`test_read_file_truncation_recovery_with_pagination`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L646) — `tests/evals/test_file_operations.py:646` -- [`test_read_file_empty_file_reports_empty`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L684) — `tests/evals/test_file_operations.py:684` -- [`test_delete_simple`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L701) — `tests/evals/test_file_operations.py:701` -- [`test_delete_one_of_several_files`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L730) — `tests/evals/test_file_operations.py:730` -- [`test_deletes_in_parallel`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L765) — `tests/evals/test_file_operations.py:765` -- [`test_write_then_delete_same_file`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L798) — `tests/evals/test_file_operations.py:798` -- [`test_delete_missing_file_reports_absence`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L828) — `tests/evals/test_file_operations.py:828` +- [`test_read_file_truncation_recovery_with_pagination`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L742) — `tests/evals/test_file_operations.py:742` +- [`test_read_file_empty_file_reports_empty`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L780) — `tests/evals/test_file_operations.py:780` +- [`test_delete_simple`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L797) — `tests/evals/test_file_operations.py:797` +- [`test_delete_one_of_several_files`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L826) — `tests/evals/test_file_operations.py:826` +- [`test_deletes_in_parallel`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L861) — `tests/evals/test_file_operations.py:861` +- [`test_write_then_delete_same_file`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L894) — `tests/evals/test_file_operations.py:894` +- [`test_delete_missing_file_reports_absence`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L924) — `tests/evals/test_file_operations.py:924` -## Retrieval (`retrieval`) (6 evals) +## Retrieval (`retrieval`) (9 evals) - [`test_frames`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_external_benchmarks.py#L67) — `tests/evals/test_external_benchmarks.py:67` - [`test_grep_finds_matching_paths`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L406) — `tests/evals/test_file_operations.py:406` -- [`test_glob_lists_markdown_files`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L434) — `tests/evals/test_file_operations.py:434` -- [`test_find_magic_phrase_deep_nesting`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L462) — `tests/evals/test_file_operations.py:462` -- [`test_identify_quote_author_from_directory_parallel_reads`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L498) — `tests/evals/test_file_operations.py:498` -- [`test_identify_quote_author_from_directory_unprompted_efficiency`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L573) — `tests/evals/test_file_operations.py:573` +- [`test_grep_alternation_regex_recovers`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L434) — `tests/evals/test_file_operations.py:434` +- [`test_grep_wildcard_regex_recovers`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L468) — `tests/evals/test_file_operations.py:468` +- [`test_grep_escaped_metachar_regex_recovers`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L499) — `tests/evals/test_file_operations.py:499` +- [`test_glob_lists_markdown_files`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L530) — `tests/evals/test_file_operations.py:530` +- [`test_find_magic_phrase_deep_nesting`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L558) — `tests/evals/test_file_operations.py:558` +- [`test_identify_quote_author_from_directory_parallel_reads`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L594) — `tests/evals/test_file_operations.py:594` +- [`test_identify_quote_author_from_directory_unprompted_efficiency`](https://github.com/langchain-ai/deepagents/blob/main/libs/evals/tests/evals/test_file_operations.py#L669) — `tests/evals/test_file_operations.py:669` ## Tool Use (`tool_use`) (53 evals) diff --git a/libs/evals/tests/evals/test_file_operations.py b/libs/evals/tests/evals/test_file_operations.py index 19a42437eeb..c7de0f4c479 100644 --- a/libs/evals/tests/evals/test_file_operations.py +++ b/libs/evals/tests/evals/test_file_operations.py @@ -428,6 +428,102 @@ def test_grep_finds_matching_paths(model: BaseChatModel) -> None: ) +@pytest.mark.eval_tier("baseline") +@pytest.mark.eval_category("retrieval") +@pytest.mark.langsmith +def test_grep_alternation_regex_recovers(model: BaseChatModel) -> None: + """Recovers when an `|` alternation grep misses, finding both literal terms. + + `grep` matches literal text, so `cat|dog` is searched verbatim and misses. + A model that reaches for regex should be steered by the no-match hint into + running a separate literal search per term instead of burning calls on + regex variants. Either way it must end up reporting both matching files. + """ + agent = create_deep_agent(model=model) + run_agent( + agent, + model=model, + initial_files={ + "/pets/a.txt": "the cat sat\n", + "/pets/b.txt": "the dog ran\n", + "/pets/c.txt": "the bird flew\n", + }, + query="Using grep, find which files mention either 'cat' or 'dog'. Answer with the matching file paths only.", + # 1st step (ideal): two literal greps, one per term. + # 2nd step: answer with the matching paths. + # A single `cat|dog` regex attempt misses and surfaces the literal hint. + scorer=TrajectoryScorer() + .expect(agent_steps=2, tool_call_requests=2) + .success( + final_text_contains("/pets/a.txt"), + final_text_contains("/pets/b.txt"), + final_text_excludes("/pets/c.txt"), + ), + ) + + +@pytest.mark.eval_tier("baseline") +@pytest.mark.eval_category("retrieval") +@pytest.mark.langsmith +def test_grep_wildcard_regex_recovers(model: BaseChatModel) -> None: + """Recovers when a `.*` wildcard grep misses, locating the literal line. + + A model may search `TODO.*cache`; literal grep treats `.*` verbatim and + misses. The no-match hint should steer it back to a plain literal search + (e.g. `TODO`) so it still finds and reports the file. + """ + agent = create_deep_agent(model=model) + run_agent( + agent, + model=model, + initial_files={ + "/src/app.py": "x = 1\n# TODO: add a cache layer here\n", + "/src/util.py": "y = 2\n", + }, + query="Using grep, find which file has a TODO comment about a cache. Answer with the matching file path only.", + # 1st step (ideal): literal grep for 'TODO'. + # 2nd step: answer with the matching path. + # A `TODO.*cache` regex attempt misses and surfaces the literal hint. + scorer=TrajectoryScorer() + .expect(agent_steps=2, tool_call_requests=1) + .success( + final_text_contains("/src/app.py"), + final_text_excludes("/src/util.py"), + ), + ) + + +@pytest.mark.eval_tier("baseline") +@pytest.mark.eval_category("retrieval") +@pytest.mark.langsmith +def test_grep_escaped_metachar_regex_recovers(model: BaseChatModel) -> None: + """Recovers when an escaped-regex grep misses a literal filename. + + A model may search `config\\.yaml`; literal grep matches the backslash + verbatim and misses `config.yaml`. The no-match hint should steer it back + to a plain literal search so it still finds and reports the file. + """ + agent = create_deep_agent(model=model) + run_agent( + agent, + model=model, + initial_files={ + "/svc/readme.md": "Edit config.yaml to set up the service.\n", + "/svc/notes.txt": "nothing relevant here\n", + }, + query="Using grep, find which file mentions the config.yaml file. Answer with the matching file path only.", + # 1st step (ideal): literal grep for 'config.yaml'. + # 2nd step: answer with the matching path. + # A `config\\.yaml` escaped-regex attempt misses and surfaces the literal hint. + scorer=TrajectoryScorer() + .expect(agent_steps=2, tool_call_requests=1) + .success( + final_text_contains("/svc/readme.md"), + final_text_excludes("/svc/notes.txt"), + ), + ) + + @pytest.mark.eval_tier("baseline") @pytest.mark.eval_category("retrieval") @pytest.mark.langsmith