From 9d4e7effe8e96c3e91048de4841aa5693ce60d94 Mon Sep 17 00:00:00 2001 From: deepagents <[email protected]> Date: Fri, 19 Jun 2026 16:04:33 -0700 Subject: [PATCH 01/36] feat(deepagents): extract sampled frames from video reads read_file on .mp4 / .mov / .webm / .mkv now routes through a PyAV-based extractor that reinterprets the existing offset/limit args as seconds: offset = seconds to skip, limit = seconds of source to sample. Frames are sampled at a constructor-defined rate (FilesystemMiddleware( video_sampling_rate=...), default 0.5 fps). The tool result is interleaved text+image content blocks with a leading row describing the window, and binary reads of the original payload are no longer leaked to the model. The PyAV import is lazy so the core package stays lightweight when the optional [video] extra is not installed; missing-dep errors surface a clear install hint. PyAV >= 17 is pinned for the av.error.InvalidDataError exception type we rely on. --- .../deepagents/middleware/_video.py | 209 ++++++++++++++++++ .../deepagents/middleware/filesystem.py | 131 ++++++++++- .../tests/unit_tests/middleware/test_video.py | 129 +++++++++++ 3 files changed, 458 insertions(+), 11 deletions(-) create mode 100644 libs/deepagents/deepagents/middleware/_video.py create mode 100644 libs/deepagents/tests/unit_tests/middleware/test_video.py diff --git a/libs/deepagents/deepagents/middleware/_video.py b/libs/deepagents/deepagents/middleware/_video.py new file mode 100644 index 00000000000..32eac9f2ad8 --- /dev/null +++ b/libs/deepagents/deepagents/middleware/_video.py @@ -0,0 +1,209 @@ +"""Video frame extraction for the `read_file` tool. + +This module is the boundary between `read_file` and the optional video +backend (PyAV). It is imported lazily so a `deepagents` install without the +``[video]`` extra stays lightweight; the import only fires when the agent +actually tries to read a video file. + +For each video read the module decodes a contiguous slice of the source +(``offset``-seconds skip, ``limit``-seconds window) and emits sampled +frames at the configured sampling rate. The output is a list of interleaved +text+image content blocks so the model sees per-frame timestamps alongside +the JPEGs. + +Sampling rate is a deploy-time knob on `FilesystemMiddleware`, not an +agent-facing argument — the public `read_file` surface stays narrow and +operators trade frame density for token cost without prompting changes. +""" + +import base64 +import io +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from langchain_core.messages.content import ContentBlock +else: + ContentBlock = dict # only used at runtime by the agent runtime + + +MISSING_AV_HINT = "Reading video files requires the optional `av` dependency. Install it with `pip install 'deepagents[video]'`." + + +class VideoExtractionError(RuntimeError): + """Raised when PyAV cannot produce frames for the requested window.""" + + +def _import_av() -> Any: # noqa: ANN401 + """Import PyAV lazily so the dep stays optional. + + Returns: + The imported `av` module. + + Raises: + VideoExtractionError: If PyAV is not installed, with installation + guidance in the message. + """ + try: + import av # noqa: PLC0415 - lazy import keeps the extra optional + except ImportError as exc: # pragma: no cover - exercised only when `av` is absent + msg = f"{MISSING_AV_HINT} (underlying error: {exc})" + raise VideoExtractionError(msg) from exc + return av + + +def _select_sampling_rate(value: float) -> float: + """Validate and normalize the constructor-supplied sampling rate.""" + if value <= 0: + msg = f"video_sampling_rate must be > 0, got {value!r}" + raise ValueError(msg) + return float(value) + + +def _format_timestamp(seconds: float) -> str: + """Format a frame timestamp as `HH:MM:SS.mmm` for the text header block.""" + if seconds < 0: + seconds = 0.0 + total_ms = round(seconds * 1000) + hours, rem_ms = divmod(total_ms, 3_600_000) + minutes, rem_ms = divmod(rem_ms, 60_000) + secs, ms = divmod(rem_ms, 1000) + return f"{hours:02d}:{minutes:02d}:{secs:02d}.{ms:03d}" + + +def extract_video_frames( + content: bytes, + *, + offset_seconds: float, + duration_seconds: float, + sampling_rate: float, +) -> list[ContentBlock]: + """Decode sampled frames from a video byte payload. + + Args: + content: Raw bytes of the video file (as returned by the backend). + offset_seconds: Seconds into the source to start sampling. Must be + non-negative. + duration_seconds: Seconds of source to sample. Must be > 0. + sampling_rate: Frames per second to emit. > 0. + + Returns: + Interleaved content blocks: a text header introducing each frame + followed by a JPEG `image` content block. Raw video bytes are never + returned. + + Raises: + VideoExtractionError: If PyAV cannot open the payload or the + requested window yields no decodable frames. + ValueError: If argument validation fails before opening the file. + """ + if offset_seconds < 0: + msg = f"offset_seconds must be >= 0, got {offset_seconds!r}" + raise ValueError(msg) + if duration_seconds <= 0: + msg = f"duration_seconds must be > 0, got {duration_seconds!r}" + raise ValueError(msg) + rate = _select_sampling_rate(sampling_rate) + + av = _import_av() + container = _open_video_container(av, content) + try: + video_stream = _find_video_stream(container) + time_base = float(video_stream.time_base) + if offset_seconds > 0: + # `seek` keeps the math correct across containers that already sit + # at a non-zero timeline (e.g. trimmed clips). + start_pts = int(offset_seconds / time_base) + container.seek(start_pts, any_frame=False, backward=True, stream=video_stream) + + blocks = list( + _sample_frames_in_window( + container.decode(video_stream), + offset_seconds=offset_seconds, + duration_seconds=duration_seconds, + sampling_rate=rate, + time_base=time_base, + ) + ) + finally: + container.close() + + if not blocks: + end_seconds = offset_seconds + duration_seconds + msg = f"No frames decoded for window [{offset_seconds:.3f}s, {end_seconds:.3f}s)" + raise VideoExtractionError(msg) + return blocks + + +def _open_video_container(av: Any, content: bytes) -> Any: # noqa: ANN401 + """Open a video byte payload, normalizing PyAV's failure modes. + + PyAV typically raises `av.error.InvalidDataError` for malformed inputs, + but it falls back to `OSError` when the system ffmpeg library is missing + or incompatible. Both surface to callers as `VideoExtractionError` so + the middleware does not have to distinguish between them. + """ + try: + return av.open(io.BytesIO(content)) + except av.error.InvalidDataError as exc: # pragma: no cover - depends on the input + msg = f"Failed to open video payload: {exc}" + raise VideoExtractionError(msg) from exc + except OSError as exc: # pragma: no cover - dependency ffmpeg missing on host + msg = f"Failed to open video payload: {exc}" + raise VideoExtractionError(msg) from exc + + +def _find_video_stream(container: Any) -> Any: # noqa: ANN401 + """Return the first video stream in `container` or raise.""" + video_stream = next((s for s in container.streams if s.type == "video"), None) + if video_stream is None: + msg = "Video payload contains no video stream" + raise VideoExtractionError(msg) + return video_stream + + +def _sample_frames_in_window( + decoded_frames: Any, # noqa: ANN401 + *, + offset_seconds: float, + duration_seconds: float, + sampling_rate: float, + time_base: float, +) -> list[ContentBlock]: + """Pick JPEG+timestamp content blocks for frames inside the requested window.""" + frame_interval_seconds = 1.0 / sampling_rate + end_seconds = offset_seconds + duration_seconds + next_emit_seconds = offset_seconds + blocks: list[ContentBlock] = [] + for frame in decoded_frames: + frame_seconds = float(frame.pts) * time_base + if frame_seconds >= end_seconds: + break + if frame_seconds + 1e-6 < next_emit_seconds: + continue + jpeg_bytes = _encode_jpeg(frame) + ts = _format_timestamp(frame_seconds) + blocks.append({"type": "text", "text": f"Frame at t={ts}"}) + blocks.append( + { + "type": "image", + "base64": base64.b64encode(jpeg_bytes).decode("ascii"), + "mime_type": "image/jpeg", + } + ) + emitted_index = round((frame_seconds - offset_seconds) / frame_interval_seconds) + 1 + next_emit_seconds = offset_seconds + frame_interval_seconds * emitted_index + return blocks + + +def _encode_jpeg(frame: Any) -> bytes: # noqa: ANN401 + """Encode a decoded PyAV frame as JPEG bytes via Pillow. + + Pillow ships with PyAV's standard install (it is its image bridge), but + we keep the import lazy so module load is independent of optional deps. + """ + from PIL import Image # noqa: PLC0415 - lazy import; PyAV pulls Pillow transitively + + img = frame.to_image() if hasattr(frame, "to_image") else Image.fromarray(frame.to_ndarray(format="rgb24")) + buf = io.BytesIO() + img.save(buf, format="JPEG", quality=85) + return buf.getvalue() diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index c973bfda1f7..c1fe6cb659d 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -2,6 +2,7 @@ # ruff: noqa: E501 import asyncio +import base64 import concurrent.futures import contextlib import contextvars @@ -75,6 +76,11 @@ _offload_tool_message_content, ) from deepagents.middleware._utils import append_to_system_message +from deepagents.middleware._video import ( + VideoExtractionError, + _select_sampling_rate as _select_video_sampling_rate, + extract_video_frames, +) _FS_WCMATCH_FLAGS = wcglob.BRACE | wcglob.GLOBSTAR _SYNC_GLOB_WORKERS = 4 @@ -91,6 +97,93 @@ "delete": "write", } +# Reinterpret offset/limit as seconds for video reads. +_VIDEO_DEFAULT_LIMIT_SECONDS = 30.0 + + +def _handle_video_read( + content: str, + validated_path: str, + tool_call_id: str | None, + offset: int, + limit: int, + sampling_rate: float, +) -> ToolMessage: + """Slice a video byte payload into a sampled frame window for the model. + + The agent's `offset` is reinterpreted as seconds to skip into the + source, and `limit` as seconds of source to sample. Missing `offset`/ + `limit` fall back to reading from the start of the source up to + `_VIDEO_DEFAULT_LIMIT_SECONDS`. + + Errors are returned as success-shaped `ToolMessage` instances carrying + a plain text error, so the turn still completes and the agent can + recover (e.g. by retrying with a smaller window). + """ + rate = _select_video_sampling_rate(sampling_rate) + offset_seconds = max(0.0, float(offset)) + if limit <= 0: + duration_seconds = float(_VIDEO_DEFAULT_LIMIT_SECONDS) + header = f"Reading first {int(duration_seconds)}s of {validated_path} at {rate} fps." + else: + duration_seconds = float(limit) + header = f"Reading [{offset_seconds:.3f}s, {offset_seconds + duration_seconds:.3f}s) of {validated_path} at {rate} fps." + + def _err(msg: str) -> ToolMessage: + return ToolMessage( + content=f"Error reading video {validated_path}: {msg}\n{header}", + name="read_file", + tool_call_id=tool_call_id, + status="error", + ) + + try: + raw_bytes = base64.b64decode(content) if isinstance(content, str) else content + except (ValueError, TypeError) as exc: + return _err(f"video bytes are not valid base64 ({exc})") + + try: + blocks = extract_video_frames( + raw_bytes, + offset_seconds=offset_seconds, + duration_seconds=duration_seconds, + sampling_rate=rate, + ) + except VideoExtractionError as exc: + return _err(str(exc)) + else: + blocks.insert(0, {"type": "text", "text": header}) + return ToolMessage( + content_blocks=blocks, + name="read_file", + tool_call_id=tool_call_id, + additional_kwargs={"read_file_path": validated_path}, + status="success", + ) + + +def _emit_binary_block( + content: str, + validated_path: str, + tool_call_id: str | None, + file_type: str, +) -> ToolMessage: + """Emit a base64 content block for non-video binary reads (image/audio/PDF/etc). + + Routes on `file_type` so the multimodal block picks the right block kind + (`image`, `audio`, `file`); falls back to the generic `file` block when the + backend-returned file extension is not in `_EXTENSION_TO_FILE_TYPE`. + """ + block_type = file_type if file_type != "text" else "file" + mime_type = mimetypes.guess_type("file" + Path(validated_path).suffix)[0] or "application/octet-stream" + return ToolMessage( + content_blocks=cast("list[ContentBlock]", [{"type": block_type, "base64": content, "mime_type": mime_type}]), + name="read_file", + tool_call_id=tool_call_id, + additional_kwargs={"read_file_path": validated_path, "read_file_media_type": mime_type}, + status="success", + ) + @dataclass class FilesystemPermission: @@ -483,7 +576,8 @@ class ExecuteSchema(BaseModel): For multimodal reads (image, audio, video, PDF, etc.): - Use `read_file(file_path=...)` -- Do NOT use `offset`/`limit` for images (pagination is text-only) +- For images and PDFs, pagination via `offset`/`limit` is text-only — supply `file_path` only +- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). - If file details were compacted from history, call `read_file` again on the same path - You should ALWAYS make sure a file has been read before editing it.""" @@ -886,6 +980,7 @@ def __init__( tool_token_limit_before_evict: int | None = 20000, human_message_token_limit_before_evict: int | None = 50000, max_execute_timeout: int = 3600, + video_sampling_rate: float = 0.5, _permissions: list[FilesystemPermission] | None = None, ) -> None: """Initialize the filesystem middleware. @@ -903,6 +998,11 @@ def __init__( Defaults to 3600 seconds (1 hour). Any per-command timeout exceeding this value will be rejected with an error message. + video_sampling_rate: Frames-per-second emitted by `read_file` when + reading a video file. For video reads, `offset` is reinterpreted + as seconds into the source to skip and `limit` as seconds of source + to sample. Must be > 0. Defaults to 0.5 (one frame every two + seconds). _permissions: Optional filesystem permission rules enforced directly by this middleware's tool implementations. @@ -913,6 +1013,9 @@ def __init__( if max_execute_timeout <= 0: msg = f"max_execute_timeout must be positive, got {max_execute_timeout}" raise ValueError(msg) + if video_sampling_rate <= 0: + msg = f"video_sampling_rate must be > 0, got {video_sampling_rate!r}" + raise ValueError(msg) # Use provided backend or default to StateBackend instance self.backend = backend if backend is not None else StateBackend() if ( @@ -945,6 +1048,7 @@ def __init__( self._tool_token_limit_before_evict = tool_token_limit_before_evict self._human_message_token_limit_before_evict = human_message_token_limit_before_evict self._max_execute_timeout = max_execute_timeout + self._video_sampling_rate = float(video_sampling_rate) self._permissions = list(_permissions or []) # Shared executor for enforcing GLOB_TIMEOUT on the sync glob tool. @@ -1119,7 +1223,7 @@ def _truncate(content: str, file_path: str, *, line_limit: int | None = None) -> return content - def _handle_read_result( + def _handle_read_result( # noqa: PLR0911 read_result: ReadResult | str, validated_path: str, tool_call_id: str | None, @@ -1177,21 +1281,26 @@ def _handle_read_result( status="success", ) + # Video reads must be sliced into a sampled frame window before the + # generic base64 branch runs; otherwise raw video bytes would reach + # the model. + if file_type == "video": + return _handle_video_read( + content, + validated_path, + tool_call_id, + offset, + limit, + self._video_sampling_rate, + ) + # Route on the backend-declared encoding first: `"base64"` means the # content is binary and must never be line-numbered as text, even # when the extension is absent from `_EXTENSION_TO_FILE_TYPE`. # The extension map is only consulted to pick the multimodal block # type; unknown binary extensions fall back to the generic `"file"`. if encoding == "base64" or file_type != "text": - block_type = file_type if file_type != "text" else "file" - mime_type = mimetypes.guess_type("file" + Path(validated_path).suffix)[0] or "application/octet-stream" - return ToolMessage( - content_blocks=cast("list[ContentBlock]", [{"type": block_type, "base64": content, "mime_type": mime_type}]), - name="read_file", - tool_call_id=tool_call_id, - additional_kwargs={"read_file_path": validated_path, "read_file_media_type": mime_type}, - status="success", - ) + return _emit_binary_block(content, validated_path, tool_call_id, file_type) content = format_content_with_line_numbers(content, start_line=offset + 1) # `limit` already bounded raw source lines at the backend; do not diff --git a/libs/deepagents/tests/unit_tests/middleware/test_video.py b/libs/deepagents/tests/unit_tests/middleware/test_video.py new file mode 100644 index 00000000000..0052a110d38 --- /dev/null +++ b/libs/deepagents/tests/unit_tests/middleware/test_video.py @@ -0,0 +1,129 @@ +"""Unit tests for the PyAV-based video frame extractor. + +The tests exercise the real PyAV decoder against a synthetic 3-second clip so +the offset/limit -> seconds reinterpretation, frame sampling, validation, and +error mapping all run end-to-end. They are skipped automatically when the +``av`` (PyAV) extra is not installed, which keeps the default unit suite +lightweight. +""" + +import base64 +import importlib.util +from collections.abc import Iterator + +import pytest + +av = pytest.importorskip("av", reason="`av` extra not installed (pip install deepagents[video])") +np = pytest.importorskip("numpy", reason="numpy not installed") + +# Lazy source resolution keeps the unit-test process free of `av` on the +# import path until pytest.importorskip above succeeds. +_VIDEO_MODULE_PATH = "/Users/john/git/deepagents/libs/deepagents/deepagents/middleware/_video.py" +_VVIDEO_SPEC = importlib.util.spec_from_file_location("_vvideo_under_test", _VIDEO_MODULE_PATH) + + +@pytest.fixture(scope="module") +def video_module() -> Iterator[object]: + """Load the `_video` module from disk for the unit suite.""" + if _VVIDEO_SPEC is None or _VVIDEO_SPEC.loader is None: # pragma: no cover + pytest.skip("Could not resolve PyAV module spec") + module = importlib.util.module_from_spec(_VVIDEO_SPEC) + _VVIDEO_SPEC.loader.exec_module(module) + return module + + +@pytest.fixture(scope="module") +def synthetic_video_bytes(tmp_path_factory: pytest.TempPathFactory) -> bytes: + """Build a synthetic 3-second test clip with distinct consecutive frames.""" + tmp = tmp_path_factory.mktemp("video") + path = tmp / "clip.mp4" + container = av.open(str(path), mode="w") + stream = container.add_stream("libx264", rate=5) + stream.width, stream.height, stream.pix_fmt = 32, 32, "yuv420p" + for i in range(15): + img = np.tile( + np.array([(i * 8) % 255, 128, 64], dtype=np.uint8), + (32, 32, 1), + ) + frame = av.VideoFrame.from_ndarray(img) + frame.pts = i + for pkt in stream.encode(frame): + container.mux(pkt) + for pkt in stream.encode(): + container.mux(pkt) + container.close() + return path.read_bytes() + + +def test_select_sampling_rate_accepts_positive(video_module: object) -> None: + assert video_module._select_sampling_rate(0.5) == 0.5 + assert video_module._select_sampling_rate(2.0) == 2.0 + + +def test_select_sampling_rate_rejects_non_positive(video_module: object) -> None: + for bad in (0, -0.1, -1): + with pytest.raises(ValueError, match="must be > 0"): + video_module._select_sampling_rate(bad) + + +def test_extract_rejects_negative_offset(video_module: object, synthetic_video_bytes: bytes) -> None: + with pytest.raises(ValueError, match="offset_seconds must be >= 0"): + video_module.extract_video_frames( + synthetic_video_bytes, + offset_seconds=-0.1, + duration_seconds=1, + sampling_rate=0.5, + ) + + +def test_extract_rejects_zero_or_negative_duration(video_module: object, synthetic_video_bytes: bytes) -> None: + with pytest.raises(ValueError, match="duration_seconds must be > 0"): + video_module.extract_video_frames( + synthetic_video_bytes, + offset_seconds=0, + duration_seconds=0, + sampling_rate=0.5, + ) + + +def test_extract_returns_interleaved_text_and_image_blocks(video_module: object, synthetic_video_bytes: bytes) -> None: + """Two frames (t=0 and t=2) interleaved with text headers at 0.5 fps over a 3s clip.""" + blocks = video_module.extract_video_frames( + synthetic_video_bytes, + offset_seconds=0, + duration_seconds=30, + sampling_rate=0.5, + ) + assert [b["type"] for b in blocks] == ["text", "image", "text", "image"] + assert blocks[0]["text"].startswith("Frame at t=") + assert blocks[1]["mime_type"] == "image/jpeg" + assert blocks[2]["text"].startswith("Frame at t=") + assert blocks[3]["mime_type"] == "image/jpeg" + assert blocks[0]["text"] != blocks[2]["text"] + jpeg = base64.b64decode(blocks[1]["base64"]) + assert jpeg[:3] == b"\xff\xd8\xff" + + +def test_extract_offset_skips_into_source(video_module: object, synthetic_video_bytes: bytes) -> None: + """`offset_seconds` seeks into the clip; every emitted frame is at or after that point.""" + blocks = video_module.extract_video_frames( + synthetic_video_bytes, + offset_seconds=1.0, + duration_seconds=1.5, + sampling_rate=1.0, + ) + headers = [b["text"] for b in blocks if b["type"] == "text"] + assert headers, "expected at least one frame in [1s, 2.5s) at 1fps" + for h in headers: + assert h.startswith("Frame at t=") + + +def test_extract_no_frames_in_window_raises(video_module: object, synthetic_video_bytes: bytes) -> None: + """Reading past the end of the clip yields a `VideoExtractionError`.""" + with pytest.raises(video_module.VideoExtractionError, match="No frames decoded"): + video_module.extract_video_frames( + synthetic_video_bytes, + offset_seconds=10, + duration_seconds=5, + sampling_rate=0.5, + ) From da5069f59e5b88203248ce906308a55277f37993 Mon Sep 17 00:00:00 2001 From: deepagents <[email protected]> Date: Fri, 19 Jun 2026 16:04:38 -0700 Subject: [PATCH 02/36] test(deepagents): cover video read_file dispatch and snapshots Adds constructor and dispatch tests for FilesystemMiddleware's video_sampling_rate knob and the read_file video branch: parser rejects non-positive rates, offset/limit are reinterpreted as seconds, raw video payloads never reach the model, and VideoExtractionError surfaces as a ToolMessage status='error' instead of an unhandled exception. Smoke test snapshots refreshed to match the updated read_file description. --- .../custom_system_message_tools.json | 2 +- .../system_prompt_with_execute_tools.json | 2 +- ...m_prompt_with_memory_and_skills_tools.json | 2 +- ...stem_prompt_with_routed_backend_tools.json | 2 +- ...t_with_sync_and_async_subagents_tools.json | 2 +- .../system_prompt_without_execute_tools.json | 2 +- .../tests/unit_tests/test_middleware.py | 155 ++++++++++++++++++ 7 files changed, 161 insertions(+), 6 deletions(-) diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json index 7aae2a75014..f82562d744b 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- By default, it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Only omit limit (read full file) when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- Do NOT use `offset`/`limit` for images (pagination is text-only)\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- By default, it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Only omit limit (read full file) when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only \u2014 supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps).\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json index fa8d4c03c6d..72899bbeaf9 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- By default, it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Only omit limit (read full file) when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- Do NOT use `offset`/`limit` for images (pagination is text-only)\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- By default, it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Only omit limit (read full file) when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only \u2014 supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps).\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json index 7aae2a75014..f82562d744b 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- By default, it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Only omit limit (read full file) when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- Do NOT use `offset`/`limit` for images (pagination is text-only)\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- By default, it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Only omit limit (read full file) when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only \u2014 supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps).\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json index fa8d4c03c6d..72899bbeaf9 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- By default, it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Only omit limit (read full file) when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- Do NOT use `offset`/`limit` for images (pagination is text-only)\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- By default, it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Only omit limit (read full file) when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only \u2014 supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps).\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json index 0b692ca9755..bd0000b0b42 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- By default, it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Only omit limit (read full file) when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- Do NOT use `offset`/`limit` for images (pagination is text-only)\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- By default, it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Only omit limit (read full file) when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only \u2014 supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps).\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json index 7aae2a75014..f82562d744b 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- By default, it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Only omit limit (read full file) when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- Do NOT use `offset`/`limit` for images (pagination is text-only)\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- By default, it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Only omit limit (read full file) when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only \u2014 supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps).\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/test_middleware.py b/libs/deepagents/tests/unit_tests/test_middleware.py index fb5d4bbf153..70b47647f6f 100644 --- a/libs/deepagents/tests/unit_tests/test_middleware.py +++ b/libs/deepagents/tests/unit_tests/test_middleware.py @@ -1467,6 +1467,149 @@ def test_mixed_content_small_text_large_image_not_evicted(self): assert result == tool_message + def test_init_video_sampling_rate_default(self): + """Default sampling rate is 0.5 fps.""" + middleware = FilesystemMiddleware() + assert middleware._video_sampling_rate == 0.5 + + def test_init_video_sampling_rate_override(self): + """Constructor value is honored verbatim.""" + middleware = FilesystemMiddleware(video_sampling_rate=2.0) + assert middleware._video_sampling_rate == 2.0 + + @pytest.mark.parametrize("bad", [0, -0.5, -1]) + def test_init_video_sampling_rate_rejects_non_positive(self, bad: float) -> None: + """Constructor rejects zero and negative sampling rates up front.""" + with pytest.raises(ValueError, match="video_sampling_rate must be > 0"): + FilesystemMiddleware(video_sampling_rate=bad) + + def test_read_file_video_routes_to_frame_extractor(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Video reads route through `_video.extract_video_frames`, not the generic base64 branch. + + The middleware must interpret `offset`/`limit` as seconds for video + reads, return interleaved text+image content blocks, and never leak the + raw video bytes to the model. + """ + + class VideoBackend(StateBackend): + def __init__(self, raw: bytes) -> None: + super().__init__() + self._raw = raw + + def read(self, path, *, offset=0, limit=100): # type: ignore[override] + return ReadResult(file_data={"content": self._raw, "encoding": "base64"}) + + sentinel = [ + {"type": "text", "text": "Frame at t=00:00:00.000"}, + {"type": "image", "base64": "AAAA", "mime_type": "image/jpeg"}, + {"type": "text", "text": "Frame at t=00:00:02.000"}, + {"type": "image", "base64": "BBBB", "mime_type": "image/jpeg"}, + ] + + def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): # type: ignore[no-untyped-def] + # Capture the call arguments so the test asserts the + # offset/limit -> seconds reinterpretation. + fake_extract.calls.append( + { + "len": len(content), + "offset_seconds": offset_seconds, + "duration_seconds": duration_seconds, + "sampling_rate": sampling_rate, + } + ) + return sentinel + + fake_extract.calls = [] + + monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) + + middleware = FilesystemMiddleware( + backend=VideoBackend(b"\x00\x01\x02 fake video bytes"), + video_sampling_rate=0.5, + ) + state = FilesystemState(messages=[], files={}) + runtime = _build_runtime(state, "video-read-1") + read_file_tool = next(t for t in middleware.tools if t.name == "read_file") + + result = read_file_tool.invoke( + { + "file_path": "/clips/intro.mp4", + "offset": 0, + "limit": 30, + "runtime": runtime, + } + ) + + assert isinstance(result, ToolMessage) + assert result.status == "success" + assert result.additional_kwargs["read_file_path"] == "/clips/intro.mp4" + # Raw video bytes must NOT leak to the model: the content_blocks are + # the extracted frames, not the original base64 from the backend. + assert isinstance(result.content, list) + assert result.content == sentinel + assert b"\x00\x01\x02" not in str(result.content).encode() + + # The middleware must convert offset/limit to seconds with the + # constructor-defined sampling rate forwarded verbatim. + assert len(fake_extract.calls) == 1 + call = fake_extract.calls[0] + assert call["offset_seconds"] == 0 + assert call["duration_seconds"] == 30 + assert call["sampling_rate"] == 0.5 + + def test_read_file_video_offset_and_limit_reinterpreted_as_seconds(self, monkeypatch: pytest.MonkeyPatch) -> None: + """`offset`skip `limit`window seconds for video reads.""" + + class VideoBackend(StateBackend): + def read(self, path, *, offset=0, limit=100): # type: ignore[override] + return ReadResult(file_data={"content": b"raw", "encoding": "base64"}) + + captured: dict[str, float] = {} + + def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): # type: ignore[no-untyped-def] # noqa: ARG001 + captured.update(offset_seconds=offset_seconds, duration_seconds=duration_seconds, sampling_rate=sampling_rate) + return [{"type": "text", "text": "ok"}] + + monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) + middleware = FilesystemMiddleware(backend=VideoBackend()) + state = FilesystemState(messages=[], files={}) + runtime = _build_runtime(state, "video-read-2") + read_file_tool = next(t for t in middleware.tools if t.name == "read_file") + + read_file_tool.invoke( + { + "file_path": "/c.mp4", + "offset": 12, # -> seek 12 seconds into the source + "limit": 90, # -> sample a 90-second window + "runtime": runtime, + } + ) + + assert captured == {"offset_seconds": 12.0, "duration_seconds": 90.0, "sampling_rate": 0.5} + + def test_read_file_video_extraction_error_surfaces_as_error_message(self, monkeypatch: pytest.MonkeyPatch) -> None: + """PyAV failures and missing-dep errors render as a tool error, not an exception.""" + + class VideoBackend(StateBackend): + def read(self, path, *, offset=0, limit=100): # type: ignore[override] + return ReadResult(file_data={"content": b"corrupt", "encoding": "base64"}) + + monkeypatch.setattr( + filesystem_middleware, + "extract_video_frames", + lambda *_args, **_kwargs: (_ for _ in ()).throw(filesystem_middleware.VideoExtractionError("av not installed")), + ) + middleware = FilesystemMiddleware(backend=VideoBackend()) + state = FilesystemState(messages=[], files={}) + runtime = _build_runtime(state, "video-read-err") + read_file_tool = next(t for t in middleware.tools if t.name == "read_file") + + result = read_file_tool.invoke({"file_path": "/c.mp4", "runtime": runtime}) + + assert isinstance(result, ToolMessage) + assert result.status == "error" + assert "av not installed" in result.content # type: ignore[operator] + def test_read_file_image_returns_standard_image_content_block(self): """Test image reads return standard image blocks with base64 + mime_type.""" @@ -2580,3 +2723,15 @@ def test_max_execute_timeout_init_validation(self): with pytest.raises(ValueError, match="max_execute_timeout must be positive"): FilesystemMiddleware(max_execute_timeout=-1) + + +def _build_runtime(state: FilesystemState, tool_call_id: str) -> ToolRuntime: + """Build a `ToolRuntime` for the binary/video read_file tests.""" + return ToolRuntime( + state=state, + context=None, + tool_call_id=tool_call_id, + store=None, + stream_writer=lambda _: None, + config={}, + ) From 508c0a69cb4ca61a6daafe9e947c7929d525d61d Mon Sep 17 00:00:00 2001 From: deepagents <[email protected]> Date: Fri, 19 Jun 2026 16:04:42 -0700 Subject: [PATCH 03/36] build(deepagents): add [video] extra and document it uv add "deepagents[video]" pulls in PyAV (>=17) for the new frame extractor. Adds av + pillow to the dev toolchain ty/ruff run from so the type-checker can resolve the lazy-imported names even though core users don't install them. --- libs/deepagents/README.md | 14 ++++ libs/deepagents/pyproject.toml | 8 ++- libs/deepagents/uv.lock | 125 ++++++++++++++++++++++++++++++++- 3 files changed, 144 insertions(+), 3 deletions(-) diff --git a/libs/deepagents/README.md b/libs/deepagents/README.md index ecde4dad9b4..27b8cba36fb 100644 --- a/libs/deepagents/README.md +++ b/libs/deepagents/README.md @@ -16,6 +16,20 @@ LangSmith is a unified developer platform for building, testing, and monitoring uv add deepagents ``` +### Optional: video frame extraction + +`read_file` can sample video files into image content blocks (`.mp4`, `.mov`, `.webm`, etc.). +The agent uses `offset` to skip seconds into the source and `limit` to control how many +seconds of source to sample; the frames-per-second is a deploy-time knob. +Install with: + +```bash +uv add "deepagents[video]" +``` + +The extra pulls in [PyAV](https://pyav.basswood-io.com/). Without it, video reads surface +a clear "install `deepagents[video]`" error instead of failing silently. + ## 🤔 What is this? Deep Agents is an open source agent harness — an opinionated agent that runs out of the box. Extend, override, or replace any piece. diff --git a/libs/deepagents/pyproject.toml b/libs/deepagents/pyproject.toml index 739d96f3b4f..a3be5ff168b 100644 --- a/libs/deepagents/pyproject.toml +++ b/libs/deepagents/pyproject.toml @@ -31,6 +31,7 @@ dependencies = [ [project.optional-dependencies] aws = ["langchain-aws>=1.6.1,<2.0.0"] quickjs = ["langchain-quickjs>=0.3.2"] +video = ["av>=17.0.0"] [project.urls] Homepage = "https://docs.langchain.com/oss/python/deepagents/overview" @@ -59,9 +60,12 @@ test = [ "langchain-openai", "twine", "build", + # Lazy-imported at runtime so the core package does + # not depend on them. ty still sees the import names and + # needs stubs, so we install them into the type-checker venv. + "av>=17.0.0", + "pillow>=10.0.0", ] - - [build-system] requires = ["setuptools>=82.0.1", "wheel"] build-backend = "setuptools.build_meta" diff --git a/libs/deepagents/uv.lock b/libs/deepagents/uv.lock index 2177fff06cd..6828bfeafe0 100644 --- a/libs/deepagents/uv.lock +++ b/libs/deepagents/uv.lock @@ -54,6 +54,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/38/0e/27be9fdef66e72d64c0cdc3cc2823101b80585f8119b5c112c2e8f5f7dab/anyio-4.12.1-py3-none-any.whl", hash = "sha256:d405828884fc140aa80a3c667b8beed277f1dfedec42ba031bd6ac3db606ab6c", size = 113592, upload-time = "2026-01-06T11:45:19.497Z" }, ] +[[package]] +name = "av" +version = "17.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/e3/477fa20578c284abeda08d91b63ee9abaebc93445d8feeb989d3d444bae1/av-17.1.0.tar.gz", hash = "sha256:7f1e71ff621b66253333926f948e00faae11d855b2442133c65128bca64cdeb3", size = 4288546, upload-time = "2026-06-07T05:52:55.999Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/87/8036b5c781bc3639ea04ef42d4e26da253bd4bd4311d8705b6a1c8824047/av-17.1.0-cp311-abi3-macosx_11_0_x86_64.whl", hash = "sha256:ad7b4aa011093324b7118245f50ac6db244cfe9900d4072508a5245a2b0d3f41", size = 22460847, upload-time = "2026-06-07T05:52:04.261Z" }, + { url = "https://files.pythonhosted.org/packages/6d/af/dfdf6fc7b17814b50d0aa9e7a7e37b87be91be3890f44b0d525433cd1fd1/av-17.1.0-cp311-abi3-macosx_14_0_arm64.whl", hash = "sha256:43ebbe977f19a7f2d2bd1a4e119675a0b15e05852cf7309846b6ab922ba7ffe9", size = 18159115, upload-time = "2026-06-07T05:52:06.64Z" }, + { url = "https://files.pythonhosted.org/packages/ad/13/64f6c466471cea225b8b2f4cdc51a571f8a286984b55a08d169b932fda5d/av-17.1.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6a20658ec7d96a70e14b1196eff00b7cdd8831ac3b99868e16b8ba8b24090847", size = 33224427, upload-time = "2026-06-07T05:52:09.165Z" }, + { url = "https://files.pythonhosted.org/packages/77/43/96b35170bf2e64e00a41748c6400ff73232dc0fc62ded283679fb07c7fe0/av-17.1.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f9a65d1f48b818323fb411e80358f89d77dec340b01d27c6b2dfbb9cbf4b779f", size = 35370183, upload-time = "2026-06-07T05:52:11.959Z" }, + { url = "https://files.pythonhosted.org/packages/2e/b3/8e8b4b6498731bfbd88e8399a756543f8088f1bd33d08eab678b5aebe728/av-17.1.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:58f7593726437cda5bd19793027e027768450b5c4a594777bf487798a33db702", size = 24459265, upload-time = "2026-06-07T05:52:14.66Z" }, + { url = "https://files.pythonhosted.org/packages/14/ac/ceb84b7553db21f1143d817245c560d9267168e1e58b1a8eeae2b62c4d04/av-17.1.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:bbab058bd965309f39962e53caac8126987c68c0be094fc4f9427e5615b0218f", size = 34283709, upload-time = "2026-06-07T05:52:17.389Z" }, + { url = "https://files.pythonhosted.org/packages/59/f9/4115fd84148c9a1cf365096694be6ac882fd3cd3cdb7a2f35e71fecf1631/av-17.1.0-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:9514cfda85180554c430695282faf4be3ffdf95775d8519733821244eecb58e0", size = 25397573, upload-time = "2026-06-07T05:52:20.012Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ac/92e52d5ed0e0b84d9d93e52b4338c2713d8a44082b8696e6516fdae7c4e4/av-17.1.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e1c90f85cd7431ede95b11e8e711571a896ebea433f298849c2c0f1594c8d86e", size = 36451495, upload-time = "2026-06-07T05:52:22.581Z" }, + { url = "https://files.pythonhosted.org/packages/6b/f2/53a7cd34adb6a971d7e6d99663e74db286966c9db8afdca17472fdf0f98e/av-17.1.0-cp311-abi3-win_amd64.whl", hash = "sha256:5df5c1172ef1cf65a1529d612f7da7798ce2cf82c1ff7212466b538a6cc7214c", size = 28036393, upload-time = "2026-06-07T05:52:25.657Z" }, + { url = "https://files.pythonhosted.org/packages/66/47/cd9ae0edf2206351c1251bb94b5ec58728e42c5f6ee16c03c412f3a1bb3e/av-17.1.0-cp311-abi3-win_arm64.whl", hash = "sha256:ee98534242a74da847af78624779ac5a3177dc7c69f956a4da9e6f0fdb37d7f6", size = 21174601, upload-time = "2026-06-07T05:52:28.077Z" }, + { url = "https://files.pythonhosted.org/packages/36/90/b5668cddb3c401fcf22553bc495d5b0c6d8a01d118624b26f0db1d0b8653/av-17.1.0-cp314-cp314t-macosx_11_0_x86_64.whl", hash = "sha256:5327807c1219293803ef0c5d1578ff3ae1cf638c09e5998962026e1a554ec240", size = 22699499, upload-time = "2026-06-07T05:52:30.335Z" }, + { url = "https://files.pythonhosted.org/packages/e0/7e/7be6bfddb823d045ff9fd5d4deb922ee3847605e162c3882e6c45b4c35ff/av-17.1.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:6c9b71fe5c0c5a8d303b1588d4d8ce9397d6b023f467cfef95000ba1f75507fa", size = 18366696, upload-time = "2026-06-07T05:52:32.645Z" }, + { url = "https://files.pythonhosted.org/packages/a2/23/391dcfa75c1ae1977efca44b753a11b929399b558826670c16a8808dd0e3/av-17.1.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:f997e3351bdf51127c07a74e21741a2996e9230cbeb2d81c14acde761b116c9c", size = 36582649, upload-time = "2026-06-07T05:52:35.218Z" }, + { url = "https://files.pythonhosted.org/packages/fb/32/7312854868b318b9d1b1dcbd1bddb460aaaeac7d57f816e11efec3bef5b1/av-17.1.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:efe9b1397300b67b644ad220c89df4892a76f2debe70f16bae1749fa20526e63", size = 38479390, upload-time = "2026-06-07T05:52:37.968Z" }, + { url = "https://files.pythonhosted.org/packages/2a/72/af47f59b4458e81ca7d89f477698dbfb3d5a0cd8ae6c1e4441d01074af8a/av-17.1.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:fa64e1f1500d01c4a98e7a41dc1a9a35fb4dfe71f5de0389264ec1192200c76a", size = 27127432, upload-time = "2026-06-07T05:52:40.371Z" }, + { url = "https://files.pythonhosted.org/packages/88/85/c2e6861baf0f8c7d21c4ce811d4d424fedac915e3910d3570ce4377717dc/av-17.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ffbd78d73d2c9bf31e9a007c992faec3991428b2941a3b085b84fb82e8c32d19", size = 37406592, upload-time = "2026-06-07T05:52:43.215Z" }, + { url = "https://files.pythonhosted.org/packages/ba/40/3cc13125aea976101c0858af99ac47257c0654411aa199b5d8e81eea7002/av-17.1.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:bff8896454b38fcb785a70e5ae0485d7021cb776303a5849393128a30b8f850b", size = 28336228, upload-time = "2026-06-07T05:52:46.134Z" }, + { url = "https://files.pythonhosted.org/packages/a2/38/c7d9c3e746209a1a695c13e3aa7d817229e84a85d0a84271f313d1befdd3/av-17.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:1284addf3c0dd939887a9722dc30df2241a97471ad52c3c507e31583ae22ff02", size = 39490680, upload-time = "2026-06-07T05:52:48.887Z" }, + { url = "https://files.pythonhosted.org/packages/a1/25/9d42da561b7b8f7dabdfaebba07b52977bee58c5c7e4285ac991abcfaa72/av-17.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:ec630be6321b04e317862f6082e84812bbd801e55a3c2298312e3fc8a0a4af4f", size = 28355673, upload-time = "2026-06-07T05:52:51.614Z" }, + { url = "https://files.pythonhosted.org/packages/a8/41/562a61d5a61fba3ffb273a115e249f1d8471b9515c59fcc38b4b9deda238/av-17.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:b41647e42884bf543b8e8d0a1dabd4d1b006c99183eb1a2d7afc5b01f73eeff4", size = 21324700, upload-time = "2026-06-07T05:52:53.972Z" }, +] + [[package]] name = "backports-tarfile" version = "1.2.0" @@ -508,12 +536,17 @@ aws = [ quickjs = [ { name = "langchain-quickjs" }, ] +video = [ + { name = "av" }, +] [package.dev-dependencies] test = [ + { name = "av" }, { name = "build" }, { name = "langchain-openai" }, { name = "langchain-tests" }, + { name = "pillow" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-benchmark" }, @@ -530,6 +563,7 @@ test = [ [package.metadata] requires-dist = [ + { name = "av", marker = "extra == 'video'", specifier = ">=17.0.0" }, { name = "langchain", specifier = ">=1.3.11,<2.0.0" }, { name = "langchain-anthropic", specifier = ">=1.4.7,<2.0.0" }, { name = "langchain-aws", marker = "extra == 'aws'", specifier = ">=1.6.1,<2.0.0" }, @@ -539,13 +573,15 @@ requires-dist = [ { name = "langsmith", specifier = ">=0.9.3" }, { name = "wcmatch", specifier = ">=10.1" }, ] -provides-extras = ["aws", "quickjs"] +provides-extras = ["aws", "quickjs", "video"] [package.metadata.requires-dev] test = [ + { name = "av", specifier = ">=17.0.0" }, { name = "build" }, { name = "langchain-openai" }, { name = "langchain-tests", specifier = ">=1.1.9" }, + { name = "pillow", specifier = ">=10.0.0" }, { name = "pytest" }, { name = "pytest-asyncio", specifier = ">=1.4.0" }, { name = "pytest-benchmark" }, @@ -1415,6 +1451,93 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, ] +[[package]] +name = "pillow" +version = "12.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/21/c2bcdd5906101a30244eaffc1b6e6ce71a31bd0742a01eb89e660ebfac2d/pillow-12.2.0.tar.gz", hash = "sha256:a830b1a40919539d07806aa58e1b114df53ddd43213d9c8b75847eee6c0182b5", size = 46987819, upload-time = "2026-04-01T14:46:17.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/e1/748f5663efe6edcfc4e74b2b93edfb9b8b99b67f21a854c3ae416500a2d9/pillow-12.2.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:8be29e59487a79f173507c30ddf57e733a357f67881430449bb32614075a40ab", size = 5354347, upload-time = "2026-04-01T14:42:44.255Z" }, + { url = "https://files.pythonhosted.org/packages/47/a1/d5ff69e747374c33a3b53b9f98cca7889fce1fd03d79cdc4e1bccc6c5a87/pillow-12.2.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:71cde9a1e1551df7d34a25462fc60325e8a11a82cc2e2f54578e5e9a1e153d65", size = 4695873, upload-time = "2026-04-01T14:42:46.452Z" }, + { url = "https://files.pythonhosted.org/packages/df/21/e3fbdf54408a973c7f7f89a23b2cb97a7ef30c61ab4142af31eee6aebc88/pillow-12.2.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f490f9368b6fc026f021db16d7ec2fbf7d89e2edb42e8ec09d2c60505f5729c7", size = 6280168, upload-time = "2026-04-01T14:42:49.228Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f1/00b7278c7dd52b17ad4329153748f87b6756ec195ff786c2bdf12518337d/pillow-12.2.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8bd7903a5f2a4545f6fd5935c90058b89d30045568985a71c79f5fd6edf9b91e", size = 8088188, upload-time = "2026-04-01T14:42:51.735Z" }, + { url = "https://files.pythonhosted.org/packages/ad/cf/220a5994ef1b10e70e85748b75649d77d506499352be135a4989c957b701/pillow-12.2.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3997232e10d2920a68d25191392e3a4487d8183039e1c74c2297f00ed1c50705", size = 6394401, upload-time = "2026-04-01T14:42:54.343Z" }, + { url = "https://files.pythonhosted.org/packages/e9/bd/e51a61b1054f09437acfbc2ff9106c30d1eb76bc1453d428399946781253/pillow-12.2.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e74473c875d78b8e9d5da2a70f7099549f9eb37ded4e2f6a463e60125bccd176", size = 7079655, upload-time = "2026-04-01T14:42:56.954Z" }, + { url = "https://files.pythonhosted.org/packages/6b/3d/45132c57d5fb4b5744567c3817026480ac7fc3ce5d4c47902bc0e7f6f853/pillow-12.2.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:56a3f9c60a13133a98ecff6197af34d7824de9b7b38c3654861a725c970c197b", size = 6503105, upload-time = "2026-04-01T14:42:59.847Z" }, + { url = "https://files.pythonhosted.org/packages/7d/2e/9df2fc1e82097b1df3dce58dc43286aa01068e918c07574711fcc53e6fb4/pillow-12.2.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:90e6f81de50ad6b534cab6e5aef77ff6e37722b2f5d908686f4a5c9eba17a909", size = 7203402, upload-time = "2026-04-01T14:43:02.664Z" }, + { url = "https://files.pythonhosted.org/packages/bd/2e/2941e42858ebb67e50ae741473de81c2984e6eff7b397017623c676e2e8d/pillow-12.2.0-cp311-cp311-win32.whl", hash = "sha256:8c984051042858021a54926eb597d6ee3012393ce9c181814115df4c60b9a808", size = 6378149, upload-time = "2026-04-01T14:43:05.274Z" }, + { url = "https://files.pythonhosted.org/packages/69/42/836b6f3cd7f3e5fa10a1f1a5420447c17966044c8fbf589cc0452d5502db/pillow-12.2.0-cp311-cp311-win_amd64.whl", hash = "sha256:6e6b2a0c538fc200b38ff9eb6628228b77908c319a005815f2dde585a0664b60", size = 7082626, upload-time = "2026-04-01T14:43:08.557Z" }, + { url = "https://files.pythonhosted.org/packages/c2/88/549194b5d6f1f494b485e493edc6693c0a16f4ada488e5bd974ed1f42fad/pillow-12.2.0-cp311-cp311-win_arm64.whl", hash = "sha256:9a8a34cc89c67a65ea7437ce257cea81a9dad65b29805f3ecee8c8fe8ff25ffe", size = 2463531, upload-time = "2026-04-01T14:43:10.743Z" }, + { url = "https://files.pythonhosted.org/packages/58/be/7482c8a5ebebbc6470b3eb791812fff7d5e0216c2be3827b30b8bb6603ed/pillow-12.2.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2d192a155bbcec180f8564f693e6fd9bccff5a7af9b32e2e4bf8c9c69dbad6b5", size = 5308279, upload-time = "2026-04-01T14:43:13.246Z" }, + { url = "https://files.pythonhosted.org/packages/d8/95/0a351b9289c2b5cbde0bacd4a83ebc44023e835490a727b2a3bd60ddc0f4/pillow-12.2.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f3f40b3c5a968281fd507d519e444c35f0ff171237f4fdde090dd60699458421", size = 4695490, upload-time = "2026-04-01T14:43:15.584Z" }, + { url = "https://files.pythonhosted.org/packages/de/af/4e8e6869cbed569d43c416fad3dc4ecb944cb5d9492defaed89ddd6fe871/pillow-12.2.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:03e7e372d5240cc23e9f07deca4d775c0817bffc641b01e9c3af208dbd300987", size = 6284462, upload-time = "2026-04-01T14:43:18.268Z" }, + { url = "https://files.pythonhosted.org/packages/e9/9e/c05e19657fd57841e476be1ab46c4d501bffbadbafdc31a6d665f8b737b6/pillow-12.2.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b86024e52a1b269467a802258c25521e6d742349d760728092e1bc2d135b4d76", size = 8094744, upload-time = "2026-04-01T14:43:20.716Z" }, + { url = "https://files.pythonhosted.org/packages/2b/54/1789c455ed10176066b6e7e6da1b01e50e36f94ba584dc68d9eebfe9156d/pillow-12.2.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7371b48c4fa448d20d2714c9a1f775a81155050d383333e0a6c15b1123dda005", size = 6398371, upload-time = "2026-04-01T14:43:23.443Z" }, + { url = "https://files.pythonhosted.org/packages/43/e3/fdc657359e919462369869f1c9f0e973f353f9a9ee295a39b1fea8ee1a77/pillow-12.2.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62f5409336adb0663b7caa0da5c7d9e7bdbaae9ce761d34669420c2a801b2780", size = 7087215, upload-time = "2026-04-01T14:43:26.758Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f8/2f6825e441d5b1959d2ca5adec984210f1ec086435b0ed5f52c19b3b8a6e/pillow-12.2.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:01afa7cf67f74f09523699b4e88c73fb55c13346d212a59a2db1f86b0a63e8c5", size = 6509783, upload-time = "2026-04-01T14:43:29.56Z" }, + { url = "https://files.pythonhosted.org/packages/67/f9/029a27095ad20f854f9dba026b3ea6428548316e057e6fc3545409e86651/pillow-12.2.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fc3d34d4a8fbec3e88a79b92e5465e0f9b842b628675850d860b8bd300b159f5", size = 7212112, upload-time = "2026-04-01T14:43:32.091Z" }, + { url = "https://files.pythonhosted.org/packages/be/42/025cfe05d1be22dbfdb4f264fe9de1ccda83f66e4fc3aac94748e784af04/pillow-12.2.0-cp312-cp312-win32.whl", hash = "sha256:58f62cc0f00fd29e64b29f4fd923ffdb3859c9f9e6105bfc37ba1d08994e8940", size = 6378489, upload-time = "2026-04-01T14:43:34.601Z" }, + { url = "https://files.pythonhosted.org/packages/5d/7b/25a221d2c761c6a8ae21bfa3874988ff2583e19cf8a27bf2fee358df7942/pillow-12.2.0-cp312-cp312-win_amd64.whl", hash = "sha256:7f84204dee22a783350679a0333981df803dac21a0190d706a50475e361c93f5", size = 7084129, upload-time = "2026-04-01T14:43:37.213Z" }, + { url = "https://files.pythonhosted.org/packages/10/e1/542a474affab20fd4a0f1836cb234e8493519da6b76899e30bcc5d990b8b/pillow-12.2.0-cp312-cp312-win_arm64.whl", hash = "sha256:af73337013e0b3b46f175e79492d96845b16126ddf79c438d7ea7ff27783a414", size = 2463612, upload-time = "2026-04-01T14:43:39.421Z" }, + { url = "https://files.pythonhosted.org/packages/4a/01/53d10cf0dbad820a8db274d259a37ba50b88b24768ddccec07355382d5ad/pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:8297651f5b5679c19968abefd6bb84d95fe30ef712eb1b2d9b2d31ca61267f4c", size = 4100837, upload-time = "2026-04-01T14:43:41.506Z" }, + { url = "https://files.pythonhosted.org/packages/0f/98/f3a6657ecb698c937f6c76ee564882945f29b79bad496abcba0e84659ec5/pillow-12.2.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:50d8520da2a6ce0af445fa6d648c4273c3eeefbc32d7ce049f22e8b5c3daecc2", size = 4176528, upload-time = "2026-04-01T14:43:43.773Z" }, + { url = "https://files.pythonhosted.org/packages/69/bc/8986948f05e3ea490b8442ea1c1d4d990b24a7e43d8a51b2c7d8b1dced36/pillow-12.2.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:766cef22385fa1091258ad7e6216792b156dc16d8d3fa607e7545b2b72061f1c", size = 3640401, upload-time = "2026-04-01T14:43:45.87Z" }, + { url = "https://files.pythonhosted.org/packages/34/46/6c717baadcd62bc8ed51d238d521ab651eaa74838291bda1f86fe1f864c9/pillow-12.2.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5d2fd0fa6b5d9d1de415060363433f28da8b1526c1c129020435e186794b3795", size = 5308094, upload-time = "2026-04-01T14:43:48.438Z" }, + { url = "https://files.pythonhosted.org/packages/71/43/905a14a8b17fdb1ccb58d282454490662d2cb89a6bfec26af6d3520da5ec/pillow-12.2.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:56b25336f502b6ed02e889f4ece894a72612fe885889a6e8c4c80239ff6e5f5f", size = 4695402, upload-time = "2026-04-01T14:43:51.292Z" }, + { url = "https://files.pythonhosted.org/packages/73/dd/42107efcb777b16fa0393317eac58f5b5cf30e8392e266e76e51cff28c3d/pillow-12.2.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f1c943e96e85df3d3478f7b691f229887e143f81fedab9b20205349ab04d73ed", size = 6280005, upload-time = "2026-04-01T14:43:54.242Z" }, + { url = "https://files.pythonhosted.org/packages/a8/68/b93e09e5e8549019e61acf49f65b1a8530765a7f812c77a7461bca7e4494/pillow-12.2.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:03f6fab9219220f041c74aeaa2939ff0062bd5c364ba9ce037197f4c6d498cd9", size = 8090669, upload-time = "2026-04-01T14:43:57.335Z" }, + { url = "https://files.pythonhosted.org/packages/4b/6e/3ccb54ce8ec4ddd1accd2d89004308b7b0b21c4ac3d20fa70af4760a4330/pillow-12.2.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5cdfebd752ec52bf5bb4e35d9c64b40826bc5b40a13df7c3cda20a2c03a0f5ed", size = 6395194, upload-time = "2026-04-01T14:43:59.864Z" }, + { url = "https://files.pythonhosted.org/packages/67/ee/21d4e8536afd1a328f01b359b4d3997b291ffd35a237c877b331c1c3b71c/pillow-12.2.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eedf4b74eda2b5a4b2b2fb4c006d6295df3bf29e459e198c90ea48e130dc75c3", size = 7082423, upload-time = "2026-04-01T14:44:02.74Z" }, + { url = "https://files.pythonhosted.org/packages/78/5f/e9f86ab0146464e8c133fe85df987ed9e77e08b29d8d35f9f9f4d6f917ba/pillow-12.2.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:00a2865911330191c0b818c59103b58a5e697cae67042366970a6b6f1b20b7f9", size = 6505667, upload-time = "2026-04-01T14:44:05.381Z" }, + { url = "https://files.pythonhosted.org/packages/ed/1e/409007f56a2fdce61584fd3acbc2bbc259857d555196cedcadc68c015c82/pillow-12.2.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1e1757442ed87f4912397c6d35a0db6a7b52592156014706f17658ff58bbf795", size = 7208580, upload-time = "2026-04-01T14:44:08.39Z" }, + { url = "https://files.pythonhosted.org/packages/23/c4/7349421080b12fb35414607b8871e9534546c128a11965fd4a7002ccfbee/pillow-12.2.0-cp313-cp313-win32.whl", hash = "sha256:144748b3af2d1b358d41286056d0003f47cb339b8c43a9ea42f5fea4d8c66b6e", size = 6375896, upload-time = "2026-04-01T14:44:11.197Z" }, + { url = "https://files.pythonhosted.org/packages/3f/82/8a3739a5e470b3c6cbb1d21d315800d8e16bff503d1f16b03a4ec3212786/pillow-12.2.0-cp313-cp313-win_amd64.whl", hash = "sha256:390ede346628ccc626e5730107cde16c42d3836b89662a115a921f28440e6a3b", size = 7081266, upload-time = "2026-04-01T14:44:13.947Z" }, + { url = "https://files.pythonhosted.org/packages/c3/25/f968f618a062574294592f668218f8af564830ccebdd1fa6200f598e65c5/pillow-12.2.0-cp313-cp313-win_arm64.whl", hash = "sha256:8023abc91fba39036dbce14a7d6535632f99c0b857807cbbbf21ecc9f4717f06", size = 2463508, upload-time = "2026-04-01T14:44:16.312Z" }, + { url = "https://files.pythonhosted.org/packages/4d/a4/b342930964e3cb4dce5038ae34b0eab4653334995336cd486c5a8c25a00c/pillow-12.2.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:042db20a421b9bafecc4b84a8b6e444686bd9d836c7fd24542db3e7df7baad9b", size = 5309927, upload-time = "2026-04-01T14:44:18.89Z" }, + { url = "https://files.pythonhosted.org/packages/9f/de/23198e0a65a9cf06123f5435a5d95cea62a635697f8f03d134d3f3a96151/pillow-12.2.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:dd025009355c926a84a612fecf58bb315a3f6814b17ead51a8e48d3823d9087f", size = 4698624, upload-time = "2026-04-01T14:44:21.115Z" }, + { url = "https://files.pythonhosted.org/packages/01/a6/1265e977f17d93ea37aa28aa81bad4fa597933879fac2520d24e021c8da3/pillow-12.2.0-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:88ddbc66737e277852913bd1e07c150cc7bb124539f94c4e2df5344494e0a612", size = 6321252, upload-time = "2026-04-01T14:44:23.663Z" }, + { url = "https://files.pythonhosted.org/packages/3c/83/5982eb4a285967baa70340320be9f88e57665a387e3a53a7f0db8231a0cd/pillow-12.2.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d362d1878f00c142b7e1a16e6e5e780f02be8195123f164edf7eddd911eefe7c", size = 8126550, upload-time = "2026-04-01T14:44:26.772Z" }, + { url = "https://files.pythonhosted.org/packages/4e/48/6ffc514adce69f6050d0753b1a18fd920fce8cac87620d5a31231b04bfc5/pillow-12.2.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c727a6d53cb0018aadd8018c2b938376af27914a68a492f59dfcaca650d5eea", size = 6433114, upload-time = "2026-04-01T14:44:29.615Z" }, + { url = "https://files.pythonhosted.org/packages/36/a3/f9a77144231fb8d40ee27107b4463e205fa4677e2ca2548e14da5cf18dce/pillow-12.2.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:efd8c21c98c5cc60653bcb311bef2ce0401642b7ce9d09e03a7da87c878289d4", size = 7115667, upload-time = "2026-04-01T14:44:32.773Z" }, + { url = "https://files.pythonhosted.org/packages/c1/fc/ac4ee3041e7d5a565e1c4fd72a113f03b6394cc72ab7089d27608f8aaccb/pillow-12.2.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9f08483a632889536b8139663db60f6724bfcb443c96f1b18855860d7d5c0fd4", size = 6538966, upload-time = "2026-04-01T14:44:35.252Z" }, + { url = "https://files.pythonhosted.org/packages/c0/a8/27fb307055087f3668f6d0a8ccb636e7431d56ed0750e07a60547b1e083e/pillow-12.2.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:dac8d77255a37e81a2efcbd1fc05f1c15ee82200e6c240d7e127e25e365c39ea", size = 7238241, upload-time = "2026-04-01T14:44:37.875Z" }, + { url = "https://files.pythonhosted.org/packages/ad/4b/926ab182c07fccae9fcb120043464e1ff1564775ec8864f21a0ebce6ac25/pillow-12.2.0-cp313-cp313t-win32.whl", hash = "sha256:ee3120ae9dff32f121610bb08e4313be87e03efeadfc6c0d18f89127e24d0c24", size = 6379592, upload-time = "2026-04-01T14:44:40.336Z" }, + { url = "https://files.pythonhosted.org/packages/c2/c4/f9e476451a098181b30050cc4c9a3556b64c02cf6497ea421ac047e89e4b/pillow-12.2.0-cp313-cp313t-win_amd64.whl", hash = "sha256:325ca0528c6788d2a6c3d40e3568639398137346c3d6e66bb61db96b96511c98", size = 7085542, upload-time = "2026-04-01T14:44:43.251Z" }, + { url = "https://files.pythonhosted.org/packages/00/a4/285f12aeacbe2d6dc36c407dfbbe9e96d4a80b0fb710a337f6d2ad978c75/pillow-12.2.0-cp313-cp313t-win_arm64.whl", hash = "sha256:2e5a76d03a6c6dcef67edabda7a52494afa4035021a79c8558e14af25313d453", size = 2465765, upload-time = "2026-04-01T14:44:45.996Z" }, + { url = "https://files.pythonhosted.org/packages/bf/98/4595daa2365416a86cb0d495248a393dfc84e96d62ad080c8546256cb9c0/pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:3adc9215e8be0448ed6e814966ecf3d9952f0ea40eb14e89a102b87f450660d8", size = 4100848, upload-time = "2026-04-01T14:44:48.48Z" }, + { url = "https://files.pythonhosted.org/packages/0b/79/40184d464cf89f6663e18dfcf7ca21aae2491fff1a16127681bf1fa9b8cf/pillow-12.2.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:6a9adfc6d24b10f89588096364cc726174118c62130c817c2837c60cf08a392b", size = 4176515, upload-time = "2026-04-01T14:44:51.353Z" }, + { url = "https://files.pythonhosted.org/packages/b0/63/703f86fd4c422a9cf722833670f4f71418fb116b2853ff7da722ea43f184/pillow-12.2.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:6a6e67ea2e6feda684ed370f9a1c52e7a243631c025ba42149a2cc5934dec295", size = 3640159, upload-time = "2026-04-01T14:44:53.588Z" }, + { url = "https://files.pythonhosted.org/packages/71/e0/fb22f797187d0be2270f83500aab851536101b254bfa1eae10795709d283/pillow-12.2.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2bb4a8d594eacdfc59d9e5ad972aa8afdd48d584ffd5f13a937a664c3e7db0ed", size = 5312185, upload-time = "2026-04-01T14:44:56.039Z" }, + { url = "https://files.pythonhosted.org/packages/ba/8c/1a9e46228571de18f8e28f16fabdfc20212a5d019f3e3303452b3f0a580d/pillow-12.2.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:80b2da48193b2f33ed0c32c38140f9d3186583ce7d516526d462645fd98660ae", size = 4695386, upload-time = "2026-04-01T14:44:58.663Z" }, + { url = "https://files.pythonhosted.org/packages/70/62/98f6b7f0c88b9addd0e87c217ded307b36be024d4ff8869a812b241d1345/pillow-12.2.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22db17c68434de69d8ecfc2fe821569195c0c373b25cccb9cbdacf2c6e53c601", size = 6280384, upload-time = "2026-04-01T14:45:01.5Z" }, + { url = "https://files.pythonhosted.org/packages/5e/03/688747d2e91cfbe0e64f316cd2e8005698f76ada3130d0194664174fa5de/pillow-12.2.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7b14cc0106cd9aecda615dd6903840a058b4700fcb817687d0ee4fc8b6e389be", size = 8091599, upload-time = "2026-04-01T14:45:04.5Z" }, + { url = "https://files.pythonhosted.org/packages/f6/35/577e22b936fcdd66537329b33af0b4ccfefaeabd8aec04b266528cddb33c/pillow-12.2.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8cbeb542b2ebc6fcdacabf8aca8c1a97c9b3ad3927d46b8723f9d4f033288a0f", size = 6396021, upload-time = "2026-04-01T14:45:07.117Z" }, + { url = "https://files.pythonhosted.org/packages/11/8d/d2532ad2a603ca2b93ad9f5135732124e57811d0168155852f37fbce2458/pillow-12.2.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4bfd07bc812fbd20395212969e41931001fd59eb55a60658b0e5710872e95286", size = 7083360, upload-time = "2026-04-01T14:45:09.763Z" }, + { url = "https://files.pythonhosted.org/packages/5e/26/d325f9f56c7e039034897e7380e9cc202b1e368bfd04d4cbe6a441f02885/pillow-12.2.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9aba9a17b623ef750a4d11b742cbafffeb48a869821252b30ee21b5e91392c50", size = 6507628, upload-time = "2026-04-01T14:45:12.378Z" }, + { url = "https://files.pythonhosted.org/packages/5f/f7/769d5632ffb0988f1c5e7660b3e731e30f7f8ec4318e94d0a5d674eb65a4/pillow-12.2.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:deede7c263feb25dba4e82ea23058a235dcc2fe1f6021025dc71f2b618e26104", size = 7209321, upload-time = "2026-04-01T14:45:15.122Z" }, + { url = "https://files.pythonhosted.org/packages/6a/7a/c253e3c645cd47f1aceea6a8bacdba9991bf45bb7dfe927f7c893e89c93c/pillow-12.2.0-cp314-cp314-win32.whl", hash = "sha256:632ff19b2778e43162304d50da0181ce24ac5bb8180122cbe1bf4673428328c7", size = 6479723, upload-time = "2026-04-01T14:45:17.797Z" }, + { url = "https://files.pythonhosted.org/packages/cd/8b/601e6566b957ca50e28725cb6c355c59c2c8609751efbecd980db44e0349/pillow-12.2.0-cp314-cp314-win_amd64.whl", hash = "sha256:4e6c62e9d237e9b65fac06857d511e90d8461a32adcc1b9065ea0c0fa3a28150", size = 7217400, upload-time = "2026-04-01T14:45:20.529Z" }, + { url = "https://files.pythonhosted.org/packages/d6/94/220e46c73065c3e2951bb91c11a1fb636c8c9ad427ac3ce7d7f3359b9b2f/pillow-12.2.0-cp314-cp314-win_arm64.whl", hash = "sha256:b1c1fbd8a5a1af3412a0810d060a78b5136ec0836c8a4ef9aa11807f2a22f4e1", size = 2554835, upload-time = "2026-04-01T14:45:23.162Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ab/1b426a3974cb0e7da5c29ccff4807871d48110933a57207b5a676cccc155/pillow-12.2.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:57850958fe9c751670e49b2cecf6294acc99e562531f4bd317fa5ddee2068463", size = 5314225, upload-time = "2026-04-01T14:45:25.637Z" }, + { url = "https://files.pythonhosted.org/packages/19/1e/dce46f371be2438eecfee2a1960ee2a243bbe5e961890146d2dee1ff0f12/pillow-12.2.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:d5d38f1411c0ed9f97bcb49b7bd59b6b7c314e0e27420e34d99d844b9ce3b6f3", size = 4698541, upload-time = "2026-04-01T14:45:28.355Z" }, + { url = "https://files.pythonhosted.org/packages/55/c3/7fbecf70adb3a0c33b77a300dc52e424dc22ad8cdc06557a2e49523b703d/pillow-12.2.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c0a9f29ca8e79f09de89293f82fc9b0270bb4af1d58bc98f540cc4aedf03166", size = 6322251, upload-time = "2026-04-01T14:45:30.924Z" }, + { url = "https://files.pythonhosted.org/packages/1c/3c/7fbc17cfb7e4fe0ef1642e0abc17fc6c94c9f7a16be41498e12e2ba60408/pillow-12.2.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1610dd6c61621ae1cf811bef44d77e149ce3f7b95afe66a4512f8c59f25d9ebe", size = 8127807, upload-time = "2026-04-01T14:45:33.908Z" }, + { url = "https://files.pythonhosted.org/packages/ff/c3/a8ae14d6defd2e448493ff512fae903b1e9bd40b72efb6ec55ce0048c8ce/pillow-12.2.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a34329707af4f73cf1782a36cd2289c0368880654a2c11f027bcee9052d35dd", size = 6433935, upload-time = "2026-04-01T14:45:36.623Z" }, + { url = "https://files.pythonhosted.org/packages/6e/32/2880fb3a074847ac159d8f902cb43278a61e85f681661e7419e6596803ed/pillow-12.2.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e9c4f5b3c546fa3458a29ab22646c1c6c787ea8f5ef51300e5a60300736905e", size = 7116720, upload-time = "2026-04-01T14:45:39.258Z" }, + { url = "https://files.pythonhosted.org/packages/46/87/495cc9c30e0129501643f24d320076f4cc54f718341df18cc70ec94c44e1/pillow-12.2.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fb043ee2f06b41473269765c2feae53fc2e2fbf96e5e22ca94fb5ad677856f06", size = 6540498, upload-time = "2026-04-01T14:45:41.879Z" }, + { url = "https://files.pythonhosted.org/packages/18/53/773f5edca692009d883a72211b60fdaf8871cbef075eaa9d577f0a2f989e/pillow-12.2.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f278f034eb75b4e8a13a54a876cc4a5ab39173d2cdd93a638e1b467fc545ac43", size = 7239413, upload-time = "2026-04-01T14:45:44.705Z" }, + { url = "https://files.pythonhosted.org/packages/c9/e4/4b64a97d71b2a83158134abbb2f5bd3f8a2ea691361282f010998f339ec7/pillow-12.2.0-cp314-cp314t-win32.whl", hash = "sha256:6bb77b2dcb06b20f9f4b4a8454caa581cd4dd0643a08bacf821216a16d9c8354", size = 6482084, upload-time = "2026-04-01T14:45:47.568Z" }, + { url = "https://files.pythonhosted.org/packages/ba/13/306d275efd3a3453f72114b7431c877d10b1154014c1ebbedd067770d629/pillow-12.2.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6562ace0d3fb5f20ed7290f1f929cae41b25ae29528f2af1722966a0a02e2aa1", size = 7225152, upload-time = "2026-04-01T14:45:50.032Z" }, + { url = "https://files.pythonhosted.org/packages/ff/6e/cf826fae916b8658848d7b9f38d88da6396895c676e8086fc0988073aaf8/pillow-12.2.0-cp314-cp314t-win_arm64.whl", hash = "sha256:aa88ccfe4e32d362816319ed727a004423aab09c5cea43c01a4b435643fa34eb", size = 2556579, upload-time = "2026-04-01T14:45:52.529Z" }, + { url = "https://files.pythonhosted.org/packages/4e/b7/2437044fb910f499610356d1352e3423753c98e34f915252aafecc64889f/pillow-12.2.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0538bd5e05efec03ae613fd89c4ce0368ecd2ba239cc25b9f9be7ed426b0af1f", size = 5273969, upload-time = "2026-04-01T14:45:55.538Z" }, + { url = "https://files.pythonhosted.org/packages/f6/f4/8316e31de11b780f4ac08ef3654a75555e624a98db1056ecb2122d008d5a/pillow-12.2.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:394167b21da716608eac917c60aa9b969421b5dcbbe02ae7f013e7b85811c69d", size = 4659674, upload-time = "2026-04-01T14:45:58.093Z" }, + { url = "https://files.pythonhosted.org/packages/d4/37/664fca7201f8bb2aa1d20e2c3d5564a62e6ae5111741966c8319ca802361/pillow-12.2.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5d04bfa02cc2d23b497d1e90a0f927070043f6cbf303e738300532379a4b4e0f", size = 5288479, upload-time = "2026-04-01T14:46:01.141Z" }, + { url = "https://files.pythonhosted.org/packages/49/62/5b0ed78fce87346be7a5cfcfaaad91f6a1f98c26f86bdbafa2066c647ef6/pillow-12.2.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0c838a5125cee37e68edec915651521191cef1e6aa336b855f495766e77a366e", size = 7032230, upload-time = "2026-04-01T14:46:03.874Z" }, + { url = "https://files.pythonhosted.org/packages/c3/28/ec0fc38107fc32536908034e990c47914c57cd7c5a3ece4d8d8f7ffd7e27/pillow-12.2.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a6c9fa44005fa37a91ebfc95d081e8079757d2e904b27103f4f5fa6f0bf78c0", size = 5355404, upload-time = "2026-04-01T14:46:06.33Z" }, + { url = "https://files.pythonhosted.org/packages/5e/8b/51b0eddcfa2180d60e41f06bd6d0a62202b20b59c68f5a132e615b75aecf/pillow-12.2.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:25373b66e0dd5905ed63fa3cae13c82fbddf3079f2c8bf15c6fb6a35586324c1", size = 6002215, upload-time = "2026-04-01T14:46:08.83Z" }, + { url = "https://files.pythonhosted.org/packages/bc/60/5382c03e1970de634027cee8e1b7d39776b778b81812aaf45b694dfe9e28/pillow-12.2.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:bfa9c230d2fe991bed5318a5f119bd6780cda2915cca595393649fc118ab895e", size = 7080946, upload-time = "2026-04-01T14:46:11.734Z" }, +] + [[package]] name = "pluggy" version = "1.6.0" From ba9e923493d14bf649e5293b7b32932d2a1022a0 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Fri, 19 Jun 2026 16:15:15 -0700 Subject: [PATCH 04/36] fix(sdk): address video read review findings --- libs/deepagents/README.md | 7 ++- .../deepagents/middleware/_video.py | 14 +++--- .../deepagents/middleware/filesystem.py | 40 ++++++++++------- libs/deepagents/pyproject.toml | 2 +- .../tests/unit_tests/middleware/test_video.py | 33 ++++---------- .../custom_system_message_tools.json | 15 +++++-- .../system_prompt_with_execute_tools.json | 15 +++++-- ...m_prompt_with_memory_and_skills_tools.json | 15 +++++-- ...stem_prompt_with_routed_backend_tools.json | 15 +++++-- ...t_with_sync_and_async_subagents_tools.json | 15 +++++-- .../system_prompt_without_execute_tools.json | 15 +++++-- .../tests/unit_tests/test_middleware.py | 43 ++++++++++++++++--- libs/deepagents/uv.lock | 2 + 13 files changed, 155 insertions(+), 76 deletions(-) diff --git a/libs/deepagents/README.md b/libs/deepagents/README.md index 27b8cba36fb..137caffc8cb 100644 --- a/libs/deepagents/README.md +++ b/libs/deepagents/README.md @@ -27,8 +27,11 @@ Install with: uv add "deepagents[video]" ``` -The extra pulls in [PyAV](https://pyav.basswood-io.com/). Without it, video reads surface -a clear "install `deepagents[video]`" error instead of failing silently. +The extra pulls in [PyAV](https://pyav.basswood-io.com/) and Pillow. Without it, video +reads surface a clear "install `deepagents[video]`" error instead of failing silently. + +Sandbox-backed video reads currently inherit the backend's binary preview size cap. +Support for larger sandbox videos is deferred to follow-up backend work. ## 🤔 What is this? diff --git a/libs/deepagents/deepagents/middleware/_video.py b/libs/deepagents/deepagents/middleware/_video.py index 32eac9f2ad8..3fa287a7a09 100644 --- a/libs/deepagents/deepagents/middleware/_video.py +++ b/libs/deepagents/deepagents/middleware/_video.py @@ -26,7 +26,7 @@ ContentBlock = dict # only used at runtime by the agent runtime -MISSING_AV_HINT = "Reading video files requires the optional `av` dependency. Install it with `pip install 'deepagents[video]'`." +MISSING_VIDEO_HINT = "Reading video files requires the optional video dependencies. Install them with `pip install 'deepagents[video]'`." class VideoExtractionError(RuntimeError): @@ -46,7 +46,7 @@ def _import_av() -> Any: # noqa: ANN401 try: import av # noqa: PLC0415 - lazy import keeps the extra optional except ImportError as exc: # pragma: no cover - exercised only when `av` is absent - msg = f"{MISSING_AV_HINT} (underlying error: {exc})" + msg = f"{MISSING_VIDEO_HINT} (underlying error: {exc})" raise VideoExtractionError(msg) from exc return av @@ -198,10 +198,14 @@ def _sample_frames_in_window( def _encode_jpeg(frame: Any) -> bytes: # noqa: ANN401 """Encode a decoded PyAV frame as JPEG bytes via Pillow. - Pillow ships with PyAV's standard install (it is its image bridge), but - we keep the import lazy so module load is independent of optional deps. + Pillow is part of the `video` extra, and the import stays lazy so module + load is independent of optional deps. """ - from PIL import Image # noqa: PLC0415 - lazy import; PyAV pulls Pillow transitively + try: + from PIL import Image # noqa: PLC0415 - lazy import keeps the extra optional + except ImportError as exc: # pragma: no cover - exercised only when Pillow is absent + msg = f"{MISSING_VIDEO_HINT} (underlying error: {exc})" + raise VideoExtractionError(msg) from exc img = frame.to_image() if hasattr(frame, "to_image") else Image.fromarray(frame.to_ndarray(format="rgb24")) buf = io.BytesIO() diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index c1fe6cb659d..5d653919462 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -106,7 +106,7 @@ def _handle_video_read( validated_path: str, tool_call_id: str | None, offset: int, - limit: int, + limit: int | None, sampling_rate: float, ) -> ToolMessage: """Slice a video byte payload into a sampled frame window for the model. @@ -122,7 +122,7 @@ def _handle_video_read( """ rate = _select_video_sampling_rate(sampling_rate) offset_seconds = max(0.0, float(offset)) - if limit <= 0: + if limit is None or limit <= 0: duration_seconds = float(_VIDEO_DEFAULT_LIMIT_SECONDS) header = f"Reading first {int(duration_seconds)}s of {validated_path} at {rate} fps." else: @@ -480,12 +480,12 @@ class ReadFileSchema(BaseModel): offset: int = Field( default=DEFAULT_READ_OFFSET, - description="Line number to start reading from (0-indexed). Use for pagination of large files.", + description="Line number to start reading text files from (0-indexed), or seconds to skip into video files.", ) - limit: int = Field( - default=DEFAULT_READ_LIMIT, - description="Maximum number of lines to read. Use for pagination of large files.", + limit: int | None = Field( + default=None, + description="Maximum number of text lines to read, or video seconds to sample. Omit to use the default window.", ) @@ -562,11 +562,11 @@ class ExecuteSchema(BaseModel): Assume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned. Usage: -- By default, it reads up to 100 lines starting from the beginning of the file +- For text files, by default it reads up to 100 lines starting from the beginning of the file - **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow - First scan: read_file(file_path="...", limit=100) to see file structure - Read more sections: read_file(file_path="...", offset=100, limit=200) for next 200 lines - - Only omit limit (read full file) when necessary for editing + - Omit `limit` to use the default window; increase it only when necessary for editing - Specify offset and limit: read_file(file_path="...", offset=0, limit=100) reads first 100 lines - Results are returned using cat -n format, with line numbers starting at 1 - Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget. @@ -576,8 +576,9 @@ class ExecuteSchema(BaseModel): For multimodal reads (image, audio, video, PDF, etc.): - Use `read_file(file_path=...)` -- For images and PDFs, pagination via `offset`/`limit` is text-only — supply `file_path` only +- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only - For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). +- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work. - If file details were compacted from history, call `read_file` again on the same path - You should ALWAYS make sure a file has been read before editing it.""" @@ -1228,8 +1229,9 @@ def _handle_read_result( # noqa: PLR0911 validated_path: str, tool_call_id: str | None, offset: int, - limit: int, + limit: int | None, ) -> ToolMessage: + effective_limit = DEFAULT_READ_LIMIT if limit is None else limit if isinstance(read_result, str): warn_deprecated( since="0.5.0", @@ -1243,7 +1245,7 @@ def _handle_read_result( # noqa: PLR0911 ) # Legacy backends already format with line numbers return ToolMessage( - content=_truncate(read_result, validated_path, line_limit=limit), + content=_truncate(read_result, validated_path, line_limit=effective_limit), name="read_file", tool_call_id=tool_call_id, status="success", @@ -1317,7 +1319,10 @@ def sync_read_file( file_path: Annotated[str, "Absolute path to the file to read. Must be absolute, not relative."], runtime: ToolRuntime[None, FilesystemState], offset: Annotated[int, "Line number to start reading from (0-indexed). Use for pagination of large files."] = DEFAULT_READ_OFFSET, - limit: Annotated[int, "Maximum number of lines to read. Use for pagination of large files."] = DEFAULT_READ_LIMIT, + limit: Annotated[ + int | None, + "Maximum number of lines to read for text files. Omit to use the default window: 100 text lines or 30 video seconds.", + ] = None, ) -> ToolMessage: """Synchronous wrapper for read_file tool.""" resolved_backend = self._get_backend(runtime) @@ -1337,14 +1342,18 @@ def sync_read_file( tool_call_id=runtime.tool_call_id, status="error", ) - read_result = resolved_backend.read(validated_path, offset=offset, limit=limit) + backend_limit = DEFAULT_READ_LIMIT if limit is None else limit + read_result = resolved_backend.read(validated_path, offset=offset, limit=backend_limit) return _handle_read_result(read_result, validated_path, runtime.tool_call_id, offset, limit) async def async_read_file( file_path: Annotated[str, "Absolute path to the file to read. Must be absolute, not relative."], runtime: ToolRuntime[None, FilesystemState], offset: Annotated[int, "Line number to start reading from (0-indexed). Use for pagination of large files."] = DEFAULT_READ_OFFSET, - limit: Annotated[int, "Maximum number of lines to read. Use for pagination of large files."] = DEFAULT_READ_LIMIT, + limit: Annotated[ + int | None, + "Maximum number of lines to read for text files. Omit to use the default window: 100 text lines or 30 video seconds.", + ] = None, ) -> ToolMessage: """Asynchronous wrapper for read_file tool.""" resolved_backend = self._get_backend(runtime) @@ -1364,7 +1373,8 @@ async def async_read_file( tool_call_id=runtime.tool_call_id, status="error", ) - read_result = await resolved_backend.aread(validated_path, offset=offset, limit=limit) + backend_limit = DEFAULT_READ_LIMIT if limit is None else limit + read_result = await resolved_backend.aread(validated_path, offset=offset, limit=backend_limit) return _handle_read_result(read_result, validated_path, runtime.tool_call_id, offset, limit) return StructuredTool.from_function( diff --git a/libs/deepagents/pyproject.toml b/libs/deepagents/pyproject.toml index a3be5ff168b..9f8e5f984b1 100644 --- a/libs/deepagents/pyproject.toml +++ b/libs/deepagents/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ [project.optional-dependencies] aws = ["langchain-aws>=1.6.1,<2.0.0"] quickjs = ["langchain-quickjs>=0.3.2"] -video = ["av>=17.0.0"] +video = ["av>=17.0.0", "pillow>=10.0.0"] [project.urls] Homepage = "https://docs.langchain.com/oss/python/deepagents/overview" diff --git a/libs/deepagents/tests/unit_tests/middleware/test_video.py b/libs/deepagents/tests/unit_tests/middleware/test_video.py index 0052a110d38..cbcb40272c4 100644 --- a/libs/deepagents/tests/unit_tests/middleware/test_video.py +++ b/libs/deepagents/tests/unit_tests/middleware/test_video.py @@ -8,29 +8,14 @@ """ import base64 -import importlib.util -from collections.abc import Iterator import pytest +import deepagents.middleware._video as video_module + av = pytest.importorskip("av", reason="`av` extra not installed (pip install deepagents[video])") np = pytest.importorskip("numpy", reason="numpy not installed") -# Lazy source resolution keeps the unit-test process free of `av` on the -# import path until pytest.importorskip above succeeds. -_VIDEO_MODULE_PATH = "/Users/john/git/deepagents/libs/deepagents/deepagents/middleware/_video.py" -_VVIDEO_SPEC = importlib.util.spec_from_file_location("_vvideo_under_test", _VIDEO_MODULE_PATH) - - -@pytest.fixture(scope="module") -def video_module() -> Iterator[object]: - """Load the `_video` module from disk for the unit suite.""" - if _VVIDEO_SPEC is None or _VVIDEO_SPEC.loader is None: # pragma: no cover - pytest.skip("Could not resolve PyAV module spec") - module = importlib.util.module_from_spec(_VVIDEO_SPEC) - _VVIDEO_SPEC.loader.exec_module(module) - return module - @pytest.fixture(scope="module") def synthetic_video_bytes(tmp_path_factory: pytest.TempPathFactory) -> bytes: @@ -55,18 +40,18 @@ def synthetic_video_bytes(tmp_path_factory: pytest.TempPathFactory) -> bytes: return path.read_bytes() -def test_select_sampling_rate_accepts_positive(video_module: object) -> None: +def test_select_sampling_rate_accepts_positive() -> None: assert video_module._select_sampling_rate(0.5) == 0.5 assert video_module._select_sampling_rate(2.0) == 2.0 -def test_select_sampling_rate_rejects_non_positive(video_module: object) -> None: +def test_select_sampling_rate_rejects_non_positive() -> None: for bad in (0, -0.1, -1): with pytest.raises(ValueError, match="must be > 0"): video_module._select_sampling_rate(bad) -def test_extract_rejects_negative_offset(video_module: object, synthetic_video_bytes: bytes) -> None: +def test_extract_rejects_negative_offset(synthetic_video_bytes: bytes) -> None: with pytest.raises(ValueError, match="offset_seconds must be >= 0"): video_module.extract_video_frames( synthetic_video_bytes, @@ -76,7 +61,7 @@ def test_extract_rejects_negative_offset(video_module: object, synthetic_video_b ) -def test_extract_rejects_zero_or_negative_duration(video_module: object, synthetic_video_bytes: bytes) -> None: +def test_extract_rejects_zero_or_negative_duration(synthetic_video_bytes: bytes) -> None: with pytest.raises(ValueError, match="duration_seconds must be > 0"): video_module.extract_video_frames( synthetic_video_bytes, @@ -86,7 +71,7 @@ def test_extract_rejects_zero_or_negative_duration(video_module: object, synthet ) -def test_extract_returns_interleaved_text_and_image_blocks(video_module: object, synthetic_video_bytes: bytes) -> None: +def test_extract_returns_interleaved_text_and_image_blocks(synthetic_video_bytes: bytes) -> None: """Two frames (t=0 and t=2) interleaved with text headers at 0.5 fps over a 3s clip.""" blocks = video_module.extract_video_frames( synthetic_video_bytes, @@ -104,7 +89,7 @@ def test_extract_returns_interleaved_text_and_image_blocks(video_module: object, assert jpeg[:3] == b"\xff\xd8\xff" -def test_extract_offset_skips_into_source(video_module: object, synthetic_video_bytes: bytes) -> None: +def test_extract_offset_skips_into_source(synthetic_video_bytes: bytes) -> None: """`offset_seconds` seeks into the clip; every emitted frame is at or after that point.""" blocks = video_module.extract_video_frames( synthetic_video_bytes, @@ -118,7 +103,7 @@ def test_extract_offset_skips_into_source(video_module: object, synthetic_video_ assert h.startswith("Frame at t=") -def test_extract_no_frames_in_window_raises(video_module: object, synthetic_video_bytes: bytes) -> None: +def test_extract_no_frames_in_window_raises(synthetic_video_bytes: bytes) -> None: """Reading past the end of the clip yields a `VideoExtractionError`.""" with pytest.raises(video_module.VideoExtractionError, match="No frames decoded"): video_module.extract_video_frames( diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json index f82562d744b..eb5163fb8d7 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- By default, it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Only omit limit (read full file) when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only \u2014 supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps).\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps).\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { @@ -68,9 +68,16 @@ "type": "string" }, "limit": { - "default": 100, - "description": "Maximum number of lines to read. Use for pagination of large files.", - "type": "integer" + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum number of lines to read for text files. Omit to use the default window: 100 text lines or 30 video seconds." }, "offset": { "default": 0, diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json index 72899bbeaf9..c3b85248bca 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- By default, it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Only omit limit (read full file) when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only \u2014 supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps).\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps).\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { @@ -68,9 +68,16 @@ "type": "string" }, "limit": { - "default": 100, - "description": "Maximum number of lines to read. Use for pagination of large files.", - "type": "integer" + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum number of lines to read for text files. Omit to use the default window: 100 text lines or 30 video seconds." }, "offset": { "default": 0, diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json index f82562d744b..eb5163fb8d7 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- By default, it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Only omit limit (read full file) when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only \u2014 supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps).\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps).\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { @@ -68,9 +68,16 @@ "type": "string" }, "limit": { - "default": 100, - "description": "Maximum number of lines to read. Use for pagination of large files.", - "type": "integer" + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum number of lines to read for text files. Omit to use the default window: 100 text lines or 30 video seconds." }, "offset": { "default": 0, diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json index 72899bbeaf9..c3b85248bca 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- By default, it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Only omit limit (read full file) when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only \u2014 supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps).\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps).\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { @@ -68,9 +68,16 @@ "type": "string" }, "limit": { - "default": 100, - "description": "Maximum number of lines to read. Use for pagination of large files.", - "type": "integer" + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum number of lines to read for text files. Omit to use the default window: 100 text lines or 30 video seconds." }, "offset": { "default": 0, diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json index bd0000b0b42..3c9f3d9d0bc 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- By default, it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Only omit limit (read full file) when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only \u2014 supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps).\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps).\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { @@ -68,9 +68,16 @@ "type": "string" }, "limit": { - "default": 100, - "description": "Maximum number of lines to read. Use for pagination of large files.", - "type": "integer" + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum number of lines to read for text files. Omit to use the default window: 100 text lines or 30 video seconds." }, "offset": { "default": 0, diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json index f82562d744b..eb5163fb8d7 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- By default, it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Only omit limit (read full file) when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only \u2014 supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps).\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps).\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { @@ -68,9 +68,16 @@ "type": "string" }, "limit": { - "default": 100, - "description": "Maximum number of lines to read. Use for pagination of large files.", - "type": "integer" + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Maximum number of lines to read for text files. Omit to use the default window: 100 text lines or 30 video seconds." }, "offset": { "default": 0, diff --git a/libs/deepagents/tests/unit_tests/test_middleware.py b/libs/deepagents/tests/unit_tests/test_middleware.py index 70b47647f6f..c4d9e86c541 100644 --- a/libs/deepagents/tests/unit_tests/test_middleware.py +++ b/libs/deepagents/tests/unit_tests/test_middleware.py @@ -1,3 +1,4 @@ +import base64 import mimetypes import time from unittest.mock import MagicMock, patch @@ -1497,7 +1498,7 @@ def __init__(self, raw: bytes) -> None: self._raw = raw def read(self, path, *, offset=0, limit=100): # type: ignore[override] - return ReadResult(file_data={"content": self._raw, "encoding": "base64"}) + return ReadResult(file_data={"content": base64.b64encode(self._raw).decode("ascii"), "encoding": "base64"}) sentinel = [ {"type": "text", "text": "Frame at t=00:00:00.000"}, @@ -1517,7 +1518,7 @@ def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): "sampling_rate": sampling_rate, } ) - return sentinel + return list(sentinel) fake_extract.calls = [] @@ -1546,7 +1547,10 @@ def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): # Raw video bytes must NOT leak to the model: the content_blocks are # the extracted frames, not the original base64 from the backend. assert isinstance(result.content, list) - assert result.content == sentinel + assert result.content == [ + {"type": "text", "text": "Reading [0.000s, 30.000s) of /clips/intro.mp4 at 0.5 fps."}, + *sentinel, + ] assert b"\x00\x01\x02" not in str(result.content).encode() # The middleware must convert offset/limit to seconds with the @@ -1556,13 +1560,14 @@ def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): assert call["offset_seconds"] == 0 assert call["duration_seconds"] == 30 assert call["sampling_rate"] == 0.5 + assert call["len"] == len(b"\x00\x01\x02 fake video bytes") def test_read_file_video_offset_and_limit_reinterpreted_as_seconds(self, monkeypatch: pytest.MonkeyPatch) -> None: """`offset`skip `limit`window seconds for video reads.""" class VideoBackend(StateBackend): def read(self, path, *, offset=0, limit=100): # type: ignore[override] - return ReadResult(file_data={"content": b"raw", "encoding": "base64"}) + return ReadResult(file_data={"content": base64.b64encode(b"raw").decode("ascii"), "encoding": "base64"}) captured: dict[str, float] = {} @@ -1587,12 +1592,40 @@ def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): assert captured == {"offset_seconds": 12.0, "duration_seconds": 90.0, "sampling_rate": 0.5} + def test_read_file_video_omitted_limit_uses_video_default(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Omitting `limit` samples the first 30 seconds for video reads.""" + + class VideoBackend(StateBackend): + def read(self, path, *, offset=0, limit=100): # type: ignore[override] + return ReadResult(file_data={"content": base64.b64encode(b"raw").decode("ascii"), "encoding": "base64"}) + + captured: dict[str, float] = {} + + def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): # type: ignore[no-untyped-def] # noqa: ARG001 + captured.update(offset_seconds=offset_seconds, duration_seconds=duration_seconds, sampling_rate=sampling_rate) + return [{"type": "text", "text": "ok"}] + + monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) + middleware = FilesystemMiddleware(backend=VideoBackend()) + state = FilesystemState(messages=[], files={}) + runtime = _build_runtime(state, "video-read-default") + read_file_tool = next(t for t in middleware.tools if t.name == "read_file") + + read_file_tool.invoke( + { + "file_path": "/c.mp4", + "runtime": runtime, + } + ) + + assert captured == {"offset_seconds": 0.0, "duration_seconds": 30.0, "sampling_rate": 0.5} + def test_read_file_video_extraction_error_surfaces_as_error_message(self, monkeypatch: pytest.MonkeyPatch) -> None: """PyAV failures and missing-dep errors render as a tool error, not an exception.""" class VideoBackend(StateBackend): def read(self, path, *, offset=0, limit=100): # type: ignore[override] - return ReadResult(file_data={"content": b"corrupt", "encoding": "base64"}) + return ReadResult(file_data={"content": base64.b64encode(b"corrupt").decode("ascii"), "encoding": "base64"}) monkeypatch.setattr( filesystem_middleware, diff --git a/libs/deepagents/uv.lock b/libs/deepagents/uv.lock index 6828bfeafe0..a9812602d5f 100644 --- a/libs/deepagents/uv.lock +++ b/libs/deepagents/uv.lock @@ -538,6 +538,7 @@ quickjs = [ ] video = [ { name = "av" }, + { name = "pillow" }, ] [package.dev-dependencies] @@ -571,6 +572,7 @@ requires-dist = [ { name = "langchain-google-genai", specifier = ">=4.2.6,<5.0.0" }, { name = "langchain-quickjs", marker = "extra == 'quickjs'", specifier = ">=0.3.2" }, { name = "langsmith", specifier = ">=0.9.3" }, + { name = "pillow", marker = "extra == 'video'", specifier = ">=10.0.0" }, { name = "wcmatch", specifier = ">=10.1" }, ] provides-extras = ["aws", "quickjs", "video"] From adefabd592fcd3343b325e3779dfe827dd28515d Mon Sep 17 00:00:00 2001 From: Claude Code Date: Fri, 19 Jun 2026 16:35:58 -0700 Subject: [PATCH 05/36] fix(sdk): address video read feedback --- libs/deepagents/README.md | 5 +- libs/deepagents/deepagents/backends/utils.py | 1 + .../deepagents/middleware/_video.py | 113 +++++++++++++++-- .../deepagents/middleware/filesystem.py | 8 +- .../tests/unit_tests/backends/test_utils.py | 4 + .../tests/unit_tests/middleware/test_video.py | 119 +++++++++++++++++- .../custom_system_message_tools.json | 2 +- .../system_prompt_with_execute_tools.json | 2 +- ...m_prompt_with_memory_and_skills_tools.json | 2 +- ...stem_prompt_with_routed_backend_tools.json | 2 +- ...t_with_sync_and_async_subagents_tools.json | 2 +- .../system_prompt_without_execute_tools.json | 2 +- .../tests/unit_tests/test_middleware.py | 12 +- 13 files changed, 240 insertions(+), 34 deletions(-) diff --git a/libs/deepagents/README.md b/libs/deepagents/README.md index 137caffc8cb..90c4ec79bef 100644 --- a/libs/deepagents/README.md +++ b/libs/deepagents/README.md @@ -18,9 +18,10 @@ uv add deepagents ### Optional: video frame extraction -`read_file` can sample video files into image content blocks (`.mp4`, `.mov`, `.webm`, etc.). +`read_file` can sample video files into image content blocks (`.mp4`, `.mov`, `.webm`, `.mkv`, etc.). The agent uses `offset` to skip seconds into the source and `limit` to control how many -seconds of source to sample; the frames-per-second is a deploy-time knob. +seconds of source to sample, capped at 30 seconds per read; the frames-per-second is +a deploy-time knob. Install with: ```bash diff --git a/libs/deepagents/deepagents/backends/utils.py b/libs/deepagents/deepagents/backends/utils.py index aeb87d7753f..fda00cc0abc 100644 --- a/libs/deepagents/deepagents/backends/utils.py +++ b/libs/deepagents/deepagents/backends/utils.py @@ -40,6 +40,7 @@ ".flv": "video", ".mpg": "video", ".webm": "video", + ".mkv": "video", ".wmv": "video", ".3gpp": "video", # Audio (https://ai.google.dev/gemini-api/docs/audio) diff --git a/libs/deepagents/deepagents/middleware/_video.py b/libs/deepagents/deepagents/middleware/_video.py index 3fa287a7a09..6510803a0a8 100644 --- a/libs/deepagents/deepagents/middleware/_video.py +++ b/libs/deepagents/deepagents/middleware/_video.py @@ -18,7 +18,8 @@ import base64 import io -from typing import TYPE_CHECKING, Any +import time +from typing import TYPE_CHECKING, Any, Final if TYPE_CHECKING: from langchain_core.messages.content import ContentBlock @@ -27,6 +28,13 @@ MISSING_VIDEO_HINT = "Reading video files requires the optional video dependencies. Install them with `pip install 'deepagents[video]'`." +MAX_VIDEO_SAMPLE_DURATION_SECONDS: Final = 30.0 +MAX_VIDEO_SAMPLED_FRAMES: Final = 64 +MAX_VIDEO_FRAME_PIXELS: Final = 1920 * 1080 +MAX_VIDEO_FRAME_SIDE: Final = 4096 +MAX_VIDEO_EMITTED_BYTES: Final = 4 * 1024 * 1024 +MAX_VIDEO_DECODE_SECONDS: Final = 10.0 +_JPEG_QUALITY: Final = 85 class VideoExtractionError(RuntimeError): @@ -59,6 +67,14 @@ def _select_sampling_rate(value: float) -> float: return float(value) +def _select_duration(value: float) -> float: + """Validate and cap the requested video sampling window.""" + if value <= 0: + msg = f"duration_seconds must be > 0, got {value!r}" + raise ValueError(msg) + return min(float(value), MAX_VIDEO_SAMPLE_DURATION_SECONDS) + + def _format_timestamp(seconds: float) -> str: """Format a frame timestamp as `HH:MM:SS.mmm` for the text header block.""" if seconds < 0: @@ -99,36 +115,37 @@ def extract_video_frames( if offset_seconds < 0: msg = f"offset_seconds must be >= 0, got {offset_seconds!r}" raise ValueError(msg) - if duration_seconds <= 0: - msg = f"duration_seconds must be > 0, got {duration_seconds!r}" - raise ValueError(msg) rate = _select_sampling_rate(sampling_rate) + duration = _select_duration(duration_seconds) av = _import_av() container = _open_video_container(av, content) try: video_stream = _find_video_stream(container) time_base = float(video_stream.time_base) + stream_start_seconds = _stream_start_seconds(video_stream, time_base) if offset_seconds > 0: # `seek` keeps the math correct across containers that already sit # at a non-zero timeline (e.g. trimmed clips). - start_pts = int(offset_seconds / time_base) + start_pts = _stream_start_pts(video_stream) + int(offset_seconds / time_base) container.seek(start_pts, any_frame=False, backward=True, stream=video_stream) blocks = list( _sample_frames_in_window( container.decode(video_stream), offset_seconds=offset_seconds, - duration_seconds=duration_seconds, + duration_seconds=duration, sampling_rate=rate, time_base=time_base, + stream_start_seconds=stream_start_seconds, + deadline_seconds=time.monotonic() + MAX_VIDEO_DECODE_SECONDS, ) ) finally: container.close() if not blocks: - end_seconds = offset_seconds + duration_seconds + end_seconds = offset_seconds + duration msg = f"No frames decoded for window [{offset_seconds:.3f}s, {end_seconds:.3f}s)" raise VideoExtractionError(msg) return blocks @@ -161,6 +178,54 @@ def _find_video_stream(container: Any) -> Any: # noqa: ANN401 return video_stream +def _stream_start_pts(video_stream: Any) -> int: # noqa: ANN401 + """Return the stream start timestamp in stream time-base units.""" + start_time = getattr(video_stream, "start_time", None) + return int(start_time) if start_time is not None else 0 + + +def _stream_start_seconds(video_stream: Any, time_base: float) -> float: # noqa: ANN401 + """Return the stream start timestamp in seconds.""" + return _stream_start_pts(video_stream) * time_base + + +def _frame_seconds(frame: Any, *, time_base: float, stream_start_seconds: float) -> float | None: # noqa: ANN401 + """Return a frame timestamp normalized to seconds from the video start.""" + pts = getattr(frame, "pts", None) + if pts is not None: + return max(0.0, float(pts) * time_base - stream_start_seconds) + frame_time = getattr(frame, "time", None) + if frame_time is not None: + return max(0.0, float(frame_time) - stream_start_seconds) + return None + + +def _frame_dimensions(frame: Any) -> tuple[int, int] | None: # noqa: ANN401 + """Return frame dimensions when the decoder exposes them.""" + width = getattr(frame, "width", None) + height = getattr(frame, "height", None) + if width is None or height is None: + return None + return int(width), int(height) + + +def _validate_dimensions(width: int, height: int) -> None: + """Reject frames that exceed the fixed decode/encode budget.""" + if width <= 0 or height <= 0: + return + pixels = width * height + if width > MAX_VIDEO_FRAME_SIDE or height > MAX_VIDEO_FRAME_SIDE or pixels > MAX_VIDEO_FRAME_PIXELS: + msg = f"Video frame dimensions {width}x{height} exceed the maximum {MAX_VIDEO_FRAME_PIXELS} pixels / {MAX_VIDEO_FRAME_SIDE}px side" + raise VideoExtractionError(msg) + + +def _check_decode_deadline(deadline_seconds: float | None) -> None: + """Raise when best-effort video decoding has exceeded its time budget.""" + if deadline_seconds is not None and time.monotonic() > deadline_seconds: + msg = f"Video decoding exceeded the {MAX_VIDEO_DECODE_SECONDS:.1f}s safety budget" + raise VideoExtractionError(msg) + + def _sample_frames_in_window( decoded_frames: Any, # noqa: ANN401 *, @@ -168,28 +233,50 @@ def _sample_frames_in_window( duration_seconds: float, sampling_rate: float, time_base: float, + stream_start_seconds: float = 0.0, + deadline_seconds: float | None = None, ) -> list[ContentBlock]: """Pick JPEG+timestamp content blocks for frames inside the requested window.""" frame_interval_seconds = 1.0 / sampling_rate - end_seconds = offset_seconds + duration_seconds + duration = _select_duration(duration_seconds) + end_seconds = offset_seconds + duration next_emit_seconds = offset_seconds blocks: list[ContentBlock] = [] + emitted_frames = 0 + emitted_bytes = 0 for frame in decoded_frames: - frame_seconds = float(frame.pts) * time_base + _check_decode_deadline(deadline_seconds) + frame_seconds = _frame_seconds(frame, time_base=time_base, stream_start_seconds=stream_start_seconds) + if frame_seconds is None: + continue if frame_seconds >= end_seconds: break if frame_seconds + 1e-6 < next_emit_seconds: continue + jpeg_bytes = _encode_jpeg(frame) + image_base64 = base64.b64encode(jpeg_bytes) ts = _format_timestamp(frame_seconds) + text = f"Frame at t={ts}" + next_block_bytes = len(text.encode()) + len(image_base64) + if emitted_bytes + next_block_bytes > MAX_VIDEO_EMITTED_BYTES: + if emitted_frames == 0: + msg = f"Video frame output exceeded the {MAX_VIDEO_EMITTED_BYTES} byte safety budget before emitting a frame" + raise VideoExtractionError(msg) + break + blocks.append({"type": "text", "text": f"Frame at t={ts}"}) blocks.append( { "type": "image", - "base64": base64.b64encode(jpeg_bytes).decode("ascii"), + "base64": image_base64.decode("ascii"), "mime_type": "image/jpeg", } ) + emitted_frames += 1 + emitted_bytes += next_block_bytes + if emitted_frames >= MAX_VIDEO_SAMPLED_FRAMES: + break emitted_index = round((frame_seconds - offset_seconds) / frame_interval_seconds) + 1 next_emit_seconds = offset_seconds + frame_interval_seconds * emitted_index return blocks @@ -207,7 +294,11 @@ def _encode_jpeg(frame: Any) -> bytes: # noqa: ANN401 msg = f"{MISSING_VIDEO_HINT} (underlying error: {exc})" raise VideoExtractionError(msg) from exc + if dimensions := _frame_dimensions(frame): + _validate_dimensions(*dimensions) + img = frame.to_image() if hasattr(frame, "to_image") else Image.fromarray(frame.to_ndarray(format="rgb24")) + _validate_dimensions(*img.size) buf = io.BytesIO() - img.save(buf, format="JPEG", quality=85) + img.save(buf, format="JPEG", quality=_JPEG_QUALITY) return buf.getvalue() diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index 5d653919462..c68e063e278 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -77,7 +77,9 @@ ) from deepagents.middleware._utils import append_to_system_message from deepagents.middleware._video import ( + MAX_VIDEO_SAMPLE_DURATION_SECONDS, VideoExtractionError, + _select_duration as _select_video_duration, _select_sampling_rate as _select_video_sampling_rate, extract_video_frames, ) @@ -98,7 +100,7 @@ } # Reinterpret offset/limit as seconds for video reads. -_VIDEO_DEFAULT_LIMIT_SECONDS = 30.0 +_VIDEO_DEFAULT_LIMIT_SECONDS = MAX_VIDEO_SAMPLE_DURATION_SECONDS def _handle_video_read( @@ -126,7 +128,7 @@ def _handle_video_read( duration_seconds = float(_VIDEO_DEFAULT_LIMIT_SECONDS) header = f"Reading first {int(duration_seconds)}s of {validated_path} at {rate} fps." else: - duration_seconds = float(limit) + duration_seconds = _select_video_duration(float(limit)) header = f"Reading [{offset_seconds:.3f}s, {offset_seconds + duration_seconds:.3f}s) of {validated_path} at {rate} fps." def _err(msg: str) -> ToolMessage: @@ -577,7 +579,7 @@ class ExecuteSchema(BaseModel): For multimodal reads (image, audio, video, PDF, etc.): - Use `read_file(file_path=...)` - For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only -- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). +- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default and maximum `limit=30` reads a 30-second window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows. - Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work. - If file details were compacted from history, call `read_file` again on the same path diff --git a/libs/deepagents/tests/unit_tests/backends/test_utils.py b/libs/deepagents/tests/unit_tests/backends/test_utils.py index ee14eb130fd..63fee062e45 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_utils.py +++ b/libs/deepagents/tests/unit_tests/backends/test_utils.py @@ -204,6 +204,10 @@ def test_get_file_type_returns_text_for_unknown_extensions() -> None: assert _get_file_type("/foo/bar") == "text" +def test_get_file_type_recognizes_mkv_as_video() -> None: + assert _get_file_type("/foo/bar.mkv") == "video" + + def test_get_file_type_non_text_values_are_valid_content_block_types() -> None: """Every non-text file type must be accepted as a ContentBlock `type`.""" for file_type in _EXTENSION_TO_FILE_TYPE.values(): diff --git a/libs/deepagents/tests/unit_tests/middleware/test_video.py b/libs/deepagents/tests/unit_tests/middleware/test_video.py index cbcb40272c4..e2907877d3d 100644 --- a/libs/deepagents/tests/unit_tests/middleware/test_video.py +++ b/libs/deepagents/tests/unit_tests/middleware/test_video.py @@ -14,7 +14,14 @@ import deepagents.middleware._video as video_module av = pytest.importorskip("av", reason="`av` extra not installed (pip install deepagents[video])") -np = pytest.importorskip("numpy", reason="numpy not installed") +Image = pytest.importorskip("PIL.Image", reason="Pillow not installed") + + +class _FakeFrame: + def __init__(self, pts: int, *, width: int = 32, height: int = 32) -> None: + self.pts = pts + self.width = width + self.height = height @pytest.fixture(scope="module") @@ -26,11 +33,8 @@ def synthetic_video_bytes(tmp_path_factory: pytest.TempPathFactory) -> bytes: stream = container.add_stream("libx264", rate=5) stream.width, stream.height, stream.pix_fmt = 32, 32, "yuv420p" for i in range(15): - img = np.tile( - np.array([(i * 8) % 255, 128, 64], dtype=np.uint8), - (32, 32, 1), - ) - frame = av.VideoFrame.from_ndarray(img) + img = Image.new("RGB", (32, 32), ((i * 8) % 255, 128, 64)) + frame = av.VideoFrame.from_image(img) frame.pts = i for pkt in stream.encode(frame): container.mux(pkt) @@ -112,3 +116,106 @@ def test_extract_no_frames_in_window_raises(synthetic_video_bytes: bytes) -> Non duration_seconds=5, sampling_rate=0.5, ) + + +def test_sample_frames_normalizes_non_zero_stream_start(monkeypatch: pytest.MonkeyPatch) -> None: + """Frame timestamps are measured from the video start, not raw container PTS.""" + monkeypatch.setattr(video_module, "_encode_jpeg", lambda _frame: b"jpeg") + + blocks = video_module._sample_frames_in_window( + [_FakeFrame(3600), _FakeFrame(3601), _FakeFrame(3602)], + offset_seconds=0, + duration_seconds=3, + sampling_rate=1, + time_base=1, + stream_start_seconds=3600, + ) + + headers = [block["text"] for block in blocks if block["type"] == "text"] + assert headers == [ + "Frame at t=00:00:00.000", + "Frame at t=00:00:01.000", + "Frame at t=00:00:02.000", + ] + + +def test_sample_frames_caps_requested_duration(monkeypatch: pytest.MonkeyPatch) -> None: + """Huge requested windows are capped before sampling frames.""" + monkeypatch.setattr(video_module, "_encode_jpeg", lambda _frame: b"jpeg") + monkeypatch.setattr(video_module, "MAX_VIDEO_SAMPLE_DURATION_SECONDS", 2.0) + + blocks = video_module._sample_frames_in_window( + [_FakeFrame(0), _FakeFrame(1), _FakeFrame(2)], + offset_seconds=0, + duration_seconds=999, + sampling_rate=1, + time_base=1, + ) + + headers = [block["text"] for block in blocks if block["type"] == "text"] + assert headers == ["Frame at t=00:00:00.000", "Frame at t=00:00:01.000"] + + +def test_sample_frames_caps_frame_count(monkeypatch: pytest.MonkeyPatch) -> None: + """Sampling stops at the fixed frame-count budget.""" + monkeypatch.setattr(video_module, "_encode_jpeg", lambda _frame: b"jpeg") + monkeypatch.setattr(video_module, "MAX_VIDEO_SAMPLED_FRAMES", 2) + + blocks = video_module._sample_frames_in_window( + [_FakeFrame(0), _FakeFrame(1), _FakeFrame(2)], + offset_seconds=0, + duration_seconds=3, + sampling_rate=1, + time_base=1, + ) + + headers = [block["text"] for block in blocks if block["type"] == "text"] + assert headers == ["Frame at t=00:00:00.000", "Frame at t=00:00:01.000"] + + +def test_sample_frames_rejects_oversized_output(monkeypatch: pytest.MonkeyPatch) -> None: + """Extraction fails before retaining an oversized first image block.""" + monkeypatch.setattr(video_module, "_encode_jpeg", lambda _frame: b"jpeg") + monkeypatch.setattr(video_module, "MAX_VIDEO_EMITTED_BYTES", 1) + + with pytest.raises(video_module.VideoExtractionError, match="safety budget"): + video_module._sample_frames_in_window( + [_FakeFrame(0)], + offset_seconds=0, + duration_seconds=1, + sampling_rate=1, + time_base=1, + ) + + +def test_encode_jpeg_rejects_oversized_dimensions_before_conversion() -> None: + """Oversized frames are rejected before Pillow conversion retains them.""" + + class OversizedFrame(_FakeFrame): + def __init__(self) -> None: + super().__init__(0, width=video_module.MAX_VIDEO_FRAME_SIDE + 1, height=1) + self.converted = False + + def to_image(self): # type: ignore[no-untyped-def] + self.converted = True + msg = "oversized frame should not be converted" + raise AssertionError(msg) + + frame = OversizedFrame() + + with pytest.raises(video_module.VideoExtractionError, match="dimensions"): + video_module._encode_jpeg(frame) + assert not frame.converted + + +def test_sample_frames_enforces_decode_deadline() -> None: + """Best-effort decode timeout raises instead of continuing indefinitely.""" + with pytest.raises(video_module.VideoExtractionError, match="decoding exceeded"): + video_module._sample_frames_in_window( + [_FakeFrame(0)], + offset_seconds=0, + duration_seconds=1, + sampling_rate=1, + time_base=1, + deadline_seconds=0, + ) diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json index eb5163fb8d7..fa91106133d 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps).\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default and maximum `limit=30` reads a 30-second window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows.\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json index c3b85248bca..a20cbb99966 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps).\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default and maximum `limit=30` reads a 30-second window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows.\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json index eb5163fb8d7..fa91106133d 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps).\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default and maximum `limit=30` reads a 30-second window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows.\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json index c3b85248bca..a20cbb99966 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps).\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default and maximum `limit=30` reads a 30-second window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows.\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json index 3c9f3d9d0bc..1ef161e3d1d 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps).\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default and maximum `limit=30` reads a 30-second window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows.\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json index eb5163fb8d7..fa91106133d 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default `limit=30` reads the first 30 seconds). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps).\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default and maximum `limit=30` reads a 30-second window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows.\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/test_middleware.py b/libs/deepagents/tests/unit_tests/test_middleware.py index c4d9e86c541..3991a5ed4f9 100644 --- a/libs/deepagents/tests/unit_tests/test_middleware.py +++ b/libs/deepagents/tests/unit_tests/test_middleware.py @@ -1534,7 +1534,7 @@ def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): result = read_file_tool.invoke( { - "file_path": "/clips/intro.mp4", + "file_path": "/clips/intro.mkv", "offset": 0, "limit": 30, "runtime": runtime, @@ -1543,12 +1543,12 @@ def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): assert isinstance(result, ToolMessage) assert result.status == "success" - assert result.additional_kwargs["read_file_path"] == "/clips/intro.mp4" + assert result.additional_kwargs["read_file_path"] == "/clips/intro.mkv" # Raw video bytes must NOT leak to the model: the content_blocks are # the extracted frames, not the original base64 from the backend. assert isinstance(result.content, list) assert result.content == [ - {"type": "text", "text": "Reading [0.000s, 30.000s) of /clips/intro.mp4 at 0.5 fps."}, + {"type": "text", "text": "Reading [0.000s, 30.000s) of /clips/intro.mkv at 0.5 fps."}, *sentinel, ] assert b"\x00\x01\x02" not in str(result.content).encode() @@ -1563,7 +1563,7 @@ def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): assert call["len"] == len(b"\x00\x01\x02 fake video bytes") def test_read_file_video_offset_and_limit_reinterpreted_as_seconds(self, monkeypatch: pytest.MonkeyPatch) -> None: - """`offset`skip `limit`window seconds for video reads.""" + """`offset` skips into the source and oversized video windows are capped.""" class VideoBackend(StateBackend): def read(self, path, *, offset=0, limit=100): # type: ignore[override] @@ -1585,12 +1585,12 @@ def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): { "file_path": "/c.mp4", "offset": 12, # -> seek 12 seconds into the source - "limit": 90, # -> sample a 90-second window + "limit": 90, # -> sample a capped 30-second window "runtime": runtime, } ) - assert captured == {"offset_seconds": 12.0, "duration_seconds": 90.0, "sampling_rate": 0.5} + assert captured == {"offset_seconds": 12.0, "duration_seconds": 30.0, "sampling_rate": 0.5} def test_read_file_video_omitted_limit_uses_video_default(self, monkeypatch: pytest.MonkeyPatch) -> None: """Omitting `limit` samples the first 30 seconds for video reads.""" From a032baa18bca1468d814078b1d26a43def7a686a Mon Sep 17 00:00:00 2001 From: Claude Code Date: Fri, 19 Jun 2026 16:59:58 -0700 Subject: [PATCH 06/36] fix(sdk): wrap lazy video decode errors --- .../deepagents/middleware/_video.py | 96 +++++++++++-------- .../tests/unit_tests/middleware/test_video.py | 22 +++++ 2 files changed, 79 insertions(+), 39 deletions(-) diff --git a/libs/deepagents/deepagents/middleware/_video.py b/libs/deepagents/deepagents/middleware/_video.py index 6510803a0a8..43bf6c59c20 100644 --- a/libs/deepagents/deepagents/middleware/_video.py +++ b/libs/deepagents/deepagents/middleware/_video.py @@ -139,6 +139,7 @@ def extract_video_frames( time_base=time_base, stream_start_seconds=stream_start_seconds, deadline_seconds=time.monotonic() + MAX_VIDEO_DECODE_SECONDS, + decode_error_types=_video_backend_error_types(av), ) ) finally: @@ -161,14 +162,26 @@ def _open_video_container(av: Any, content: bytes) -> Any: # noqa: ANN401 """ try: return av.open(io.BytesIO(content)) - except av.error.InvalidDataError as exc: # pragma: no cover - depends on the input - msg = f"Failed to open video payload: {exc}" - raise VideoExtractionError(msg) from exc - except OSError as exc: # pragma: no cover - dependency ffmpeg missing on host + except _video_backend_error_types(av) as exc: # pragma: no cover - depends on host/input msg = f"Failed to open video payload: {exc}" raise VideoExtractionError(msg) from exc +def _video_backend_error_types(av: Any) -> tuple[type[BaseException], ...]: # noqa: ANN401 + """Return backend failures that should surface as `VideoExtractionError`.""" + errors: list[type[BaseException]] = [OSError] + av_error = getattr(av, "error", None) + for name in ("FFmpegError", "InvalidDataError"): + error_type = getattr(av_error, name, None) + if ( + isinstance(error_type, type) + and issubclass(error_type, BaseException) + and not any(issubclass(error_type, existing) for existing in errors) + ): + errors.append(error_type) + return tuple(errors) + + def _find_video_stream(container: Any) -> Any: # noqa: ANN401 """Return the first video stream in `container` or raise.""" video_stream = next((s for s in container.streams if s.type == "video"), None) @@ -235,6 +248,7 @@ def _sample_frames_in_window( time_base: float, stream_start_seconds: float = 0.0, deadline_seconds: float | None = None, + decode_error_types: tuple[type[BaseException], ...] = (), ) -> list[ContentBlock]: """Pick JPEG+timestamp content blocks for frames inside the requested window.""" frame_interval_seconds = 1.0 / sampling_rate @@ -244,41 +258,45 @@ def _sample_frames_in_window( blocks: list[ContentBlock] = [] emitted_frames = 0 emitted_bytes = 0 - for frame in decoded_frames: - _check_decode_deadline(deadline_seconds) - frame_seconds = _frame_seconds(frame, time_base=time_base, stream_start_seconds=stream_start_seconds) - if frame_seconds is None: - continue - if frame_seconds >= end_seconds: - break - if frame_seconds + 1e-6 < next_emit_seconds: - continue - - jpeg_bytes = _encode_jpeg(frame) - image_base64 = base64.b64encode(jpeg_bytes) - ts = _format_timestamp(frame_seconds) - text = f"Frame at t={ts}" - next_block_bytes = len(text.encode()) + len(image_base64) - if emitted_bytes + next_block_bytes > MAX_VIDEO_EMITTED_BYTES: - if emitted_frames == 0: - msg = f"Video frame output exceeded the {MAX_VIDEO_EMITTED_BYTES} byte safety budget before emitting a frame" - raise VideoExtractionError(msg) - break - - blocks.append({"type": "text", "text": f"Frame at t={ts}"}) - blocks.append( - { - "type": "image", - "base64": image_base64.decode("ascii"), - "mime_type": "image/jpeg", - } - ) - emitted_frames += 1 - emitted_bytes += next_block_bytes - if emitted_frames >= MAX_VIDEO_SAMPLED_FRAMES: - break - emitted_index = round((frame_seconds - offset_seconds) / frame_interval_seconds) + 1 - next_emit_seconds = offset_seconds + frame_interval_seconds * emitted_index + try: + for frame in decoded_frames: + _check_decode_deadline(deadline_seconds) + frame_seconds = _frame_seconds(frame, time_base=time_base, stream_start_seconds=stream_start_seconds) + if frame_seconds is None: + continue + if frame_seconds >= end_seconds: + break + if frame_seconds + 1e-6 < next_emit_seconds: + continue + + jpeg_bytes = _encode_jpeg(frame) + image_base64 = base64.b64encode(jpeg_bytes) + ts = _format_timestamp(frame_seconds) + text = f"Frame at t={ts}" + next_block_bytes = len(text.encode()) + len(image_base64) + if emitted_bytes + next_block_bytes > MAX_VIDEO_EMITTED_BYTES: + if emitted_frames == 0: + msg = f"Video frame output exceeded the {MAX_VIDEO_EMITTED_BYTES} byte safety budget before emitting a frame" + raise VideoExtractionError(msg) + break + + blocks.append({"type": "text", "text": f"Frame at t={ts}"}) + blocks.append( + { + "type": "image", + "base64": image_base64.decode("ascii"), + "mime_type": "image/jpeg", + } + ) + emitted_frames += 1 + emitted_bytes += next_block_bytes + if emitted_frames >= MAX_VIDEO_SAMPLED_FRAMES: + break + emitted_index = round((frame_seconds - offset_seconds) / frame_interval_seconds) + 1 + next_emit_seconds = offset_seconds + frame_interval_seconds * emitted_index + except decode_error_types as exc: + msg = f"Failed to decode video frames: {exc}" + raise VideoExtractionError(msg) from exc return blocks diff --git a/libs/deepagents/tests/unit_tests/middleware/test_video.py b/libs/deepagents/tests/unit_tests/middleware/test_video.py index e2907877d3d..555d3748645 100644 --- a/libs/deepagents/tests/unit_tests/middleware/test_video.py +++ b/libs/deepagents/tests/unit_tests/middleware/test_video.py @@ -8,6 +8,7 @@ """ import base64 +from collections.abc import Iterator import pytest @@ -118,6 +119,27 @@ def test_extract_no_frames_in_window_raises(synthetic_video_bytes: bytes) -> Non ) +def test_sample_frames_wraps_decode_iteration_errors(monkeypatch: pytest.MonkeyPatch) -> None: + """PyAV decode errors raised after iteration starts surface as `VideoExtractionError`.""" + monkeypatch.setattr(video_module, "_encode_jpeg", lambda _frame: b"jpeg") + + def decoded_frames() -> Iterator[_FakeFrame]: + yield _FakeFrame(0) + raise av.error.InvalidDataError(1094995529, "invalid data") + + with pytest.raises(video_module.VideoExtractionError, match="Failed to decode video frames") as exc_info: + video_module._sample_frames_in_window( + decoded_frames(), + offset_seconds=0, + duration_seconds=3, + sampling_rate=1, + time_base=1, + decode_error_types=video_module._video_backend_error_types(av), + ) + + assert isinstance(exc_info.value.__cause__, av.error.InvalidDataError) + + def test_sample_frames_normalizes_non_zero_stream_start(monkeypatch: pytest.MonkeyPatch) -> None: """Frame timestamps are measured from the video start, not raw container PTS.""" monkeypatch.setattr(video_module, "_encode_jpeg", lambda _frame: b"jpeg") From 358e8927ececae87c583405f3ff191b72c154bb0 Mon Sep 17 00:00:00 2001 From: Claude Code Date: Fri, 19 Jun 2026 18:59:14 -0700 Subject: [PATCH 07/36] fix(sdk): drop agent-input limit clamp on video read_file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The agent's `limit` is authoritative over how much of the source is sampled; the silent `min(limit, MAX_VIDEO_SAMPLE_DURATION_SECONDS = 30.0)` clamp in `_video._select_duration` prevented the agent from widening the window beyond 30 s without warning. - Delete the `MAX_VIDEO_SAMPLE_DURATION_SECONDS` constant. - Drop the clamp; `_select_duration` is now a pure `> 0` validator. - `_VIDEO_DEFAULT_LIMIT_SECONDS` becomes `100` (mirrors `DEFAULT_READ_LIMIT` for text reads) so the omit-args default is consistent across file types. - System-prompt wording updated to "default 100 s, no upper clamp — set `limit` explicitly to widen the window" and smoke-test snapshots regenerated to match. - Tests: replace `test_sample_frames_caps_requested_duration` with `test_sample_frames_does_not_cap_requested_duration`, asserting a 999 s request is honored and only the layered `MAX_VIDEO_SAMPLED_FRAMES` cap trims output; update the two `test_middleware.py` video cases to assert the new defaults (`100` for omitted `limit`, passthrough for supplied `limit`). No change to layered output caps (`MAX_VIDEO_SAMPLED_FRAMES`, `MAX_VIDEO_EMITTED_BYTES`, `MAX_VIDEO_DECODE_SECONDS`, `MAX_VIDEO_FRAME_PIXELS`, `MAX_VIDEO_FRAME_SIDE`) — they remain the defence-in-depth bounds on the emitted output regardless of which window the agent requests. --- .../deepagents/middleware/_video.py | 12 +++++++++--- .../deepagents/middleware/filesystem.py | 19 ++++++++++++------- .../tests/unit_tests/middleware/test_video.py | 15 +++++++++++---- .../custom_system_message_tools.json | 2 +- .../system_prompt_with_execute_tools.json | 2 +- ...m_prompt_with_memory_and_skills_tools.json | 2 +- ...stem_prompt_with_routed_backend_tools.json | 2 +- ...t_with_sync_and_async_subagents_tools.json | 2 +- .../system_prompt_without_execute_tools.json | 2 +- .../tests/unit_tests/test_middleware.py | 10 +++++----- 10 files changed, 43 insertions(+), 25 deletions(-) diff --git a/libs/deepagents/deepagents/middleware/_video.py b/libs/deepagents/deepagents/middleware/_video.py index 43bf6c59c20..6761e07f636 100644 --- a/libs/deepagents/deepagents/middleware/_video.py +++ b/libs/deepagents/deepagents/middleware/_video.py @@ -28,7 +28,6 @@ MISSING_VIDEO_HINT = "Reading video files requires the optional video dependencies. Install them with `pip install 'deepagents[video]'`." -MAX_VIDEO_SAMPLE_DURATION_SECONDS: Final = 30.0 MAX_VIDEO_SAMPLED_FRAMES: Final = 64 MAX_VIDEO_FRAME_PIXELS: Final = 1920 * 1080 MAX_VIDEO_FRAME_SIDE: Final = 4096 @@ -68,11 +67,18 @@ def _select_sampling_rate(value: float) -> float: def _select_duration(value: float) -> float: - """Validate and cap the requested video sampling window.""" + """Validate the agent-supplied video window length. + + The agent's `limit` is authoritative over how much of the source is + sampled; this function only checks that the request is positive. Bounds + on emitted output are enforced elsewhere in the pipeline + (`MAX_VIDEO_DECODE_SECONDS`, `MAX_VIDEO_SAMPLED_FRAMES`, + `MAX_VIDEO_EMITTED_BYTES`). + """ if value <= 0: msg = f"duration_seconds must be > 0, got {value!r}" raise ValueError(msg) - return min(float(value), MAX_VIDEO_SAMPLE_DURATION_SECONDS) + return float(value) def _format_timestamp(seconds: float) -> str: diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index c68e063e278..2268b15e5f7 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -77,7 +77,6 @@ ) from deepagents.middleware._utils import append_to_system_message from deepagents.middleware._video import ( - MAX_VIDEO_SAMPLE_DURATION_SECONDS, VideoExtractionError, _select_duration as _select_video_duration, _select_sampling_rate as _select_video_sampling_rate, @@ -99,8 +98,10 @@ "delete": "write", } -# Reinterpret offset/limit as seconds for video reads. -_VIDEO_DEFAULT_LIMIT_SECONDS = MAX_VIDEO_SAMPLE_DURATION_SECONDS +# Reinterpret offset/limit as seconds for video reads. The agent's `limit` +# is authoritative; this is only the fallback when the agent omits the +# argument. Mirrors `DEFAULT_READ_LIMIT` (the text-read default). +_VIDEO_DEFAULT_LIMIT_SECONDS = 100 def _handle_video_read( @@ -114,9 +115,13 @@ def _handle_video_read( """Slice a video byte payload into a sampled frame window for the model. The agent's `offset` is reinterpreted as seconds to skip into the - source, and `limit` as seconds of source to sample. Missing `offset`/ - `limit` fall back to reading from the start of the source up to - `_VIDEO_DEFAULT_LIMIT_SECONDS`. + source, and `limit` as seconds of source to sample. The agent's + supplied `limit` is authoritative — there is no per-call upper clamp on + it. When `limit` is omitted or non-positive, the fallback is + `_VIDEO_DEFAULT_LIMIT_SECONDS`. Output volume is still bounded by the + layered caps on the extractor (`MAX_VIDEO_DECODE_SECONDS`, + `MAX_VIDEO_SAMPLED_FRAMES`, `MAX_VIDEO_EMITTED_BYTES`, + `MAX_VIDEO_FRAME_PIXELS`, `MAX_VIDEO_FRAME_SIDE`). Errors are returned as success-shaped `ToolMessage` instances carrying a plain text error, so the turn still completes and the agent can @@ -579,7 +584,7 @@ class ExecuteSchema(BaseModel): For multimodal reads (image, audio, video, PDF, etc.): - Use `read_file(file_path=...)` - For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only -- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default and maximum `limit=30` reads a 30-second window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows. +- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default 100 s, no upper clamp — set `limit` explicitly to widen the window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows. - Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work. - If file details were compacted from history, call `read_file` again on the same path diff --git a/libs/deepagents/tests/unit_tests/middleware/test_video.py b/libs/deepagents/tests/unit_tests/middleware/test_video.py index 555d3748645..b176a63eab9 100644 --- a/libs/deepagents/tests/unit_tests/middleware/test_video.py +++ b/libs/deepagents/tests/unit_tests/middleware/test_video.py @@ -161,13 +161,18 @@ def test_sample_frames_normalizes_non_zero_stream_start(monkeypatch: pytest.Monk ] -def test_sample_frames_caps_requested_duration(monkeypatch: pytest.MonkeyPatch) -> None: - """Huge requested windows are capped before sampling frames.""" +def test_sample_frames_does_not_cap_requested_duration(monkeypatch: pytest.MonkeyPatch) -> None: + """Huge requested windows are passed through unchanged. + + Output is still bounded by the layered output caps (`MAX_VIDEO_SAMPLED_FRAMES`, + etc.); the agent's `limit` is the only authority on how much of the source is + sampled. + """ monkeypatch.setattr(video_module, "_encode_jpeg", lambda _frame: b"jpeg") - monkeypatch.setattr(video_module, "MAX_VIDEO_SAMPLE_DURATION_SECONDS", 2.0) + monkeypatch.setattr(video_module, "MAX_VIDEO_SAMPLED_FRAMES", 2) blocks = video_module._sample_frames_in_window( - [_FakeFrame(0), _FakeFrame(1), _FakeFrame(2)], + [_FakeFrame(0), _FakeFrame(1), _FakeFrame(2), _FakeFrame(3)], offset_seconds=0, duration_seconds=999, sampling_rate=1, @@ -175,6 +180,8 @@ def test_sample_frames_caps_requested_duration(monkeypatch: pytest.MonkeyPatch) ) headers = [block["text"] for block in blocks if block["type"] == "text"] + # Three frames fit inside the 999 s window without being silently clamped; + # only the frame-count cap (`MAX_VIDEO_SAMPLED_FRAMES = 2`) trims the output. assert headers == ["Frame at t=00:00:00.000", "Frame at t=00:00:01.000"] diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json index fa91106133d..ca959c7457d 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default and maximum `limit=30` reads a 30-second window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows.\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default 100 s, no upper clamp \u2014 set `limit` explicitly to widen the window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows.\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json index a20cbb99966..d21e8989909 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default and maximum `limit=30` reads a 30-second window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows.\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default 100 s, no upper clamp \u2014 set `limit` explicitly to widen the window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows.\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json index fa91106133d..ca959c7457d 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default and maximum `limit=30` reads a 30-second window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows.\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default 100 s, no upper clamp \u2014 set `limit` explicitly to widen the window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows.\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json index a20cbb99966..d21e8989909 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default and maximum `limit=30` reads a 30-second window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows.\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default 100 s, no upper clamp \u2014 set `limit` explicitly to widen the window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows.\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json index 1ef161e3d1d..bcd6c693c57 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default and maximum `limit=30` reads a 30-second window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows.\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default 100 s, no upper clamp \u2014 set `limit` explicitly to widen the window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows.\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json index fa91106133d..ca959c7457d 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default and maximum `limit=30` reads a 30-second window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows.\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default 100 s, no upper clamp \u2014 set `limit` explicitly to widen the window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows.\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/test_middleware.py b/libs/deepagents/tests/unit_tests/test_middleware.py index 3991a5ed4f9..285f58e6434 100644 --- a/libs/deepagents/tests/unit_tests/test_middleware.py +++ b/libs/deepagents/tests/unit_tests/test_middleware.py @@ -1563,7 +1563,7 @@ def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): assert call["len"] == len(b"\x00\x01\x02 fake video bytes") def test_read_file_video_offset_and_limit_reinterpreted_as_seconds(self, monkeypatch: pytest.MonkeyPatch) -> None: - """`offset` skips into the source and oversized video windows are capped.""" + """`offset` skips into the source and the agent-supplied `limit` is authoritative.""" class VideoBackend(StateBackend): def read(self, path, *, offset=0, limit=100): # type: ignore[override] @@ -1585,15 +1585,15 @@ def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): { "file_path": "/c.mp4", "offset": 12, # -> seek 12 seconds into the source - "limit": 90, # -> sample a capped 30-second window + "limit": 90, # -> sample a 90-second window, no per-call clamp "runtime": runtime, } ) - assert captured == {"offset_seconds": 12.0, "duration_seconds": 30.0, "sampling_rate": 0.5} + assert captured == {"offset_seconds": 12.0, "duration_seconds": 90.0, "sampling_rate": 0.5} def test_read_file_video_omitted_limit_uses_video_default(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Omitting `limit` samples the first 30 seconds for video reads.""" + """Omitting `limit` samples the first 100 seconds (video default) for video reads.""" class VideoBackend(StateBackend): def read(self, path, *, offset=0, limit=100): # type: ignore[override] @@ -1618,7 +1618,7 @@ def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): } ) - assert captured == {"offset_seconds": 0.0, "duration_seconds": 30.0, "sampling_rate": 0.5} + assert captured == {"offset_seconds": 0.0, "duration_seconds": 100.0, "sampling_rate": 0.5} def test_read_file_video_extraction_error_surfaces_as_error_message(self, monkeypatch: pytest.MonkeyPatch) -> None: """PyAV failures and missing-dep errors render as a tool error, not an exception.""" From e28555a9008b566d851a1a7c3f53dbaceb92f059 Mon Sep 17 00:00:00 2001 From: Deep Agent Date: Fri, 19 Jun 2026 19:17:28 -0700 Subject: [PATCH 08/36] refactor(sdk): tighten video read_file review findings Address Open SWE review items against the video read_file PR: - Collapse sync/async read_file duplication via a shared _resolve_read_inputs helper for path/permission validation and backend_limit resolution. - Stop importing private _video validation helpers; filesystem.py now uses only extract_video_frames + VideoExtractionError as the public video boundary. _handle_video_read no longer revalidates the sampling rate (constructor + extractor are sufficient). - Remove duplicate duration capping in _sample_frames_in_window; only extract_video_frames validates (one layer at the boundary). - Use strict base64 decoding (validate=True with binascii.Error caught) so malformed payloads surface as a clearly-labeled tool error at the intended boundary. - Treat a non-positive video limit as a tool error instead of silently falling back to the default 100s window. - Add a small _tool_error helper to deduplicate ToolMessage errors across read_file and its wrappers. - Consolidate the four near-identical test VideoBackend classes into a single _video_backend(raw=...) factory. - Switch install guidance to `uv add 'deepagents[video]'` for README/runtime consistency. - Trim the model-facing read_file description and drop the duplicated sandbox-size-cap line; point at the deepagents docs. Refresh smoke snapshots accordingly. - Add test_read_file_video_non_positive_limit_rejected covering the new error path. All 1897 unit tests pass; ruff and ty are clean. --- .../deepagents/middleware/_video.py | 7 +- .../deepagents/middleware/filesystem.py | 157 +++++++++--------- .../custom_system_message_tools.json | 4 +- .../system_prompt_with_execute_tools.json | 4 +- ...m_prompt_with_memory_and_skills_tools.json | 4 +- ...stem_prompt_with_routed_backend_tools.json | 4 +- ...t_with_sync_and_async_subagents_tools.json | 4 +- .../system_prompt_without_execute_tools.json | 4 +- .../tests/unit_tests/test_middleware.py | 69 ++++---- 9 files changed, 129 insertions(+), 128 deletions(-) diff --git a/libs/deepagents/deepagents/middleware/_video.py b/libs/deepagents/deepagents/middleware/_video.py index 6761e07f636..daf8a21816b 100644 --- a/libs/deepagents/deepagents/middleware/_video.py +++ b/libs/deepagents/deepagents/middleware/_video.py @@ -27,7 +27,7 @@ ContentBlock = dict # only used at runtime by the agent runtime -MISSING_VIDEO_HINT = "Reading video files requires the optional video dependencies. Install them with `pip install 'deepagents[video]'`." +MISSING_VIDEO_HINT = "Reading video files requires the optional video dependencies. Install them with `uv add 'deepagents[video]'`." MAX_VIDEO_SAMPLED_FRAMES: Final = 64 MAX_VIDEO_FRAME_PIXELS: Final = 1920 * 1080 MAX_VIDEO_FRAME_SIDE: Final = 4096 @@ -140,7 +140,7 @@ def extract_video_frames( _sample_frames_in_window( container.decode(video_stream), offset_seconds=offset_seconds, - duration_seconds=duration, + duration_seconds=float(duration), sampling_rate=rate, time_base=time_base, stream_start_seconds=stream_start_seconds, @@ -258,8 +258,7 @@ def _sample_frames_in_window( ) -> list[ContentBlock]: """Pick JPEG+timestamp content blocks for frames inside the requested window.""" frame_interval_seconds = 1.0 / sampling_rate - duration = _select_duration(duration_seconds) - end_seconds = offset_seconds + duration + end_seconds = offset_seconds + float(duration_seconds) next_emit_seconds = offset_seconds blocks: list[ContentBlock] = [] emitted_frames = 0 diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index 2268b15e5f7..92375d47e9c 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -3,6 +3,7 @@ import asyncio import base64 +import binascii import concurrent.futures import contextlib import contextvars @@ -78,8 +79,6 @@ from deepagents.middleware._utils import append_to_system_message from deepagents.middleware._video import ( VideoExtractionError, - _select_duration as _select_video_duration, - _select_sampling_rate as _select_video_sampling_rate, extract_video_frames, ) @@ -104,6 +103,11 @@ _VIDEO_DEFAULT_LIMIT_SECONDS = 100 +def _tool_error(name: str, tool_call_id: str | None, content: str) -> ToolMessage: + """Build a success-shaped `ToolMessage` carrying a plain text error.""" + return ToolMessage(content=content, name=name, tool_call_id=tool_call_id, status="error") + + def _handle_video_read( content: str, validated_path: str, @@ -114,39 +118,30 @@ def _handle_video_read( ) -> ToolMessage: """Slice a video byte payload into a sampled frame window for the model. - The agent's `offset` is reinterpreted as seconds to skip into the - source, and `limit` as seconds of source to sample. The agent's - supplied `limit` is authoritative — there is no per-call upper clamp on - it. When `limit` is omitted or non-positive, the fallback is - `_VIDEO_DEFAULT_LIMIT_SECONDS`. Output volume is still bounded by the - layered caps on the extractor (`MAX_VIDEO_DECODE_SECONDS`, - `MAX_VIDEO_SAMPLED_FRAMES`, `MAX_VIDEO_EMITTED_BYTES`, - `MAX_VIDEO_FRAME_PIXELS`, `MAX_VIDEO_FRAME_SIDE`). - - Errors are returned as success-shaped `ToolMessage` instances carrying - a plain text error, so the turn still completes and the agent can - recover (e.g. by retrying with a smaller window). + `offset` is reinterpreted as seconds into the source to skip; `limit` as + seconds of source to sample. The agent's supplied `limit` is authoritative + (no per-call upper clamp); omitting it falls back to + `_VIDEO_DEFAULT_LIMIT_SECONDS` and supplying a non-positive value is + rejected as a tool error. Output volume is bounded by the layered caps on + the extractor (`MAX_VIDEO_DECODE_SECONDS`, `MAX_VIDEO_SAMPLED_FRAMES`, + `MAX_VIDEO_EMITTED_BYTES`, `MAX_VIDEO_FRAME_PIXELS`, `MAX_VIDEO_FRAME_SIDE`). + + Errors are returned as `ToolMessage` errors so the turn still completes and + the agent can recover (e.g. by retrying with a smaller window). """ - rate = _select_video_sampling_rate(sampling_rate) + if limit is not None and limit <= 0: + return _tool_error("read_file", tool_call_id, f"Error reading video {validated_path}: limit must be > 0, got {limit!r}") + rate = float(sampling_rate) offset_seconds = max(0.0, float(offset)) - if limit is None or limit <= 0: - duration_seconds = float(_VIDEO_DEFAULT_LIMIT_SECONDS) - header = f"Reading first {int(duration_seconds)}s of {validated_path} at {rate} fps." - else: - duration_seconds = _select_video_duration(float(limit)) - header = f"Reading [{offset_seconds:.3f}s, {offset_seconds + duration_seconds:.3f}s) of {validated_path} at {rate} fps." + duration_seconds = float(_VIDEO_DEFAULT_LIMIT_SECONDS if limit is None else limit) + header = _video_window_header(validated_path, offset_seconds, duration_seconds, rate) def _err(msg: str) -> ToolMessage: - return ToolMessage( - content=f"Error reading video {validated_path}: {msg}\n{header}", - name="read_file", - tool_call_id=tool_call_id, - status="error", - ) + return _tool_error("read_file", tool_call_id, f"Error reading video {validated_path}: {msg}\n{header}") try: - raw_bytes = base64.b64decode(content) if isinstance(content, str) else content - except (ValueError, TypeError) as exc: + raw_bytes = base64.b64decode(content, validate=True) if isinstance(content, str) else content + except (ValueError, TypeError, binascii.Error) as exc: return _err(f"video bytes are not valid base64 ({exc})") try: @@ -158,15 +153,22 @@ def _err(msg: str) -> ToolMessage: ) except VideoExtractionError as exc: return _err(str(exc)) - else: - blocks.insert(0, {"type": "text", "text": header}) - return ToolMessage( - content_blocks=blocks, - name="read_file", - tool_call_id=tool_call_id, - additional_kwargs={"read_file_path": validated_path}, - status="success", - ) + blocks.insert(0, {"type": "text", "text": header}) + return ToolMessage( + content_blocks=blocks, + name="read_file", + tool_call_id=tool_call_id, + additional_kwargs={"read_file_path": validated_path}, + status="success", + ) + + +def _video_window_header(path: str, offset_seconds: float, duration_seconds: float, rate: float) -> str: + """Render the model-facing text header introducing a sampled frame window.""" + end = offset_seconds + duration_seconds + if offset_seconds <= 0.0: + return f"Reading first {int(duration_seconds)}s of {path} at {rate} fps." + return f"Reading [{offset_seconds:.3f}s, {end:.3f}s) of {path} at {rate} fps." def _emit_binary_block( @@ -487,12 +489,12 @@ class ReadFileSchema(BaseModel): offset: int = Field( default=DEFAULT_READ_OFFSET, - description="Line number to start reading text files from (0-indexed), or seconds to skip into video files.", + description="Line number to start reading from (0-indexed). Use for pagination of large files.", ) limit: int | None = Field( default=None, - description="Maximum number of text lines to read, or video seconds to sample. Omit to use the default window.", + description="Maximum number of lines to read for text files. Omit to use the default window.", ) @@ -584,8 +586,7 @@ class ExecuteSchema(BaseModel): For multimodal reads (image, audio, video, PDF, etc.): - Use `read_file(file_path=...)` - For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only -- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default 100 s, no upper clamp — set `limit` explicitly to widen the window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows. -- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work. +- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s; sampling rate configured on `FilesystemMiddleware`). Detailed semantics live in the deepagents docs. - If file details were compacted from history, call `read_file` again on the same path - You should ALWAYS make sure a file has been read before editing it.""" @@ -1322,35 +1323,40 @@ def _handle_read_result( # noqa: PLR0911 status="success", ) + def _resolve_read_inputs( + file_path: str, + tool_call_id: str | None, + limit: int | None, + ) -> tuple[str, int] | ToolMessage: + """Validate path + permission and resolve the effective backend limit. + + Returns `(validated_path, backend_limit)` on success or a ready-to- + return `ToolMessage` error. + """ + try: + validated_path = validate_path(file_path) + except ValueError as e: + return _tool_error("read_file", tool_call_id, f"Error: {e}") + if _check_fs_permission(self._permissions, "read", validated_path) == "deny": + return _tool_error("read_file", tool_call_id, f"Error: permission denied for read on {validated_path}") + return validated_path, DEFAULT_READ_LIMIT if limit is None else limit + def sync_read_file( file_path: Annotated[str, "Absolute path to the file to read. Must be absolute, not relative."], runtime: ToolRuntime[None, FilesystemState], offset: Annotated[int, "Line number to start reading from (0-indexed). Use for pagination of large files."] = DEFAULT_READ_OFFSET, limit: Annotated[ int | None, - "Maximum number of lines to read for text files. Omit to use the default window: 100 text lines or 30 video seconds.", + "Maximum number of lines to read for text files. Omit to use the default window.", ] = None, ) -> ToolMessage: """Synchronous wrapper for read_file tool.""" - resolved_backend = self._get_backend(runtime) - try: - validated_path = validate_path(file_path) - except ValueError as e: - return ToolMessage( - content=f"Error: {e}", - name="read_file", - tool_call_id=runtime.tool_call_id, - status="error", - ) - if _check_fs_permission(self._permissions, "read", validated_path) == "deny": - return ToolMessage( - content=f"Error: permission denied for read on {validated_path}", - name="read_file", - tool_call_id=runtime.tool_call_id, - status="error", - ) - backend_limit = DEFAULT_READ_LIMIT if limit is None else limit - read_result = resolved_backend.read(validated_path, offset=offset, limit=backend_limit) + backend = self._get_backend(runtime) + prepared = _resolve_read_inputs(file_path, runtime.tool_call_id, limit) + if isinstance(prepared, ToolMessage): + return prepared + validated_path, backend_limit = prepared + read_result = backend.read(validated_path, offset=offset, limit=backend_limit) return _handle_read_result(read_result, validated_path, runtime.tool_call_id, offset, limit) async def async_read_file( @@ -1359,29 +1365,16 @@ async def async_read_file( offset: Annotated[int, "Line number to start reading from (0-indexed). Use for pagination of large files."] = DEFAULT_READ_OFFSET, limit: Annotated[ int | None, - "Maximum number of lines to read for text files. Omit to use the default window: 100 text lines or 30 video seconds.", + "Maximum number of lines to read for text files. Omit to use the default window.", ] = None, ) -> ToolMessage: """Asynchronous wrapper for read_file tool.""" - resolved_backend = self._get_backend(runtime) - try: - validated_path = validate_path(file_path) - except ValueError as e: - return ToolMessage( - content=f"Error: {e}", - name="read_file", - tool_call_id=runtime.tool_call_id, - status="error", - ) - if _check_fs_permission(self._permissions, "read", validated_path) == "deny": - return ToolMessage( - content=f"Error: permission denied for read on {validated_path}", - name="read_file", - tool_call_id=runtime.tool_call_id, - status="error", - ) - backend_limit = DEFAULT_READ_LIMIT if limit is None else limit - read_result = await resolved_backend.aread(validated_path, offset=offset, limit=backend_limit) + backend = self._get_backend(runtime) + prepared = _resolve_read_inputs(file_path, runtime.tool_call_id, limit) + if isinstance(prepared, ToolMessage): + return prepared + validated_path, backend_limit = prepared + read_result = await backend.aread(validated_path, offset=offset, limit=backend_limit) return _handle_read_result(read_result, validated_path, runtime.tool_call_id, offset, limit) return StructuredTool.from_function( diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json index ca959c7457d..1b3f93540e2 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default 100 s, no upper clamp \u2014 set `limit` explicitly to widen the window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows.\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s; sampling rate configured on `FilesystemMiddleware`). Detailed semantics live in the deepagents docs.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { @@ -77,7 +77,7 @@ } ], "default": null, - "description": "Maximum number of lines to read for text files. Omit to use the default window: 100 text lines or 30 video seconds." + "description": "Maximum number of lines to read for text files. Omit to use the default window." }, "offset": { "default": 0, diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json index d21e8989909..17826903909 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default 100 s, no upper clamp \u2014 set `limit` explicitly to widen the window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows.\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s; sampling rate configured on `FilesystemMiddleware`). Detailed semantics live in the deepagents docs.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { @@ -77,7 +77,7 @@ } ], "default": null, - "description": "Maximum number of lines to read for text files. Omit to use the default window: 100 text lines or 30 video seconds." + "description": "Maximum number of lines to read for text files. Omit to use the default window." }, "offset": { "default": 0, diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json index ca959c7457d..1b3f93540e2 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default 100 s, no upper clamp \u2014 set `limit` explicitly to widen the window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows.\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s; sampling rate configured on `FilesystemMiddleware`). Detailed semantics live in the deepagents docs.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { @@ -77,7 +77,7 @@ } ], "default": null, - "description": "Maximum number of lines to read for text files. Omit to use the default window: 100 text lines or 30 video seconds." + "description": "Maximum number of lines to read for text files. Omit to use the default window." }, "offset": { "default": 0, diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json index d21e8989909..17826903909 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default 100 s, no upper clamp \u2014 set `limit` explicitly to widen the window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows.\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s; sampling rate configured on `FilesystemMiddleware`). Detailed semantics live in the deepagents docs.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { @@ -77,7 +77,7 @@ } ], "default": null, - "description": "Maximum number of lines to read for text files. Omit to use the default window: 100 text lines or 30 video seconds." + "description": "Maximum number of lines to read for text files. Omit to use the default window." }, "offset": { "default": 0, diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json index bcd6c693c57..c05f7a6ac04 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default 100 s, no upper clamp \u2014 set `limit` explicitly to widen the window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows.\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s; sampling rate configured on `FilesystemMiddleware`). Detailed semantics live in the deepagents docs.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { @@ -77,7 +77,7 @@ } ], "default": null, - "description": "Maximum number of lines to read for text files. Omit to use the default window: 100 text lines or 30 video seconds." + "description": "Maximum number of lines to read for text files. Omit to use the default window." }, "offset": { "default": 0, diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json index ca959c7457d..1b3f93540e2 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset` is interpreted as seconds into the source to skip, `limit` as seconds of source to sample (default 100 s, no upper clamp \u2014 set `limit` explicitly to widen the window). Frames are sampled at the rate configured by `FilesystemMiddleware(video_sampling_rate=...)` (default 0.5 fps). Use `offset` to read later windows.\n- Sandbox-backed video reads currently inherit the backend's binary preview size cap; larger sandbox videos require follow-up backend work.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s; sampling rate configured on `FilesystemMiddleware`). Detailed semantics live in the deepagents docs.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { @@ -77,7 +77,7 @@ } ], "default": null, - "description": "Maximum number of lines to read for text files. Omit to use the default window: 100 text lines or 30 video seconds." + "description": "Maximum number of lines to read for text files. Omit to use the default window." }, "offset": { "default": 0, diff --git a/libs/deepagents/tests/unit_tests/test_middleware.py b/libs/deepagents/tests/unit_tests/test_middleware.py index 285f58e6434..7bb00cfcbb2 100644 --- a/libs/deepagents/tests/unit_tests/test_middleware.py +++ b/libs/deepagents/tests/unit_tests/test_middleware.py @@ -1491,15 +1491,7 @@ def test_read_file_video_routes_to_frame_extractor(self, monkeypatch: pytest.Mon reads, return interleaved text+image content blocks, and never leak the raw video bytes to the model. """ - - class VideoBackend(StateBackend): - def __init__(self, raw: bytes) -> None: - super().__init__() - self._raw = raw - - def read(self, path, *, offset=0, limit=100): # type: ignore[override] - return ReadResult(file_data={"content": base64.b64encode(self._raw).decode("ascii"), "encoding": "base64"}) - + raw_bytes = b"\x00\x01\x02 fake video bytes" sentinel = [ {"type": "text", "text": "Frame at t=00:00:00.000"}, {"type": "image", "base64": "AAAA", "mime_type": "image/jpeg"}, @@ -1525,7 +1517,7 @@ def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) middleware = FilesystemMiddleware( - backend=VideoBackend(b"\x00\x01\x02 fake video bytes"), + backend=_video_backend(raw_bytes), video_sampling_rate=0.5, ) state = FilesystemState(messages=[], files={}) @@ -1548,7 +1540,7 @@ def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): # the extracted frames, not the original base64 from the backend. assert isinstance(result.content, list) assert result.content == [ - {"type": "text", "text": "Reading [0.000s, 30.000s) of /clips/intro.mkv at 0.5 fps."}, + {"type": "text", "text": "Reading first 30s of /clips/intro.mkv at 0.5 fps."}, *sentinel, ] assert b"\x00\x01\x02" not in str(result.content).encode() @@ -1560,15 +1552,10 @@ def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): assert call["offset_seconds"] == 0 assert call["duration_seconds"] == 30 assert call["sampling_rate"] == 0.5 - assert call["len"] == len(b"\x00\x01\x02 fake video bytes") + assert call["len"] == len(raw_bytes) def test_read_file_video_offset_and_limit_reinterpreted_as_seconds(self, monkeypatch: pytest.MonkeyPatch) -> None: """`offset` skips into the source and the agent-supplied `limit` is authoritative.""" - - class VideoBackend(StateBackend): - def read(self, path, *, offset=0, limit=100): # type: ignore[override] - return ReadResult(file_data={"content": base64.b64encode(b"raw").decode("ascii"), "encoding": "base64"}) - captured: dict[str, float] = {} def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): # type: ignore[no-untyped-def] # noqa: ARG001 @@ -1576,7 +1563,7 @@ def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): return [{"type": "text", "text": "ok"}] monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) - middleware = FilesystemMiddleware(backend=VideoBackend()) + middleware = FilesystemMiddleware(backend=_video_backend()) state = FilesystemState(messages=[], files={}) runtime = _build_runtime(state, "video-read-2") read_file_tool = next(t for t in middleware.tools if t.name == "read_file") @@ -1594,11 +1581,6 @@ def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): def test_read_file_video_omitted_limit_uses_video_default(self, monkeypatch: pytest.MonkeyPatch) -> None: """Omitting `limit` samples the first 100 seconds (video default) for video reads.""" - - class VideoBackend(StateBackend): - def read(self, path, *, offset=0, limit=100): # type: ignore[override] - return ReadResult(file_data={"content": base64.b64encode(b"raw").decode("ascii"), "encoding": "base64"}) - captured: dict[str, float] = {} def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): # type: ignore[no-untyped-def] # noqa: ARG001 @@ -1606,7 +1588,7 @@ def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): return [{"type": "text", "text": "ok"}] monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) - middleware = FilesystemMiddleware(backend=VideoBackend()) + middleware = FilesystemMiddleware(backend=_video_backend()) state = FilesystemState(messages=[], files={}) runtime = _build_runtime(state, "video-read-default") read_file_tool = next(t for t in middleware.tools if t.name == "read_file") @@ -1622,17 +1604,12 @@ def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): def test_read_file_video_extraction_error_surfaces_as_error_message(self, monkeypatch: pytest.MonkeyPatch) -> None: """PyAV failures and missing-dep errors render as a tool error, not an exception.""" - - class VideoBackend(StateBackend): - def read(self, path, *, offset=0, limit=100): # type: ignore[override] - return ReadResult(file_data={"content": base64.b64encode(b"corrupt").decode("ascii"), "encoding": "base64"}) - monkeypatch.setattr( filesystem_middleware, "extract_video_frames", lambda *_args, **_kwargs: (_ for _ in ()).throw(filesystem_middleware.VideoExtractionError("av not installed")), ) - middleware = FilesystemMiddleware(backend=VideoBackend()) + middleware = FilesystemMiddleware(backend=_video_backend(b"corrupt")) state = FilesystemState(messages=[], files={}) runtime = _build_runtime(state, "video-read-err") read_file_tool = next(t for t in middleware.tools if t.name == "read_file") @@ -1643,6 +1620,27 @@ def read(self, path, *, offset=0, limit=100): # type: ignore[override] assert result.status == "error" assert "av not installed" in result.content # type: ignore[operator] + def test_read_file_video_non_positive_limit_rejected(self, monkeypatch: pytest.MonkeyPatch) -> None: + """A non-positive `limit` is rejected as a tool error, not silently clamped.""" + called = {"count": 0} + + def fake_extract(*args: "object", **kwargs: "object") -> "list[dict[str, object]]": # noqa: ARG001 + called["count"] += 1 + return [{"type": "text", "text": "ok"}] + + monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) + middleware = FilesystemMiddleware(backend=_video_backend()) + state = FilesystemState(messages=[], files={}) + runtime = _build_runtime(state, "video-read-bad-limit") + read_file_tool = next(t for t in middleware.tools if t.name == "read_file") + + result = read_file_tool.invoke({"file_path": "/c.mp4", "limit": 0, "runtime": runtime}) + + assert isinstance(result, ToolMessage) + assert result.status == "error" + assert "limit must be > 0" in result.content # type: ignore[operator] + assert called["count"] == 0 + def test_read_file_image_returns_standard_image_content_block(self): """Test image reads return standard image blocks with base64 + mime_type.""" @@ -2768,3 +2766,14 @@ def _build_runtime(state: FilesystemState, tool_call_id: str) -> ToolRuntime: stream_writer=lambda _: None, config={}, ) + + +def _video_backend(raw: bytes = b"raw") -> StateBackend: + """Backend stub that returns `raw` base64-encoded for any video read.""" + payload = base64.b64encode(raw).decode("ascii") + + class VideoBackend(StateBackend): + def read(self, path, *, offset=0, limit=100): # type: ignore[override] + return ReadResult(file_data={"content": payload, "encoding": "base64"}) + + return VideoBackend() From 17075e92624731961f0778972c4d32dfb53997ea Mon Sep 17 00:00:00 2001 From: Deep Agent Date: Sun, 21 Jun 2026 20:49:45 -0700 Subject: [PATCH 09/36] fix(sdk): tighten video read bounds and schema --- libs/deepagents/README.md | 5 +- .../deepagents/backends/filesystem.py | 29 ++++--- libs/deepagents/deepagents/backends/utils.py | 4 +- .../deepagents/middleware/_video.py | 8 +- .../deepagents/middleware/filesystem.py | 47 +++++------ .../backends/test_filesystem_backend.py | 13 +++ .../tests/unit_tests/middleware/test_video.py | 20 +++++ .../custom_system_message_tools.json | 13 +-- .../system_prompt_with_execute_tools.json | 13 +-- ...m_prompt_with_memory_and_skills_tools.json | 13 +-- ...stem_prompt_with_routed_backend_tools.json | 13 +-- ...t_with_sync_and_async_subagents_tools.json | 13 +-- .../system_prompt_without_execute_tools.json | 13 +-- .../tests/unit_tests/test_middleware.py | 81 +++++++++++-------- 14 files changed, 147 insertions(+), 138 deletions(-) diff --git a/libs/deepagents/README.md b/libs/deepagents/README.md index 90c4ec79bef..964df8116c3 100644 --- a/libs/deepagents/README.md +++ b/libs/deepagents/README.md @@ -20,8 +20,9 @@ uv add deepagents `read_file` can sample video files into image content blocks (`.mp4`, `.mov`, `.webm`, `.mkv`, etc.). The agent uses `offset` to skip seconds into the source and `limit` to control how many -seconds of source to sample, capped at 30 seconds per read; the frames-per-second is -a deploy-time knob. +seconds of source to sample; the frames-per-second is a deploy-time knob. Video reads +are bounded by fixed input size, decode time, frame count, frame size, and emitted-byte +budgets. Install with: ```bash diff --git a/libs/deepagents/deepagents/backends/filesystem.py b/libs/deepagents/deepagents/backends/filesystem.py index 3d1382fb274..1fbef959d5f 100644 --- a/libs/deepagents/deepagents/backends/filesystem.py +++ b/libs/deepagents/deepagents/backends/filesystem.py @@ -37,6 +37,7 @@ WriteResult, ) from deepagents.backends.utils import ( + MAX_VIDEO_INPUT_BYTES, _get_file_type, check_empty_content, perform_string_replacement, @@ -434,15 +435,25 @@ def read( return ReadResult(error=f"File '{file_path}' not found") fd = os.open(resolved_path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) - if _get_file_type(file_path) != "text": - with os.fdopen(fd, "rb") as f: - raw = f.read() - encoded = base64.standard_b64encode(raw).decode("ascii") - file_data = FileData(content=encoded, encoding="base64") - else: - with os.fdopen(fd, "r", encoding="utf-8") as f: - content = f.read() - + try: + file_type = _get_file_type(file_path) + if file_type != "text": + if file_type == "video" and os.fstat(fd).st_size > MAX_VIDEO_INPUT_BYTES: + return ReadResult(error=f"Video file exceeds maximum input size of {MAX_VIDEO_INPUT_BYTES} bytes") + with os.fdopen(fd, "rb") as f: + fd = -1 + raw = f.read() + encoded = base64.standard_b64encode(raw).decode("ascii") + file_data = FileData(content=encoded, encoding="base64") + else: + with os.fdopen(fd, "r", encoding="utf-8") as f: + fd = -1 + content = f.read() + finally: + if fd >= 0: + os.close(fd) + + if file_type == "text": empty_msg = check_empty_content(content) if empty_msg: file_data = FileData(content=empty_msg, encoding="utf-8") diff --git a/libs/deepagents/deepagents/backends/utils.py b/libs/deepagents/deepagents/backends/utils.py index fda00cc0abc..895f4a7a7e9 100644 --- a/libs/deepagents/deepagents/backends/utils.py +++ b/libs/deepagents/deepagents/backends/utils.py @@ -11,7 +11,7 @@ from collections.abc import Sequence from datetime import UTC, datetime from pathlib import Path, PurePosixPath -from typing import Any, Literal, overload +from typing import Any, Final, Literal, overload import wcmatch.glob as wcglob @@ -19,6 +19,8 @@ from deepagents.backends.protocol import FileData, FileInfo as _FileInfo, GrepMatch as _GrepMatch, GrepResult, ReadResult EMPTY_CONTENT_WARNING = "System reminder: File exists but has empty contents" +MAX_VIDEO_INPUT_BYTES: Final = 50 * 1024 * 1024 +"""Maximum raw video payload size accepted by `read_file` frame extraction.""" FileType = Literal["text", "image", "audio", "video", "file"] """Classification of a file by extension.""" diff --git a/libs/deepagents/deepagents/middleware/_video.py b/libs/deepagents/deepagents/middleware/_video.py index daf8a21816b..53cfd2cef61 100644 --- a/libs/deepagents/deepagents/middleware/_video.py +++ b/libs/deepagents/deepagents/middleware/_video.py @@ -18,6 +18,7 @@ import base64 import io +import math import time from typing import TYPE_CHECKING, Any, Final @@ -297,8 +298,11 @@ def _sample_frames_in_window( emitted_bytes += next_block_bytes if emitted_frames >= MAX_VIDEO_SAMPLED_FRAMES: break - emitted_index = round((frame_seconds - offset_seconds) / frame_interval_seconds) + 1 - next_emit_seconds = offset_seconds + frame_interval_seconds * emitted_index + emitted_index = math.floor((frame_seconds - offset_seconds) / frame_interval_seconds) + 1 + next_emit_seconds = max( + next_emit_seconds + frame_interval_seconds, + offset_seconds + frame_interval_seconds * emitted_index, + ) except decode_error_types as exc: msg = f"Failed to decode video frames: {exc}" raise VideoExtractionError(msg) from exc diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index 92375d47e9c..a926d39671b 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -59,6 +59,7 @@ execute_accepts_timeout, ) from deepagents.backends.utils import ( + MAX_VIDEO_INPUT_BYTES, _get_file_type, _glob_anchor, _paths_overlap, @@ -97,12 +98,6 @@ "delete": "write", } -# Reinterpret offset/limit as seconds for video reads. The agent's `limit` -# is authoritative; this is only the fallback when the agent omits the -# argument. Mirrors `DEFAULT_READ_LIMIT` (the text-read default). -_VIDEO_DEFAULT_LIMIT_SECONDS = 100 - - def _tool_error(name: str, tool_call_id: str | None, content: str) -> ToolMessage: """Build a success-shaped `ToolMessage` carrying a plain text error.""" return ToolMessage(content=content, name=name, tool_call_id=tool_call_id, status="error") @@ -113,27 +108,26 @@ def _handle_video_read( validated_path: str, tool_call_id: str | None, offset: int, - limit: int | None, + limit: int, sampling_rate: float, ) -> ToolMessage: """Slice a video byte payload into a sampled frame window for the model. `offset` is reinterpreted as seconds into the source to skip; `limit` as seconds of source to sample. The agent's supplied `limit` is authoritative - (no per-call upper clamp); omitting it falls back to - `_VIDEO_DEFAULT_LIMIT_SECONDS` and supplying a non-positive value is - rejected as a tool error. Output volume is bounded by the layered caps on - the extractor (`MAX_VIDEO_DECODE_SECONDS`, `MAX_VIDEO_SAMPLED_FRAMES`, + (no per-call upper clamp), and supplying a non-positive value is rejected + as a tool error. Output volume is bounded by the layered caps on the + extractor (`MAX_VIDEO_DECODE_SECONDS`, `MAX_VIDEO_SAMPLED_FRAMES`, `MAX_VIDEO_EMITTED_BYTES`, `MAX_VIDEO_FRAME_PIXELS`, `MAX_VIDEO_FRAME_SIDE`). Errors are returned as `ToolMessage` errors so the turn still completes and the agent can recover (e.g. by retrying with a smaller window). """ - if limit is not None and limit <= 0: + if limit <= 0: return _tool_error("read_file", tool_call_id, f"Error reading video {validated_path}: limit must be > 0, got {limit!r}") rate = float(sampling_rate) offset_seconds = max(0.0, float(offset)) - duration_seconds = float(_VIDEO_DEFAULT_LIMIT_SECONDS if limit is None else limit) + duration_seconds = float(limit) header = _video_window_header(validated_path, offset_seconds, duration_seconds, rate) def _err(msg: str) -> ToolMessage: @@ -143,6 +137,8 @@ def _err(msg: str) -> ToolMessage: raw_bytes = base64.b64decode(content, validate=True) if isinstance(content, str) else content except (ValueError, TypeError, binascii.Error) as exc: return _err(f"video bytes are not valid base64 ({exc})") + if len(raw_bytes) > MAX_VIDEO_INPUT_BYTES: + return _err(f"video payload exceeds maximum input size of {MAX_VIDEO_INPUT_BYTES} bytes") try: blocks = extract_video_frames( @@ -492,9 +488,9 @@ class ReadFileSchema(BaseModel): description="Line number to start reading from (0-indexed). Use for pagination of large files.", ) - limit: int | None = Field( - default=None, - description="Maximum number of lines to read for text files. Omit to use the default window.", + limit: int = Field( + default=DEFAULT_READ_LIMIT, + description="Maximum number of lines to read for text files. For videos, seconds of source to sample.", ) @@ -1237,9 +1233,8 @@ def _handle_read_result( # noqa: PLR0911 validated_path: str, tool_call_id: str | None, offset: int, - limit: int | None, + limit: int, ) -> ToolMessage: - effective_limit = DEFAULT_READ_LIMIT if limit is None else limit if isinstance(read_result, str): warn_deprecated( since="0.5.0", @@ -1253,7 +1248,7 @@ def _handle_read_result( # noqa: PLR0911 ) # Legacy backends already format with line numbers return ToolMessage( - content=_truncate(read_result, validated_path, line_limit=effective_limit), + content=_truncate(read_result, validated_path, line_limit=limit), name="read_file", tool_call_id=tool_call_id, status="success", @@ -1326,7 +1321,7 @@ def _handle_read_result( # noqa: PLR0911 def _resolve_read_inputs( file_path: str, tool_call_id: str | None, - limit: int | None, + limit: int, ) -> tuple[str, int] | ToolMessage: """Validate path + permission and resolve the effective backend limit. @@ -1339,16 +1334,13 @@ def _resolve_read_inputs( return _tool_error("read_file", tool_call_id, f"Error: {e}") if _check_fs_permission(self._permissions, "read", validated_path) == "deny": return _tool_error("read_file", tool_call_id, f"Error: permission denied for read on {validated_path}") - return validated_path, DEFAULT_READ_LIMIT if limit is None else limit + return validated_path, limit def sync_read_file( file_path: Annotated[str, "Absolute path to the file to read. Must be absolute, not relative."], runtime: ToolRuntime[None, FilesystemState], offset: Annotated[int, "Line number to start reading from (0-indexed). Use for pagination of large files."] = DEFAULT_READ_OFFSET, - limit: Annotated[ - int | None, - "Maximum number of lines to read for text files. Omit to use the default window.", - ] = None, + limit: Annotated[int, "Maximum number of lines to read for text files. For videos, seconds of source to sample."] = DEFAULT_READ_LIMIT, ) -> ToolMessage: """Synchronous wrapper for read_file tool.""" backend = self._get_backend(runtime) @@ -1363,10 +1355,7 @@ async def async_read_file( file_path: Annotated[str, "Absolute path to the file to read. Must be absolute, not relative."], runtime: ToolRuntime[None, FilesystemState], offset: Annotated[int, "Line number to start reading from (0-indexed). Use for pagination of large files."] = DEFAULT_READ_OFFSET, - limit: Annotated[ - int | None, - "Maximum number of lines to read for text files. Omit to use the default window.", - ] = None, + limit: Annotated[int, "Maximum number of lines to read for text files. For videos, seconds of source to sample."] = DEFAULT_READ_LIMIT, ) -> ToolMessage: """Asynchronous wrapper for read_file tool.""" backend = self._get_backend(runtime) diff --git a/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py b/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py index 43088d4483c..8e9af57e0aa 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py +++ b/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py @@ -257,6 +257,19 @@ def test_filesystem_backend_read_non_utf8_file(tmp_path: Path): assert "chinese.txt" in result.error +def test_filesystem_backend_rejects_oversized_video_before_read(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Local video reads fail before loading oversized files into memory.""" + target = tmp_path / "clip.mp4" + target.write_bytes(b"abcd") + monkeypatch.setattr(fs_module, "MAX_VIDEO_INPUT_BYTES", 3) + + be = FilesystemBackend(root_dir=str(tmp_path), virtual_mode=False) + result = be.read(str(target)) + + assert isinstance(result, ReadResult) + assert result.error == "Video file exceeds maximum input size of 3 bytes" + + def test_filesystem_backend_intercept_large_tool_result(tmp_path: Path): """Test that FilesystemBackend properly handles large tool result interception.""" root = tmp_path diff --git a/libs/deepagents/tests/unit_tests/middleware/test_video.py b/libs/deepagents/tests/unit_tests/middleware/test_video.py index b176a63eab9..37bb840c2bd 100644 --- a/libs/deepagents/tests/unit_tests/middleware/test_video.py +++ b/libs/deepagents/tests/unit_tests/middleware/test_video.py @@ -161,6 +161,26 @@ def test_sample_frames_normalizes_non_zero_stream_start(monkeypatch: pytest.Monk ] +def test_sample_frames_keeps_next_target_after_late_frame(monkeypatch: pytest.MonkeyPatch) -> None: + """A late frame should not skip the next sampling target.""" + monkeypatch.setattr(video_module, "_encode_jpeg", lambda _frame: b"jpeg") + + blocks = video_module._sample_frames_in_window( + [_FakeFrame(1), _FakeFrame(2), _FakeFrame(3)], + offset_seconds=0.4, + duration_seconds=4, + sampling_rate=1, + time_base=1, + ) + + headers = [block["text"] for block in blocks if block["type"] == "text"] + assert headers == [ + "Frame at t=00:00:01.000", + "Frame at t=00:00:02.000", + "Frame at t=00:00:03.000", + ] + + def test_sample_frames_does_not_cap_requested_duration(monkeypatch: pytest.MonkeyPatch) -> None: """Huge requested windows are passed through unchanged. diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json index 1b3f93540e2..ca94d010d50 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json @@ -68,16 +68,9 @@ "type": "string" }, "limit": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Maximum number of lines to read for text files. Omit to use the default window." + "default": 100, + "description": "Maximum number of lines to read for text files. For videos, seconds of source to sample.", + "type": "integer" }, "offset": { "default": 0, diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json index 17826903909..31ed5424ddb 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json @@ -68,16 +68,9 @@ "type": "string" }, "limit": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Maximum number of lines to read for text files. Omit to use the default window." + "default": 100, + "description": "Maximum number of lines to read for text files. For videos, seconds of source to sample.", + "type": "integer" }, "offset": { "default": 0, diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json index 1b3f93540e2..ca94d010d50 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json @@ -68,16 +68,9 @@ "type": "string" }, "limit": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Maximum number of lines to read for text files. Omit to use the default window." + "default": 100, + "description": "Maximum number of lines to read for text files. For videos, seconds of source to sample.", + "type": "integer" }, "offset": { "default": 0, diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json index 17826903909..31ed5424ddb 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json @@ -68,16 +68,9 @@ "type": "string" }, "limit": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Maximum number of lines to read for text files. Omit to use the default window." + "default": 100, + "description": "Maximum number of lines to read for text files. For videos, seconds of source to sample.", + "type": "integer" }, "offset": { "default": 0, diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json index c05f7a6ac04..c9cc109214c 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json @@ -68,16 +68,9 @@ "type": "string" }, "limit": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Maximum number of lines to read for text files. Omit to use the default window." + "default": 100, + "description": "Maximum number of lines to read for text files. For videos, seconds of source to sample.", + "type": "integer" }, "offset": { "default": 0, diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json index 1b3f93540e2..ca94d010d50 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json @@ -68,16 +68,9 @@ "type": "string" }, "limit": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Maximum number of lines to read for text files. Omit to use the default window." + "default": 100, + "description": "Maximum number of lines to read for text files. For videos, seconds of source to sample.", + "type": "integer" }, "offset": { "default": 0, diff --git a/libs/deepagents/tests/unit_tests/test_middleware.py b/libs/deepagents/tests/unit_tests/test_middleware.py index 7bb00cfcbb2..bfe12d12113 100644 --- a/libs/deepagents/tests/unit_tests/test_middleware.py +++ b/libs/deepagents/tests/unit_tests/test_middleware.py @@ -1554,33 +1554,27 @@ def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): assert call["sampling_rate"] == 0.5 assert call["len"] == len(raw_bytes) - def test_read_file_video_offset_and_limit_reinterpreted_as_seconds(self, monkeypatch: pytest.MonkeyPatch) -> None: - """`offset` skips into the source and the agent-supplied `limit` is authoritative.""" - captured: dict[str, float] = {} - - def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): # type: ignore[no-untyped-def] # noqa: ARG001 - captured.update(offset_seconds=offset_seconds, duration_seconds=duration_seconds, sampling_rate=sampling_rate) - return [{"type": "text", "text": "ok"}] - - monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) - middleware = FilesystemMiddleware(backend=_video_backend()) - state = FilesystemState(messages=[], files={}) - runtime = _build_runtime(state, "video-read-2") - read_file_tool = next(t for t in middleware.tools if t.name == "read_file") - - read_file_tool.invoke( - { - "file_path": "/c.mp4", - "offset": 12, # -> seek 12 seconds into the source - "limit": 90, # -> sample a 90-second window, no per-call clamp - "runtime": runtime, - } - ) - - assert captured == {"offset_seconds": 12.0, "duration_seconds": 90.0, "sampling_rate": 0.5} - - def test_read_file_video_omitted_limit_uses_video_default(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Omitting `limit` samples the first 100 seconds (video default) for video reads.""" + @pytest.mark.parametrize( + ("tool_input", "expected"), + [ + ( + {"offset": 12, "limit": 90}, + {"offset_seconds": 12.0, "duration_seconds": 90.0, "sampling_rate": 0.5}, + ), + ( + {}, + {"offset_seconds": 0.0, "duration_seconds": 100.0, "sampling_rate": 0.5}, + ), + ], + ids=["explicit-window", "default-window"], + ) + def test_read_file_video_window_forwards_seconds( + self, + monkeypatch: pytest.MonkeyPatch, + tool_input: dict[str, int], + expected: dict[str, float], + ) -> None: + """`offset`/`limit` become seconds, with the default window preserved.""" captured: dict[str, float] = {} def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): # type: ignore[no-untyped-def] # noqa: ARG001 @@ -1590,17 +1584,12 @@ def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) middleware = FilesystemMiddleware(backend=_video_backend()) state = FilesystemState(messages=[], files={}) - runtime = _build_runtime(state, "video-read-default") + runtime = _build_runtime(state, "video-read-window") read_file_tool = next(t for t in middleware.tools if t.name == "read_file") - read_file_tool.invoke( - { - "file_path": "/c.mp4", - "runtime": runtime, - } - ) + read_file_tool.invoke({"file_path": "/c.mp4", "runtime": runtime, **tool_input}) - assert captured == {"offset_seconds": 0.0, "duration_seconds": 100.0, "sampling_rate": 0.5} + assert captured == expected def test_read_file_video_extraction_error_surfaces_as_error_message(self, monkeypatch: pytest.MonkeyPatch) -> None: """PyAV failures and missing-dep errors render as a tool error, not an exception.""" @@ -1641,6 +1630,28 @@ def fake_extract(*args: "object", **kwargs: "object") -> "list[dict[str, object] assert "limit must be > 0" in result.content # type: ignore[operator] assert called["count"] == 0 + def test_read_file_video_payload_size_cap_rejected(self, monkeypatch: pytest.MonkeyPatch) -> None: + """State/store/custom video payloads are capped after base64 decoding.""" + called = {"count": 0} + + def fake_extract(*args: "object", **kwargs: "object") -> "list[dict[str, object]]": # noqa: ARG001 + called["count"] += 1 + return [{"type": "text", "text": "ok"}] + + monkeypatch.setattr(filesystem_middleware, "MAX_VIDEO_INPUT_BYTES", 3) + monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) + middleware = FilesystemMiddleware(backend=_video_backend(b"abcd")) + state = FilesystemState(messages=[], files={}) + runtime = _build_runtime(state, "video-read-big-payload") + read_file_tool = next(t for t in middleware.tools if t.name == "read_file") + + result = read_file_tool.invoke({"file_path": "/c.mp4", "runtime": runtime}) + + assert isinstance(result, ToolMessage) + assert result.status == "error" + assert "video payload exceeds maximum input size of 3 bytes" in result.content # type: ignore[operator] + assert called["count"] == 0 + def test_read_file_image_returns_standard_image_content_block(self): """Test image reads return standard image blocks with base64 + mime_type.""" From b2b972dd49b9cf3d7fef0e663f71e4d9a7b9c1cc Mon Sep 17 00:00:00 2001 From: Deep Agent Date: Sun, 21 Jun 2026 21:31:31 -0700 Subject: [PATCH 10/36] fix(sdk): bound video extra dependencies --- libs/deepagents/deepagents/middleware/filesystem.py | 1 + libs/deepagents/pyproject.toml | 6 +++--- libs/deepagents/uv.lock | 8 ++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index a926d39671b..1a6e2018376 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -98,6 +98,7 @@ "delete": "write", } + def _tool_error(name: str, tool_call_id: str | None, content: str) -> ToolMessage: """Build a success-shaped `ToolMessage` carrying a plain text error.""" return ToolMessage(content=content, name=name, tool_call_id=tool_call_id, status="error") diff --git a/libs/deepagents/pyproject.toml b/libs/deepagents/pyproject.toml index 9f8e5f984b1..1eff01b4619 100644 --- a/libs/deepagents/pyproject.toml +++ b/libs/deepagents/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ [project.optional-dependencies] aws = ["langchain-aws>=1.6.1,<2.0.0"] quickjs = ["langchain-quickjs>=0.3.2"] -video = ["av>=17.0.0", "pillow>=10.0.0"] +video = ["av>=17.0.0,<18.0.0", "pillow>=10.0.0,<13.0.0"] [project.urls] Homepage = "https://docs.langchain.com/oss/python/deepagents/overview" @@ -63,8 +63,8 @@ test = [ # Lazy-imported at runtime so the core package does # not depend on them. ty still sees the import names and # needs stubs, so we install them into the type-checker venv. - "av>=17.0.0", - "pillow>=10.0.0", + "av>=17.0.0,<18.0.0", + "pillow>=10.0.0,<13.0.0", ] [build-system] requires = ["setuptools>=82.0.1", "wheel"] diff --git a/libs/deepagents/uv.lock b/libs/deepagents/uv.lock index a9812602d5f..ae48813ebde 100644 --- a/libs/deepagents/uv.lock +++ b/libs/deepagents/uv.lock @@ -564,7 +564,7 @@ test = [ [package.metadata] requires-dist = [ - { name = "av", marker = "extra == 'video'", specifier = ">=17.0.0" }, + { name = "av", marker = "extra == 'video'", specifier = ">=17.0.0,<18.0.0" }, { name = "langchain", specifier = ">=1.3.11,<2.0.0" }, { name = "langchain-anthropic", specifier = ">=1.4.7,<2.0.0" }, { name = "langchain-aws", marker = "extra == 'aws'", specifier = ">=1.6.1,<2.0.0" }, @@ -572,18 +572,18 @@ requires-dist = [ { name = "langchain-google-genai", specifier = ">=4.2.6,<5.0.0" }, { name = "langchain-quickjs", marker = "extra == 'quickjs'", specifier = ">=0.3.2" }, { name = "langsmith", specifier = ">=0.9.3" }, - { name = "pillow", marker = "extra == 'video'", specifier = ">=10.0.0" }, + { name = "pillow", marker = "extra == 'video'", specifier = ">=10.0.0,<13.0.0" }, { name = "wcmatch", specifier = ">=10.1" }, ] provides-extras = ["aws", "quickjs", "video"] [package.metadata.requires-dev] test = [ - { name = "av", specifier = ">=17.0.0" }, + { name = "av", specifier = ">=17.0.0,<18.0.0" }, { name = "build" }, { name = "langchain-openai" }, { name = "langchain-tests", specifier = ">=1.1.9" }, - { name = "pillow", specifier = ">=10.0.0" }, + { name = "pillow", specifier = ">=10.0.0,<13.0.0" }, { name = "pytest" }, { name = "pytest-asyncio", specifier = ">=1.4.0" }, { name = "pytest-benchmark" }, From 08bfe73c0ff73cb52c4ac91feae0a9c062b08f6e Mon Sep 17 00:00:00 2001 From: Deep Agent Date: Mon, 22 Jun 2026 23:59:52 -0700 Subject: [PATCH 11/36] fix(sdk): move video frames out of tool output --- .../deepagents/middleware/filesystem.py | 80 +++++++++++++++++-- .../tests/unit_tests/test_end_to_end.py | 52 ++++++++++++ .../tests/unit_tests/test_middleware.py | 49 ++++++++++-- 3 files changed, 165 insertions(+), 16 deletions(-) diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index 1a6e2018376..c0a4ef10390 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -30,7 +30,7 @@ ) from langchain.tools import ToolRuntime from langchain.tools.tool_node import ToolCallRequest -from langchain_core.messages import AnyMessage, HumanMessage, RemoveMessage, ToolMessage +from langchain_core.messages import AIMessage, AnyMessage, HumanMessage, RemoveMessage, ToolMessage from langchain_core.messages.content import ContentBlock from langchain_core.tools import BaseTool, StructuredTool from langgraph.channels.delta import DeltaChannel @@ -98,12 +98,50 @@ "delete": "write", } +_READ_FILE_MEDIA_RESULT = "read_file_media_result" + def _tool_error(name: str, tool_call_id: str | None, content: str) -> ToolMessage: """Build a success-shaped `ToolMessage` carrying a plain text error.""" return ToolMessage(content=content, name=name, tool_call_id=tool_call_id, status="error") +def _is_read_file_media_result(message: AnyMessage) -> bool: + """Return whether `message` carries media emitted by a `read_file` tool result.""" + return isinstance(message, HumanMessage) and message.additional_kwargs.get(_READ_FILE_MEDIA_RESULT) is True + + +def _move_media_results_after_tool_results(messages: list[AnyMessage]) -> list[AnyMessage]: + """Keep synthetic media messages after the tool-result batch they describe. + + Tool-call providers require every `ToolMessage` for an assistant tool-call + batch to arrive before any non-tool message. Video reads attach sampled + frames as a synthetic `HumanMessage`; when multiple tools run in the same + turn this helper keeps those attachments behind the full batch. + """ + reordered: list[AnyMessage] = [] + i = 0 + while i < len(messages): + message = messages[i] + reordered.append(message) + i += 1 + if not isinstance(message, AIMessage) or not message.tool_calls: + continue + + batch: list[AnyMessage] = [] + while i < len(messages): + next_message = messages[i] + if isinstance(next_message, ToolMessage) or _is_read_file_media_result(next_message): + batch.append(next_message) + i += 1 + continue + break + if batch: + reordered.extend(message for message in batch if isinstance(message, ToolMessage)) + reordered.extend(message for message in batch if _is_read_file_media_result(message)) + return reordered + + def _handle_video_read( content: str, validated_path: str, @@ -111,7 +149,7 @@ def _handle_video_read( offset: int, limit: int, sampling_rate: float, -) -> ToolMessage: +) -> ToolMessage | Command: """Slice a video byte payload into a sampled frame window for the model. `offset` is reinterpreted as seconds into the source to skip; `limit` as @@ -151,13 +189,31 @@ def _err(msg: str) -> ToolMessage: except VideoExtractionError as exc: return _err(str(exc)) blocks.insert(0, {"type": "text", "text": header}) - return ToolMessage( - content_blocks=blocks, + frame_count = sum(1 for block in blocks if isinstance(block, dict) and block.get("type") == "image") + frame_label = "frame" if frame_count == 1 else "frames" + tool_message = ToolMessage( + content=f"Read video {validated_path}: sampled {frame_count} {frame_label}. The sampled frames are attached in the following message.", name="read_file", tool_call_id=tool_call_id, - additional_kwargs={"read_file_path": validated_path}, + additional_kwargs={"read_file_path": validated_path, "read_file_frame_count": frame_count}, status="success", ) + media_message = HumanMessage( + content_blocks=blocks, + additional_kwargs={ + _READ_FILE_MEDIA_RESULT: True, + "read_file_path": validated_path, + "read_file_tool_call_id": tool_call_id, + }, + ) + return Command( + update={ + "messages": [ + tool_message, + media_message, + ], + } + ) def _video_window_header(path: str, offset_seconds: float, duration_seconds: float, rate: float) -> str: @@ -1235,7 +1291,7 @@ def _handle_read_result( # noqa: PLR0911 tool_call_id: str | None, offset: int, limit: int, - ) -> ToolMessage: + ) -> ToolMessage | Command: if isinstance(read_result, str): warn_deprecated( since="0.5.0", @@ -1342,7 +1398,7 @@ def sync_read_file( runtime: ToolRuntime[None, FilesystemState], offset: Annotated[int, "Line number to start reading from (0-indexed). Use for pagination of large files."] = DEFAULT_READ_OFFSET, limit: Annotated[int, "Maximum number of lines to read for text files. For videos, seconds of source to sample."] = DEFAULT_READ_LIMIT, - ) -> ToolMessage: + ) -> ToolMessage | Command: """Synchronous wrapper for read_file tool.""" backend = self._get_backend(runtime) prepared = _resolve_read_inputs(file_path, runtime.tool_call_id, limit) @@ -1357,7 +1413,7 @@ async def async_read_file( runtime: ToolRuntime[None, FilesystemState], offset: Annotated[int, "Line number to start reading from (0-indexed). Use for pagination of large files."] = DEFAULT_READ_OFFSET, limit: Annotated[int, "Maximum number of lines to read for text files. For videos, seconds of source to sample."] = DEFAULT_READ_LIMIT, - ) -> ToolMessage: + ) -> ToolMessage | Command: """Asynchronous wrapper for read_file tool.""" backend = self._get_backend(runtime) prepared = _resolve_read_inputs(file_path, runtime.tool_call_id, limit) @@ -2202,6 +2258,10 @@ def wrap_model_call( """ request = self._filter_unsupported_tools_and_apply_prompt(request) + request_messages = _move_media_results_after_tool_results(list(request.messages)) + if request_messages != list(request.messages): + request = request.override(messages=request_messages) + eviction_result = self._evict_and_truncate_messages(request) if eviction_result is not None: messages, state_command = eviction_result @@ -2233,6 +2293,10 @@ async def awrap_model_call( """ request = self._filter_unsupported_tools_and_apply_prompt(request) + request_messages = _move_media_results_after_tool_results(list(request.messages)) + if request_messages != list(request.messages): + request = request.override(messages=request_messages) + eviction_result = await self._aevict_and_truncate_messages(request) if eviction_result is not None: messages, state_command = eviction_result diff --git a/libs/deepagents/tests/unit_tests/test_end_to_end.py b/libs/deepagents/tests/unit_tests/test_end_to_end.py index fdf466b1015..5ead4b46ed6 100644 --- a/libs/deepagents/tests/unit_tests/test_end_to_end.py +++ b/libs/deepagents/tests/unit_tests/test_end_to_end.py @@ -26,6 +26,7 @@ from langgraph.types import Command from pydantic import Field +import deepagents.middleware.filesystem as filesystem_middleware from deepagents.backends import CompositeBackend, FilesystemBackend from deepagents.backends.protocol import BackendProtocol, ExecuteResponse, SandboxBackendProtocol from deepagents.backends.state import StateBackend @@ -3948,6 +3949,57 @@ def agent_b_node(_state: dict) -> dict: assert visited == ["agent_b"] +def test_read_file_video_frames_attached_after_tool_message(monkeypatch: pytest.MonkeyPatch) -> None: + """Video frame media is sent as input after the required tool result.""" + monkeypatch.setattr( + filesystem_middleware, + "extract_video_frames", + lambda *_args, **_kwargs: [ + {"type": "text", "text": "Frame at t=00:00:00.000"}, + {"type": "image", "base64": "AAAA", "mime_type": "image/jpeg"}, + ], + ) + fake_model = FakeChatModelWithHistory( + messages=iter( + [ + AIMessage( + content="", + tool_calls=[ + { + "id": "call_video", + "name": "read_file", + "args": {"file_path": "/clip.mp4"}, + } + ], + ), + AIMessage(content="saw frame"), + ] + ) + ) + fake_model.call_history.clear() + video = base64.b64encode(b"video bytes").decode("ascii") + agent = create_deep_agent(model=fake_model, backend=StateBackend()) + + result = agent.invoke( + { + "messages": [HumanMessage(content="Read the video")], + "files": {"/clip.mp4": {**create_file_data(video, encoding="base64")}}, + } + ) + + assert result["messages"][-1].content == "saw frame" + second_call = fake_model.call_history[1]["messages"] + tool_index = next(i for i, message in enumerate(second_call) if isinstance(message, ToolMessage)) + media_message = second_call[tool_index + 1] + assert isinstance(media_message, HumanMessage) + assert media_message.additional_kwargs["read_file_media_result"] is True + assert media_message.content == [ + {"type": "text", "text": "Reading first 100s of /clip.mp4 at 0.5 fps."}, + {"type": "text", "text": "Frame at t=00:00:00.000"}, + {"type": "image", "base64": "AAAA", "mime_type": "image/jpeg"}, + ] + + def test_invalid_tool_call_patched_on_next_turn() -> None: # Turn 1: model truncates and emits an invalid tool call (no matching ToolMessage # will be produced because agents only route on `tool_calls`). diff --git a/libs/deepagents/tests/unit_tests/test_middleware.py b/libs/deepagents/tests/unit_tests/test_middleware.py index bfe12d12113..656be6a159c 100644 --- a/libs/deepagents/tests/unit_tests/test_middleware.py +++ b/libs/deepagents/tests/unit_tests/test_middleware.py @@ -1533,17 +1533,26 @@ def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): } ) - assert isinstance(result, ToolMessage) - assert result.status == "success" - assert result.additional_kwargs["read_file_path"] == "/clips/intro.mkv" - # Raw video bytes must NOT leak to the model: the content_blocks are - # the extracted frames, not the original base64 from the backend. - assert isinstance(result.content, list) - assert result.content == [ + assert isinstance(result, Command) + messages = result.update["messages"] # type: ignore[index] + assert len(messages) == 2 + tool_message, media_message = messages + assert isinstance(tool_message, ToolMessage) + assert tool_message.status == "success" + assert tool_message.content == "Read video /clips/intro.mkv: sampled 2 frames. The sampled frames are attached in the following message." + assert tool_message.additional_kwargs["read_file_path"] == "/clips/intro.mkv" + assert tool_message.additional_kwargs["read_file_frame_count"] == 2 + assert isinstance(media_message, HumanMessage) + assert media_message.additional_kwargs["read_file_media_result"] is True + assert media_message.additional_kwargs["read_file_path"] == "/clips/intro.mkv" + assert media_message.additional_kwargs["read_file_tool_call_id"] == "video-read-1" + # Raw video bytes must NOT leak to the model: the media message contains + # extracted frames, not the original base64 from the backend. + assert media_message.content == [ {"type": "text", "text": "Reading first 30s of /clips/intro.mkv at 0.5 fps."}, *sentinel, ] - assert b"\x00\x01\x02" not in str(result.content).encode() + assert b"\x00\x01\x02" not in str(media_message.content).encode() # The middleware must convert offset/limit to seconds with the # constructor-defined sampling rate forwarded verbatim. @@ -1591,6 +1600,30 @@ def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): assert captured == expected + def test_read_file_video_media_result_ordered_after_parallel_tool_results(self) -> None: + """Video frame attachments do not split a provider-required tool result batch.""" + ai_message = AIMessage( + content="", + tool_calls=[ + {"id": "call_video", "name": "read_file", "args": {"file_path": "/c.mp4"}}, + {"id": "call_ls", "name": "ls", "args": {"path": "/"}}, + ], + ) + video_tool = ToolMessage(content="sampled frames", name="read_file", tool_call_id="call_video") + video_media = HumanMessage( + content=[ + {"type": "text", "text": "Reading first 100s of /c.mp4 at 0.5 fps."}, + {"type": "image", "base64": "AAAA", "mime_type": "image/jpeg"}, + ], + additional_kwargs={"read_file_media_result": True}, + ) + ls_tool = ToolMessage(content="[]", name="ls", tool_call_id="call_ls") + + user_message = HumanMessage(content="read and list") + reordered = filesystem_middleware._move_media_results_after_tool_results([user_message, ai_message, video_tool, video_media, ls_tool]) + + assert reordered == [user_message, ai_message, video_tool, ls_tool, video_media] + def test_read_file_video_extraction_error_surfaces_as_error_message(self, monkeypatch: pytest.MonkeyPatch) -> None: """PyAV failures and missing-dep errors render as a tool error, not an exception.""" monkeypatch.setattr( From acef9e5e15a9bf1fd8b20f1ebf6e6fcc113122ed Mon Sep 17 00:00:00 2001 From: Deep Agent Date: Tue, 23 Jun 2026 00:03:14 -0700 Subject: [PATCH 12/36] docs(sdk): remove video readme update --- libs/deepagents/README.md | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/libs/deepagents/README.md b/libs/deepagents/README.md index 964df8116c3..ecde4dad9b4 100644 --- a/libs/deepagents/README.md +++ b/libs/deepagents/README.md @@ -16,25 +16,6 @@ LangSmith is a unified developer platform for building, testing, and monitoring uv add deepagents ``` -### Optional: video frame extraction - -`read_file` can sample video files into image content blocks (`.mp4`, `.mov`, `.webm`, `.mkv`, etc.). -The agent uses `offset` to skip seconds into the source and `limit` to control how many -seconds of source to sample; the frames-per-second is a deploy-time knob. Video reads -are bounded by fixed input size, decode time, frame count, frame size, and emitted-byte -budgets. -Install with: - -```bash -uv add "deepagents[video]" -``` - -The extra pulls in [PyAV](https://pyav.basswood-io.com/) and Pillow. Without it, video -reads surface a clear "install `deepagents[video]`" error instead of failing silently. - -Sandbox-backed video reads currently inherit the backend's binary preview size cap. -Support for larger sandbox videos is deferred to follow-up backend work. - ## 🤔 What is this? Deep Agents is an open source agent harness — an opinionated agent that runs out of the box. Extend, override, or replace any piece. From 3bc6b7f6970a1b058fc1ade024101cc0c165a240 Mon Sep 17 00:00:00 2001 From: Deep Agent Date: Tue, 23 Jun 2026 00:18:55 -0700 Subject: [PATCH 13/36] fix(sdk): keep video reads out of tool output --- .../deepagents/middleware/filesystem.py | 122 ++++++++---------- .../tests/unit_tests/test_middleware.py | 41 +++--- 2 files changed, 70 insertions(+), 93 deletions(-) diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index c0a4ef10390..e597b81ffbd 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -99,10 +99,11 @@ } _READ_FILE_MEDIA_RESULT = "read_file_media_result" +_VIDEO_SAMPLING_RATE = 0.5 def _tool_error(name: str, tool_call_id: str | None, content: str) -> ToolMessage: - """Build a success-shaped `ToolMessage` carrying a plain text error.""" + """Build a `ToolMessage` carrying a plain text error.""" return ToolMessage(content=content, name=name, tool_call_id=tool_call_id, status="error") @@ -148,7 +149,6 @@ def _handle_video_read( tool_call_id: str | None, offset: int, limit: int, - sampling_rate: float, ) -> ToolMessage | Command: """Slice a video byte payload into a sampled frame window for the model. @@ -164,7 +164,7 @@ def _handle_video_read( """ if limit <= 0: return _tool_error("read_file", tool_call_id, f"Error reading video {validated_path}: limit must be > 0, got {limit!r}") - rate = float(sampling_rate) + rate = _VIDEO_SAMPLING_RATE offset_seconds = max(0.0, float(offset)) duration_seconds = float(limit) header = _video_window_header(validated_path, offset_seconds, duration_seconds, rate) @@ -224,29 +224,6 @@ def _video_window_header(path: str, offset_seconds: float, duration_seconds: flo return f"Reading [{offset_seconds:.3f}s, {end:.3f}s) of {path} at {rate} fps." -def _emit_binary_block( - content: str, - validated_path: str, - tool_call_id: str | None, - file_type: str, -) -> ToolMessage: - """Emit a base64 content block for non-video binary reads (image/audio/PDF/etc). - - Routes on `file_type` so the multimodal block picks the right block kind - (`image`, `audio`, `file`); falls back to the generic `file` block when the - backend-returned file extension is not in `_EXTENSION_TO_FILE_TYPE`. - """ - block_type = file_type if file_type != "text" else "file" - mime_type = mimetypes.guess_type("file" + Path(validated_path).suffix)[0] or "application/octet-stream" - return ToolMessage( - content_blocks=cast("list[ContentBlock]", [{"type": block_type, "base64": content, "mime_type": mime_type}]), - name="read_file", - tool_call_id=tool_call_id, - additional_kwargs={"read_file_path": validated_path, "read_file_media_type": mime_type}, - status="success", - ) - - @dataclass class FilesystemPermission: """A single access rule for filesystem operations.""" @@ -628,7 +605,7 @@ class ExecuteSchema(BaseModel): - **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow - First scan: read_file(file_path="...", limit=100) to see file structure - Read more sections: read_file(file_path="...", offset=100, limit=200) for next 200 lines - - Omit `limit` to use the default window; increase it only when necessary for editing + - Only omit limit (read full file) when necessary for editing - Specify offset and limit: read_file(file_path="...", offset=0, limit=100) reads first 100 lines - Results are returned using cat -n format, with line numbers starting at 1 - Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget. @@ -638,8 +615,8 @@ class ExecuteSchema(BaseModel): For multimodal reads (image, audio, video, PDF, etc.): - Use `read_file(file_path=...)` -- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only -- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s; sampling rate configured on `FilesystemMiddleware`). Detailed semantics live in the deepagents docs. +- Do NOT use `offset`/`limit` for images (pagination is text-only) +- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s). - If file details were compacted from history, call `read_file` again on the same path - You should ALWAYS make sure a file has been read before editing it.""" @@ -1042,7 +1019,6 @@ def __init__( tool_token_limit_before_evict: int | None = 20000, human_message_token_limit_before_evict: int | None = 50000, max_execute_timeout: int = 3600, - video_sampling_rate: float = 0.5, _permissions: list[FilesystemPermission] | None = None, ) -> None: """Initialize the filesystem middleware. @@ -1060,11 +1036,6 @@ def __init__( Defaults to 3600 seconds (1 hour). Any per-command timeout exceeding this value will be rejected with an error message. - video_sampling_rate: Frames-per-second emitted by `read_file` when - reading a video file. For video reads, `offset` is reinterpreted - as seconds into the source to skip and `limit` as seconds of source - to sample. Must be > 0. Defaults to 0.5 (one frame every two - seconds). _permissions: Optional filesystem permission rules enforced directly by this middleware's tool implementations. @@ -1075,9 +1046,6 @@ def __init__( if max_execute_timeout <= 0: msg = f"max_execute_timeout must be positive, got {max_execute_timeout}" raise ValueError(msg) - if video_sampling_rate <= 0: - msg = f"video_sampling_rate must be > 0, got {video_sampling_rate!r}" - raise ValueError(msg) # Use provided backend or default to StateBackend instance self.backend = backend if backend is not None else StateBackend() if ( @@ -1110,7 +1078,6 @@ def __init__( self._tool_token_limit_before_evict = tool_token_limit_before_evict self._human_message_token_limit_before_evict = human_message_token_limit_before_evict self._max_execute_timeout = max_execute_timeout - self._video_sampling_rate = float(video_sampling_rate) self._permissions = list(_permissions or []) # Shared executor for enforcing GLOB_TIMEOUT on the sync glob tool. @@ -1353,7 +1320,6 @@ def _handle_read_result( # noqa: PLR0911 tool_call_id, offset, limit, - self._video_sampling_rate, ) # Route on the backend-declared encoding first: `"base64"` means the @@ -1362,7 +1328,15 @@ def _handle_read_result( # noqa: PLR0911 # The extension map is only consulted to pick the multimodal block # type; unknown binary extensions fall back to the generic `"file"`. if encoding == "base64" or file_type != "text": - return _emit_binary_block(content, validated_path, tool_call_id, file_type) + block_type = file_type if file_type != "text" else "file" + mime_type = mimetypes.guess_type("file" + Path(validated_path).suffix)[0] or "application/octet-stream" + return ToolMessage( + content_blocks=cast("list[ContentBlock]", [{"type": block_type, "base64": content, "mime_type": mime_type}]), + name="read_file", + tool_call_id=tool_call_id, + additional_kwargs={"read_file_path": validated_path, "read_file_media_type": mime_type}, + status="success", + ) content = format_content_with_line_numbers(content, start_line=offset + 1) # `limit` already bounded raw source lines at the backend; do not @@ -1375,24 +1349,6 @@ def _handle_read_result( # noqa: PLR0911 status="success", ) - def _resolve_read_inputs( - file_path: str, - tool_call_id: str | None, - limit: int, - ) -> tuple[str, int] | ToolMessage: - """Validate path + permission and resolve the effective backend limit. - - Returns `(validated_path, backend_limit)` on success or a ready-to- - return `ToolMessage` error. - """ - try: - validated_path = validate_path(file_path) - except ValueError as e: - return _tool_error("read_file", tool_call_id, f"Error: {e}") - if _check_fs_permission(self._permissions, "read", validated_path) == "deny": - return _tool_error("read_file", tool_call_id, f"Error: permission denied for read on {validated_path}") - return validated_path, limit - def sync_read_file( file_path: Annotated[str, "Absolute path to the file to read. Must be absolute, not relative."], runtime: ToolRuntime[None, FilesystemState], @@ -1400,12 +1356,24 @@ def sync_read_file( limit: Annotated[int, "Maximum number of lines to read for text files. For videos, seconds of source to sample."] = DEFAULT_READ_LIMIT, ) -> ToolMessage | Command: """Synchronous wrapper for read_file tool.""" - backend = self._get_backend(runtime) - prepared = _resolve_read_inputs(file_path, runtime.tool_call_id, limit) - if isinstance(prepared, ToolMessage): - return prepared - validated_path, backend_limit = prepared - read_result = backend.read(validated_path, offset=offset, limit=backend_limit) + resolved_backend = self._get_backend(runtime) + try: + validated_path = validate_path(file_path) + except ValueError as e: + return ToolMessage( + content=f"Error: {e}", + name="read_file", + tool_call_id=runtime.tool_call_id, + status="error", + ) + if _check_fs_permission(self._permissions, "read", validated_path) == "deny": + return ToolMessage( + content=f"Error: permission denied for read on {validated_path}", + name="read_file", + tool_call_id=runtime.tool_call_id, + status="error", + ) + read_result = resolved_backend.read(validated_path, offset=offset, limit=limit) return _handle_read_result(read_result, validated_path, runtime.tool_call_id, offset, limit) async def async_read_file( @@ -1415,12 +1383,24 @@ async def async_read_file( limit: Annotated[int, "Maximum number of lines to read for text files. For videos, seconds of source to sample."] = DEFAULT_READ_LIMIT, ) -> ToolMessage | Command: """Asynchronous wrapper for read_file tool.""" - backend = self._get_backend(runtime) - prepared = _resolve_read_inputs(file_path, runtime.tool_call_id, limit) - if isinstance(prepared, ToolMessage): - return prepared - validated_path, backend_limit = prepared - read_result = await backend.aread(validated_path, offset=offset, limit=backend_limit) + resolved_backend = self._get_backend(runtime) + try: + validated_path = validate_path(file_path) + except ValueError as e: + return ToolMessage( + content=f"Error: {e}", + name="read_file", + tool_call_id=runtime.tool_call_id, + status="error", + ) + if _check_fs_permission(self._permissions, "read", validated_path) == "deny": + return ToolMessage( + content=f"Error: permission denied for read on {validated_path}", + name="read_file", + tool_call_id=runtime.tool_call_id, + status="error", + ) + read_result = await resolved_backend.aread(validated_path, offset=offset, limit=limit) return _handle_read_result(read_result, validated_path, runtime.tool_call_id, offset, limit) return StructuredTool.from_function( diff --git a/libs/deepagents/tests/unit_tests/test_middleware.py b/libs/deepagents/tests/unit_tests/test_middleware.py index 656be6a159c..09dfb0ae43a 100644 --- a/libs/deepagents/tests/unit_tests/test_middleware.py +++ b/libs/deepagents/tests/unit_tests/test_middleware.py @@ -15,6 +15,7 @@ ToolCall, ToolMessage, ) +from langchain_core.messages.utils import convert_to_openai_messages from langgraph.graph.message import REMOVE_ALL_MESSAGES from langgraph.store.memory import InMemoryStore from langgraph.types import Command @@ -1468,22 +1469,6 @@ def test_mixed_content_small_text_large_image_not_evicted(self): assert result == tool_message - def test_init_video_sampling_rate_default(self): - """Default sampling rate is 0.5 fps.""" - middleware = FilesystemMiddleware() - assert middleware._video_sampling_rate == 0.5 - - def test_init_video_sampling_rate_override(self): - """Constructor value is honored verbatim.""" - middleware = FilesystemMiddleware(video_sampling_rate=2.0) - assert middleware._video_sampling_rate == 2.0 - - @pytest.mark.parametrize("bad", [0, -0.5, -1]) - def test_init_video_sampling_rate_rejects_non_positive(self, bad: float) -> None: - """Constructor rejects zero and negative sampling rates up front.""" - with pytest.raises(ValueError, match="video_sampling_rate must be > 0"): - FilesystemMiddleware(video_sampling_rate=bad) - def test_read_file_video_routes_to_frame_extractor(self, monkeypatch: pytest.MonkeyPatch) -> None: """Video reads route through `_video.extract_video_frames`, not the generic base64 branch. @@ -1516,10 +1501,7 @@ def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) - middleware = FilesystemMiddleware( - backend=_video_backend(raw_bytes), - video_sampling_rate=0.5, - ) + middleware = FilesystemMiddleware(backend=_video_backend(raw_bytes)) state = FilesystemState(messages=[], files={}) runtime = _build_runtime(state, "video-read-1") read_file_tool = next(t for t in middleware.tools if t.name == "read_file") @@ -1554,8 +1536,23 @@ def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): ] assert b"\x00\x01\x02" not in str(media_message.content).encode() - # The middleware must convert offset/limit to seconds with the - # constructor-defined sampling rate forwarded verbatim. + openai_messages = convert_to_openai_messages( + [ + HumanMessage(content="read video"), + AIMessage(content="", tool_calls=[{"id": "video-read-1", "name": "read_file", "args": {"file_path": "/clips/intro.mkv"}}]), + *messages, + ] + ) + openai_tool_message = next(message for message in openai_messages if message["role"] == "tool") + assert isinstance(openai_tool_message["content"], str) + assert all( + block.get("type") != "video" + for message in openai_messages + for block in (message["content"] if isinstance(message.get("content"), list) else []) + ) + + # The middleware must convert offset/limit to seconds and use the + # internal sampling rate. assert len(fake_extract.calls) == 1 call = fake_extract.calls[0] assert call["offset_seconds"] == 0 From d21beb3724e9867361a52f61305d2ef52fd5f519 Mon Sep 17 00:00:00 2001 From: Deep Agent Date: Tue, 23 Jun 2026 00:24:59 -0700 Subject: [PATCH 14/36] fix(sdk): sync `read_file` prompt snapshot wording --- libs/deepagents/deepagents/middleware/filesystem.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index e597b81ffbd..a635ef6e769 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -605,7 +605,7 @@ class ExecuteSchema(BaseModel): - **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow - First scan: read_file(file_path="...", limit=100) to see file structure - Read more sections: read_file(file_path="...", offset=100, limit=200) for next 200 lines - - Only omit limit (read full file) when necessary for editing + - Omit `limit` to use the default window; increase it only when necessary for editing - Specify offset and limit: read_file(file_path="...", offset=0, limit=100) reads first 100 lines - Results are returned using cat -n format, with line numbers starting at 1 - Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget. @@ -615,8 +615,8 @@ class ExecuteSchema(BaseModel): For multimodal reads (image, audio, video, PDF, etc.): - Use `read_file(file_path=...)` -- Do NOT use `offset`/`limit` for images (pagination is text-only) -- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s). +- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only +- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s; sampling rate configured on `FilesystemMiddleware`). Detailed semantics live in the deepagents docs. - If file details were compacted from history, call `read_file` again on the same path - You should ALWAYS make sure a file has been read before editing it.""" From da025df5c4b10138100ba21531a1cb77a863ec55 Mon Sep 17 00:00:00 2001 From: Deep Agent Date: Tue, 23 Jun 2026 00:41:46 -0700 Subject: [PATCH 15/36] fix(sdk): preserve native video pass-through Add request-time conversion for user-supplied video blocks on non-video-capable models while leaving Gemini-style native video requests unchanged. Keep read_file video extraction on the existing PyAV path and update prompt wording for fixed sampling. --- .../deepagents/middleware/_video.py | 18 +- .../deepagents/middleware/filesystem.py | 185 +++++++++++++++++- .../custom_system_message_tools.json | 2 +- .../system_prompt_with_execute_tools.json | 2 +- ...m_prompt_with_memory_and_skills_tools.json | 2 +- ...stem_prompt_with_routed_backend_tools.json | 2 +- ...t_with_sync_and_async_subagents_tools.json | 2 +- .../system_prompt_without_execute_tools.json | 2 +- .../tests/unit_tests/test_middleware.py | 102 ++++++++++ 9 files changed, 301 insertions(+), 16 deletions(-) diff --git a/libs/deepagents/deepagents/middleware/_video.py b/libs/deepagents/deepagents/middleware/_video.py index 53cfd2cef61..50410355dcb 100644 --- a/libs/deepagents/deepagents/middleware/_video.py +++ b/libs/deepagents/deepagents/middleware/_video.py @@ -1,9 +1,9 @@ -"""Video frame extraction for the `read_file` tool. +"""Video frame extraction for filesystem and request media handling. -This module is the boundary between `read_file` and the optional video -backend (PyAV). It is imported lazily so a `deepagents` install without the -``[video]`` extra stays lightweight; the import only fires when the agent -actually tries to read a video file. +This module is the boundary between Deep Agents middleware and the optional +video backend (PyAV). It imports PyAV lazily so a `deepagents` install without +the ``[video]`` extra stays lightweight; the import only fires when the agent +actually tries to read or transform a video. For each video read the module decodes a contiguous slice of the source (``offset``-seconds skip, ``limit``-seconds window) and emits sampled @@ -11,9 +11,9 @@ text+image content blocks so the model sees per-frame timestamps alongside the JPEGs. -Sampling rate is a deploy-time knob on `FilesystemMiddleware`, not an -agent-facing argument — the public `read_file` surface stays narrow and -operators trade frame density for token cost without prompting changes. +Sampling rate is intentionally fixed by the middleware. Agents control the +window they inspect through `read_file`'s existing `offset`/`limit` arguments, +which are interpreted as seconds for video reads. """ import base64 @@ -60,7 +60,7 @@ def _import_av() -> Any: # noqa: ANN401 def _select_sampling_rate(value: float) -> float: - """Validate and normalize the constructor-supplied sampling rate.""" + """Validate and normalize an internal sampling rate.""" if value <= 0: msg = f"video_sampling_rate must be > 0, got {value!r}" raise ValueError(msg) diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index a635ef6e769..d75c2ab3a70 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -7,6 +7,7 @@ import concurrent.futures import contextlib import contextvars +import json import mimetypes import threading import uuid @@ -40,6 +41,7 @@ from pydantic import BaseModel, Field from deepagents._api.deprecation import warn_deprecated +from deepagents._models import get_model_identifier, get_model_provider from deepagents.backends import CompositeBackend, FilesystemBackend, LocalShellBackend, StateBackend from deepagents.backends.protocol import ( BACKEND_TYPES as BACKEND_TYPES, # Re-export type here for backwards compatibility @@ -100,6 +102,9 @@ _READ_FILE_MEDIA_RESULT = "read_file_media_result" _VIDEO_SAMPLING_RATE = 0.5 +_VIDEO_DEFAULT_DURATION_SECONDS = 100.0 +_VIDEO_CAPABLE_PATTERNS = ("gemini-",) +_VIDEO_CAPABLE_PROVIDERS = frozenset({"google_genai", "google_vertexai"}) def _tool_error(name: str, tool_call_id: str | None, content: str) -> ToolMessage: @@ -143,6 +148,157 @@ def _move_media_results_after_tool_results(messages: list[AnyMessage]) -> list[A return reordered +def _model_accepts_native_video(model: Any) -> bool: # noqa: ANN401 + """Return whether `model` can receive native video blocks without conversion.""" + model_name = get_model_identifier(model) + if model_name: + lowered = model_name.lower() + return any(lowered.startswith(prefix) for prefix in _VIDEO_CAPABLE_PATTERNS) + provider = get_model_provider(model) + return bool(provider and provider.lower() in _VIDEO_CAPABLE_PROVIDERS) + + +def _is_video_block(block: object) -> bool: + """Return whether a content block represents video media.""" + if not isinstance(block, dict): + return False + video_block = cast("dict[str, Any]", block) + if video_block.get("type") == "video": + return True + mime_type = video_block.get("mime_type") + return video_block.get("type") == "media" and isinstance(mime_type, str) and mime_type.startswith("video/") + + +def _maybe_parse_stringified_video_blocks(content: str) -> list[Any] | str: + """Recover video content blocks stringified by tool-node serialization.""" + if not content.startswith("[") or ('"type": "video"' not in content and '"mime_type": "video/' not in content): + return content + try: + parsed = json.loads(content) + except ValueError: + return content + if isinstance(parsed, list) and all(isinstance(block, dict) for block in parsed): + return parsed + return content + + +def _video_block_filename(block: dict[str, Any]) -> str: + """Resolve a best-effort label for a video content block.""" + source_metadata = block.get("source_metadata") + if isinstance(source_metadata, dict): + for key in ("filename", "name", "read_file_path"): + value = source_metadata.get(key) + if isinstance(value, str) and value: + return value + extras = block.get("extras") + if isinstance(extras, dict): + placeholder = extras.get("placeholder") + if isinstance(placeholder, str) and placeholder: + return placeholder + url = block.get("url") + if isinstance(url, str) and url and not url.startswith("data:"): + return url + return "video" + + +def _video_error_block(filename: str, msg: str) -> ContentBlock: + """Build a model-facing replacement block for a video extraction failure.""" + return cast( + "ContentBlock", + { + "type": "text", + "text": f"[Video '{filename}' could not be processed: {msg}. Please ask the user to describe it or retry.]", + }, + ) + + +def _decode_data_url(data_url: str) -> bytes: + """Decode a base64 `data:video/...` URL payload.""" + header, payload = data_url.split(",", 1) + if "base64" not in header.lower().split(";"): + msg = "video data URL is not base64-encoded" + raise ValueError(msg) + return base64.b64decode(payload, validate=True) + + +def _decode_video_block(block: dict[str, Any]) -> bytes: + """Decode supported inline video block shapes to raw bytes.""" + data = block.get("base64") + if not isinstance(data, str) and block.get("source_type") == "base64": + data = block.get("data") + if isinstance(data, str): + return base64.b64decode(data, validate=True) + + url = block.get("url") + if isinstance(url, str) and url.startswith("data:"): + return _decode_data_url(url) + + msg = "video block does not contain inline base64 data" + raise ValueError(msg) + + +def _expand_video_block_for_model(block: dict[str, Any]) -> list[ContentBlock]: + """Replace a video content block with sampled frame blocks for the model.""" + filename = _video_block_filename(block) + try: + raw_bytes = _decode_video_block(block) + except (ValueError, TypeError, binascii.Error) as exc: + return [_video_error_block(filename, str(exc))] + + if len(raw_bytes) > MAX_VIDEO_INPUT_BYTES: + return [_video_error_block(filename, f"video payload exceeds maximum input size of {MAX_VIDEO_INPUT_BYTES} bytes")] + + try: + blocks = extract_video_frames( + raw_bytes, + offset_seconds=0.0, + duration_seconds=_VIDEO_DEFAULT_DURATION_SECONDS, + sampling_rate=_VIDEO_SAMPLING_RATE, + ) + except (ValueError, VideoExtractionError) as exc: + return [_video_error_block(filename, str(exc))] + + header = _video_window_header(filename, 0.0, _VIDEO_DEFAULT_DURATION_SECONDS, _VIDEO_SAMPLING_RATE) + return [cast("ContentBlock", {"type": "text", "text": header}), *blocks] + + +def _transform_video_blocks_for_model( + messages: list[AnyMessage], + *, + native_video: bool, +) -> tuple[list[AnyMessage], bool]: + """Convert native video blocks to sampled frames unless the model supports video.""" + if native_video: + return messages, False + + transformed: list[AnyMessage] = [] + mutated = False + for message in messages: + content = getattr(message, "content", None) + if isinstance(content, str): + content = _maybe_parse_stringified_video_blocks(content) + if not isinstance(content, list): + transformed.append(message) + continue + + message_mutated = False + blocks: list[Any] = [] + for block in content: + if _is_video_block(block): + blocks.extend(_expand_video_block_for_model(cast("dict[str, Any]", block))) + message_mutated = True + else: + blocks.append(block) + + if message_mutated: + transformed.append(message.model_copy(update={"content": blocks})) + mutated = True + else: + transformed.append(message) + + return transformed, mutated + + def _handle_video_read( content: str, validated_path: str, @@ -616,7 +772,7 @@ class ExecuteSchema(BaseModel): For multimodal reads (image, audio, video, PDF, etc.): - Use `read_file(file_path=...)` - For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only -- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s; sampling rate configured on `FilesystemMiddleware`). Detailed semantics live in the deepagents docs. +- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s; sampled at a fixed rate). Use smaller windows when you need more temporal detail. - If file details were compacted from history, call `read_file` again on the same path - You should ALWAYS make sure a file has been read before editing it.""" @@ -2211,6 +2367,29 @@ def _filter_unsupported_tools_and_apply_prompt(self, request: ModelRequest[Conte return request + @staticmethod + def _apply_video_block_transform(request: ModelRequest[ContextT]) -> ModelRequest[ContextT]: + """Convert request video blocks unless the target model accepts native video.""" + messages, mutated = _transform_video_blocks_for_model( + list(request.messages), + native_video=_model_accepts_native_video(request.model), + ) + if not mutated: + return request + return request.override(messages=messages) + + @staticmethod + async def _aapply_video_block_transform(request: ModelRequest[ContextT]) -> ModelRequest[ContextT]: + """Async variant of `_apply_video_block_transform`.""" + messages, mutated = await asyncio.to_thread( + _transform_video_blocks_for_model, + list(request.messages), + native_video=_model_accepts_native_video(request.model), + ) + if not mutated: + return request + return request.override(messages=messages) + def wrap_model_call( self, request: ModelRequest[ContextT], @@ -2242,6 +2421,8 @@ def wrap_model_call( if request_messages != list(request.messages): request = request.override(messages=request_messages) + request = self._apply_video_block_transform(request) + eviction_result = self._evict_and_truncate_messages(request) if eviction_result is not None: messages, state_command = eviction_result @@ -2277,6 +2458,8 @@ async def awrap_model_call( if request_messages != list(request.messages): request = request.override(messages=request_messages) + request = await self._aapply_video_block_transform(request) + eviction_result = await self._aevict_and_truncate_messages(request) if eviction_result is not None: messages, state_command = eviction_result diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json index ca94d010d50..0f20a4c7364 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s; sampling rate configured on `FilesystemMiddleware`). Detailed semantics live in the deepagents docs.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s; sampled at a fixed rate). Use smaller windows when you need more temporal detail.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json index 31ed5424ddb..482a8020a0c 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s; sampling rate configured on `FilesystemMiddleware`). Detailed semantics live in the deepagents docs.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s; sampled at a fixed rate). Use smaller windows when you need more temporal detail.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json index ca94d010d50..0f20a4c7364 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s; sampling rate configured on `FilesystemMiddleware`). Detailed semantics live in the deepagents docs.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s; sampled at a fixed rate). Use smaller windows when you need more temporal detail.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json index 31ed5424ddb..482a8020a0c 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s; sampling rate configured on `FilesystemMiddleware`). Detailed semantics live in the deepagents docs.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s; sampled at a fixed rate). Use smaller windows when you need more temporal detail.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json index c9cc109214c..e3d25b07a67 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s; sampling rate configured on `FilesystemMiddleware`). Detailed semantics live in the deepagents docs.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s; sampled at a fixed rate). Use smaller windows when you need more temporal detail.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json index ca94d010d50..0f20a4c7364 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json @@ -59,7 +59,7 @@ }, { "function": { - "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s; sampling rate configured on `FilesystemMiddleware`). Detailed semantics live in the deepagents docs.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", + "description": "Reads a file from the filesystem.\n\nAssume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned.\n\nUsage:\n- For text files, by default it reads up to 100 lines starting from the beginning of the file\n- **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow\n - First scan: read_file(file_path=\"...\", limit=100) to see file structure\n - Read more sections: read_file(file_path=\"...\", offset=100, limit=200) for next 200 lines\n - Omit `limit` to use the default window; increase it only when necessary for editing\n- Specify offset and limit: read_file(file_path=\"...\", offset=0, limit=100) reads first 100 lines\n- Results are returned using cat -n format, with line numbers starting at 1\n- Lines longer than 5,000 characters will be split into multiple lines with continuation markers (e.g., 5.1, 5.2, etc.). `limit` applies to source lines, so continuation rows do not consume the budget.\n- You have the capability to call multiple tools in a single response. It is always better to speculatively read multiple files as a batch that are potentially useful.\n- If you read a file that exists but has empty contents you will receive a system reminder warning in place of file contents.\n- Image files (`.png`, `.jpg`, `.jpeg`, `.gif`, `.webp`, etc.), audio and video files, and PDFs are returned as multimodal content blocks (see https://docs.langchain.com/oss/python/langchain/messages#multimodal).\n\nFor multimodal reads (image, audio, video, PDF, etc.):\n- Use `read_file(file_path=...)`\n- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only\n- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s; sampled at a fixed rate). Use smaller windows when you need more temporal detail.\n- If file details were compacted from history, call `read_file` again on the same path\n\n- You should ALWAYS make sure a file has been read before editing it.", "name": "read_file", "parameters": { "properties": { diff --git a/libs/deepagents/tests/unit_tests/test_middleware.py b/libs/deepagents/tests/unit_tests/test_middleware.py index 09dfb0ae43a..2073b97b14a 100644 --- a/libs/deepagents/tests/unit_tests/test_middleware.py +++ b/libs/deepagents/tests/unit_tests/test_middleware.py @@ -1,4 +1,5 @@ import base64 +import json import mimetypes import time from unittest.mock import MagicMock, patch @@ -1682,6 +1683,86 @@ def fake_extract(*args: "object", **kwargs: "object") -> "list[dict[str, object] assert "video payload exceeds maximum input size of 3 bytes" in result.content # type: ignore[operator] assert called["count"] == 0 + def test_user_supplied_video_block_transforms_for_non_video_model(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Native video blocks are converted to sampled frames for non-video models.""" + raw_bytes = b"user supplied video" + sentinel = [ + {"type": "text", "text": "Frame at t=00:00:00.000"}, + {"type": "image", "base64": "AAAA", "mime_type": "image/jpeg"}, + ] + calls: list[dict[str, object]] = [] + + def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): # type: ignore[no-untyped-def] + calls.append( + { + "content": content, + "offset_seconds": offset_seconds, + "duration_seconds": duration_seconds, + "sampling_rate": sampling_rate, + } + ) + return list(sentinel) + + monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) + video_block = _video_content_block(raw_bytes, filename="upload.mp4") + messages = [HumanMessage(content=[{"type": "text", "text": "watch this"}, video_block])] + + transformed, mutated = filesystem_middleware._transform_video_blocks_for_model(messages, native_video=False) + + assert mutated is True + assert transformed[0] is not messages[0] + assert messages[0].content == [{"type": "text", "text": "watch this"}, video_block] + assert transformed[0].content == [ + {"type": "text", "text": "watch this"}, + {"type": "text", "text": "Reading first 100s of upload.mp4 at 0.5 fps."}, + *sentinel, + ] + assert calls == [ + { + "content": raw_bytes, + "offset_seconds": 0.0, + "duration_seconds": 100.0, + "sampling_rate": 0.5, + } + ] + assert video_block["base64"] not in str(transformed[0].content) + + def test_user_supplied_video_block_passes_through_for_video_model(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Video-capable models keep native video blocks unchanged.""" + called = {"count": 0} + + def fake_extract(*args: object, **kwargs: object) -> list[dict[str, object]]: # noqa: ARG001 + called["count"] += 1 + return [{"type": "text", "text": "unexpected"}] + + monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) + messages = [HumanMessage(content=[_video_content_block(b"native", filename="gemini.mov")])] + native_video = filesystem_middleware._model_accepts_native_video(_VideoCapableModel("google_genai", "gemini-2.0-flash")) + + transformed, mutated = filesystem_middleware._transform_video_blocks_for_model(messages, native_video=native_video) + + assert native_video is True + assert transformed is messages + assert mutated is False + assert called["count"] == 0 + + def test_stringified_video_blocks_are_recovered_for_non_video_model(self, monkeypatch: pytest.MonkeyPatch) -> None: + """Stringified video tool content is still converted before non-video model calls.""" + monkeypatch.setattr( + filesystem_middleware, + "extract_video_frames", + lambda *_args, **_kwargs: [{"type": "image", "base64": "AAAA", "mime_type": "image/jpeg"}], + ) + payload = json.dumps([_video_content_block(b"tool video", filename="tool.mp4")]) + messages = [ToolMessage(content=payload, name="custom_tool", tool_call_id="call_1")] + + transformed, mutated = filesystem_middleware._transform_video_blocks_for_model(messages, native_video=False) + + assert mutated is True + assert isinstance(transformed[0].content, list) + assert transformed[0].content[0] == {"type": "text", "text": "Reading first 100s of tool.mp4 at 0.5 fps."} + assert all(block.get("type") != "video" for block in transformed[0].content) + def test_read_file_image_returns_standard_image_content_block(self): """Test image reads return standard image blocks with base64 + mime_type.""" @@ -2818,3 +2899,24 @@ def read(self, path, *, offset=0, limit=100): # type: ignore[override] return ReadResult(file_data={"content": payload, "encoding": "base64"}) return VideoBackend() + + +def _video_content_block(raw: bytes, *, filename: str) -> dict[str, object]: + """Build an inline video content block for request-time transformation tests.""" + return { + "type": "video", + "base64": base64.b64encode(raw).decode("ascii"), + "mime_type": "video/mp4", + "source_metadata": {"filename": filename}, + } + + +class _VideoCapableModel: + """Small model stub exposing the provider metadata used for video capability.""" + + def __init__(self, provider: str, model_name: str) -> None: + self.model_name = model_name + self._provider = provider + + def _get_ls_params(self) -> dict[str, str]: + return {"ls_provider": self._provider} From ead192f3ee40b580779f1c4938d184a4d64ff4d0 Mon Sep 17 00:00:00 2001 From: Deep Agent Date: Tue, 23 Jun 2026 00:55:45 -0700 Subject: [PATCH 16/36] fix(sdk): raise video input cap and downscale frames --- libs/deepagents/deepagents/backends/utils.py | 2 +- .../deepagents/middleware/_video.py | 13 ++++++++---- .../tests/unit_tests/middleware/test_video.py | 20 +++++++++++++++++++ 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/libs/deepagents/deepagents/backends/utils.py b/libs/deepagents/deepagents/backends/utils.py index 895f4a7a7e9..27722c2f0bf 100644 --- a/libs/deepagents/deepagents/backends/utils.py +++ b/libs/deepagents/deepagents/backends/utils.py @@ -19,7 +19,7 @@ from deepagents.backends.protocol import FileData, FileInfo as _FileInfo, GrepMatch as _GrepMatch, GrepResult, ReadResult EMPTY_CONTENT_WARNING = "System reminder: File exists but has empty contents" -MAX_VIDEO_INPUT_BYTES: Final = 50 * 1024 * 1024 +MAX_VIDEO_INPUT_BYTES: Final = 1024 * 1024 * 1024 """Maximum raw video payload size accepted by `read_file` frame extraction.""" FileType = Literal["text", "image", "audio", "video", "file"] diff --git a/libs/deepagents/deepagents/middleware/_video.py b/libs/deepagents/deepagents/middleware/_video.py index 50410355dcb..c60fff32778 100644 --- a/libs/deepagents/deepagents/middleware/_video.py +++ b/libs/deepagents/deepagents/middleware/_video.py @@ -32,6 +32,8 @@ MAX_VIDEO_SAMPLED_FRAMES: Final = 64 MAX_VIDEO_FRAME_PIXELS: Final = 1920 * 1080 MAX_VIDEO_FRAME_SIDE: Final = 4096 +MAX_VIDEO_OUTPUT_WIDTH: Final = 1920 +MAX_VIDEO_OUTPUT_HEIGHT: Final = 1080 MAX_VIDEO_EMITTED_BYTES: Final = 4 * 1024 * 1024 MAX_VIDEO_DECODE_SECONDS: Final = 10.0 _JPEG_QUALITY: Final = 85 @@ -230,12 +232,11 @@ def _frame_dimensions(frame: Any) -> tuple[int, int] | None: # noqa: ANN401 def _validate_dimensions(width: int, height: int) -> None: - """Reject frames that exceed the fixed decode/encode budget.""" + """Reject frame dimensions that are too large to safely convert.""" if width <= 0 or height <= 0: return - pixels = width * height - if width > MAX_VIDEO_FRAME_SIDE or height > MAX_VIDEO_FRAME_SIDE or pixels > MAX_VIDEO_FRAME_PIXELS: - msg = f"Video frame dimensions {width}x{height} exceed the maximum {MAX_VIDEO_FRAME_PIXELS} pixels / {MAX_VIDEO_FRAME_SIDE}px side" + if width > MAX_VIDEO_FRAME_SIDE or height > MAX_VIDEO_FRAME_SIDE: + msg = f"Video frame dimensions {width}x{height} exceed the maximum {MAX_VIDEO_FRAME_SIDE}px side" raise VideoExtractionError(msg) @@ -326,6 +327,10 @@ def _encode_jpeg(frame: Any) -> bytes: # noqa: ANN401 img = frame.to_image() if hasattr(frame, "to_image") else Image.fromarray(frame.to_ndarray(format="rgb24")) _validate_dimensions(*img.size) + width, height = img.size + if width * height > MAX_VIDEO_FRAME_PIXELS or width > MAX_VIDEO_OUTPUT_WIDTH or height > MAX_VIDEO_OUTPUT_HEIGHT: + img = img.copy() + img.thumbnail((MAX_VIDEO_OUTPUT_WIDTH, MAX_VIDEO_OUTPUT_HEIGHT), Image.Resampling.LANCZOS) buf = io.BytesIO() img.save(buf, format="JPEG", quality=_JPEG_QUALITY) return buf.getvalue() diff --git a/libs/deepagents/tests/unit_tests/middleware/test_video.py b/libs/deepagents/tests/unit_tests/middleware/test_video.py index 37bb840c2bd..2e09c7aae04 100644 --- a/libs/deepagents/tests/unit_tests/middleware/test_video.py +++ b/libs/deepagents/tests/unit_tests/middleware/test_video.py @@ -8,6 +8,7 @@ """ import base64 +import io from collections.abc import Iterator import pytest @@ -257,6 +258,25 @@ def to_image(self): # type: ignore[no-untyped-def] assert not frame.converted +def test_encode_jpeg_downscales_high_resolution_frames() -> None: + """Normal high-resolution video frames are resized instead of rejected.""" + + class HighResolutionFrame(_FakeFrame): + def __init__(self) -> None: + super().__init__(0, width=2218, height=1440) + + def to_image(self): # type: ignore[no-untyped-def] + return Image.new("RGB", (2218, 1440), "black") + + jpeg = video_module._encode_jpeg(HighResolutionFrame()) + with Image.open(io.BytesIO(jpeg)) as img: + width, height = img.size + + assert width <= video_module.MAX_VIDEO_OUTPUT_WIDTH + assert height <= video_module.MAX_VIDEO_OUTPUT_HEIGHT + assert width * height <= video_module.MAX_VIDEO_FRAME_PIXELS + + def test_sample_frames_enforces_decode_deadline() -> None: """Best-effort decode timeout raises instead of continuing indefinitely.""" with pytest.raises(video_module.VideoExtractionError, match="decoding exceeded"): From 9c2d5d1cb35b7b1522b37c0de2fa065247d187b5 Mon Sep 17 00:00:00 2001 From: Deep Agent Date: Tue, 23 Jun 2026 01:01:56 -0700 Subject: [PATCH 17/36] fix(code): install video extra for CLI reads --- libs/code/pyproject.toml | 2 +- libs/code/uv.lock | 44 +++++++++++++++++++++++++++++++++++++--- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/libs/code/pyproject.toml b/libs/code/pyproject.toml index 09a13a62bb0..7694702386e 100644 --- a/libs/code/pyproject.toml +++ b/libs/code/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ ] dependencies = [ # Framework - "deepagents==0.7.0a2", + "deepagents[video]==0.7.0a2", "langchain>=1.3.11,<2.0.0", "langgraph-checkpoint-sqlite>=3.1.0,<4.0.0", diff --git a/libs/code/uv.lock b/libs/code/uv.lock index 92f7c8a6e45..15d3eac9e8b 100644 --- a/libs/code/uv.lock +++ b/libs/code/uv.lock @@ -249,6 +249,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, ] +[[package]] +name = "av" +version = "17.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/e3/477fa20578c284abeda08d91b63ee9abaebc93445d8feeb989d3d444bae1/av-17.1.0.tar.gz", hash = "sha256:7f1e71ff621b66253333926f948e00faae11d855b2442133c65128bca64cdeb3", size = 4288546, upload-time = "2026-06-07T05:52:55.999Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/87/8036b5c781bc3639ea04ef42d4e26da253bd4bd4311d8705b6a1c8824047/av-17.1.0-cp311-abi3-macosx_11_0_x86_64.whl", hash = "sha256:ad7b4aa011093324b7118245f50ac6db244cfe9900d4072508a5245a2b0d3f41", size = 22460847, upload-time = "2026-06-07T05:52:04.261Z" }, + { url = "https://files.pythonhosted.org/packages/6d/af/dfdf6fc7b17814b50d0aa9e7a7e37b87be91be3890f44b0d525433cd1fd1/av-17.1.0-cp311-abi3-macosx_14_0_arm64.whl", hash = "sha256:43ebbe977f19a7f2d2bd1a4e119675a0b15e05852cf7309846b6ab922ba7ffe9", size = 18159115, upload-time = "2026-06-07T05:52:06.64Z" }, + { url = "https://files.pythonhosted.org/packages/ad/13/64f6c466471cea225b8b2f4cdc51a571f8a286984b55a08d169b932fda5d/av-17.1.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6a20658ec7d96a70e14b1196eff00b7cdd8831ac3b99868e16b8ba8b24090847", size = 33224427, upload-time = "2026-06-07T05:52:09.165Z" }, + { url = "https://files.pythonhosted.org/packages/77/43/96b35170bf2e64e00a41748c6400ff73232dc0fc62ded283679fb07c7fe0/av-17.1.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f9a65d1f48b818323fb411e80358f89d77dec340b01d27c6b2dfbb9cbf4b779f", size = 35370183, upload-time = "2026-06-07T05:52:11.959Z" }, + { url = "https://files.pythonhosted.org/packages/2e/b3/8e8b4b6498731bfbd88e8399a756543f8088f1bd33d08eab678b5aebe728/av-17.1.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:58f7593726437cda5bd19793027e027768450b5c4a594777bf487798a33db702", size = 24459265, upload-time = "2026-06-07T05:52:14.66Z" }, + { url = "https://files.pythonhosted.org/packages/14/ac/ceb84b7553db21f1143d817245c560d9267168e1e58b1a8eeae2b62c4d04/av-17.1.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:bbab058bd965309f39962e53caac8126987c68c0be094fc4f9427e5615b0218f", size = 34283709, upload-time = "2026-06-07T05:52:17.389Z" }, + { url = "https://files.pythonhosted.org/packages/59/f9/4115fd84148c9a1cf365096694be6ac882fd3cd3cdb7a2f35e71fecf1631/av-17.1.0-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:9514cfda85180554c430695282faf4be3ffdf95775d8519733821244eecb58e0", size = 25397573, upload-time = "2026-06-07T05:52:20.012Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ac/92e52d5ed0e0b84d9d93e52b4338c2713d8a44082b8696e6516fdae7c4e4/av-17.1.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e1c90f85cd7431ede95b11e8e711571a896ebea433f298849c2c0f1594c8d86e", size = 36451495, upload-time = "2026-06-07T05:52:22.581Z" }, + { url = "https://files.pythonhosted.org/packages/6b/f2/53a7cd34adb6a971d7e6d99663e74db286966c9db8afdca17472fdf0f98e/av-17.1.0-cp311-abi3-win_amd64.whl", hash = "sha256:5df5c1172ef1cf65a1529d612f7da7798ce2cf82c1ff7212466b538a6cc7214c", size = 28036393, upload-time = "2026-06-07T05:52:25.657Z" }, + { url = "https://files.pythonhosted.org/packages/66/47/cd9ae0edf2206351c1251bb94b5ec58728e42c5f6ee16c03c412f3a1bb3e/av-17.1.0-cp311-abi3-win_arm64.whl", hash = "sha256:ee98534242a74da847af78624779ac5a3177dc7c69f956a4da9e6f0fdb37d7f6", size = 21174601, upload-time = "2026-06-07T05:52:28.077Z" }, + { url = "https://files.pythonhosted.org/packages/36/90/b5668cddb3c401fcf22553bc495d5b0c6d8a01d118624b26f0db1d0b8653/av-17.1.0-cp314-cp314t-macosx_11_0_x86_64.whl", hash = "sha256:5327807c1219293803ef0c5d1578ff3ae1cf638c09e5998962026e1a554ec240", size = 22699499, upload-time = "2026-06-07T05:52:30.335Z" }, + { url = "https://files.pythonhosted.org/packages/e0/7e/7be6bfddb823d045ff9fd5d4deb922ee3847605e162c3882e6c45b4c35ff/av-17.1.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:6c9b71fe5c0c5a8d303b1588d4d8ce9397d6b023f467cfef95000ba1f75507fa", size = 18366696, upload-time = "2026-06-07T05:52:32.645Z" }, + { url = "https://files.pythonhosted.org/packages/a2/23/391dcfa75c1ae1977efca44b753a11b929399b558826670c16a8808dd0e3/av-17.1.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:f997e3351bdf51127c07a74e21741a2996e9230cbeb2d81c14acde761b116c9c", size = 36582649, upload-time = "2026-06-07T05:52:35.218Z" }, + { url = "https://files.pythonhosted.org/packages/fb/32/7312854868b318b9d1b1dcbd1bddb460aaaeac7d57f816e11efec3bef5b1/av-17.1.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:efe9b1397300b67b644ad220c89df4892a76f2debe70f16bae1749fa20526e63", size = 38479390, upload-time = "2026-06-07T05:52:37.968Z" }, + { url = "https://files.pythonhosted.org/packages/2a/72/af47f59b4458e81ca7d89f477698dbfb3d5a0cd8ae6c1e4441d01074af8a/av-17.1.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:fa64e1f1500d01c4a98e7a41dc1a9a35fb4dfe71f5de0389264ec1192200c76a", size = 27127432, upload-time = "2026-06-07T05:52:40.371Z" }, + { url = "https://files.pythonhosted.org/packages/88/85/c2e6861baf0f8c7d21c4ce811d4d424fedac915e3910d3570ce4377717dc/av-17.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ffbd78d73d2c9bf31e9a007c992faec3991428b2941a3b085b84fb82e8c32d19", size = 37406592, upload-time = "2026-06-07T05:52:43.215Z" }, + { url = "https://files.pythonhosted.org/packages/ba/40/3cc13125aea976101c0858af99ac47257c0654411aa199b5d8e81eea7002/av-17.1.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:bff8896454b38fcb785a70e5ae0485d7021cb776303a5849393128a30b8f850b", size = 28336228, upload-time = "2026-06-07T05:52:46.134Z" }, + { url = "https://files.pythonhosted.org/packages/a2/38/c7d9c3e746209a1a695c13e3aa7d817229e84a85d0a84271f313d1befdd3/av-17.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:1284addf3c0dd939887a9722dc30df2241a97471ad52c3c507e31583ae22ff02", size = 39490680, upload-time = "2026-06-07T05:52:48.887Z" }, + { url = "https://files.pythonhosted.org/packages/a1/25/9d42da561b7b8f7dabdfaebba07b52977bee58c5c7e4285ac991abcfaa72/av-17.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:ec630be6321b04e317862f6082e84812bbd801e55a3c2298312e3fc8a0a4af4f", size = 28355673, upload-time = "2026-06-07T05:52:51.614Z" }, + { url = "https://files.pythonhosted.org/packages/a8/41/562a61d5a61fba3ffb273a115e249f1d8471b9515c59fcc38b4b9deda238/av-17.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:b41647e42884bf543b8e8d0a1dabd4d1b006c99183eb1a2d7afc5b01f73eeff4", size = 21324700, upload-time = "2026-06-07T05:52:53.972Z" }, +] + [[package]] name = "backports-tarfile" version = "1.2.0" @@ -1015,8 +1043,15 @@ dependencies = [ { name = "wcmatch" }, ] +[package.optional-dependencies] +video = [ + { name = "av" }, + { name = "pillow" }, +] + [package.metadata] requires-dist = [ + { name = "av", marker = "extra == 'video'", specifier = ">=17.0.0,<18.0.0" }, { name = "langchain", specifier = ">=1.3.11,<2.0.0" }, { name = "langchain-anthropic", specifier = ">=1.4.7,<2.0.0" }, { name = "langchain-aws", marker = "extra == 'aws'", specifier = ">=1.6.1,<2.0.0" }, @@ -1024,15 +1059,18 @@ requires-dist = [ { name = "langchain-google-genai", specifier = ">=4.2.6,<5.0.0" }, { name = "langchain-quickjs", marker = "extra == 'quickjs'", specifier = ">=0.3.2" }, { name = "langsmith", specifier = ">=0.9.3" }, + { name = "pillow", marker = "extra == 'video'", specifier = ">=10.0.0,<13.0.0" }, { name = "wcmatch", specifier = ">=10.1" }, ] -provides-extras = ["aws", "quickjs"] +provides-extras = ["aws", "quickjs", "video"] [package.metadata.requires-dev] test = [ + { name = "av", specifier = ">=17.0.0,<18.0.0" }, { name = "build" }, { name = "langchain-openai" }, { name = "langchain-tests", specifier = ">=1.1.9" }, + { name = "pillow", specifier = ">=10.0.0,<13.0.0" }, { name = "pytest" }, { name = "pytest-asyncio", specifier = ">=1.4.0" }, { name = "pytest-benchmark" }, @@ -1067,7 +1105,7 @@ version = "0.1.27" source = { editable = "." } dependencies = [ { name = "aiosqlite" }, - { name = "deepagents" }, + { name = "deepagents", extra = ["video"] }, { name = "deepagents-acp" }, { name = "httpx" }, { name = "langchain" }, @@ -1230,7 +1268,7 @@ test = [ [package.metadata] requires-dist = [ { name = "aiosqlite", specifier = ">=0.22.1,<1.0.0" }, - { name = "deepagents", editable = "../deepagents" }, + { name = "deepagents", extras = ["video"], editable = "../deepagents" }, { name = "deepagents-acp", specifier = ">=0.0.8,<1.0.0" }, { name = "deepagents-code", extras = ["agentcore", "daytona", "modal", "runloop", "vercel"], marker = "extra == 'all-sandboxes'" }, { name = "deepagents-code", extras = ["anthropic", "baseten", "bedrock", "cohere", "deepseek", "fireworks", "google-genai", "groq", "huggingface", "ibm", "litellm", "mistralai", "nvidia", "ollama", "openai", "openrouter", "perplexity", "together", "vertex", "xai"], marker = "extra == 'all-providers'" }, From b5b2c2298567db311f2b560c2478623b06609a0c Mon Sep 17 00:00:00 2001 From: Deep Agent Date: Tue, 23 Jun 2026 01:03:59 -0700 Subject: [PATCH 18/36] fix(talon): enable video extra in WhatsApp image --- examples/talon/Dockerfile | 2 +- libs/talon/pyproject.toml | 3 +++ libs/talon/uv.lock | 50 +++++++++++++++++++++++++++++++++++---- 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/examples/talon/Dockerfile b/examples/talon/Dockerfile index 39dbb7aa8ff..1f4a319809d 100644 --- a/examples/talon/Dockerfile +++ b/examples/talon/Dockerfile @@ -57,7 +57,7 @@ RUN mkdir -p /opt/whatsapp-bridge && \ cd /opt/whatsapp-bridge && \ npm install --omit=dev -RUN cd libs/talon && uv sync --frozen --extra speech +RUN cd libs/talon && uv sync --frozen --extra speech --extra video WORKDIR /workspace diff --git a/libs/talon/pyproject.toml b/libs/talon/pyproject.toml index 2b16738fae8..34429e5d8a2 100644 --- a/libs/talon/pyproject.toml +++ b/libs/talon/pyproject.toml @@ -41,6 +41,9 @@ speech = [ "torchaudio>=2.11.0,<2.12.0", "transformers>=5.12.1,<6.0.0", ] +video = [ + "deepagents[video]>=0.6.8,<0.7.0", +] [project.scripts] deepagents-talon = "deepagents_talon.__main__:main" diff --git a/libs/talon/uv.lock b/libs/talon/uv.lock index de7eca0f0e6..62a84bf45a2 100644 --- a/libs/talon/uv.lock +++ b/libs/talon/uv.lock @@ -156,6 +156,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/16/fbe8e1e185a45042f7cd3a282def5bb8d95bb69ab9e9ef6a5368aa17e426/audioread-3.1.0-py3-none-any.whl", hash = "sha256:b30d1df6c5d3de5dcef0fb0e256f6ea17bdcf5f979408df0297d8a408e2971b4", size = 23143, upload-time = "2025-10-26T19:44:12.016Z" }, ] +[[package]] +name = "av" +version = "17.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/e3/477fa20578c284abeda08d91b63ee9abaebc93445d8feeb989d3d444bae1/av-17.1.0.tar.gz", hash = "sha256:7f1e71ff621b66253333926f948e00faae11d855b2442133c65128bca64cdeb3", size = 4288546, upload-time = "2026-06-07T05:52:55.999Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/87/8036b5c781bc3639ea04ef42d4e26da253bd4bd4311d8705b6a1c8824047/av-17.1.0-cp311-abi3-macosx_11_0_x86_64.whl", hash = "sha256:ad7b4aa011093324b7118245f50ac6db244cfe9900d4072508a5245a2b0d3f41", size = 22460847, upload-time = "2026-06-07T05:52:04.261Z" }, + { url = "https://files.pythonhosted.org/packages/6d/af/dfdf6fc7b17814b50d0aa9e7a7e37b87be91be3890f44b0d525433cd1fd1/av-17.1.0-cp311-abi3-macosx_14_0_arm64.whl", hash = "sha256:43ebbe977f19a7f2d2bd1a4e119675a0b15e05852cf7309846b6ab922ba7ffe9", size = 18159115, upload-time = "2026-06-07T05:52:06.64Z" }, + { url = "https://files.pythonhosted.org/packages/ad/13/64f6c466471cea225b8b2f4cdc51a571f8a286984b55a08d169b932fda5d/av-17.1.0-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6a20658ec7d96a70e14b1196eff00b7cdd8831ac3b99868e16b8ba8b24090847", size = 33224427, upload-time = "2026-06-07T05:52:09.165Z" }, + { url = "https://files.pythonhosted.org/packages/77/43/96b35170bf2e64e00a41748c6400ff73232dc0fc62ded283679fb07c7fe0/av-17.1.0-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f9a65d1f48b818323fb411e80358f89d77dec340b01d27c6b2dfbb9cbf4b779f", size = 35370183, upload-time = "2026-06-07T05:52:11.959Z" }, + { url = "https://files.pythonhosted.org/packages/2e/b3/8e8b4b6498731bfbd88e8399a756543f8088f1bd33d08eab678b5aebe728/av-17.1.0-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:58f7593726437cda5bd19793027e027768450b5c4a594777bf487798a33db702", size = 24459265, upload-time = "2026-06-07T05:52:14.66Z" }, + { url = "https://files.pythonhosted.org/packages/14/ac/ceb84b7553db21f1143d817245c560d9267168e1e58b1a8eeae2b62c4d04/av-17.1.0-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:bbab058bd965309f39962e53caac8126987c68c0be094fc4f9427e5615b0218f", size = 34283709, upload-time = "2026-06-07T05:52:17.389Z" }, + { url = "https://files.pythonhosted.org/packages/59/f9/4115fd84148c9a1cf365096694be6ac882fd3cd3cdb7a2f35e71fecf1631/av-17.1.0-cp311-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:9514cfda85180554c430695282faf4be3ffdf95775d8519733821244eecb58e0", size = 25397573, upload-time = "2026-06-07T05:52:20.012Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ac/92e52d5ed0e0b84d9d93e52b4338c2713d8a44082b8696e6516fdae7c4e4/av-17.1.0-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e1c90f85cd7431ede95b11e8e711571a896ebea433f298849c2c0f1594c8d86e", size = 36451495, upload-time = "2026-06-07T05:52:22.581Z" }, + { url = "https://files.pythonhosted.org/packages/6b/f2/53a7cd34adb6a971d7e6d99663e74db286966c9db8afdca17472fdf0f98e/av-17.1.0-cp311-abi3-win_amd64.whl", hash = "sha256:5df5c1172ef1cf65a1529d612f7da7798ce2cf82c1ff7212466b538a6cc7214c", size = 28036393, upload-time = "2026-06-07T05:52:25.657Z" }, + { url = "https://files.pythonhosted.org/packages/66/47/cd9ae0edf2206351c1251bb94b5ec58728e42c5f6ee16c03c412f3a1bb3e/av-17.1.0-cp311-abi3-win_arm64.whl", hash = "sha256:ee98534242a74da847af78624779ac5a3177dc7c69f956a4da9e6f0fdb37d7f6", size = 21174601, upload-time = "2026-06-07T05:52:28.077Z" }, + { url = "https://files.pythonhosted.org/packages/36/90/b5668cddb3c401fcf22553bc495d5b0c6d8a01d118624b26f0db1d0b8653/av-17.1.0-cp314-cp314t-macosx_11_0_x86_64.whl", hash = "sha256:5327807c1219293803ef0c5d1578ff3ae1cf638c09e5998962026e1a554ec240", size = 22699499, upload-time = "2026-06-07T05:52:30.335Z" }, + { url = "https://files.pythonhosted.org/packages/e0/7e/7be6bfddb823d045ff9fd5d4deb922ee3847605e162c3882e6c45b4c35ff/av-17.1.0-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:6c9b71fe5c0c5a8d303b1588d4d8ce9397d6b023f467cfef95000ba1f75507fa", size = 18366696, upload-time = "2026-06-07T05:52:32.645Z" }, + { url = "https://files.pythonhosted.org/packages/a2/23/391dcfa75c1ae1977efca44b753a11b929399b558826670c16a8808dd0e3/av-17.1.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:f997e3351bdf51127c07a74e21741a2996e9230cbeb2d81c14acde761b116c9c", size = 36582649, upload-time = "2026-06-07T05:52:35.218Z" }, + { url = "https://files.pythonhosted.org/packages/fb/32/7312854868b318b9d1b1dcbd1bddb460aaaeac7d57f816e11efec3bef5b1/av-17.1.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:efe9b1397300b67b644ad220c89df4892a76f2debe70f16bae1749fa20526e63", size = 38479390, upload-time = "2026-06-07T05:52:37.968Z" }, + { url = "https://files.pythonhosted.org/packages/2a/72/af47f59b4458e81ca7d89f477698dbfb3d5a0cd8ae6c1e4441d01074af8a/av-17.1.0-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:fa64e1f1500d01c4a98e7a41dc1a9a35fb4dfe71f5de0389264ec1192200c76a", size = 27127432, upload-time = "2026-06-07T05:52:40.371Z" }, + { url = "https://files.pythonhosted.org/packages/88/85/c2e6861baf0f8c7d21c4ce811d4d424fedac915e3910d3570ce4377717dc/av-17.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:ffbd78d73d2c9bf31e9a007c992faec3991428b2941a3b085b84fb82e8c32d19", size = 37406592, upload-time = "2026-06-07T05:52:43.215Z" }, + { url = "https://files.pythonhosted.org/packages/ba/40/3cc13125aea976101c0858af99ac47257c0654411aa199b5d8e81eea7002/av-17.1.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:bff8896454b38fcb785a70e5ae0485d7021cb776303a5849393128a30b8f850b", size = 28336228, upload-time = "2026-06-07T05:52:46.134Z" }, + { url = "https://files.pythonhosted.org/packages/a2/38/c7d9c3e746209a1a695c13e3aa7d817229e84a85d0a84271f313d1befdd3/av-17.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:1284addf3c0dd939887a9722dc30df2241a97471ad52c3c507e31583ae22ff02", size = 39490680, upload-time = "2026-06-07T05:52:48.887Z" }, + { url = "https://files.pythonhosted.org/packages/a1/25/9d42da561b7b8f7dabdfaebba07b52977bee58c5c7e4285ac991abcfaa72/av-17.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:ec630be6321b04e317862f6082e84812bbd801e55a3c2298312e3fc8a0a4af4f", size = 28355673, upload-time = "2026-06-07T05:52:51.614Z" }, + { url = "https://files.pythonhosted.org/packages/a8/41/562a61d5a61fba3ffb273a115e249f1d8471b9515c59fcc38b4b9deda238/av-17.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:b41647e42884bf543b8e8d0a1dabd4d1b006c99183eb1a2d7afc5b01f73eeff4", size = 21324700, upload-time = "2026-06-07T05:52:53.972Z" }, +] + [[package]] name = "babel" version = "2.18.0" @@ -732,8 +760,15 @@ dependencies = [ { name = "wcmatch" }, ] +[package.optional-dependencies] +video = [ + { name = "av" }, + { name = "pillow" }, +] + [package.metadata] requires-dist = [ + { name = "av", marker = "extra == 'video'", specifier = ">=17.0.0,<18.0.0" }, { name = "langchain", specifier = ">=1.3.11,<2.0.0" }, { name = "langchain-anthropic", specifier = ">=1.4.7,<2.0.0" }, { name = "langchain-aws", marker = "extra == 'aws'", specifier = ">=1.6.1,<2.0.0" }, @@ -741,15 +776,18 @@ requires-dist = [ { name = "langchain-google-genai", specifier = ">=4.2.6,<5.0.0" }, { name = "langchain-quickjs", marker = "extra == 'quickjs'", specifier = ">=0.3.2" }, { name = "langsmith", specifier = ">=0.9.3" }, + { name = "pillow", marker = "extra == 'video'", specifier = ">=10.0.0,<13.0.0" }, { name = "wcmatch", specifier = ">=10.1" }, ] -provides-extras = ["aws", "quickjs"] +provides-extras = ["aws", "quickjs", "video"] [package.metadata.requires-dev] test = [ + { name = "av", specifier = ">=17.0.0,<18.0.0" }, { name = "build" }, { name = "langchain-openai" }, { name = "langchain-tests", specifier = ">=1.1.9" }, + { name = "pillow", specifier = ">=10.0.0,<13.0.0" }, { name = "pytest" }, { name = "pytest-asyncio", specifier = ">=1.4.0" }, { name = "pytest-benchmark" }, @@ -784,7 +822,7 @@ version = "0.1.27" source = { editable = "../code" } dependencies = [ { name = "aiosqlite" }, - { name = "deepagents" }, + { name = "deepagents", extra = ["video"] }, { name = "deepagents-acp" }, { name = "httpx" }, { name = "langchain" }, @@ -820,7 +858,7 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "aiosqlite", specifier = ">=0.22.1,<1.0.0" }, - { name = "deepagents", editable = "../deepagents" }, + { name = "deepagents", extras = ["video"], editable = "../deepagents" }, { name = "deepagents-acp", specifier = ">=0.0.8,<1.0.0" }, { name = "deepagents-code", extras = ["agentcore", "daytona", "modal", "runloop", "vercel"], marker = "extra == 'all-sandboxes'" }, { name = "deepagents-code", extras = ["anthropic", "baseten", "bedrock", "cohere", "deepseek", "fireworks", "google-genai", "groq", "huggingface", "ibm", "litellm", "mistralai", "nvidia", "ollama", "openai", "openrouter", "perplexity", "together", "vertex", "xai"], marker = "extra == 'all-providers'" }, @@ -922,6 +960,9 @@ speech = [ { name = "torchaudio" }, { name = "transformers" }, ] +video = [ + { name = "deepagents", extra = ["video"] }, +] [package.dev-dependencies] test = [ @@ -938,6 +979,7 @@ test = [ [package.metadata] requires-dist = [ { name = "deepagents", editable = "../deepagents" }, + { name = "deepagents", extras = ["video"], marker = "extra == 'video'", editable = "../deepagents" }, { name = "deepagents-code", editable = "../code" }, { name = "fleet-deepagents-export", specifier = ">=0.0.1,<1.0.0" }, { name = "langchain-mcp-adapters", specifier = ">=0.3.0,<1.0.0" }, @@ -948,7 +990,7 @@ requires-dist = [ { name = "torchaudio", marker = "extra == 'speech'", specifier = ">=2.11.0,<2.12.0" }, { name = "transformers", marker = "extra == 'speech'", specifier = ">=5.12.1,<6.0.0" }, ] -provides-extras = ["speech"] +provides-extras = ["speech", "video"] [package.metadata.requires-dev] test = [ From 54c755d481a78499d27c9634dfdc32823939a8fe Mon Sep 17 00:00:00 2001 From: Deep Agent Date: Tue, 23 Jun 2026 01:41:18 -0700 Subject: [PATCH 19/36] fix(sdk): simplify video read handling --- .../deepagents/middleware/_video.py | 78 ++-- .../deepagents/middleware/filesystem.py | 195 +--------- .../middleware/test_filesystem_video.py | 258 +++++++++++++ .../tests/unit_tests/middleware/test_video.py | 156 +++++--- .../tests/unit_tests/test_middleware.py | 340 ------------------ 5 files changed, 410 insertions(+), 617 deletions(-) create mode 100644 libs/deepagents/tests/unit_tests/middleware/test_filesystem_video.py diff --git a/libs/deepagents/deepagents/middleware/_video.py b/libs/deepagents/deepagents/middleware/_video.py index c60fff32778..3b6e99bf664 100644 --- a/libs/deepagents/deepagents/middleware/_video.py +++ b/libs/deepagents/deepagents/middleware/_video.py @@ -61,29 +61,6 @@ def _import_av() -> Any: # noqa: ANN401 return av -def _select_sampling_rate(value: float) -> float: - """Validate and normalize an internal sampling rate.""" - if value <= 0: - msg = f"video_sampling_rate must be > 0, got {value!r}" - raise ValueError(msg) - return float(value) - - -def _select_duration(value: float) -> float: - """Validate the agent-supplied video window length. - - The agent's `limit` is authoritative over how much of the source is - sampled; this function only checks that the request is positive. Bounds - on emitted output are enforced elsewhere in the pipeline - (`MAX_VIDEO_DECODE_SECONDS`, `MAX_VIDEO_SAMPLED_FRAMES`, - `MAX_VIDEO_EMITTED_BYTES`). - """ - if value <= 0: - msg = f"duration_seconds must be > 0, got {value!r}" - raise ValueError(msg) - return float(value) - - def _format_timestamp(seconds: float) -> str: """Format a frame timestamp as `HH:MM:SS.mmm` for the text header block.""" if seconds < 0: @@ -124,33 +101,44 @@ def extract_video_frames( if offset_seconds < 0: msg = f"offset_seconds must be >= 0, got {offset_seconds!r}" raise ValueError(msg) - rate = _select_sampling_rate(sampling_rate) - duration = _select_duration(duration_seconds) + if sampling_rate <= 0: + msg = f"sampling_rate must be > 0, got {sampling_rate!r}" + raise ValueError(msg) + if duration_seconds <= 0: + msg = f"duration_seconds must be > 0, got {duration_seconds!r}" + raise ValueError(msg) + rate = float(sampling_rate) + duration = float(duration_seconds) av = _import_av() container = _open_video_container(av, content) + backend_error_types = _video_backend_error_types(av) try: - video_stream = _find_video_stream(container) - time_base = float(video_stream.time_base) - stream_start_seconds = _stream_start_seconds(video_stream, time_base) - if offset_seconds > 0: - # `seek` keeps the math correct across containers that already sit - # at a non-zero timeline (e.g. trimmed clips). - start_pts = _stream_start_pts(video_stream) + int(offset_seconds / time_base) - container.seek(start_pts, any_frame=False, backward=True, stream=video_stream) - - blocks = list( - _sample_frames_in_window( - container.decode(video_stream), - offset_seconds=offset_seconds, - duration_seconds=float(duration), - sampling_rate=rate, - time_base=time_base, - stream_start_seconds=stream_start_seconds, - deadline_seconds=time.monotonic() + MAX_VIDEO_DECODE_SECONDS, - decode_error_types=_video_backend_error_types(av), + try: + video_stream = _find_video_stream(container) + time_base = float(video_stream.time_base) + stream_start_seconds = _stream_start_seconds(video_stream, time_base) + if offset_seconds > 0: + # `seek` keeps the math correct across containers that already sit + # at a non-zero timeline (e.g. trimmed clips). + start_pts = _stream_start_pts(video_stream) + int(offset_seconds / time_base) + container.seek(start_pts, any_frame=False, backward=True, stream=video_stream) + + blocks = list( + _sample_frames_in_window( + container.decode(video_stream), + offset_seconds=offset_seconds, + duration_seconds=float(duration), + sampling_rate=rate, + time_base=time_base, + stream_start_seconds=stream_start_seconds, + deadline_seconds=time.monotonic() + MAX_VIDEO_DECODE_SECONDS, + decode_error_types=backend_error_types, + ) ) - ) + except backend_error_types as exc: + msg = f"Failed to decode video frames: {exc}" + raise VideoExtractionError(msg) from exc finally: container.close() diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index d75c2ab3a70..341192ae2fe 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -7,7 +7,6 @@ import concurrent.futures import contextlib import contextvars -import json import mimetypes import threading import uuid @@ -41,7 +40,6 @@ from pydantic import BaseModel, Field from deepagents._api.deprecation import warn_deprecated -from deepagents._models import get_model_identifier, get_model_provider from deepagents.backends import CompositeBackend, FilesystemBackend, LocalShellBackend, StateBackend from deepagents.backends.protocol import ( BACKEND_TYPES as BACKEND_TYPES, # Re-export type here for backwards compatibility @@ -102,9 +100,6 @@ _READ_FILE_MEDIA_RESULT = "read_file_media_result" _VIDEO_SAMPLING_RATE = 0.5 -_VIDEO_DEFAULT_DURATION_SECONDS = 100.0 -_VIDEO_CAPABLE_PATTERNS = ("gemini-",) -_VIDEO_CAPABLE_PROVIDERS = frozenset({"google_genai", "google_vertexai"}) def _tool_error(name: str, tool_call_id: str | None, content: str) -> ToolMessage: @@ -148,157 +143,6 @@ def _move_media_results_after_tool_results(messages: list[AnyMessage]) -> list[A return reordered -def _model_accepts_native_video(model: Any) -> bool: # noqa: ANN401 - """Return whether `model` can receive native video blocks without conversion.""" - model_name = get_model_identifier(model) - if model_name: - lowered = model_name.lower() - return any(lowered.startswith(prefix) for prefix in _VIDEO_CAPABLE_PATTERNS) - provider = get_model_provider(model) - return bool(provider and provider.lower() in _VIDEO_CAPABLE_PROVIDERS) - - -def _is_video_block(block: object) -> bool: - """Return whether a content block represents video media.""" - if not isinstance(block, dict): - return False - video_block = cast("dict[str, Any]", block) - if video_block.get("type") == "video": - return True - mime_type = video_block.get("mime_type") - return video_block.get("type") == "media" and isinstance(mime_type, str) and mime_type.startswith("video/") - - -def _maybe_parse_stringified_video_blocks(content: str) -> list[Any] | str: - """Recover video content blocks stringified by tool-node serialization.""" - if not content.startswith("[") or ('"type": "video"' not in content and '"mime_type": "video/' not in content): - return content - try: - parsed = json.loads(content) - except ValueError: - return content - if isinstance(parsed, list) and all(isinstance(block, dict) for block in parsed): - return parsed - return content - - -def _video_block_filename(block: dict[str, Any]) -> str: - """Resolve a best-effort label for a video content block.""" - source_metadata = block.get("source_metadata") - if isinstance(source_metadata, dict): - for key in ("filename", "name", "read_file_path"): - value = source_metadata.get(key) - if isinstance(value, str) and value: - return value - extras = block.get("extras") - if isinstance(extras, dict): - placeholder = extras.get("placeholder") - if isinstance(placeholder, str) and placeholder: - return placeholder - url = block.get("url") - if isinstance(url, str) and url and not url.startswith("data:"): - return url - return "video" - - -def _video_error_block(filename: str, msg: str) -> ContentBlock: - """Build a model-facing replacement block for a video extraction failure.""" - return cast( - "ContentBlock", - { - "type": "text", - "text": f"[Video '{filename}' could not be processed: {msg}. Please ask the user to describe it or retry.]", - }, - ) - - -def _decode_data_url(data_url: str) -> bytes: - """Decode a base64 `data:video/...` URL payload.""" - header, payload = data_url.split(",", 1) - if "base64" not in header.lower().split(";"): - msg = "video data URL is not base64-encoded" - raise ValueError(msg) - return base64.b64decode(payload, validate=True) - - -def _decode_video_block(block: dict[str, Any]) -> bytes: - """Decode supported inline video block shapes to raw bytes.""" - data = block.get("base64") - if not isinstance(data, str) and block.get("source_type") == "base64": - data = block.get("data") - if isinstance(data, str): - return base64.b64decode(data, validate=True) - - url = block.get("url") - if isinstance(url, str) and url.startswith("data:"): - return _decode_data_url(url) - - msg = "video block does not contain inline base64 data" - raise ValueError(msg) - - -def _expand_video_block_for_model(block: dict[str, Any]) -> list[ContentBlock]: - """Replace a video content block with sampled frame blocks for the model.""" - filename = _video_block_filename(block) - try: - raw_bytes = _decode_video_block(block) - except (ValueError, TypeError, binascii.Error) as exc: - return [_video_error_block(filename, str(exc))] - - if len(raw_bytes) > MAX_VIDEO_INPUT_BYTES: - return [_video_error_block(filename, f"video payload exceeds maximum input size of {MAX_VIDEO_INPUT_BYTES} bytes")] - - try: - blocks = extract_video_frames( - raw_bytes, - offset_seconds=0.0, - duration_seconds=_VIDEO_DEFAULT_DURATION_SECONDS, - sampling_rate=_VIDEO_SAMPLING_RATE, - ) - except (ValueError, VideoExtractionError) as exc: - return [_video_error_block(filename, str(exc))] - - header = _video_window_header(filename, 0.0, _VIDEO_DEFAULT_DURATION_SECONDS, _VIDEO_SAMPLING_RATE) - return [cast("ContentBlock", {"type": "text", "text": header}), *blocks] - - -def _transform_video_blocks_for_model( - messages: list[AnyMessage], - *, - native_video: bool, -) -> tuple[list[AnyMessage], bool]: - """Convert native video blocks to sampled frames unless the model supports video.""" - if native_video: - return messages, False - - transformed: list[AnyMessage] = [] - mutated = False - for message in messages: - content = getattr(message, "content", None) - if isinstance(content, str): - content = _maybe_parse_stringified_video_blocks(content) - if not isinstance(content, list): - transformed.append(message) - continue - - message_mutated = False - blocks: list[Any] = [] - for block in content: - if _is_video_block(block): - blocks.extend(_expand_video_block_for_model(cast("dict[str, Any]", block))) - message_mutated = True - else: - blocks.append(block) - - if message_mutated: - transformed.append(message.model_copy(update={"content": blocks})) - mutated = True - else: - transformed.append(message) - - return transformed, mutated - - def _handle_video_read( content: str, validated_path: str, @@ -675,7 +519,7 @@ class ReadFileSchema(BaseModel): offset: int = Field( default=DEFAULT_READ_OFFSET, - description="Line number to start reading from (0-indexed). Use for pagination of large files.", + description="Line number to start reading from for text files (0-indexed). For videos, seconds into the source to start sampling.", ) limit: int = Field( @@ -1508,7 +1352,10 @@ def _handle_read_result( # noqa: PLR0911 def sync_read_file( file_path: Annotated[str, "Absolute path to the file to read. Must be absolute, not relative."], runtime: ToolRuntime[None, FilesystemState], - offset: Annotated[int, "Line number to start reading from (0-indexed). Use for pagination of large files."] = DEFAULT_READ_OFFSET, + offset: Annotated[ + int, + "Line number to start reading from for text files (0-indexed). For videos, seconds into the source to start sampling.", + ] = DEFAULT_READ_OFFSET, limit: Annotated[int, "Maximum number of lines to read for text files. For videos, seconds of source to sample."] = DEFAULT_READ_LIMIT, ) -> ToolMessage | Command: """Synchronous wrapper for read_file tool.""" @@ -1535,7 +1382,10 @@ def sync_read_file( async def async_read_file( file_path: Annotated[str, "Absolute path to the file to read. Must be absolute, not relative."], runtime: ToolRuntime[None, FilesystemState], - offset: Annotated[int, "Line number to start reading from (0-indexed). Use for pagination of large files."] = DEFAULT_READ_OFFSET, + offset: Annotated[ + int, + "Line number to start reading from for text files (0-indexed). For videos, seconds into the source to start sampling.", + ] = DEFAULT_READ_OFFSET, limit: Annotated[int, "Maximum number of lines to read for text files. For videos, seconds of source to sample."] = DEFAULT_READ_LIMIT, ) -> ToolMessage | Command: """Asynchronous wrapper for read_file tool.""" @@ -2367,29 +2217,6 @@ def _filter_unsupported_tools_and_apply_prompt(self, request: ModelRequest[Conte return request - @staticmethod - def _apply_video_block_transform(request: ModelRequest[ContextT]) -> ModelRequest[ContextT]: - """Convert request video blocks unless the target model accepts native video.""" - messages, mutated = _transform_video_blocks_for_model( - list(request.messages), - native_video=_model_accepts_native_video(request.model), - ) - if not mutated: - return request - return request.override(messages=messages) - - @staticmethod - async def _aapply_video_block_transform(request: ModelRequest[ContextT]) -> ModelRequest[ContextT]: - """Async variant of `_apply_video_block_transform`.""" - messages, mutated = await asyncio.to_thread( - _transform_video_blocks_for_model, - list(request.messages), - native_video=_model_accepts_native_video(request.model), - ) - if not mutated: - return request - return request.override(messages=messages) - def wrap_model_call( self, request: ModelRequest[ContextT], @@ -2421,8 +2248,6 @@ def wrap_model_call( if request_messages != list(request.messages): request = request.override(messages=request_messages) - request = self._apply_video_block_transform(request) - eviction_result = self._evict_and_truncate_messages(request) if eviction_result is not None: messages, state_command = eviction_result @@ -2458,8 +2283,6 @@ async def awrap_model_call( if request_messages != list(request.messages): request = request.override(messages=request_messages) - request = await self._aapply_video_block_transform(request) - eviction_result = await self._aevict_and_truncate_messages(request) if eviction_result is not None: messages, state_command = eviction_result diff --git a/libs/deepagents/tests/unit_tests/middleware/test_filesystem_video.py b/libs/deepagents/tests/unit_tests/middleware/test_filesystem_video.py new file mode 100644 index 00000000000..12f6e53221a --- /dev/null +++ b/libs/deepagents/tests/unit_tests/middleware/test_filesystem_video.py @@ -0,0 +1,258 @@ +import base64 + +import pytest +from langchain.tools import ToolRuntime +from langchain_core.messages import AIMessage, HumanMessage, ToolMessage +from langchain_core.messages.utils import convert_to_openai_messages +from langgraph.types import Command + +import deepagents.middleware.filesystem as filesystem_middleware +from deepagents.backends import StateBackend +from deepagents.backends.protocol import ReadResult +from deepagents.middleware.filesystem import FilesystemMiddleware, FilesystemState + + +def test_read_file_video_routes_to_frame_extractor(monkeypatch: pytest.MonkeyPatch) -> None: + """Video reads route through the frame extractor, not generic base64 media.""" + raw_bytes = b"\x00\x01\x02 fake video bytes" + sentinel = [ + {"type": "text", "text": "Frame at t=00:00:00.000"}, + {"type": "image", "base64": "AAAA", "mime_type": "image/jpeg"}, + {"type": "text", "text": "Frame at t=00:00:02.000"}, + {"type": "image", "base64": "BBBB", "mime_type": "image/jpeg"}, + ] + calls: list[dict[str, float | int]] = [] + + def fake_extract( + content: bytes, + *, + offset_seconds: float, + duration_seconds: float, + sampling_rate: float, + ) -> list[dict[str, str]]: + calls.append( + { + "len": len(content), + "offset_seconds": offset_seconds, + "duration_seconds": duration_seconds, + "sampling_rate": sampling_rate, + } + ) + return list(sentinel) + + monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) + + middleware = FilesystemMiddleware(backend=_video_backend(raw_bytes)) + state = FilesystemState(messages=[], files={}) + runtime = _build_runtime(state, "video-read-1") + read_file_tool = next(t for t in middleware.tools if t.name == "read_file") + + result = read_file_tool.invoke( + { + "file_path": "/clips/intro.mkv", + "offset": 0, + "limit": 30, + "runtime": runtime, + } + ) + + assert isinstance(result, Command) + assert isinstance(result.update, dict) + messages = result.update["messages"] + assert isinstance(messages, list) + assert len(messages) == 2 + tool_message, media_message = messages + assert isinstance(tool_message, ToolMessage) + assert tool_message.status == "success" + assert tool_message.content == "Read video /clips/intro.mkv: sampled 2 frames. The sampled frames are attached in the following message." + assert tool_message.additional_kwargs["read_file_path"] == "/clips/intro.mkv" + assert tool_message.additional_kwargs["read_file_frame_count"] == 2 + assert isinstance(media_message, HumanMessage) + assert media_message.additional_kwargs["read_file_media_result"] is True + assert media_message.additional_kwargs["read_file_path"] == "/clips/intro.mkv" + assert media_message.additional_kwargs["read_file_tool_call_id"] == "video-read-1" + assert media_message.content == [ + {"type": "text", "text": "Reading first 30s of /clips/intro.mkv at 0.5 fps."}, + *sentinel, + ] + assert b"\x00\x01\x02" not in str(media_message.content).encode() + + openai_messages = convert_to_openai_messages( + [ + HumanMessage(content="read video"), + AIMessage(content="", tool_calls=[{"id": "video-read-1", "name": "read_file", "args": {"file_path": "/clips/intro.mkv"}}]), + *messages, + ] + ) + openai_tool_message = next(message for message in openai_messages if message["role"] == "tool") + assert isinstance(openai_tool_message["content"], str) + assert all( + block.get("type") != "video" + for message in openai_messages + for block in (message["content"] if isinstance(message.get("content"), list) else []) + ) + + assert calls == [ + { + "len": len(raw_bytes), + "offset_seconds": 0, + "duration_seconds": 30, + "sampling_rate": 0.5, + } + ] + + +@pytest.mark.parametrize( + ("tool_input", "expected"), + [ + ( + {"offset": 12, "limit": 90}, + {"offset_seconds": 12.0, "duration_seconds": 90.0, "sampling_rate": 0.5}, + ), + ( + {}, + {"offset_seconds": 0.0, "duration_seconds": 100.0, "sampling_rate": 0.5}, + ), + ], + ids=["explicit-window", "default-window"], +) +def test_read_file_video_window_forwards_seconds( + monkeypatch: pytest.MonkeyPatch, + tool_input: dict[str, int], + expected: dict[str, float], +) -> None: + """`offset`/`limit` become seconds, with the default window preserved.""" + captured: dict[str, float] = {} + + def fake_extract( + _content: bytes, + *, + offset_seconds: float, + duration_seconds: float, + sampling_rate: float, + ) -> list[dict[str, str]]: + captured.update(offset_seconds=offset_seconds, duration_seconds=duration_seconds, sampling_rate=sampling_rate) + return [{"type": "text", "text": "ok"}] + + monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) + middleware = FilesystemMiddleware(backend=_video_backend()) + state = FilesystemState(messages=[], files={}) + runtime = _build_runtime(state, "video-read-window") + read_file_tool = next(t for t in middleware.tools if t.name == "read_file") + + read_file_tool.invoke({"file_path": "/c.mp4", "runtime": runtime, **tool_input}) + + assert captured == expected + + +def test_read_file_video_media_result_ordered_after_parallel_tool_results() -> None: + """Video frame attachments do not split a provider-required tool result batch.""" + ai_message = AIMessage( + content="", + tool_calls=[ + {"id": "call_video", "name": "read_file", "args": {"file_path": "/c.mp4"}}, + {"id": "call_ls", "name": "ls", "args": {"path": "/"}}, + ], + ) + video_tool = ToolMessage(content="sampled frames", name="read_file", tool_call_id="call_video") + video_media = HumanMessage( + content=[ + {"type": "text", "text": "Reading first 100s of /c.mp4 at 0.5 fps."}, + {"type": "image", "base64": "AAAA", "mime_type": "image/jpeg"}, + ], + additional_kwargs={"read_file_media_result": True}, + ) + ls_tool = ToolMessage(content="[]", name="ls", tool_call_id="call_ls") + + user_message = HumanMessage(content="read and list") + reordered = filesystem_middleware._move_media_results_after_tool_results([user_message, ai_message, video_tool, video_media, ls_tool]) + + assert reordered == [user_message, ai_message, video_tool, ls_tool, video_media] + + +def test_read_file_video_extraction_error_surfaces_as_error_message(monkeypatch: pytest.MonkeyPatch) -> None: + """PyAV failures and missing-dep errors render as a tool error, not an exception.""" + + def fake_extract(*_args: object, **_kwargs: object) -> list[dict[str, str]]: + msg = "av not installed" + raise filesystem_middleware.VideoExtractionError(msg) + + monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) + middleware = FilesystemMiddleware(backend=_video_backend(b"corrupt")) + state = FilesystemState(messages=[], files={}) + runtime = _build_runtime(state, "video-read-err") + read_file_tool = next(t for t in middleware.tools if t.name == "read_file") + + result = read_file_tool.invoke({"file_path": "/c.mp4", "runtime": runtime}) + + assert isinstance(result, ToolMessage) + assert result.status == "error" + assert "av not installed" in result.content + + +def test_read_file_video_non_positive_limit_rejected(monkeypatch: pytest.MonkeyPatch) -> None: + """A non-positive `limit` is rejected as a tool error, not silently clamped.""" + called = {"count": 0} + + def fake_extract(*_args: object, **_kwargs: object) -> list[dict[str, str]]: + called["count"] += 1 + return [{"type": "text", "text": "ok"}] + + monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) + middleware = FilesystemMiddleware(backend=_video_backend()) + state = FilesystemState(messages=[], files={}) + runtime = _build_runtime(state, "video-read-bad-limit") + read_file_tool = next(t for t in middleware.tools if t.name == "read_file") + + result = read_file_tool.invoke({"file_path": "/c.mp4", "limit": 0, "runtime": runtime}) + + assert isinstance(result, ToolMessage) + assert result.status == "error" + assert "limit must be > 0" in result.content + assert called["count"] == 0 + + +def test_read_file_video_payload_size_cap_rejected(monkeypatch: pytest.MonkeyPatch) -> None: + """State/store/custom video payloads are capped after base64 decoding.""" + called = {"count": 0} + + def fake_extract(*_args: object, **_kwargs: object) -> list[dict[str, str]]: + called["count"] += 1 + return [{"type": "text", "text": "ok"}] + + monkeypatch.setattr(filesystem_middleware, "MAX_VIDEO_INPUT_BYTES", 3) + monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) + middleware = FilesystemMiddleware(backend=_video_backend(b"abcd")) + state = FilesystemState(messages=[], files={}) + runtime = _build_runtime(state, "video-read-big-payload") + read_file_tool = next(t for t in middleware.tools if t.name == "read_file") + + result = read_file_tool.invoke({"file_path": "/c.mp4", "runtime": runtime}) + + assert isinstance(result, ToolMessage) + assert result.status == "error" + assert "video payload exceeds maximum input size of 3 bytes" in result.content + assert called["count"] == 0 + + +def _build_runtime(state: FilesystemState, tool_call_id: str) -> ToolRuntime[None, FilesystemState]: + """Build a `ToolRuntime` for video `read_file` tests.""" + return ToolRuntime( + state=state, + context=None, + tool_call_id=tool_call_id, + store=None, + stream_writer=lambda _: None, + config={}, + ) + + +def _video_backend(raw: bytes = b"raw") -> StateBackend: + """Backend stub that returns `raw` base64-encoded for any video read.""" + payload = base64.b64encode(raw).decode("ascii") + + class VideoBackend(StateBackend): + def read(self, file_path: str, offset: int = 0, limit: int = 100) -> ReadResult: + return ReadResult(file_data={"content": payload, "encoding": "base64"}) + + return VideoBackend() diff --git a/libs/deepagents/tests/unit_tests/middleware/test_video.py b/libs/deepagents/tests/unit_tests/middleware/test_video.py index 2e09c7aae04..870b1d8f9a3 100644 --- a/libs/deepagents/tests/unit_tests/middleware/test_video.py +++ b/libs/deepagents/tests/unit_tests/middleware/test_video.py @@ -10,6 +10,7 @@ import base64 import io from collections.abc import Iterator +from typing import cast import pytest @@ -46,17 +47,6 @@ def synthetic_video_bytes(tmp_path_factory: pytest.TempPathFactory) -> bytes: return path.read_bytes() -def test_select_sampling_rate_accepts_positive() -> None: - assert video_module._select_sampling_rate(0.5) == 0.5 - assert video_module._select_sampling_rate(2.0) == 2.0 - - -def test_select_sampling_rate_rejects_non_positive() -> None: - for bad in (0, -0.1, -1): - with pytest.raises(ValueError, match="must be > 0"): - video_module._select_sampling_rate(bad) - - def test_extract_rejects_negative_offset(synthetic_video_bytes: bytes) -> None: with pytest.raises(ValueError, match="offset_seconds must be >= 0"): video_module.extract_video_frames( @@ -77,13 +67,26 @@ def test_extract_rejects_zero_or_negative_duration(synthetic_video_bytes: bytes) ) +def test_extract_rejects_zero_or_negative_sampling_rate(synthetic_video_bytes: bytes) -> None: + with pytest.raises(ValueError, match="sampling_rate must be > 0"): + video_module.extract_video_frames( + synthetic_video_bytes, + offset_seconds=0, + duration_seconds=1, + sampling_rate=0, + ) + + def test_extract_returns_interleaved_text_and_image_blocks(synthetic_video_bytes: bytes) -> None: """Two frames (t=0 and t=2) interleaved with text headers at 0.5 fps over a 3s clip.""" - blocks = video_module.extract_video_frames( - synthetic_video_bytes, - offset_seconds=0, - duration_seconds=30, - sampling_rate=0.5, + blocks = cast( + "list[dict[str, str]]", + video_module.extract_video_frames( + synthetic_video_bytes, + offset_seconds=0, + duration_seconds=30, + sampling_rate=0.5, + ), ) assert [b["type"] for b in blocks] == ["text", "image", "text", "image"] assert blocks[0]["text"].startswith("Frame at t=") @@ -97,11 +100,14 @@ def test_extract_returns_interleaved_text_and_image_blocks(synthetic_video_bytes def test_extract_offset_skips_into_source(synthetic_video_bytes: bytes) -> None: """`offset_seconds` seeks into the clip; every emitted frame is at or after that point.""" - blocks = video_module.extract_video_frames( - synthetic_video_bytes, - offset_seconds=1.0, - duration_seconds=1.5, - sampling_rate=1.0, + blocks = cast( + "list[dict[str, str]]", + video_module.extract_video_frames( + synthetic_video_bytes, + offset_seconds=1.0, + duration_seconds=1.5, + sampling_rate=1.0, + ), ) headers = [b["text"] for b in blocks if b["type"] == "text"] assert headers, "expected at least one frame in [1s, 2.5s) at 1fps" @@ -141,17 +147,66 @@ def decoded_frames() -> Iterator[_FakeFrame]: assert isinstance(exc_info.value.__cause__, av.error.InvalidDataError) +def test_extract_wraps_seek_errors(monkeypatch: pytest.MonkeyPatch) -> None: + """Seek failures surface as `VideoExtractionError` instead of escaping.""" + + class FakeStream: + type = "video" + time_base = 1 + start_time = 0 + + class FakeContainer: + def __init__(self) -> None: + self.closed = False + self.streams = [FakeStream()] + + def seek(self, *_args: object, **_kwargs: object) -> None: + msg = "seek failed" + raise OSError(msg) + + def decode(self, *_args: object) -> Iterator[object]: + msg = "decode should not run after seek fails" + raise AssertionError(msg) + + def close(self) -> None: + self.closed = True + + class FakeAv: + error = type("ErrorNamespace", (), {}) + + @staticmethod + def open(_payload: object) -> FakeContainer: + return container + + container = FakeContainer() + monkeypatch.setattr(video_module, "_import_av", lambda: FakeAv) + + with pytest.raises(video_module.VideoExtractionError, match="Failed to decode video frames") as exc_info: + video_module.extract_video_frames( + b"fake", + offset_seconds=1, + duration_seconds=1, + sampling_rate=1, + ) + + assert isinstance(exc_info.value.__cause__, OSError) + assert container.closed + + def test_sample_frames_normalizes_non_zero_stream_start(monkeypatch: pytest.MonkeyPatch) -> None: """Frame timestamps are measured from the video start, not raw container PTS.""" monkeypatch.setattr(video_module, "_encode_jpeg", lambda _frame: b"jpeg") - blocks = video_module._sample_frames_in_window( - [_FakeFrame(3600), _FakeFrame(3601), _FakeFrame(3602)], - offset_seconds=0, - duration_seconds=3, - sampling_rate=1, - time_base=1, - stream_start_seconds=3600, + blocks = cast( + "list[dict[str, str]]", + video_module._sample_frames_in_window( + [_FakeFrame(3600), _FakeFrame(3601), _FakeFrame(3602)], + offset_seconds=0, + duration_seconds=3, + sampling_rate=1, + time_base=1, + stream_start_seconds=3600, + ), ) headers = [block["text"] for block in blocks if block["type"] == "text"] @@ -166,12 +221,15 @@ def test_sample_frames_keeps_next_target_after_late_frame(monkeypatch: pytest.Mo """A late frame should not skip the next sampling target.""" monkeypatch.setattr(video_module, "_encode_jpeg", lambda _frame: b"jpeg") - blocks = video_module._sample_frames_in_window( - [_FakeFrame(1), _FakeFrame(2), _FakeFrame(3)], - offset_seconds=0.4, - duration_seconds=4, - sampling_rate=1, - time_base=1, + blocks = cast( + "list[dict[str, str]]", + video_module._sample_frames_in_window( + [_FakeFrame(1), _FakeFrame(2), _FakeFrame(3)], + offset_seconds=0.4, + duration_seconds=4, + sampling_rate=1, + time_base=1, + ), ) headers = [block["text"] for block in blocks if block["type"] == "text"] @@ -192,12 +250,15 @@ def test_sample_frames_does_not_cap_requested_duration(monkeypatch: pytest.Monke monkeypatch.setattr(video_module, "_encode_jpeg", lambda _frame: b"jpeg") monkeypatch.setattr(video_module, "MAX_VIDEO_SAMPLED_FRAMES", 2) - blocks = video_module._sample_frames_in_window( - [_FakeFrame(0), _FakeFrame(1), _FakeFrame(2), _FakeFrame(3)], - offset_seconds=0, - duration_seconds=999, - sampling_rate=1, - time_base=1, + blocks = cast( + "list[dict[str, str]]", + video_module._sample_frames_in_window( + [_FakeFrame(0), _FakeFrame(1), _FakeFrame(2), _FakeFrame(3)], + offset_seconds=0, + duration_seconds=999, + sampling_rate=1, + time_base=1, + ), ) headers = [block["text"] for block in blocks if block["type"] == "text"] @@ -211,12 +272,15 @@ def test_sample_frames_caps_frame_count(monkeypatch: pytest.MonkeyPatch) -> None monkeypatch.setattr(video_module, "_encode_jpeg", lambda _frame: b"jpeg") monkeypatch.setattr(video_module, "MAX_VIDEO_SAMPLED_FRAMES", 2) - blocks = video_module._sample_frames_in_window( - [_FakeFrame(0), _FakeFrame(1), _FakeFrame(2)], - offset_seconds=0, - duration_seconds=3, - sampling_rate=1, - time_base=1, + blocks = cast( + "list[dict[str, str]]", + video_module._sample_frames_in_window( + [_FakeFrame(0), _FakeFrame(1), _FakeFrame(2)], + offset_seconds=0, + duration_seconds=3, + sampling_rate=1, + time_base=1, + ), ) headers = [block["text"] for block in blocks if block["type"] == "text"] diff --git a/libs/deepagents/tests/unit_tests/test_middleware.py b/libs/deepagents/tests/unit_tests/test_middleware.py index 2073b97b14a..fb5d4bbf153 100644 --- a/libs/deepagents/tests/unit_tests/test_middleware.py +++ b/libs/deepagents/tests/unit_tests/test_middleware.py @@ -1,5 +1,3 @@ -import base64 -import json import mimetypes import time from unittest.mock import MagicMock, patch @@ -16,7 +14,6 @@ ToolCall, ToolMessage, ) -from langchain_core.messages.utils import convert_to_openai_messages from langgraph.graph.message import REMOVE_ALL_MESSAGES from langgraph.store.memory import InMemoryStore from langgraph.types import Command @@ -1470,299 +1467,6 @@ def test_mixed_content_small_text_large_image_not_evicted(self): assert result == tool_message - def test_read_file_video_routes_to_frame_extractor(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Video reads route through `_video.extract_video_frames`, not the generic base64 branch. - - The middleware must interpret `offset`/`limit` as seconds for video - reads, return interleaved text+image content blocks, and never leak the - raw video bytes to the model. - """ - raw_bytes = b"\x00\x01\x02 fake video bytes" - sentinel = [ - {"type": "text", "text": "Frame at t=00:00:00.000"}, - {"type": "image", "base64": "AAAA", "mime_type": "image/jpeg"}, - {"type": "text", "text": "Frame at t=00:00:02.000"}, - {"type": "image", "base64": "BBBB", "mime_type": "image/jpeg"}, - ] - - def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): # type: ignore[no-untyped-def] - # Capture the call arguments so the test asserts the - # offset/limit -> seconds reinterpretation. - fake_extract.calls.append( - { - "len": len(content), - "offset_seconds": offset_seconds, - "duration_seconds": duration_seconds, - "sampling_rate": sampling_rate, - } - ) - return list(sentinel) - - fake_extract.calls = [] - - monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) - - middleware = FilesystemMiddleware(backend=_video_backend(raw_bytes)) - state = FilesystemState(messages=[], files={}) - runtime = _build_runtime(state, "video-read-1") - read_file_tool = next(t for t in middleware.tools if t.name == "read_file") - - result = read_file_tool.invoke( - { - "file_path": "/clips/intro.mkv", - "offset": 0, - "limit": 30, - "runtime": runtime, - } - ) - - assert isinstance(result, Command) - messages = result.update["messages"] # type: ignore[index] - assert len(messages) == 2 - tool_message, media_message = messages - assert isinstance(tool_message, ToolMessage) - assert tool_message.status == "success" - assert tool_message.content == "Read video /clips/intro.mkv: sampled 2 frames. The sampled frames are attached in the following message." - assert tool_message.additional_kwargs["read_file_path"] == "/clips/intro.mkv" - assert tool_message.additional_kwargs["read_file_frame_count"] == 2 - assert isinstance(media_message, HumanMessage) - assert media_message.additional_kwargs["read_file_media_result"] is True - assert media_message.additional_kwargs["read_file_path"] == "/clips/intro.mkv" - assert media_message.additional_kwargs["read_file_tool_call_id"] == "video-read-1" - # Raw video bytes must NOT leak to the model: the media message contains - # extracted frames, not the original base64 from the backend. - assert media_message.content == [ - {"type": "text", "text": "Reading first 30s of /clips/intro.mkv at 0.5 fps."}, - *sentinel, - ] - assert b"\x00\x01\x02" not in str(media_message.content).encode() - - openai_messages = convert_to_openai_messages( - [ - HumanMessage(content="read video"), - AIMessage(content="", tool_calls=[{"id": "video-read-1", "name": "read_file", "args": {"file_path": "/clips/intro.mkv"}}]), - *messages, - ] - ) - openai_tool_message = next(message for message in openai_messages if message["role"] == "tool") - assert isinstance(openai_tool_message["content"], str) - assert all( - block.get("type") != "video" - for message in openai_messages - for block in (message["content"] if isinstance(message.get("content"), list) else []) - ) - - # The middleware must convert offset/limit to seconds and use the - # internal sampling rate. - assert len(fake_extract.calls) == 1 - call = fake_extract.calls[0] - assert call["offset_seconds"] == 0 - assert call["duration_seconds"] == 30 - assert call["sampling_rate"] == 0.5 - assert call["len"] == len(raw_bytes) - - @pytest.mark.parametrize( - ("tool_input", "expected"), - [ - ( - {"offset": 12, "limit": 90}, - {"offset_seconds": 12.0, "duration_seconds": 90.0, "sampling_rate": 0.5}, - ), - ( - {}, - {"offset_seconds": 0.0, "duration_seconds": 100.0, "sampling_rate": 0.5}, - ), - ], - ids=["explicit-window", "default-window"], - ) - def test_read_file_video_window_forwards_seconds( - self, - monkeypatch: pytest.MonkeyPatch, - tool_input: dict[str, int], - expected: dict[str, float], - ) -> None: - """`offset`/`limit` become seconds, with the default window preserved.""" - captured: dict[str, float] = {} - - def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): # type: ignore[no-untyped-def] # noqa: ARG001 - captured.update(offset_seconds=offset_seconds, duration_seconds=duration_seconds, sampling_rate=sampling_rate) - return [{"type": "text", "text": "ok"}] - - monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) - middleware = FilesystemMiddleware(backend=_video_backend()) - state = FilesystemState(messages=[], files={}) - runtime = _build_runtime(state, "video-read-window") - read_file_tool = next(t for t in middleware.tools if t.name == "read_file") - - read_file_tool.invoke({"file_path": "/c.mp4", "runtime": runtime, **tool_input}) - - assert captured == expected - - def test_read_file_video_media_result_ordered_after_parallel_tool_results(self) -> None: - """Video frame attachments do not split a provider-required tool result batch.""" - ai_message = AIMessage( - content="", - tool_calls=[ - {"id": "call_video", "name": "read_file", "args": {"file_path": "/c.mp4"}}, - {"id": "call_ls", "name": "ls", "args": {"path": "/"}}, - ], - ) - video_tool = ToolMessage(content="sampled frames", name="read_file", tool_call_id="call_video") - video_media = HumanMessage( - content=[ - {"type": "text", "text": "Reading first 100s of /c.mp4 at 0.5 fps."}, - {"type": "image", "base64": "AAAA", "mime_type": "image/jpeg"}, - ], - additional_kwargs={"read_file_media_result": True}, - ) - ls_tool = ToolMessage(content="[]", name="ls", tool_call_id="call_ls") - - user_message = HumanMessage(content="read and list") - reordered = filesystem_middleware._move_media_results_after_tool_results([user_message, ai_message, video_tool, video_media, ls_tool]) - - assert reordered == [user_message, ai_message, video_tool, ls_tool, video_media] - - def test_read_file_video_extraction_error_surfaces_as_error_message(self, monkeypatch: pytest.MonkeyPatch) -> None: - """PyAV failures and missing-dep errors render as a tool error, not an exception.""" - monkeypatch.setattr( - filesystem_middleware, - "extract_video_frames", - lambda *_args, **_kwargs: (_ for _ in ()).throw(filesystem_middleware.VideoExtractionError("av not installed")), - ) - middleware = FilesystemMiddleware(backend=_video_backend(b"corrupt")) - state = FilesystemState(messages=[], files={}) - runtime = _build_runtime(state, "video-read-err") - read_file_tool = next(t for t in middleware.tools if t.name == "read_file") - - result = read_file_tool.invoke({"file_path": "/c.mp4", "runtime": runtime}) - - assert isinstance(result, ToolMessage) - assert result.status == "error" - assert "av not installed" in result.content # type: ignore[operator] - - def test_read_file_video_non_positive_limit_rejected(self, monkeypatch: pytest.MonkeyPatch) -> None: - """A non-positive `limit` is rejected as a tool error, not silently clamped.""" - called = {"count": 0} - - def fake_extract(*args: "object", **kwargs: "object") -> "list[dict[str, object]]": # noqa: ARG001 - called["count"] += 1 - return [{"type": "text", "text": "ok"}] - - monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) - middleware = FilesystemMiddleware(backend=_video_backend()) - state = FilesystemState(messages=[], files={}) - runtime = _build_runtime(state, "video-read-bad-limit") - read_file_tool = next(t for t in middleware.tools if t.name == "read_file") - - result = read_file_tool.invoke({"file_path": "/c.mp4", "limit": 0, "runtime": runtime}) - - assert isinstance(result, ToolMessage) - assert result.status == "error" - assert "limit must be > 0" in result.content # type: ignore[operator] - assert called["count"] == 0 - - def test_read_file_video_payload_size_cap_rejected(self, monkeypatch: pytest.MonkeyPatch) -> None: - """State/store/custom video payloads are capped after base64 decoding.""" - called = {"count": 0} - - def fake_extract(*args: "object", **kwargs: "object") -> "list[dict[str, object]]": # noqa: ARG001 - called["count"] += 1 - return [{"type": "text", "text": "ok"}] - - monkeypatch.setattr(filesystem_middleware, "MAX_VIDEO_INPUT_BYTES", 3) - monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) - middleware = FilesystemMiddleware(backend=_video_backend(b"abcd")) - state = FilesystemState(messages=[], files={}) - runtime = _build_runtime(state, "video-read-big-payload") - read_file_tool = next(t for t in middleware.tools if t.name == "read_file") - - result = read_file_tool.invoke({"file_path": "/c.mp4", "runtime": runtime}) - - assert isinstance(result, ToolMessage) - assert result.status == "error" - assert "video payload exceeds maximum input size of 3 bytes" in result.content # type: ignore[operator] - assert called["count"] == 0 - - def test_user_supplied_video_block_transforms_for_non_video_model(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Native video blocks are converted to sampled frames for non-video models.""" - raw_bytes = b"user supplied video" - sentinel = [ - {"type": "text", "text": "Frame at t=00:00:00.000"}, - {"type": "image", "base64": "AAAA", "mime_type": "image/jpeg"}, - ] - calls: list[dict[str, object]] = [] - - def fake_extract(content, *, offset_seconds, duration_seconds, sampling_rate): # type: ignore[no-untyped-def] - calls.append( - { - "content": content, - "offset_seconds": offset_seconds, - "duration_seconds": duration_seconds, - "sampling_rate": sampling_rate, - } - ) - return list(sentinel) - - monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) - video_block = _video_content_block(raw_bytes, filename="upload.mp4") - messages = [HumanMessage(content=[{"type": "text", "text": "watch this"}, video_block])] - - transformed, mutated = filesystem_middleware._transform_video_blocks_for_model(messages, native_video=False) - - assert mutated is True - assert transformed[0] is not messages[0] - assert messages[0].content == [{"type": "text", "text": "watch this"}, video_block] - assert transformed[0].content == [ - {"type": "text", "text": "watch this"}, - {"type": "text", "text": "Reading first 100s of upload.mp4 at 0.5 fps."}, - *sentinel, - ] - assert calls == [ - { - "content": raw_bytes, - "offset_seconds": 0.0, - "duration_seconds": 100.0, - "sampling_rate": 0.5, - } - ] - assert video_block["base64"] not in str(transformed[0].content) - - def test_user_supplied_video_block_passes_through_for_video_model(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Video-capable models keep native video blocks unchanged.""" - called = {"count": 0} - - def fake_extract(*args: object, **kwargs: object) -> list[dict[str, object]]: # noqa: ARG001 - called["count"] += 1 - return [{"type": "text", "text": "unexpected"}] - - monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) - messages = [HumanMessage(content=[_video_content_block(b"native", filename="gemini.mov")])] - native_video = filesystem_middleware._model_accepts_native_video(_VideoCapableModel("google_genai", "gemini-2.0-flash")) - - transformed, mutated = filesystem_middleware._transform_video_blocks_for_model(messages, native_video=native_video) - - assert native_video is True - assert transformed is messages - assert mutated is False - assert called["count"] == 0 - - def test_stringified_video_blocks_are_recovered_for_non_video_model(self, monkeypatch: pytest.MonkeyPatch) -> None: - """Stringified video tool content is still converted before non-video model calls.""" - monkeypatch.setattr( - filesystem_middleware, - "extract_video_frames", - lambda *_args, **_kwargs: [{"type": "image", "base64": "AAAA", "mime_type": "image/jpeg"}], - ) - payload = json.dumps([_video_content_block(b"tool video", filename="tool.mp4")]) - messages = [ToolMessage(content=payload, name="custom_tool", tool_call_id="call_1")] - - transformed, mutated = filesystem_middleware._transform_video_blocks_for_model(messages, native_video=False) - - assert mutated is True - assert isinstance(transformed[0].content, list) - assert transformed[0].content[0] == {"type": "text", "text": "Reading first 100s of tool.mp4 at 0.5 fps."} - assert all(block.get("type") != "video" for block in transformed[0].content) - def test_read_file_image_returns_standard_image_content_block(self): """Test image reads return standard image blocks with base64 + mime_type.""" @@ -2876,47 +2580,3 @@ def test_max_execute_timeout_init_validation(self): with pytest.raises(ValueError, match="max_execute_timeout must be positive"): FilesystemMiddleware(max_execute_timeout=-1) - - -def _build_runtime(state: FilesystemState, tool_call_id: str) -> ToolRuntime: - """Build a `ToolRuntime` for the binary/video read_file tests.""" - return ToolRuntime( - state=state, - context=None, - tool_call_id=tool_call_id, - store=None, - stream_writer=lambda _: None, - config={}, - ) - - -def _video_backend(raw: bytes = b"raw") -> StateBackend: - """Backend stub that returns `raw` base64-encoded for any video read.""" - payload = base64.b64encode(raw).decode("ascii") - - class VideoBackend(StateBackend): - def read(self, path, *, offset=0, limit=100): # type: ignore[override] - return ReadResult(file_data={"content": payload, "encoding": "base64"}) - - return VideoBackend() - - -def _video_content_block(raw: bytes, *, filename: str) -> dict[str, object]: - """Build an inline video content block for request-time transformation tests.""" - return { - "type": "video", - "base64": base64.b64encode(raw).decode("ascii"), - "mime_type": "video/mp4", - "source_metadata": {"filename": filename}, - } - - -class _VideoCapableModel: - """Small model stub exposing the provider metadata used for video capability.""" - - def __init__(self, provider: str, model_name: str) -> None: - self.model_name = model_name - self._provider = provider - - def _get_ls_params(self) -> dict[str, str]: - return {"ls_provider": self._provider} From 646d36170444c5f5fa20d33a2ec06d0a634c73a1 Mon Sep 17 00:00:00 2001 From: Deep Agent Date: Tue, 23 Jun 2026 01:48:29 -0700 Subject: [PATCH 20/36] fix(code): make video extra opt-in --- libs/code/pyproject.toml | 3 ++- libs/code/uv.lock | 10 +++++++--- libs/deepagents/deepagents/middleware/filesystem.py | 4 ++-- libs/talon/uv.lock | 7 ++++--- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/libs/code/pyproject.toml b/libs/code/pyproject.toml index 7694702386e..5fba6861995 100644 --- a/libs/code/pyproject.toml +++ b/libs/code/pyproject.toml @@ -26,7 +26,7 @@ classifiers = [ ] dependencies = [ # Framework - "deepagents[video]==0.7.0a2", + "deepagents==0.7.0a2", "langchain>=1.3.11,<2.0.0", "langgraph-checkpoint-sqlite>=3.1.0,<4.0.0", @@ -121,6 +121,7 @@ all-sandboxes = [ # Standalone integrations (kept for backwards-compatible install commands) quickjs = [] +video = ["deepagents[video]==0.7.0a2"] [project.scripts] diff --git a/libs/code/uv.lock b/libs/code/uv.lock index 15d3eac9e8b..4cd8d3de4fa 100644 --- a/libs/code/uv.lock +++ b/libs/code/uv.lock @@ -1105,7 +1105,7 @@ version = "0.1.27" source = { editable = "." } dependencies = [ { name = "aiosqlite" }, - { name = "deepagents", extra = ["video"] }, + { name = "deepagents" }, { name = "deepagents-acp" }, { name = "httpx" }, { name = "langchain" }, @@ -1240,6 +1240,9 @@ vercel = [ vertex = [ { name = "langchain-google-vertexai" }, ] +video = [ + { name = "deepagents", extra = ["video"] }, +] xai = [ { name = "langchain-xai" }, ] @@ -1268,7 +1271,8 @@ test = [ [package.metadata] requires-dist = [ { name = "aiosqlite", specifier = ">=0.22.1,<1.0.0" }, - { name = "deepagents", extras = ["video"], editable = "../deepagents" }, + { name = "deepagents", editable = "../deepagents" }, + { name = "deepagents", extras = ["video"], marker = "extra == 'video'", editable = "../deepagents" }, { name = "deepagents-acp", specifier = ">=0.0.8,<1.0.0" }, { name = "deepagents-code", extras = ["agentcore", "daytona", "modal", "runloop", "vercel"], marker = "extra == 'all-sandboxes'" }, { name = "deepagents-code", extras = ["anthropic", "baseten", "bedrock", "cohere", "deepseek", "fireworks", "google-genai", "groq", "huggingface", "ibm", "litellm", "mistralai", "nvidia", "ollama", "openai", "openrouter", "perplexity", "together", "vertex", "xai"], marker = "extra == 'all-providers'" }, @@ -1327,7 +1331,7 @@ requires-dist = [ { name = "tomli-w", specifier = ">=1.2.0,<2.0.0" }, { name = "uuid-utils", specifier = ">=0.16.2,<1.0.0" }, ] -provides-extras = ["anthropic", "baseten", "bedrock", "cohere", "deepseek", "fireworks", "google-genai", "groq", "huggingface", "ibm", "litellm", "mistralai", "nvidia", "ollama", "openai", "openrouter", "perplexity", "together", "vertex", "xai", "all-providers", "agentcore", "daytona", "modal", "runloop", "vercel", "all-sandboxes", "quickjs"] +provides-extras = ["anthropic", "baseten", "bedrock", "cohere", "deepseek", "fireworks", "google-genai", "groq", "huggingface", "ibm", "litellm", "mistralai", "nvidia", "ollama", "openai", "openrouter", "perplexity", "together", "vertex", "xai", "all-providers", "agentcore", "daytona", "modal", "runloop", "vercel", "all-sandboxes", "quickjs", "video"] [package.metadata.requires-dev] test = [ diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index 341192ae2fe..bc8b01c8a7a 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -3,13 +3,13 @@ import asyncio import base64 -import binascii import concurrent.futures import contextlib import contextvars import mimetypes import threading import uuid +from binascii import Error as BinasciiError from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass from pathlib import Path, PurePosixPath @@ -174,7 +174,7 @@ def _err(msg: str) -> ToolMessage: try: raw_bytes = base64.b64decode(content, validate=True) if isinstance(content, str) else content - except (ValueError, TypeError, binascii.Error) as exc: + except (ValueError, TypeError, BinasciiError) as exc: return _err(f"video bytes are not valid base64 ({exc})") if len(raw_bytes) > MAX_VIDEO_INPUT_BYTES: return _err(f"video payload exceeds maximum input size of {MAX_VIDEO_INPUT_BYTES} bytes") diff --git a/libs/talon/uv.lock b/libs/talon/uv.lock index 62a84bf45a2..8c2b39986e6 100644 --- a/libs/talon/uv.lock +++ b/libs/talon/uv.lock @@ -822,7 +822,7 @@ version = "0.1.27" source = { editable = "../code" } dependencies = [ { name = "aiosqlite" }, - { name = "deepagents", extra = ["video"] }, + { name = "deepagents" }, { name = "deepagents-acp" }, { name = "httpx" }, { name = "langchain" }, @@ -858,7 +858,8 @@ dependencies = [ [package.metadata] requires-dist = [ { name = "aiosqlite", specifier = ">=0.22.1,<1.0.0" }, - { name = "deepagents", extras = ["video"], editable = "../deepagents" }, + { name = "deepagents", editable = "../deepagents" }, + { name = "deepagents", extras = ["video"], marker = "extra == 'video'", editable = "../deepagents" }, { name = "deepagents-acp", specifier = ">=0.0.8,<1.0.0" }, { name = "deepagents-code", extras = ["agentcore", "daytona", "modal", "runloop", "vercel"], marker = "extra == 'all-sandboxes'" }, { name = "deepagents-code", extras = ["anthropic", "baseten", "bedrock", "cohere", "deepseek", "fireworks", "google-genai", "groq", "huggingface", "ibm", "litellm", "mistralai", "nvidia", "ollama", "openai", "openrouter", "perplexity", "together", "vertex", "xai"], marker = "extra == 'all-providers'" }, @@ -917,7 +918,7 @@ requires-dist = [ { name = "tomli-w", specifier = ">=1.2.0,<2.0.0" }, { name = "uuid-utils", specifier = ">=0.16.2,<1.0.0" }, ] -provides-extras = ["anthropic", "baseten", "bedrock", "cohere", "deepseek", "fireworks", "google-genai", "groq", "huggingface", "ibm", "litellm", "mistralai", "nvidia", "ollama", "openai", "openrouter", "perplexity", "together", "vertex", "xai", "all-providers", "agentcore", "daytona", "modal", "runloop", "vercel", "all-sandboxes", "quickjs"] +provides-extras = ["anthropic", "baseten", "bedrock", "cohere", "deepseek", "fireworks", "google-genai", "groq", "huggingface", "ibm", "litellm", "mistralai", "nvidia", "ollama", "openai", "openrouter", "perplexity", "together", "vertex", "xai", "all-providers", "agentcore", "daytona", "modal", "runloop", "vercel", "all-sandboxes", "quickjs", "video"] [package.metadata.requires-dev] test = [ From f7762377cd7397607685497132a750efea063eb3 Mon Sep 17 00:00:00 2001 From: Deep Agent Date: Tue, 23 Jun 2026 14:47:07 -0700 Subject: [PATCH 21/36] fix(sdk): resolve video read CI failures --- libs/code/deepagents_code/extras_info.py | 4 ++-- .../smoke_tests/snapshots/custom_system_message_tools.json | 2 +- .../snapshots/system_prompt_with_execute_tools.json | 2 +- .../snapshots/system_prompt_with_memory_and_skills_tools.json | 2 +- .../snapshots/system_prompt_with_routed_backend_tools.json | 2 +- .../system_prompt_with_sync_and_async_subagents_tools.json | 2 +- .../snapshots/system_prompt_without_execute_tools.json | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/libs/code/deepagents_code/extras_info.py b/libs/code/deepagents_code/extras_info.py index eee65c8aaf4..b94553fdf95 100644 --- a/libs/code/deepagents_code/extras_info.py +++ b/libs/code/deepagents_code/extras_info.py @@ -204,8 +204,8 @@ class ExtrasIntrospectionError(RuntimeError): ) """Optional extras that add sandbox integrations.""" -STANDALONE_EXTRAS: frozenset[str] = frozenset({"quickjs"}) -"""Compatibility extras that don't fit the provider/sandbox taxonomy. +STANDALONE_EXTRAS: frozenset[str] = frozenset({"quickjs", "video"}) +"""Optional extras that don't fit the provider/sandbox taxonomy. `quickjs` is a core dependency now, but the empty extra remains installable so older `deepagents-code[quickjs]` and `/install quickjs` workflows stay harmless. diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json index 0f20a4c7364..f3329be6f63 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/custom_system_message_tools.json @@ -74,7 +74,7 @@ }, "offset": { "default": 0, - "description": "Line number to start reading from (0-indexed). Use for pagination of large files.", + "description": "Line number to start reading from for text files (0-indexed). For videos, seconds into the source to start sampling.", "type": "integer" } }, diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json index 482a8020a0c..31d4c1e3558 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_execute_tools.json @@ -74,7 +74,7 @@ }, "offset": { "default": 0, - "description": "Line number to start reading from (0-indexed). Use for pagination of large files.", + "description": "Line number to start reading from for text files (0-indexed). For videos, seconds into the source to start sampling.", "type": "integer" } }, diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json index 0f20a4c7364..f3329be6f63 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_memory_and_skills_tools.json @@ -74,7 +74,7 @@ }, "offset": { "default": 0, - "description": "Line number to start reading from (0-indexed). Use for pagination of large files.", + "description": "Line number to start reading from for text files (0-indexed). For videos, seconds into the source to start sampling.", "type": "integer" } }, diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json index 482a8020a0c..31d4c1e3558 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_routed_backend_tools.json @@ -74,7 +74,7 @@ }, "offset": { "default": 0, - "description": "Line number to start reading from (0-indexed). Use for pagination of large files.", + "description": "Line number to start reading from for text files (0-indexed). For videos, seconds into the source to start sampling.", "type": "integer" } }, diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json index e3d25b07a67..da35899b741 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_with_sync_and_async_subagents_tools.json @@ -74,7 +74,7 @@ }, "offset": { "default": 0, - "description": "Line number to start reading from (0-indexed). Use for pagination of large files.", + "description": "Line number to start reading from for text files (0-indexed). For videos, seconds into the source to start sampling.", "type": "integer" } }, diff --git a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json index 0f20a4c7364..f3329be6f63 100644 --- a/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json +++ b/libs/deepagents/tests/unit_tests/smoke_tests/snapshots/system_prompt_without_execute_tools.json @@ -74,7 +74,7 @@ }, "offset": { "default": 0, - "description": "Line number to start reading from (0-indexed). Use for pagination of large files.", + "description": "Line number to start reading from for text files (0-indexed). For videos, seconds into the source to start sampling.", "type": "integer" } }, From 978263553549cabfd4bcb4796d03050eb67afd6a Mon Sep 17 00:00:00 2001 From: Deep Agent Date: Fri, 26 Jun 2026 15:41:11 -0700 Subject: [PATCH 22/36] fix(talon): align video extra SDK range --- libs/talon/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/talon/pyproject.toml b/libs/talon/pyproject.toml index 34429e5d8a2..cc2c7ce4a1d 100644 --- a/libs/talon/pyproject.toml +++ b/libs/talon/pyproject.toml @@ -42,7 +42,7 @@ speech = [ "transformers>=5.12.1,<6.0.0", ] video = [ - "deepagents[video]>=0.6.8,<0.7.0", + "deepagents[video]>=0.6.12,<0.7.0", ] [project.scripts] From 8d03fb5bdbea07e476e6f6230ba6a1550745e466 Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Thu, 2 Jul 2026 00:10:04 -0400 Subject: [PATCH 23/36] cr --- examples/async-subagent-server/uv.lock | 6 +- examples/llm-wiki/uv.lock | 6 +- libs/code/pyproject.toml | 2 +- .../deepagents/middleware/_video.py | 4 +- .../tests/unit_tests/middleware/test_video.py | 2 +- libs/evals/uv.lock | 1089 +++++++++-------- libs/talon/pyproject.toml | 7 +- 7 files changed, 564 insertions(+), 552 deletions(-) diff --git a/examples/async-subagent-server/uv.lock b/examples/async-subagent-server/uv.lock index 80556fffc06..06e7b3ee7b5 100644 --- a/examples/async-subagent-server/uv.lock +++ b/examples/async-subagent-server/uv.lock @@ -363,6 +363,7 @@ dependencies = [ [package.metadata] requires-dist = [ + { name = "av", marker = "extra == 'video'", specifier = ">=17.0.0,<18.0.0" }, { name = "langchain", specifier = ">=1.3.11,<2.0.0" }, { name = "langchain-anthropic", specifier = ">=1.4.7,<2.0.0" }, { name = "langchain-aws", marker = "extra == 'aws'", specifier = ">=1.6.1,<2.0.0" }, @@ -370,15 +371,18 @@ requires-dist = [ { name = "langchain-google-genai", specifier = ">=4.2.6,<5.0.0" }, { name = "langchain-quickjs", marker = "extra == 'quickjs'", specifier = ">=0.3.2" }, { name = "langsmith", specifier = ">=0.9.3" }, + { name = "pillow", marker = "extra == 'video'", specifier = ">=10.0.0,<13.0.0" }, { name = "wcmatch", specifier = ">=10.1" }, ] -provides-extras = ["aws", "quickjs"] +provides-extras = ["aws", "quickjs", "video"] [package.metadata.requires-dev] test = [ + { name = "av", specifier = ">=17.0.0,<18.0.0" }, { name = "build" }, { name = "langchain-openai" }, { name = "langchain-tests", specifier = ">=1.1.9" }, + { name = "pillow", specifier = ">=10.0.0,<13.0.0" }, { name = "pytest" }, { name = "pytest-asyncio", specifier = ">=1.4.0" }, { name = "pytest-benchmark" }, diff --git a/examples/llm-wiki/uv.lock b/examples/llm-wiki/uv.lock index 12a617db37a..3c12075c70b 100644 --- a/examples/llm-wiki/uv.lock +++ b/examples/llm-wiki/uv.lock @@ -294,6 +294,7 @@ dependencies = [ [package.metadata] requires-dist = [ + { name = "av", marker = "extra == 'video'", specifier = ">=17.0.0,<18.0.0" }, { name = "langchain", specifier = ">=1.3.11,<2.0.0" }, { name = "langchain-anthropic", specifier = ">=1.4.7,<2.0.0" }, { name = "langchain-aws", marker = "extra == 'aws'", specifier = ">=1.6.1,<2.0.0" }, @@ -301,15 +302,18 @@ requires-dist = [ { name = "langchain-google-genai", specifier = ">=4.2.6,<5.0.0" }, { name = "langchain-quickjs", marker = "extra == 'quickjs'", specifier = ">=0.3.2" }, { name = "langsmith", specifier = ">=0.9.3" }, + { name = "pillow", marker = "extra == 'video'", specifier = ">=10.0.0,<13.0.0" }, { name = "wcmatch", specifier = ">=10.1" }, ] -provides-extras = ["aws", "quickjs"] +provides-extras = ["aws", "quickjs", "video"] [package.metadata.requires-dev] test = [ + { name = "av", specifier = ">=17.0.0,<18.0.0" }, { name = "build" }, { name = "langchain-openai" }, { name = "langchain-tests", specifier = ">=1.1.9" }, + { name = "pillow", specifier = ">=10.0.0,<13.0.0" }, { name = "pytest" }, { name = "pytest-asyncio", specifier = ">=1.4.0" }, { name = "pytest-benchmark" }, diff --git a/libs/code/pyproject.toml b/libs/code/pyproject.toml index 5dcc7a9a769..37a1a5189d9 100644 --- a/libs/code/pyproject.toml +++ b/libs/code/pyproject.toml @@ -124,7 +124,7 @@ all-sandboxes = [ # Standalone integrations (kept for backwards-compatible install commands) quickjs = [] -video = ["deepagents[video]==0.7.0a2"] +video = ["deepagents[video]==0.7.0a3"] [project.scripts] diff --git a/libs/deepagents/deepagents/middleware/_video.py b/libs/deepagents/deepagents/middleware/_video.py index 3b6e99bf664..0c7e0ca7af1 100644 --- a/libs/deepagents/deepagents/middleware/_video.py +++ b/libs/deepagents/deepagents/middleware/_video.py @@ -2,11 +2,11 @@ This module is the boundary between Deep Agents middleware and the optional video backend (PyAV). It imports PyAV lazily so a `deepagents` install without -the ``[video]`` extra stays lightweight; the import only fires when the agent +the `[video]` extra stays lightweight; the import only fires when the agent actually tries to read or transform a video. For each video read the module decodes a contiguous slice of the source -(``offset``-seconds skip, ``limit``-seconds window) and emits sampled +(`offset`-seconds skip, `limit`-seconds window) and emits sampled frames at the configured sampling rate. The output is a list of interleaved text+image content blocks so the model sees per-frame timestamps alongside the JPEGs. diff --git a/libs/deepagents/tests/unit_tests/middleware/test_video.py b/libs/deepagents/tests/unit_tests/middleware/test_video.py index 870b1d8f9a3..40cfaaea10f 100644 --- a/libs/deepagents/tests/unit_tests/middleware/test_video.py +++ b/libs/deepagents/tests/unit_tests/middleware/test_video.py @@ -3,7 +3,7 @@ The tests exercise the real PyAV decoder against a synthetic 3-second clip so the offset/limit -> seconds reinterpretation, frame sampling, validation, and error mapping all run end-to-end. They are skipped automatically when the -``av`` (PyAV) extra is not installed, which keeps the default unit suite +`av` (PyAV) extra is not installed, which keeps the default unit suite lightweight. """ diff --git a/libs/evals/uv.lock b/libs/evals/uv.lock index ecb9b3b919c..ac9a7d8be1b 100644 --- a/libs/evals/uv.lock +++ b/libs/evals/uv.lock @@ -27,7 +27,7 @@ name = "agent-client-protocol" version = "0.10.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pydantic" }, + { name = "pydantic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/88/a0/3b96cd8374725c69bc3dae9fcc2082f3f6cafec1be35d24d7af0f8c3265f/agent_client_protocol-0.10.1.tar.gz", hash = "sha256:355c65ca19f0568344aafc2c1552b7066a8fc491df23ab28e7e253c6c9a85a25", size = 81924, upload-time = "2026-05-24T18:46:44.444Z" } wheels = [ @@ -48,13 +48,13 @@ name = "aiohttp" version = "3.13.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "aiohappyeyeballs" }, - { name = "aiosignal" }, - { name = "attrs" }, - { name = "frozenlist" }, - { name = "multidict" }, - { name = "propcache" }, - { name = "yarl" }, + { name = "aiohappyeyeballs", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "aiosignal", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "attrs", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "frozenlist", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "multidict", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "propcache", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "yarl", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/77/9a/152096d4808df8e4268befa55fba462f440f14beab85e8ad9bf990516918/aiohttp-3.13.5.tar.gz", hash = "sha256:9d98cc980ecc96be6eb4c1994ce35d28d8b1f5e5208a23b421187d1209dbb7d1", size = 7858271, upload-time = "2026-03-31T22:01:03.343Z" } wheels = [ @@ -73,8 +73,8 @@ name = "aiosignal" version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "frozenlist" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "frozenlist", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(python_full_version < '3.13' and platform_machine == 'arm64' and sys_platform == 'darwin') or (python_full_version < '3.13' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/61/62/06741b579156360248d1ec624842ad0edf697050bbaf7c3e46394e106ad1/aiosignal-1.4.0.tar.gz", hash = "sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7", size = 25007, upload-time = "2025-07-03T22:54:43.528Z" } wheels = [ @@ -113,14 +113,14 @@ name = "anthropic" version = "0.97.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "docstring-parser" }, - { name = "httpx" }, - { name = "jiter" }, - { name = "pydantic" }, - { name = "sniffio" }, - { name = "typing-extensions" }, + { name = "anyio", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "distro", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "docstring-parser", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "jiter", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pydantic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "sniffio", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/14/93/f66ea8bfe39f2e6bb9da8e27fa5457ad2520e8f7612dfc547b17fad55c4d/anthropic-0.97.0.tar.gz", hash = "sha256:021e79fd8e21e90ad94dc5ba2bbbd8b1599f424f5b1fab6c06204009cab764be", size = 669502, upload-time = "2026-04-23T20:52:34.445Z" } wheels = [ @@ -132,8 +132,8 @@ name = "anyio" version = "4.12.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "idna" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "idna", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(python_full_version < '3.13' and platform_machine == 'arm64' and sys_platform == 'darwin') or (python_full_version < '3.13' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/96/f0/5eb65b2bb0d09ac6776f2eb54adee6abe8228ea05b20a5ad0e4945de8aac/anyio-4.12.1.tar.gz", hash = "sha256:41cfcc3a4c85d3f05c932da7c26d0201ac36f72abd4435ba90d0464a3ffed703", size = 228685, upload-time = "2026-01-06T11:45:21.246Z" } wheels = [ @@ -154,7 +154,7 @@ name = "baseten-performance-client" version = "0.1.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "requests" }, + { name = "requests", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/41/eb/dc8a0dd381cdac0132ea0f7833e707413a137f0590a849504e03d40776eb/baseten_performance_client-0.1.5.tar.gz", hash = "sha256:8e6999295fa2a5ce54aeb83f1c04417628282b66117c22101d023c746522816a", size = 92286, upload-time = "2026-01-30T06:37:30.283Z" } wheels = [ @@ -171,8 +171,8 @@ name = "beautifulsoup4" version = "4.14.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "soupsieve" }, - { name = "typing-extensions" }, + { name = "soupsieve", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c3/b0/1c6a16426d389813b48d95e26898aff79abbde42ad353958ad95cc8c9b21/beautifulsoup4-4.14.3.tar.gz", hash = "sha256:6292b1c5186d356bba669ef9f7f051757099565ad9ada5dd630bd9de5fa7fb86", size = 627737, upload-time = "2025-11-30T15:08:26.084Z" } wheels = [ @@ -184,7 +184,7 @@ name = "blockbuster" version = "1.5.26" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "forbiddenfruit", marker = "implementation_name == 'cpython'" }, + { name = "forbiddenfruit", marker = "(implementation_name == 'cpython' and platform_machine == 'arm64' and sys_platform == 'darwin') or (implementation_name == 'cpython' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/55/e0/dcbab602790a576b0b94108c07e2c048e5897df7cc83722a89582d733987/blockbuster-1.5.26.tar.gz", hash = "sha256:cc3ce8c70fa852a97ee3411155f31e4ad2665cd1c6c7d2f8bb1851dab61dc629", size = 36085, upload-time = "2025-12-05T10:43:47.735Z" } wheels = [ @@ -243,7 +243,7 @@ name = "cffi" version = "2.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pycparser", marker = "implementation_name != 'PyPy'" }, + { name = "pycparser", marker = "(implementation_name != 'PyPy' and platform_machine == 'arm64' and sys_platform == 'darwin') or (implementation_name != 'PyPy' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } wheels = [ @@ -293,7 +293,7 @@ name = "contourpy" version = "1.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, + { name = "numpy", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/58/01/1253e6698a07380cd31a736d248a3f2a50a7c88779a1813da27503cadc2a/contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880", size = 13466174, upload-time = "2025-07-26T12:03:12.549Z" } wheels = [ @@ -331,7 +331,7 @@ name = "croniter" version = "6.2.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "python-dateutil" }, + { name = "python-dateutil", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/df/de/5832661ed55107b8a09af3f0a2e71e0957226a59eb1dcf0a445cce6daf20/croniter-6.2.2.tar.gz", hash = "sha256:ba60832a5ec8e12e51b8691c3309a113d1cf6526bdf1a48150ce8ec7a532d0ab", size = 113762, upload-time = "2026-03-15T08:43:48.112Z" } wheels = [ @@ -343,7 +343,7 @@ name = "cryptography" version = "48.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, + { name = "cffi", marker = "(platform_machine == 'arm64' and platform_python_implementation != 'PyPy' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and platform_python_implementation != 'PyPy' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/12/45/870e7f4bef50e5f53b9f51d4428aee5290eedf58ba443f16b1ebb7ab8e66/cryptography-48.0.1.tar.gz", hash = "sha256:266f4ee051abb2f725b74ef8072b521ce1feacf685a3364fa6a6b45548db791a", size = 832989, upload-time = "2026-06-09T22:32:31.8Z" } wheels = [ @@ -373,20 +373,20 @@ name = "datasets" version = "5.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "dill" }, - { name = "filelock" }, - { name = "fsspec", extra = ["http"] }, - { name = "httpx" }, - { name = "huggingface-hub" }, - { name = "multiprocess" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "pandas" }, - { name = "pyarrow" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "tqdm" }, - { name = "xxhash" }, + { name = "dill", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "filelock", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "fsspec", extra = ["http"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "huggingface-hub", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "multiprocess", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "numpy", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "packaging", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pandas", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pyarrow", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pyyaml", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "requests", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "tqdm", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "xxhash", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d9/85/ce4f780c32f7e36d71257f1c27e8ba898ebe379cb54f211f5f2013f2c219/datasets-5.0.0.tar.gz", hash = "sha256:83dbbbdb07a33b82192b8c419deb18739b138ee2ce1a322d55ce6b100954ec1a", size = 631708, upload-time = "2026-06-05T13:18:26.124Z" } wheels = [ @@ -398,16 +398,17 @@ name = "deepagents" version = "0.6.12" source = { editable = "../deepagents" } dependencies = [ - { name = "langchain" }, - { name = "langchain-anthropic" }, - { name = "langchain-core" }, - { name = "langchain-google-genai" }, - { name = "langsmith" }, - { name = "wcmatch" }, + { name = "langchain", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-anthropic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-core", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-google-genai", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langsmith", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "wcmatch", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] [package.metadata] requires-dist = [ + { name = "av", marker = "extra == 'video'", specifier = ">=17.0.0,<18.0.0" }, { name = "langchain", specifier = ">=1.3.11,<2.0.0" }, { name = "langchain-anthropic", specifier = ">=1.4.7,<2.0.0" }, { name = "langchain-aws", marker = "extra == 'aws'", specifier = ">=1.6.1,<2.0.0" }, @@ -415,15 +416,18 @@ requires-dist = [ { name = "langchain-google-genai", specifier = ">=4.2.6,<5.0.0" }, { name = "langchain-quickjs", marker = "extra == 'quickjs'", specifier = ">=0.3.2" }, { name = "langsmith", specifier = ">=0.9.3" }, + { name = "pillow", marker = "extra == 'video'", specifier = ">=10.0.0,<13.0.0" }, { name = "wcmatch", specifier = ">=10.1" }, ] -provides-extras = ["aws", "quickjs"] +provides-extras = ["aws", "quickjs", "video"] [package.metadata.requires-dev] test = [ + { name = "av", specifier = ">=17.0.0,<18.0.0" }, { name = "build" }, { name = "langchain-openai" }, { name = "langchain-tests", specifier = ">=1.1.9" }, + { name = "pillow", specifier = ">=10.0.0,<13.0.0" }, { name = "pytest" }, { name = "pytest-asyncio", specifier = ">=1.4.0" }, { name = "pytest-benchmark" }, @@ -443,9 +447,9 @@ name = "deepagents-acp" version = "0.0.8" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "agent-client-protocol" }, - { name = "deepagents" }, - { name = "python-dotenv" }, + { name = "agent-client-protocol", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "deepagents", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "python-dotenv", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d4/4e/586cf7a70161e3eb0c3596ec9513f8249c0739283c2abe08b0c10537197e/deepagents_acp-0.0.8.tar.gz", hash = "sha256:9fb5cecfe9e9238de27e69ac76e7b0bc80a31cbff268c542bf5c9f4727dde1f4", size = 13010377, upload-time = "2026-06-03T18:08:17.321Z" } wheels = [ @@ -457,38 +461,38 @@ name = "deepagents-code" version = "0.1.30" source = { directory = "../code" } dependencies = [ - { name = "aiosqlite" }, - { name = "deepagents" }, - { name = "deepagents-acp" }, - { name = "httpx" }, - { name = "langchain" }, - { name = "langchain-anthropic" }, - { name = "langchain-google-genai" }, - { name = "langchain-mcp-adapters" }, - { name = "langchain-openai" }, - { name = "langchain-quickjs" }, - { name = "langgraph-checkpoint-sqlite" }, - { name = "langgraph-cli", extra = ["inmem"] }, - { name = "langgraph-runtime-inmem" }, - { name = "langgraph-sdk" }, - { name = "langsmith" }, - { name = "markdownify" }, - { name = "mcp" }, - { name = "packaging" }, - { name = "pillow" }, - { name = "prompt-toolkit" }, - { name = "pydantic" }, - { name = "pyperclip" }, - { name = "python-dotenv" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "rich" }, - { name = "tavily-python" }, - { name = "textual" }, - { name = "textual-autocomplete" }, - { name = "textual-speedups" }, - { name = "tomli-w" }, - { name = "uuid-utils" }, + { name = "aiosqlite", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "deepagents", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "deepagents-acp", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-anthropic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-google-genai", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-mcp-adapters", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-openai", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-quickjs", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langgraph-checkpoint-sqlite", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langgraph-cli", extra = ["inmem"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langgraph-runtime-inmem", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langgraph-sdk", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langsmith", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "markdownify", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "mcp", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "packaging", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pillow", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "prompt-toolkit", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pydantic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pyperclip", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "python-dotenv", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pyyaml", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "requests", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "rich", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "tavily-python", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "textual", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "textual-autocomplete", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "textual-speedups", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "tomli-w", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "uuid-utils", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] [package.metadata] @@ -496,6 +500,7 @@ requires-dist = [ { name = "aiohttp", marker = "extra == 'nvidia'", specifier = ">=3.14.1,<3.15.0" }, { name = "aiosqlite", specifier = ">=0.22.1,<1.0.0" }, { name = "deepagents", editable = "../deepagents" }, + { name = "deepagents", extras = ["video"], marker = "extra == 'video'", editable = "../deepagents" }, { name = "deepagents-acp", specifier = ">=0.0.8,<1.0.0" }, { name = "deepagents-code", extras = ["agentcore", "daytona", "modal", "runloop", "vercel"], marker = "extra == 'all-sandboxes'" }, { name = "deepagents-code", extras = ["anthropic", "baseten", "bedrock", "cohere", "deepseek", "fireworks", "google-genai", "groq", "huggingface", "ibm", "litellm", "mistralai", "nvidia", "ollama", "openai", "openrouter", "perplexity", "together", "vertex", "xai"], marker = "extra == 'all-providers'" }, @@ -554,7 +559,7 @@ requires-dist = [ { name = "tomli-w", specifier = ">=1.2.0,<2.0.0" }, { name = "uuid-utils", specifier = ">=0.16.2,<1.0.0" }, ] -provides-extras = ["anthropic", "baseten", "bedrock", "cohere", "deepseek", "fireworks", "google-genai", "groq", "huggingface", "ibm", "litellm", "mistralai", "nvidia", "ollama", "openai", "openrouter", "perplexity", "together", "vertex", "xai", "all-providers", "agentcore", "daytona", "modal", "runloop", "vercel", "all-sandboxes", "quickjs"] +provides-extras = ["anthropic", "baseten", "bedrock", "cohere", "deepseek", "fireworks", "google-genai", "groq", "huggingface", "ibm", "litellm", "mistralai", "nvidia", "ollama", "openai", "openrouter", "perplexity", "together", "vertex", "xai", "all-providers", "agentcore", "daytona", "modal", "runloop", "vercel", "all-sandboxes", "quickjs", "video"] [package.metadata.requires-dev] test = [ @@ -582,48 +587,48 @@ name = "deepagents-evals" version = "0.0.1" source = { editable = "." } dependencies = [ - { name = "datasets" }, - { name = "deepagents" }, - { name = "deepagents-code" }, - { name = "dockerfile-parse" }, - { name = "harbor", extra = ["langsmith"] }, - { name = "langchain" }, - { name = "langchain-anthropic" }, - { name = "langchain-baseten" }, - { name = "langchain-deepseek" }, - { name = "langchain-fireworks" }, - { name = "langchain-google-genai" }, - { name = "langchain-groq" }, - { name = "langchain-mistralai" }, - { name = "langchain-nvidia-ai-endpoints" }, - { name = "langchain-ollama" }, - { name = "langchain-openai" }, - { name = "langchain-openrouter" }, - { name = "langchain-quickjs" }, - { name = "langchain-xai" }, - { name = "langsmith" }, - { name = "modal" }, - { name = "nltk" }, - { name = "openevals" }, - { name = "tiktoken" }, + { name = "datasets", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "deepagents", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "deepagents-code", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "dockerfile-parse", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "harbor", extra = ["langsmith"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-anthropic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-baseten", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-deepseek", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-fireworks", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-google-genai", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-groq", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-mistralai", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-nvidia-ai-endpoints", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-ollama", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-openai", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-openrouter", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-quickjs", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-xai", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langsmith", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "modal", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "nltk", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "openevals", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "tiktoken", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] [package.optional-dependencies] charts = [ - { name = "matplotlib" }, + { name = "matplotlib", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] [package.dev-dependencies] test = [ - { name = "matplotlib" }, - { name = "pytest" }, - { name = "pytest-asyncio" }, - { name = "pytest-cov" }, - { name = "pytest-socket" }, - { name = "pytest-watcher" }, - { name = "python-dotenv" }, - { name = "ruff" }, - { name = "ty" }, + { name = "matplotlib", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pytest", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pytest-asyncio", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pytest-cov", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pytest-socket", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pytest-watcher", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "python-dotenv", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "ruff", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "ty", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] [package.metadata] @@ -674,7 +679,7 @@ name = "deprecation" version = "2.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "packaging" }, + { name = "packaging", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5a/d3/8ae2869247df154b64c1884d7346d412fed0c49df84db635aab2d1c40e62/deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff", size = 173788, upload-time = "2020-04-20T14:23:38.738Z" } wheels = [ @@ -695,7 +700,7 @@ name = "dirhash" version = "0.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "scantree" }, + { name = "scantree", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1d/70/49f93897f3a4f7ab5f20a854ebc91aad47854e9fb2cd169e3a4452fa3f5e/dirhash-0.5.0.tar.gz", hash = "sha256:e60760f0ab2e935d8cb088923ea2c6492398dca42cec785df778985fd4cd5386", size = 21377, upload-time = "2024-08-03T22:14:13.322Z" } wheels = [ @@ -734,11 +739,11 @@ name = "fastapi" version = "0.135.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "annotated-doc" }, - { name = "pydantic" }, - { name = "starlette" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, + { name = "annotated-doc", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pydantic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "starlette", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-inspection", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e7/7b/f8e0211e9380f7195ba3f3d40c292594fd81ba8ec4629e3854c353aaca45/fastapi-0.135.1.tar.gz", hash = "sha256:d04115b508d936d254cea545b7312ecaa58a7b3a0f84952535b4c9afae7668cd", size = 394962, upload-time = "2026-03-01T18:18:29.369Z" } wheels = [ @@ -784,14 +789,14 @@ name = "fireworks-ai" version = "1.2.0a78" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "aiohttp" }, - { name = "anyio" }, - { name = "distro" }, - { name = "httpx" }, - { name = "httpx-aiohttp" }, - { name = "pydantic" }, - { name = "sniffio" }, - { name = "typing-extensions" }, + { name = "aiohttp", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "anyio", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "distro", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "httpx-aiohttp", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pydantic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "sniffio", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/46/9f/4406331abaa2c30b4306a935671f82ddf46650589bb022f494b8f19870b3/fireworks_ai-1.2.0a78.tar.gz", hash = "sha256:250c8825d78d81d4211610f4e92a2052213571146a0a98b72dd3a0a11e403fc0", size = 431024, upload-time = "2026-06-09T05:58:18.217Z" } wheels = [ @@ -851,7 +856,7 @@ wheels = [ [package.optional-dependencies] http = [ - { name = "aiohttp" }, + { name = "aiohttp", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] [[package]] @@ -859,8 +864,8 @@ name = "google-auth" version = "2.49.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography" }, - { name = "pyasn1-modules" }, + { name = "cryptography", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pyasn1-modules", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ea/80/6a696a07d3d3b0a92488933532f03dbefa4a24ab80fb231395b9a2a1be77/google_auth-2.49.1.tar.gz", hash = "sha256:16d40da1c3c5a0533f57d268fe72e0ebb0ae1cc3b567024122651c045d879b64", size = 333825, upload-time = "2026-03-12T19:30:58.135Z" } wheels = [ @@ -869,7 +874,7 @@ wheels = [ [package.optional-dependencies] requests = [ - { name = "requests" }, + { name = "requests", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] [[package]] @@ -877,16 +882,16 @@ name = "google-genai" version = "1.68.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "google-auth", extra = ["requests"] }, - { name = "httpx" }, - { name = "pydantic" }, - { name = "requests" }, - { name = "sniffio" }, - { name = "tenacity" }, - { name = "typing-extensions" }, - { name = "websockets" }, + { name = "anyio", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "distro", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "google-auth", extra = ["requests"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pydantic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "requests", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "sniffio", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "tenacity", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "websockets", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9c/2c/f059982dbcb658cc535c81bbcbe7e2c040d675f4b563b03cdb01018a4bc3/google_genai-1.68.0.tar.gz", hash = "sha256:ac30c0b8bc630f9372993a97e4a11dae0e36f2e10d7c55eacdca95a9fa14ca96", size = 511285, upload-time = "2026-03-18T01:03:18.243Z" } wheels = [ @@ -898,7 +903,7 @@ name = "googleapis-common-protos" version = "1.73.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "protobuf" }, + { name = "protobuf", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/99/96/a0205167fa0154f4a542fd6925bdc63d039d88dab3588b875078107e6f06/googleapis_common_protos-1.73.0.tar.gz", hash = "sha256:778d07cd4fbeff84c6f7c72102f0daf98fa2bfd3fa8bea426edc545588da0b5a", size = 147323, upload-time = "2026-03-06T21:53:09.727Z" } wheels = [ @@ -910,12 +915,12 @@ name = "groq" version = "0.37.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "httpx" }, - { name = "pydantic" }, - { name = "sniffio" }, - { name = "typing-extensions" }, + { name = "anyio", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "distro", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pydantic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "sniffio", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e9/78/18948a9056e1509c87e10ab8316a90ecce87035fbd53342dffdf97f4de00/groq-0.37.1.tar.gz", hash = "sha256:7353d6dfb60834fd7aacbb86af106e2dc2aeaff6d0edd65fb2fd0f16bd39314c", size = 145289, upload-time = "2025-12-04T18:08:07.118Z" } wheels = [ @@ -927,7 +932,7 @@ name = "grpcio" version = "1.80.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b7/48/af6173dbca4454f4637a4678b67f52ca7e0c1ed7d5894d89d434fecede05/grpcio-1.80.0.tar.gz", hash = "sha256:29aca15edd0688c22ba01d7cc01cb000d72b2033f4a3c72a81a19b56fd143257", size = 12978905, upload-time = "2026-03-30T08:49:10.502Z" } wheels = [ @@ -944,8 +949,8 @@ name = "grpcio-health-checking" version = "1.80.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "grpcio" }, - { name = "protobuf" }, + { name = "grpcio", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "protobuf", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d1/a2/aa3cc47f19c03f8e5287b987317059753141a3af8f66b96d5a64b3be10b8/grpcio_health_checking-1.80.0.tar.gz", hash = "sha256:2cc5f08bc8b816b8655ab6f59c71450063ba20766d31e21a493e912e3560c8b1", size = 17117, upload-time = "2026-03-30T08:54:41.899Z" } wheels = [ @@ -957,9 +962,9 @@ name = "grpcio-tools" version = "1.80.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "grpcio" }, - { name = "protobuf" }, - { name = "setuptools" }, + { name = "grpcio", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "protobuf", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "setuptools", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/94/c8/1223f29c84a143ae9a56c084fc96894de0ba84b6e8d60a26241abd81d278/grpcio_tools-1.80.0.tar.gz", hash = "sha256:26052b19c6ce0dcf52d1024496aea3e2bdfa864159f06dc7b97b22d041a94b26", size = 6133212, upload-time = "2026-03-30T08:52:39.077Z" } wheels = [ @@ -976,8 +981,8 @@ name = "grpclib" version = "0.4.9" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "h2" }, - { name = "multidict" }, + { name = "h2", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "multidict", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5b/28/5a2c299ec82a876a252c5919aa895a6f1d1d35c96417c5ce4a4660dc3a80/grpclib-0.4.9.tar.gz", hash = "sha256:cc589c330fa81004c6400a52a566407574498cb5b055fa927013361e21466c46", size = 84798, upload-time = "2025-12-14T22:23:14.349Z" } wheels = [ @@ -998,8 +1003,8 @@ name = "h2" version = "4.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "hpack" }, - { name = "hyperframe" }, + { name = "hpack", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "hyperframe", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1d/17/afa56379f94ad0fe8defd37d6eb3f89a25404ffc71d4d848893d270325fc/h2-4.3.0.tar.gz", hash = "sha256:6c59efe4323fa18b47a632221a1888bd7fde6249819beda254aeca909f221bf1", size = 2152026, upload-time = "2025-08-23T18:12:19.778Z" } wheels = [ @@ -1011,27 +1016,27 @@ name = "harbor" version = "0.16.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "datasets" }, - { name = "dirhash" }, - { name = "fastapi" }, - { name = "filelock" }, - { name = "httpx" }, - { name = "jinja2" }, - { name = "litellm" }, - { name = "packaging" }, - { name = "pathspec" }, - { name = "platformdirs" }, - { name = "pydantic" }, - { name = "python-dotenv" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "rich" }, - { name = "shortuuid" }, - { name = "supabase" }, - { name = "tenacity" }, - { name = "toml" }, - { name = "typer" }, - { name = "uvicorn" }, + { name = "datasets", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "dirhash", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "fastapi", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "filelock", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "jinja2", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "litellm", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "packaging", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pathspec", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "platformdirs", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pydantic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "python-dotenv", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pyyaml", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "requests", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "rich", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "shortuuid", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "supabase", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "tenacity", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "toml", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typer", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "uvicorn", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/37/46/49008d2a07f98ce0eb333366546cac8451d6e80c8068ac7687711c56c862/harbor-0.16.1.tar.gz", hash = "sha256:115c39ad76b3d1dbfd82ff51af904adf22acac240c90b12eb34e8cb7d46f8cc4", size = 1372236, upload-time = "2026-06-28T05:13:17.962Z" } wheels = [ @@ -1040,8 +1045,8 @@ wheels = [ [package.optional-dependencies] langsmith = [ - { name = "harbor-langsmith" }, - { name = "langsmith" }, + { name = "harbor-langsmith", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langsmith", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] [[package]] @@ -1049,8 +1054,8 @@ name = "harbor-langsmith" version = "0.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "harbor" }, - { name = "requests" }, + { name = "harbor", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "requests", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/46/15/cb0f23fce7c644a3bf9a08084da0ab09cffca01f56af1a1bee2eea0ba3a0/harbor_langsmith-0.1.1.tar.gz", hash = "sha256:4ee136ea5915ebf5917ae7e7f0b2960641f6f54c6b6972bcf2bc7d54069ea1b9", size = 5196, upload-time = "2026-06-02T21:40:58.352Z" } wheels = [ @@ -1085,8 +1090,8 @@ name = "httpcore" version = "1.0.9" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "certifi" }, - { name = "h11" }, + { name = "certifi", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "h11", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } wheels = [ @@ -1114,10 +1119,10 @@ name = "httpx" version = "0.28.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio" }, - { name = "certifi" }, - { name = "httpcore" }, - { name = "idna" }, + { name = "anyio", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "certifi", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "httpcore", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "idna", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } wheels = [ @@ -1126,7 +1131,7 @@ wheels = [ [package.optional-dependencies] http2 = [ - { name = "h2" }, + { name = "h2", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] [[package]] @@ -1134,8 +1139,8 @@ name = "httpx-aiohttp" version = "0.1.12" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "aiohttp" }, - { name = "httpx" }, + { name = "aiohttp", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/63/2c/b894861cecf030fb45675ea24aa55b5722e97c602a163d872fca66c5a6d8/httpx_aiohttp-0.1.12.tar.gz", hash = "sha256:81feec51fd82c0ecfa0e9aaf1b1a6c2591260d5e2bcbeb7eb0277a78e610df2c", size = 275945, upload-time = "2025-12-12T10:12:15.283Z" } wheels = [ @@ -1156,15 +1161,15 @@ name = "huggingface-hub" version = "1.7.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "filelock" }, - { name = "fsspec" }, - { name = "hf-xet" }, - { name = "httpx" }, - { name = "packaging" }, - { name = "pyyaml" }, - { name = "tqdm" }, - { name = "typer" }, - { name = "typing-extensions" }, + { name = "filelock", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "fsspec", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "hf-xet", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "packaging", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pyyaml", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "tqdm", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typer", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b4/a8/94ccc0aec97b996a3a68f3e1fa06a4bd7185dd02bf22bfba794a0ade8440/huggingface_hub-1.7.1.tar.gz", hash = "sha256:be38fe66e9b03c027ad755cb9e4b87ff0303c98acf515b5d579690beb0bf3048", size = 722097, upload-time = "2026-03-13T09:36:07.758Z" } wheels = [ @@ -1194,7 +1199,7 @@ name = "importlib-metadata" version = "8.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "zipp" }, + { name = "zipp", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cd/12/33e59336dca5be0c398a7482335911a33aa0e20776128f038019f1a95f1b/importlib_metadata-8.5.0.tar.gz", hash = "sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7", size = 55304, upload-time = "2024-09-11T14:56:08.937Z" } wheels = [ @@ -1215,7 +1220,7 @@ name = "jinja2" version = "3.1.6" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "markupsafe" }, + { name = "markupsafe", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } wheels = [ @@ -1254,7 +1259,7 @@ name = "jsonpatch" version = "1.33" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jsonpointer" }, + { name = "jsonpointer", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/78/18813351fe5d63acad16aec57f94ec2b70a09e53ca98145589e185423873/jsonpatch-1.33.tar.gz", hash = "sha256:9fcd4009c41e6d12348b4a0ff2563ba56a2923a7dfee731d004e212e1ee5030c", size = 21699, upload-time = "2023-06-26T12:07:29.144Z" } wheels = [ @@ -1284,10 +1289,10 @@ name = "jsonschema" version = "4.23.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "attrs" }, - { name = "jsonschema-specifications" }, - { name = "referencing" }, - { name = "rpds-py" }, + { name = "attrs", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "jsonschema-specifications", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "referencing", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "rpds-py", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/38/2e/03362ee4034a4c917f697890ccd4aec0800ccf9ded7f511971c75451deec/jsonschema-4.23.0.tar.gz", hash = "sha256:d71497fef26351a33265337fa77ffeb82423f3ea21283cd9467bb03999266bc4", size = 325778, upload-time = "2024-07-08T18:40:05.546Z" } wheels = [ @@ -1313,7 +1318,7 @@ name = "jsonschema-specifications" version = "2025.9.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "referencing" }, + { name = "referencing", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } wheels = [ @@ -1347,9 +1352,9 @@ name = "langchain" version = "1.3.11" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "langchain-core" }, - { name = "langgraph" }, - { name = "pydantic" }, + { name = "langchain-core", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langgraph", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pydantic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b9/a2/91a7197c604a3ce1b774b3c10dd114c3c745c6186a304fc2573b3f94d400/langchain-1.3.11.tar.gz", hash = "sha256:f3cf9cd4d2329b1a03eb8fd92b9d73e4e58a4d52570d67725fc77fbe0f104b32", size = 633374, upload-time = "2026-06-22T23:00:33.44Z" } wheels = [ @@ -1361,9 +1366,9 @@ name = "langchain-anthropic" version = "1.4.7" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anthropic" }, - { name = "langchain-core" }, - { name = "pydantic" }, + { name = "anthropic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-core", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pydantic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/df/d0/5b117541b9e13619a387b3c829b97ef8d54f848716c0cb574df35a326dea/langchain_anthropic-1.4.7.tar.gz", hash = "sha256:34a7eb0db6bb7c92ff89b3f985de6417d5ef7a148d4595f4498fd3ca4f14aa77", size = 706692, upload-time = "2026-06-22T22:56:05.749Z" } wheels = [ @@ -1375,9 +1380,9 @@ name = "langchain-baseten" version = "0.2.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "baseten-performance-client" }, - { name = "langchain-core" }, - { name = "langchain-openai" }, + { name = "baseten-performance-client", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-core", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-openai", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e1/bc/2936168d7152835a00cd42d31115983e3390b99f5c18a0c53d1eb8122666/langchain_baseten-0.2.1.tar.gz", hash = "sha256:03175a26df9e91d49cbef4e9559be11918c56a4fda83fcef9fb2168b30279e59", size = 150249, upload-time = "2026-06-26T20:22:31.559Z" } wheels = [ @@ -1389,15 +1394,15 @@ name = "langchain-core" version = "1.4.8" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jsonpatch" }, - { name = "langchain-protocol" }, - { name = "langsmith" }, - { name = "packaging" }, - { name = "pydantic" }, - { name = "pyyaml" }, - { name = "tenacity" }, - { name = "typing-extensions" }, - { name = "uuid-utils" }, + { name = "jsonpatch", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-protocol", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langsmith", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "packaging", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pydantic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pyyaml", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "tenacity", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "uuid-utils", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/12/e3/bea6d0080acf183332f24dcd74c208aee5857cf8f783c3fb0bd86027d8fb/langchain_core-1.4.8.tar.gz", hash = "sha256:5bf1f8411077c904182ad8f975943d36adcbf579c4e017b3a118b719229ebf9a", size = 957974, upload-time = "2026-06-18T19:39:23.636Z" } wheels = [ @@ -1409,8 +1414,8 @@ name = "langchain-deepseek" version = "1.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "langchain-core" }, - { name = "langchain-openai" }, + { name = "langchain-core", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-openai", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/15/f4/200792013d86406aa02aba8d1b536f8877a832372702189aef0c8c7a8329/langchain_deepseek-1.1.0.tar.gz", hash = "sha256:5ec8128cabae3d71366e2f297dea2153649be317a4be7f135cfcad1160523bf6", size = 138078, upload-time = "2026-06-03T19:07:11.887Z" } wheels = [ @@ -1422,11 +1427,11 @@ name = "langchain-fireworks" version = "1.4.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "aiohttp" }, - { name = "fireworks-ai" }, - { name = "langchain-core" }, - { name = "openai" }, - { name = "requests" }, + { name = "aiohttp", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "fireworks-ai", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-core", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "openai", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "requests", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/47/2d/8e8f5876ee0b209214b23db9d202fd310767d306a3bfae4667eccac30881/langchain_fireworks-1.4.3.tar.gz", hash = "sha256:320e07fd7021b7c01f28ed802a88b4c7cbe66972695a57f64a3d46ba006c44ee", size = 214372, upload-time = "2026-06-26T06:51:51.681Z" } wheels = [ @@ -1438,10 +1443,10 @@ name = "langchain-google-genai" version = "4.2.6" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "filetype" }, - { name = "google-genai" }, - { name = "langchain-core" }, - { name = "pydantic" }, + { name = "filetype", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "google-genai", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-core", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pydantic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4c/08/68e4f90b273d1d8dc3a7a948e6535e60baac726adbb2a8341dcf17662f54/langchain_google_genai-4.2.6.tar.gz", hash = "sha256:653dc331e691ddd79784d9ff6a4082749e0f873394c6dc782c414eb4409850eb", size = 280074, upload-time = "2026-06-26T06:18:58.807Z" } wheels = [ @@ -1453,8 +1458,8 @@ name = "langchain-groq" version = "1.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "groq" }, - { name = "langchain-core" }, + { name = "groq", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-core", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6e/4d/64b5e57e9e43f009fcb1591f567f0095bcf38f0b705a2d63cec4c04e0d56/langchain_groq-1.1.3.tar.gz", hash = "sha256:890c099a55526bceafc3e696d123cb9d36464c6664a3ead34ae6e09e0d50caeb", size = 192228, upload-time = "2026-06-10T04:16:08.009Z" } wheels = [ @@ -1466,9 +1471,9 @@ name = "langchain-mcp-adapters" version = "0.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "langchain-core" }, - { name = "mcp" }, - { name = "typing-extensions" }, + { name = "langchain-core", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "mcp", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a9/1c/b179d8650d2349a342bc1fd1aab41b34154e79c7fc86fc42bdf0bb110d6f/langchain_mcp_adapters-0.3.0.tar.gz", hash = "sha256:fa6c9497015eb2807de5d0c341a36e1d2445cecbae1f4a24e922fc5b94f1a36c", size = 42774, upload-time = "2026-06-10T01:10:28.552Z" } wheels = [ @@ -1480,11 +1485,11 @@ name = "langchain-mistralai" version = "1.1.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "httpx" }, - { name = "httpx-sse" }, - { name = "langchain-core" }, - { name = "pydantic" }, - { name = "tokenizers" }, + { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "httpx-sse", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-core", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pydantic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "tokenizers", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6b/00/63ce73bf6a704d041f77f40487b275c72bbc15889cb26468c2fe37a294dc/langchain_mistralai-1.1.5.tar.gz", hash = "sha256:439652b31665fc8c3739a46652e3b97b5e228144e83a8f98ad9e2733b300166f", size = 133362, upload-time = "2026-06-10T21:53:21.879Z" } wheels = [ @@ -1496,9 +1501,9 @@ name = "langchain-nvidia-ai-endpoints" version = "1.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "aiohttp" }, - { name = "langchain-core" }, - { name = "requests" }, + { name = "aiohttp", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-core", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "requests", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/68/8c/cfd23ba98f57d83ca801f9312c195a40a06ea4aa5a2846186efd270386f0/langchain_nvidia_ai_endpoints-1.4.2.tar.gz", hash = "sha256:7499f622d222a4103fa8d6b35e50ff82c14ae0c9a4e2a08bd0155a7be7dfd766", size = 59062, upload-time = "2026-06-23T21:30:28.533Z" } wheels = [ @@ -1510,8 +1515,8 @@ name = "langchain-ollama" version = "1.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "langchain-core" }, - { name = "ollama" }, + { name = "langchain-core", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "ollama", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d4/9b/6641afe8a5bf807e454fd464eddfc7eb2f2df53cb0b29744381171f9c609/langchain_ollama-1.1.0.tar.gz", hash = "sha256:f776f56f6782ae4da7692579b94a6575906118318d1023b455d7207f9d059811", size = 133075, upload-time = "2026-04-07T02:48:00.873Z" } wheels = [ @@ -1523,9 +1528,9 @@ name = "langchain-openai" version = "1.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "langchain-core" }, - { name = "openai" }, - { name = "tiktoken" }, + { name = "langchain-core", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "openai", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "tiktoken", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d0/f3/b38e052f943a75ba0e6762c0a50b6f1d6bcd52a9ce63386d8803c2ff506e/langchain_openai-1.3.3.tar.gz", hash = "sha256:143769bf943820b80db769e47ca8fd0aac08ed18714519333b044c4431e9aa67", size = 3256559, upload-time = "2026-06-22T22:54:05.445Z" } wheels = [ @@ -1537,8 +1542,8 @@ name = "langchain-openrouter" version = "0.2.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "langchain-core" }, - { name = "openrouter" }, + { name = "langchain-core", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "openrouter", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/35/df/9876dc983cc9ee11e340f5e98da9f1050c5a1fd1a953f01ed020db141e57/langchain_openrouter-0.2.4.tar.gz", hash = "sha256:5af80982d8d7e9bd94bfb91837ca2aecdcb8b8b86c96472ba5a3d1f6c961b190", size = 179074, upload-time = "2026-06-23T03:45:20.726Z" } wheels = [ @@ -1550,7 +1555,7 @@ name = "langchain-protocol" version = "0.0.18" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/d2/59/b5959aea96faa9146e2e49a7a22882b3528c62efafe9a6a95beab30c2305/langchain_protocol-0.0.18.tar.gz", hash = "sha256:ec3e11782f1ed0c9db38e5a9ed01b0e7a0d3fba406faa8aef6594b73c56a63e6", size = 6150, upload-time = "2026-06-18T17:08:26.959Z" } wheels = [ @@ -1562,12 +1567,12 @@ name = "langchain-quickjs" version = "0.3.2" source = { editable = "../partners/quickjs" } dependencies = [ - { name = "bsdiff4" }, - { name = "deepagents" }, - { name = "langchain" }, - { name = "langchain-core" }, - { name = "langgraph" }, - { name = "quickjs-rs" }, + { name = "bsdiff4", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "deepagents", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-core", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langgraph", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "quickjs-rs", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] [package.metadata] @@ -1605,10 +1610,10 @@ name = "langchain-xai" version = "1.2.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "aiohttp" }, - { name = "langchain-core" }, - { name = "langchain-openai" }, - { name = "requests" }, + { name = "aiohttp", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-core", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-openai", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "requests", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1e/50/721db990f9eff5ebd07ca7c06dfa4745406980c90554a2f508bd2a178f91/langchain_xai-1.2.2.tar.gz", hash = "sha256:5d88d31ae078b211eb341d238cc6518818b8e8816efdfc4d98e5e14767629639", size = 175995, upload-time = "2026-01-26T04:26:12.978Z" } wheels = [ @@ -1620,12 +1625,12 @@ name = "langgraph" version = "1.2.6" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "langchain-core" }, - { name = "langgraph-checkpoint" }, - { name = "langgraph-prebuilt" }, - { name = "langgraph-sdk" }, - { name = "pydantic" }, - { name = "xxhash" }, + { name = "langchain-core", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langgraph-checkpoint", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langgraph-prebuilt", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langgraph-sdk", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pydantic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "xxhash", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/02/7a/ea09b05bb0cbddfa43bd34fc581357e87fc3f21a751cc0d419688c3106da/langgraph-1.2.6.tar.gz", hash = "sha256:f9b45a34f13930c94d96cdb76277447ad2cc70ec2d18cd2764d7fdadb36cdc1b", size = 714400, upload-time = "2026-06-18T20:58:21.514Z" } wheels = [ @@ -1637,38 +1642,38 @@ name = "langgraph-api" version = "0.10.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cloudpickle" }, - { name = "cryptography" }, - { name = "grpcio" }, - { name = "grpcio-health-checking" }, - { name = "grpcio-tools" }, - { name = "httptools" }, - { name = "httpx" }, - { name = "jsonschema-rs" }, - { name = "langchain-core" }, - { name = "langchain-protocol" }, - { name = "langgraph" }, - { name = "langgraph-checkpoint" }, - { name = "langgraph-runtime-inmem" }, - { name = "langgraph-sdk" }, - { name = "langsmith", extra = ["otel"] }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-exporter-otlp-proto-http" }, - { name = "opentelemetry-sdk" }, - { name = "orjson" }, - { name = "protobuf" }, - { name = "pyjwt" }, - { name = "sse-starlette" }, - { name = "starlette" }, - { name = "structlog" }, - { name = "tenacity" }, - { name = "truststore" }, - { name = "uuid-utils" }, - { name = "uvicorn" }, - { name = "uvloop" }, - { name = "watchfiles" }, - { name = "websockets" }, - { name = "zstandard" }, + { name = "cloudpickle", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "cryptography", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "grpcio", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "grpcio-health-checking", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "grpcio-tools", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "httptools", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "jsonschema-rs", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-core", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-protocol", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langgraph", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langgraph-checkpoint", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langgraph-runtime-inmem", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langgraph-sdk", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langsmith", extra = ["otel"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "opentelemetry-api", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "opentelemetry-exporter-otlp-proto-http", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "opentelemetry-sdk", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "orjson", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "protobuf", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pyjwt", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "sse-starlette", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "starlette", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "structlog", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "tenacity", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "truststore", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "uuid-utils", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "uvicorn", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "uvloop", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "watchfiles", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "websockets", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "zstandard", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7d/17/a23ecdad3de7e48209bc8d534312ba83e332126ec60da9e6487461b02597/langgraph_api-0.10.0.tar.gz", hash = "sha256:f594a160857e2cd7fb2352bba585111177a4a394625ad0cf5898814754fb7bc1", size = 713903, upload-time = "2026-06-11T14:36:50.469Z" } wheels = [ @@ -1680,8 +1685,8 @@ name = "langgraph-checkpoint" version = "4.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "langchain-core" }, - { name = "ormsgpack" }, + { name = "langchain-core", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "ormsgpack", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/83/47/886af6f886f0bff2273164a45f008694e48a96ff3cd25ff0228f2aa9480e/langgraph_checkpoint-4.1.1.tar.gz", hash = "sha256:6c2bdb530c91f91d7d9c1bd100925d0fc4f498d418c17f3587d1526279482a25", size = 184020, upload-time = "2026-05-22T16:57:38.503Z" } wheels = [ @@ -1693,9 +1698,9 @@ name = "langgraph-checkpoint-sqlite" version = "3.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "aiosqlite" }, - { name = "langgraph-checkpoint" }, - { name = "sqlite-vec" }, + { name = "aiosqlite", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langgraph-checkpoint", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "sqlite-vec", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e3/ea/83917c2369acf8a10a894d4247655fd063c07924ba5bc4e83c85d2eaeded/langgraph_checkpoint_sqlite-3.1.0.tar.gz", hash = "sha256:f926916ebc1b985d802cc9c820026036e84db9d910d62c97b57e4ba64f67d5ae", size = 147902, upload-time = "2026-05-12T03:34:52.503Z" } wheels = [ @@ -1707,11 +1712,11 @@ name = "langgraph-cli" version = "0.4.30" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click" }, - { name = "httpx" }, - { name = "langgraph-sdk" }, - { name = "pathspec" }, - { name = "python-dotenv" }, + { name = "click", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langgraph-sdk", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pathspec", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "python-dotenv", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f0/27/4b6a0f00c804f0b0831f741c0607b46a4cbddff14d1eab6bbd4ce5820837/langgraph_cli-0.4.30.tar.gz", hash = "sha256:4948fdc77ff45fc5ef3d8330d17bbecfcb26cd9c4d3a4f00da84a41a0226cd72", size = 1046771, upload-time = "2026-06-16T19:46:27.949Z" } wheels = [ @@ -1720,8 +1725,8 @@ wheels = [ [package.optional-dependencies] inmem = [ - { name = "langgraph-api" }, - { name = "langgraph-runtime-inmem" }, + { name = "langgraph-api", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langgraph-runtime-inmem", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] [[package]] @@ -1729,8 +1734,8 @@ name = "langgraph-prebuilt" version = "1.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "langchain-core" }, - { name = "langgraph-checkpoint" }, + { name = "langchain-core", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langgraph-checkpoint", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/29/66/ed9b93f56bc17ef22d551892f0ac2b225a97fe0fcf23a511b857f70d590b/langgraph_prebuilt-1.1.0.tar.gz", hash = "sha256:3c579cf6eed2d17f9c157c2d0fcaddcd8688524e7022d3b22b37a3bf4589d528", size = 178833, upload-time = "2026-05-12T03:37:49.332Z" } wheels = [ @@ -1742,13 +1747,13 @@ name = "langgraph-runtime-inmem" version = "0.30.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "blockbuster" }, - { name = "croniter" }, - { name = "langgraph" }, - { name = "langgraph-checkpoint" }, - { name = "sse-starlette" }, - { name = "starlette" }, - { name = "structlog" }, + { name = "blockbuster", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "croniter", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langgraph", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langgraph-checkpoint", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "sse-starlette", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "starlette", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "structlog", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/98/19/5d330e22e3ee2569742d3577b4ddfbb6b8e383f2edd9724b4a4b73735717/langgraph_runtime_inmem-0.30.0.tar.gz", hash = "sha256:31954e1de6cd43c284b425243d4faeae818f68b3daca4dbe4313e651b6f5e11d", size = 129940, upload-time = "2026-06-11T14:22:46.985Z" } wheels = [ @@ -1760,11 +1765,11 @@ name = "langgraph-sdk" version = "0.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "httpx" }, - { name = "langchain-core" }, - { name = "langchain-protocol" }, - { name = "orjson" }, - { name = "websockets" }, + { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-core", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-protocol", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "orjson", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "websockets", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b4/2b/bd8ac26d4e97f6df88ef05ce5b6a38945a3903e1025d926f4752aa88aa97/langgraph_sdk-0.4.2.tar.gz", hash = "sha256:b88f0f5f6328ac0680d6790614a905b2bcfa257f2276dba4e38f0e86db0aa738", size = 348327, upload-time = "2026-06-01T17:51:19.856Z" } wheels = [ @@ -1776,20 +1781,20 @@ name = "langsmith" version = "0.9.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "httpx" }, - { name = "orjson", marker = "platform_python_implementation != 'PyPy'" }, - { name = "packaging" }, - { name = "pydantic" }, - { name = "requests" }, - { name = "requests-toolbelt" }, - { name = "sniffio" }, - { name = "typing-extensions" }, - { name = "uuid-utils" }, - { name = "websockets" }, - { name = "xxhash" }, - { name = "zstandard" }, + { name = "anyio", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "distro", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "orjson", marker = "(platform_machine == 'arm64' and platform_python_implementation != 'PyPy' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and platform_python_implementation != 'PyPy' and sys_platform == 'linux')" }, + { name = "packaging", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pydantic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "requests", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "requests-toolbelt", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "sniffio", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "uuid-utils", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "websockets", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "xxhash", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "zstandard", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7e/b7/43f80940a758ceb48714ccc10c7acb9564be4f0d4786e8ed5d46deb0be39/langsmith-0.9.3.tar.gz", hash = "sha256:868a007b3ecda002914b21212a953c77fcb1d71a8a09a94562c3b57941d3df3a", size = 4577380, upload-time = "2026-06-26T15:56:30.45Z" } wheels = [ @@ -1798,9 +1803,9 @@ wheels = [ [package.optional-dependencies] otel = [ - { name = "opentelemetry-api" }, - { name = "opentelemetry-exporter-otlp-proto-http" }, - { name = "opentelemetry-sdk" }, + { name = "opentelemetry-api", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "opentelemetry-exporter-otlp-proto-http", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "opentelemetry-sdk", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] [[package]] @@ -1808,7 +1813,7 @@ name = "linkify-it-py" version = "2.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "uc-micro-py" }, + { name = "uc-micro-py", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2e/c9/06ea13676ef354f0af6169587ae292d3e2406e212876a413bf9eece4eb23/linkify_it_py-2.1.0.tar.gz", hash = "sha256:43360231720999c10e9328dc3691160e27a718e280673d444c38d7d3aaa3b98b", size = 29158, upload-time = "2026-03-01T07:48:47.683Z" } wheels = [ @@ -1820,18 +1825,18 @@ name = "litellm" version = "1.87.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "aiohttp" }, - { name = "click" }, - { name = "fastuuid" }, - { name = "httpx" }, - { name = "importlib-metadata" }, - { name = "jinja2" }, - { name = "jsonschema" }, - { name = "openai" }, - { name = "pydantic" }, - { name = "python-dotenv" }, - { name = "tiktoken" }, - { name = "tokenizers" }, + { name = "aiohttp", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "click", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "fastuuid", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "importlib-metadata", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "jinja2", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "jsonschema", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "openai", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pydantic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "python-dotenv", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "tiktoken", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "tokenizers", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/77/0d/ccdf682ccfd7f18bf0e179c39d85616b8f8ef05a798588285310412db13d/litellm-1.87.0.tar.gz", hash = "sha256:cafc1882cb0cbab8374c41180af86e4a067796e4524e15f59e99f6e689cd1bd8", size = 15453755, upload-time = "2026-06-02T03:53:29.076Z" } wheels = [ @@ -1843,7 +1848,7 @@ name = "markdown-it-py" version = "4.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "mdurl" }, + { name = "mdurl", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } wheels = [ @@ -1852,7 +1857,7 @@ wheels = [ [package.optional-dependencies] linkify = [ - { name = "linkify-it-py" }, + { name = "linkify-it-py", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] [[package]] @@ -1860,8 +1865,8 @@ name = "markdownify" version = "1.2.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "beautifulsoup4" }, - { name = "six" }, + { name = "beautifulsoup4", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "six", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3f/bc/c8c8eea5335341306b0fa7e1cb33c5e1c8d24ef70ddd684da65f41c49c92/markdownify-1.2.2.tar.gz", hash = "sha256:b274f1b5943180b031b699b199cbaeb1e2ac938b75851849a31fd0c3d6603d09", size = 18816, upload-time = "2025-11-16T19:21:18.565Z" } wheels = [ @@ -1890,15 +1895,15 @@ name = "matplotlib" version = "3.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "contourpy" }, - { name = "cycler" }, - { name = "fonttools" }, - { name = "kiwisolver" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "pillow" }, - { name = "pyparsing" }, - { name = "python-dateutil" }, + { name = "contourpy", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "cycler", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "fonttools", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "kiwisolver", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "numpy", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "packaging", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pillow", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pyparsing", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "python-dateutil", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1f/24/080c99d223d158d3a8902769269ab6da5b50f7a0e6e072513907e02b7a6c/matplotlib-3.11.0.tar.gz", hash = "sha256:68c0c7be01b30dcca3638934f7f591df73401235cbdbf0d1ab1c71e7db7f8b57", size = 33251176, upload-time = "2026-06-12T02:29:15.508Z" } wheels = [ @@ -1918,19 +1923,19 @@ name = "mcp" version = "1.26.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio" }, - { name = "httpx" }, - { name = "httpx-sse" }, - { name = "jsonschema" }, - { name = "pydantic" }, - { name = "pydantic-settings" }, - { name = "pyjwt", extra = ["crypto"] }, - { name = "python-multipart" }, - { name = "sse-starlette" }, - { name = "starlette" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, - { name = "uvicorn" }, + { name = "anyio", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "httpx-sse", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "jsonschema", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pydantic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pydantic-settings", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pyjwt", extra = ["crypto"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "python-multipart", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "sse-starlette", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "starlette", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-inspection", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "uvicorn", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fc/6d/62e76bbb8144d6ed86e202b5edd8a4cb631e7c8130f3f4893c3f90262b10/mcp-1.26.0.tar.gz", hash = "sha256:db6e2ef491eecc1a0d93711a76f28dec2e05999f93afd48795da1c1137142c66", size = 608005, upload-time = "2026-01-24T19:40:32.468Z" } wheels = [ @@ -1942,7 +1947,7 @@ name = "mdit-py-plugins" version = "0.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "markdown-it-py" }, + { name = "markdown-it-py", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b2/fd/a756d36c0bfba5f6e39a1cdbdbfdd448dc02692467d83816dff4592a1ebc/mdit_py_plugins-0.5.0.tar.gz", hash = "sha256:f4918cb50119f50446560513a8e311d574ff6aaed72606ddae6d35716fe809c6", size = 44655, upload-time = "2025-08-11T07:25:49.083Z" } wheels = [ @@ -1963,19 +1968,19 @@ name = "modal" version = "1.4.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "aiohttp" }, - { name = "cbor2" }, - { name = "certifi" }, - { name = "click" }, - { name = "grpclib" }, - { name = "protobuf" }, - { name = "rich" }, - { name = "synchronicity" }, - { name = "toml" }, - { name = "types-certifi" }, - { name = "types-toml" }, - { name = "typing-extensions" }, - { name = "watchfiles" }, + { name = "aiohttp", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "cbor2", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "certifi", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "click", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "grpclib", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "protobuf", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "rich", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "synchronicity", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "toml", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "types-certifi", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "types-toml", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "watchfiles", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c6/7d/4126d0fe879ef3e86002ca821a34cb68a2588ea2e8ccb2bfe421d0f42ffe/modal-1.4.3.tar.gz", hash = "sha256:35b2fc840f759b512e12527afb538e1ea4cc232b84cfbfcef3f5d96d5a66abaa", size = 720488, upload-time = "2026-05-18T22:34:45.842Z" } wheels = [ @@ -2008,7 +2013,7 @@ name = "multiprocess" version = "0.70.19" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "dill" }, + { name = "dill", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a2/f2/e783ac7f2aeeed14e9e12801f22529cc7e6b7ab80928d6dcce4e9f00922d/multiprocess-0.70.19.tar.gz", hash = "sha256:952021e0e6c55a4a9fe4cd787895b86e239a40e76802a789d6305398d3975897", size = 2079989, upload-time = "2026-01-19T06:47:39.744Z" } wheels = [ @@ -2024,10 +2029,10 @@ name = "nltk" version = "3.9.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click" }, - { name = "joblib" }, - { name = "regex" }, - { name = "tqdm" }, + { name = "click", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "joblib", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "regex", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "tqdm", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/74/a1/b3b4adf15585a5bc4c357adde150c01ebeeb642173ded4d871e89468767c/nltk-3.9.4.tar.gz", hash = "sha256:ed03bc098a40481310320808b2db712d95d13ca65b27372f8a403949c8b523d0", size = 2946864, upload-time = "2026-03-24T06:13:40.641Z" } wheels = [ @@ -2059,8 +2064,8 @@ name = "ollama" version = "0.6.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "httpx" }, - { name = "pydantic" }, + { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pydantic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9d/5a/652dac4b7affc2b37b95386f8ae78f22808af09d720689e3d7a86b6ed98e/ollama-0.6.1.tar.gz", hash = "sha256:478c67546836430034b415ed64fa890fd3d1ff91781a9d548b3325274e69d7c6", size = 51620, upload-time = "2025-11-13T23:02:17.416Z" } wheels = [ @@ -2072,14 +2077,14 @@ name = "openai" version = "2.44.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "httpx" }, - { name = "jiter" }, - { name = "pydantic" }, - { name = "sniffio" }, - { name = "tqdm" }, - { name = "typing-extensions" }, + { name = "anyio", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "distro", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "jiter", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pydantic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "sniffio", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "tqdm", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/49/f5/7c7cb955305cb41f7f3c5fd7e0e38bf6bbf2658468863d4b7b868a5cb8df/openai-2.44.0.tar.gz", hash = "sha256:68a5a5ffad82b8ff7d451c437529fb64f7c3b8123aaf0c021966a882d9e3947d", size = 988753, upload-time = "2026-06-24T20:56:02.293Z" } wheels = [ @@ -2091,10 +2096,10 @@ name = "openevals" version = "0.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "langchain" }, - { name = "langchain-openai" }, - { name = "langsmith" }, - { name = "rich" }, + { name = "langchain", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langchain-openai", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "langsmith", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "rich", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b0/b1/028a05846136805b29b7a3afb58a940c2d213fa2c3d0a7d7003c7fbaa115/openevals-0.2.0.tar.gz", hash = "sha256:7e95fa64625be53eaa8c657d7f69b842a52bda10bdf3bb91781c7d09a385b069", size = 140711, upload-time = "2026-04-07T19:45:22.749Z" } wheels = [ @@ -2106,10 +2111,10 @@ name = "openrouter" version = "0.10.6" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "httpcore" }, - { name = "httpx" }, - { name = "jsonpath-python" }, - { name = "pydantic" }, + { name = "httpcore", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "jsonpath-python", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pydantic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/14/ea/85eecb0108254f9e63e65d9f78fe56a6223865e106e5c20bc12e6e6de514/openrouter-0.10.6.tar.gz", hash = "sha256:cd75a85a80212c4b76480ebb4d07b02a82f56aa443e77813994df85a70ef220e", size = 339776, upload-time = "2026-06-26T18:26:03.313Z" } wheels = [ @@ -2121,8 +2126,8 @@ name = "opentelemetry-api" version = "1.40.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "importlib-metadata" }, - { name = "typing-extensions" }, + { name = "importlib-metadata", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2c/1d/4049a9e8698361cc1a1aa03a6c59e4fa4c71e0c0f94a30f988a6876a2ae6/opentelemetry_api-1.40.0.tar.gz", hash = "sha256:159be641c0b04d11e9ecd576906462773eb97ae1b657730f0ecf64d32071569f", size = 70851, upload-time = "2026-03-04T14:17:21.555Z" } wheels = [ @@ -2134,7 +2139,7 @@ name = "opentelemetry-exporter-otlp-proto-common" version = "1.40.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "opentelemetry-proto" }, + { name = "opentelemetry-proto", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/51/bc/1559d46557fe6eca0b46c88d4c2676285f1f3be2e8d06bb5d15fbffc814a/opentelemetry_exporter_otlp_proto_common-1.40.0.tar.gz", hash = "sha256:1cbee86a4064790b362a86601ee7934f368b81cd4cc2f2e163902a6e7818a0fa", size = 20416, upload-time = "2026-03-04T14:17:23.801Z" } wheels = [ @@ -2146,13 +2151,13 @@ name = "opentelemetry-exporter-otlp-proto-http" version = "1.40.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "googleapis-common-protos" }, - { name = "opentelemetry-api" }, - { name = "opentelemetry-exporter-otlp-proto-common" }, - { name = "opentelemetry-proto" }, - { name = "opentelemetry-sdk" }, - { name = "requests" }, - { name = "typing-extensions" }, + { name = "googleapis-common-protos", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "opentelemetry-api", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "opentelemetry-exporter-otlp-proto-common", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "opentelemetry-proto", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "opentelemetry-sdk", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "requests", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2e/fa/73d50e2c15c56be4d000c98e24221d494674b0cc95524e2a8cb3856d95a4/opentelemetry_exporter_otlp_proto_http-1.40.0.tar.gz", hash = "sha256:db48f5e0f33217588bbc00274a31517ba830da576e59503507c839b38fa0869c", size = 17772, upload-time = "2026-03-04T14:17:25.324Z" } wheels = [ @@ -2164,7 +2169,7 @@ name = "opentelemetry-proto" version = "1.40.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "protobuf" }, + { name = "protobuf", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4c/77/dd38991db037fdfce45849491cb61de5ab000f49824a00230afb112a4392/opentelemetry_proto-1.40.0.tar.gz", hash = "sha256:03f639ca129ba513f5819810f5b1f42bcb371391405d99c168fe6937c62febcd", size = 45667, upload-time = "2026-03-04T14:17:31.194Z" } wheels = [ @@ -2176,9 +2181,9 @@ name = "opentelemetry-sdk" version = "1.40.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "opentelemetry-api" }, - { name = "opentelemetry-semantic-conventions" }, - { name = "typing-extensions" }, + { name = "opentelemetry-api", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "opentelemetry-semantic-conventions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/58/fd/3c3125b20ba18ce2155ba9ea74acb0ae5d25f8cd39cfd37455601b7955cc/opentelemetry_sdk-1.40.0.tar.gz", hash = "sha256:18e9f5ec20d859d268c7cb3c5198c8d105d073714db3de50b593b8c1345a48f2", size = 184252, upload-time = "2026-03-04T14:17:31.87Z" } wheels = [ @@ -2190,8 +2195,8 @@ name = "opentelemetry-semantic-conventions" version = "0.61b0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "opentelemetry-api" }, - { name = "typing-extensions" }, + { name = "opentelemetry-api", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6d/c0/4ae7973f3c2cfd2b6e321f1675626f0dab0a97027cc7a297474c9c8f3d04/opentelemetry_semantic_conventions-0.61b0.tar.gz", hash = "sha256:072f65473c5d7c6dc0355b27d6c9d1a679d63b6d4b4b16a9773062cb7e31192a", size = 145755, upload-time = "2026-03-04T14:17:32.664Z" } wheels = [ @@ -2242,8 +2247,8 @@ name = "pandas" version = "3.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, - { name = "python-dateutil" }, + { name = "numpy", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "python-dateutil", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/2e/0c/b28ed414f080ee0ad153f848586d61d1878f91689950f037f976ce15f6c8/pandas-3.0.1.tar.gz", hash = "sha256:4186a699674af418f655dbd420ed87f50d56b4cd6603784279d9eef6627823c8", size = 4641901, upload-time = "2026-02-17T22:20:16.434Z" } wheels = [ @@ -2313,10 +2318,10 @@ name = "postgrest" version = "2.31.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "deprecation" }, - { name = "httpx", extra = ["http2"] }, - { name = "pydantic" }, - { name = "yarl" }, + { name = "deprecation", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "httpx", extra = ["http2"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pydantic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "yarl", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e2/22/88c470d8838d2678a44e0172d061630b8837cba3fb7fb492e28f6578c309/postgrest-2.31.0.tar.gz", hash = "sha256:2f395d84b2ee34fc57622ff2f711df603e2ede625f98e5015240741888f7bd0c", size = 14419, upload-time = "2026-06-04T13:37:20.474Z" } wheels = [ @@ -2328,7 +2333,7 @@ name = "prompt-toolkit" version = "3.0.52" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "wcwidth" }, + { name = "wcwidth", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a1/96/06e01a7b38dce6fe1db213e061a4602dd6032a8a97ef6c1a862537732421/prompt_toolkit-3.0.52.tar.gz", hash = "sha256:28cde192929c8e7321de85de1ddbe736f1375148b02f2e17edd840042b1be855", size = 434198, upload-time = "2025-08-27T15:24:02.057Z" } wheels = [ @@ -2398,7 +2403,7 @@ name = "pyasn1-modules" version = "0.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pyasn1" }, + { name = "pyasn1", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e9/e6/78ebbb10a8c8e4b61a59249394a4a594c1a7af95593dc933a349c8d00964/pyasn1_modules-0.4.2.tar.gz", hash = "sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6", size = 307892, upload-time = "2025-03-28T02:41:22.17Z" } wheels = [ @@ -2419,10 +2424,10 @@ name = "pydantic" version = "2.12.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "annotated-types" }, - { name = "pydantic-core" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, + { name = "annotated-types", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pydantic-core", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-inspection", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/69/44/36f1a6e523abc58ae5f928898e4aca2e0ea509b5aa6f6f392a5d882be928/pydantic-2.12.5.tar.gz", hash = "sha256:4d351024c75c0f085a9febbb665ce8c0c6ec5d30e903bdb6394b7ede26aebb49", size = 821591, upload-time = "2025-11-26T15:11:46.471Z" } wheels = [ @@ -2434,7 +2439,7 @@ name = "pydantic-core" version = "2.41.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/71/70/23b021c950c2addd24ec408e9ab05d59b035b39d97cdc1130e1bce647bb6/pydantic_core-2.41.5.tar.gz", hash = "sha256:08daa51ea16ad373ffd5e7606252cc32f07bc72b28284b6bc9c6df804816476e", size = 460952, upload-time = "2025-11-04T13:43:49.098Z" } wheels = [ @@ -2453,9 +2458,9 @@ name = "pydantic-settings" version = "2.14.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pydantic" }, - { name = "python-dotenv" }, - { name = "typing-inspection" }, + { name = "pydantic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "python-dotenv", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-inspection", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5c/b5/8f48e906c3e0205276e8bd8cb7512217a87b2685304d64be27cad5b3019f/pydantic_settings-2.14.2.tar.gz", hash = "sha256:c19dd64b19097f1de80184f0cc7b0272a13ae6e170cbf240a3e27e381ed14a5f", size = 237700, upload-time = "2026-06-19T13:44:56.324Z" } wheels = [ @@ -2482,7 +2487,7 @@ wheels = [ [package.optional-dependencies] crypto = [ - { name = "cryptography" }, + { name = "cryptography", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] [[package]] @@ -2508,10 +2513,10 @@ name = "pytest" version = "9.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "iniconfig" }, - { name = "packaging" }, - { name = "pluggy" }, - { name = "pygments" }, + { name = "iniconfig", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "packaging", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pluggy", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pygments", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } wheels = [ @@ -2523,8 +2528,8 @@ name = "pytest-asyncio" version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pytest" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "pytest", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(python_full_version < '3.13' and platform_machine == 'arm64' and sys_platform == 'darwin') or (python_full_version < '3.13' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/43/7c/d36d04db312ecf4298932ef77e6e4a9e8ad017906e24e34f0b0c361a2473/pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42", size = 58514, upload-time = "2026-05-26T09:56:04.083Z" } wheels = [ @@ -2536,9 +2541,9 @@ name = "pytest-cov" version = "7.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "coverage" }, - { name = "pluggy" }, - { name = "pytest" }, + { name = "coverage", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pluggy", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pytest", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } wheels = [ @@ -2550,7 +2555,7 @@ name = "pytest-socket" version = "0.8.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pytest" }, + { name = "pytest", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/59/3c/f9b58e57830e58980dbe8867d0e348f45701d3f3ea065672d448f4366da5/pytest_socket-0.8.0.tar.gz", hash = "sha256:af9bb5f487da72be63573a6194cfac033b6c7a1c1561e150521105970f9e99f2", size = 13912, upload-time = "2026-05-21T16:50:22.552Z" } wheels = [ @@ -2562,7 +2567,7 @@ name = "pytest-watcher" version = "0.6.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "watchdog" }, + { name = "watchdog", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e6/d2/80606077b7fa8784417687f494ff801d7ab817d9a17fc94305811d5919bb/pytest_watcher-0.6.3.tar.gz", hash = "sha256:842dc904264df0ad2d5264153a66bb452fccfa46598cd6e0a5ef1d19afed9b13", size = 601878, upload-time = "2026-01-10T23:28:18.805Z" } wheels = [ @@ -2574,7 +2579,7 @@ name = "python-dateutil" version = "2.9.0.post0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "six" }, + { name = "six", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } wheels = [ @@ -2618,7 +2623,7 @@ name = "quickjs-rs" version = "0.2.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "wasmtime" }, + { name = "wasmtime", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/61/34/1b63bc3db8d950139618d1d38eff5648e5aaa53b2982da5ec121fcf72c63/quickjs_rs-0.2.4.tar.gz", hash = "sha256:7226d18bfa05d97fd629348b7d71512dab4da12bfbe29bdd50486bfc10858cc9", size = 829056, upload-time = "2026-06-24T19:42:21.6Z" } wheels = [ @@ -2630,9 +2635,9 @@ name = "realtime" version = "2.31.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pydantic" }, - { name = "typing-extensions" }, - { name = "websockets" }, + { name = "pydantic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "websockets", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/df/34/54a1eaaefa24db5cb12596fd74792e08efa53ed30dc5bce2c0a68ded6146/realtime-2.31.0.tar.gz", hash = "sha256:9e641cb4d77ca0fe768515f8cf9f83550c79f49ce1550a95afc2dc0e252be8c9", size = 18716, upload-time = "2026-06-04T13:37:22.089Z" } wheels = [ @@ -2644,9 +2649,9 @@ name = "referencing" version = "0.37.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "attrs" }, - { name = "rpds-py" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "attrs", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "rpds-py", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(python_full_version < '3.13' and platform_machine == 'arm64' and sys_platform == 'darwin') or (python_full_version < '3.13' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } wheels = [ @@ -2678,10 +2683,10 @@ name = "requests" version = "2.34.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "certifi" }, - { name = "charset-normalizer" }, - { name = "idna" }, - { name = "urllib3" }, + { name = "certifi", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "charset-normalizer", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "idna", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "urllib3", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } wheels = [ @@ -2693,7 +2698,7 @@ name = "requests-toolbelt" version = "1.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "requests" }, + { name = "requests", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f3/61/d7545dafb7ac2230c70d38d31cbfe4cc64f7144dc41f6e4e4b78ecd9f5bb/requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6", size = 206888, upload-time = "2023-05-01T04:11:33.229Z" } wheels = [ @@ -2705,8 +2710,8 @@ name = "rich" version = "15.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "markdown-it-py" }, - { name = "pygments" }, + { name = "markdown-it-py", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pygments", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } wheels = [ @@ -2746,8 +2751,8 @@ name = "scantree" version = "0.0.4" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "attrs" }, - { name = "pathspec" }, + { name = "attrs", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pathspec", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b3/e4/40998faefc72ba1ddeb640a44fba92935353525dba110488806da8339c0b/scantree-0.0.4.tar.gz", hash = "sha256:15bd5cb24483b04db2c70653604e8ea3522e98087db7e38ab8482f053984c0ac", size = 24643, upload-time = "2024-08-03T20:08:59.413Z" } wheels = [ @@ -2822,8 +2827,8 @@ name = "sse-starlette" version = "3.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio" }, - { name = "starlette" }, + { name = "anyio", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "starlette", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/14/2f/9223c24f568bb7a0c03d751e609844dce0968f13b39a3f73fbb3a96cd27a/sse_starlette-3.3.3.tar.gz", hash = "sha256:72a95d7575fd5129bd0ae15275ac6432bb35ac542fdebb82889c24bb9f3f4049", size = 32420, upload-time = "2026-03-17T20:05:55.529Z" } wheels = [ @@ -2835,8 +2840,8 @@ name = "starlette" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio" }, - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "anyio", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(python_full_version < '3.13' and platform_machine == 'arm64' and sys_platform == 'darwin') or (python_full_version < '3.13' and platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/e3/7c1dc7381d9f8ab7d854328ebfa884e62cb3f3d8549ddfd37c7814f42afa/starlette-1.3.1.tar.gz", hash = "sha256:05d0213193f2fbaae60e2ecb593b4add4262ad4e46536b54abe36f11a71724e0", size = 2703240, upload-time = "2026-06-12T09:23:11.602Z" } wheels = [ @@ -2848,10 +2853,10 @@ name = "storage3" version = "2.31.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "deprecation" }, - { name = "httpx", extra = ["http2"] }, - { name = "pydantic" }, - { name = "yarl" }, + { name = "deprecation", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "httpx", extra = ["http2"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pydantic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "yarl", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/13/30/fee43d523d3f680a833a4aae5bf8094de0b9031b0c2bddb3e0bc6e829e1b/storage3-2.31.0.tar.gz", hash = "sha256:d2161e2ea650dc115a1787c30e09b118365589ac772f4dd8643e3a503ecfc667", size = 20348, upload-time = "2026-06-04T13:37:23.703Z" } wheels = [ @@ -2881,13 +2886,13 @@ name = "supabase" version = "2.31.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "httpx" }, - { name = "postgrest" }, - { name = "realtime" }, - { name = "storage3" }, - { name = "supabase-auth" }, - { name = "supabase-functions" }, - { name = "yarl" }, + { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "postgrest", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "realtime", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "storage3", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "supabase-auth", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "supabase-functions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "yarl", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fe/8e/54a2f950629689b1613434a61fc3bff5f92f84ba6b20213f5b2add05c1bb/supabase-2.31.0.tar.gz", hash = "sha256:3467b09d00482b9a0138235bdbde7a350426f93cf2a1342372eaddfc669f1206", size = 9805, upload-time = "2026-06-04T13:37:25.22Z" } wheels = [ @@ -2899,9 +2904,9 @@ name = "supabase-auth" version = "2.31.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "httpx", extra = ["http2"] }, - { name = "pydantic" }, - { name = "pyjwt", extra = ["crypto"] }, + { name = "httpx", extra = ["http2"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pydantic", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pyjwt", extra = ["crypto"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/61/8a/408689cf39820f0d46d2731d6747ff94dbefc87ae977b4b5c4066da5b070/supabase_auth-2.31.0.tar.gz", hash = "sha256:0945b33fa96239c76dc8eaf96d7d2c94991950d24b4cfe4a5c2da9aa5e909663", size = 39151, upload-time = "2026-06-04T13:37:27.375Z" } wheels = [ @@ -2913,9 +2918,9 @@ name = "supabase-functions" version = "2.31.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "httpx", extra = ["http2"] }, - { name = "strenum" }, - { name = "yarl" }, + { name = "httpx", extra = ["http2"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "strenum", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "yarl", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/84/5d/61c2446ed26a57fa5543f9c270a731320911569202340d15341f72cdba7c/supabase_functions-2.31.0.tar.gz", hash = "sha256:4ad027b3ae3bd28b31233339f4db1da6965affd3546f655b421baf40cee2690f", size = 4683, upload-time = "2026-06-04T13:37:28.878Z" } wheels = [ @@ -2927,7 +2932,7 @@ name = "synchronicity" version = "0.12.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ec/d5/e96e6082790c92480380f28aa53e111844cdac7b0f75846f4772cb535a43/synchronicity-0.12.3.tar.gz", hash = "sha256:0d4228b85eaf2805f23b4615b2039a9d24ea811646e2d9f8d0c033094eb85841", size = 60261, upload-time = "2026-05-28T12:33:50.206Z" } wheels = [ @@ -2939,9 +2944,9 @@ name = "tavily-python" version = "0.7.26" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "httpx" }, - { name = "requests" }, - { name = "tiktoken" }, + { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "requests", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "tiktoken", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/bd/29/c2d5056dbf45a80484df5a1bf0d0ae59c611aa9680372c400a278fbf0c50/tavily_python-0.7.26.tar.gz", hash = "sha256:36202728144fce1ef0cece33800f98e3cdeb5f600e6678c50b1232b866a2f509", size = 29647, upload-time = "2026-06-10T18:38:45.231Z" } wheels = [ @@ -2962,12 +2967,12 @@ name = "textual" version = "8.2.7" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "markdown-it-py", extra = ["linkify"] }, - { name = "mdit-py-plugins" }, - { name = "platformdirs" }, - { name = "pygments" }, - { name = "rich" }, - { name = "typing-extensions" }, + { name = "markdown-it-py", extra = ["linkify"], marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "mdit-py-plugins", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "platformdirs", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "pygments", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "rich", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/9b/7a/c519db0aba5024f86e71e9631810bfdd6866ed2c8695bd7fa34b90e7ef59/textual-8.2.7.tar.gz", hash = "sha256:658f568ff81e30ed43890c3e07520390e5cf1b4763822006e060656b0a88f105", size = 1859249, upload-time = "2026-05-19T10:52:49.531Z" } wheels = [ @@ -2979,8 +2984,8 @@ name = "textual-autocomplete" version = "4.0.6" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "textual" }, - { name = "typing-extensions" }, + { name = "textual", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1e/3a/80411bc7b94969eb116ad1b18db90f8dce8a1de441278c4a81fee55a27ca/textual_autocomplete-4.0.6.tar.gz", hash = "sha256:2ba2f0d767be4480ecacb3e4b130cf07340e033c3500fc424fed9125d27a4586", size = 97967, upload-time = "2025-09-24T21:19:20.213Z" } wheels = [ @@ -3007,8 +3012,8 @@ name = "tiktoken" version = "0.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "regex" }, - { name = "requests" }, + { name = "regex", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "requests", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e4/e5/5f3cb2159769d0f4324c0e9e87f9de3c4b1cd45848a96b2eb3566ad5ca77/tiktoken-0.13.0.tar.gz", hash = "sha256:c9435714c3a84c2319499de9a300c0e604449dd0799ff246458b3bb6a7f433c1", size = 38986, upload-time = "2026-05-15T04:51:27.153Z" } wheels = [ @@ -3028,7 +3033,7 @@ name = "tokenizers" version = "0.22.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "huggingface-hub" }, + { name = "huggingface-hub", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/73/6f/f80cfef4a312e1fb34baf7d85c72d4411afde10978d4657f8cdd811d3ccc/tokenizers-0.22.2.tar.gz", hash = "sha256:473b83b915e547aa366d1eee11806deaf419e17be16310ac0a14077f1e28f917", size = 372115, upload-time = "2026-01-05T10:45:15.988Z" } wheels = [ @@ -3089,10 +3094,10 @@ name = "typer" version = "0.23.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "annotated-doc" }, - { name = "click" }, - { name = "rich" }, - { name = "shellingham" }, + { name = "annotated-doc", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "click", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "rich", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "shellingham", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fd/07/b822e1b307d40e263e8253d2384cf98c51aa2368cc7ba9a07e523a1d964b/typer-0.23.1.tar.gz", hash = "sha256:2070374e4d31c83e7b61362fd859aa683576432fd5b026b060ad6b4cd3b86134", size = 120047, upload-time = "2026-02-13T10:04:30.984Z" } wheels = [ @@ -3131,7 +3136,7 @@ name = "typing-inspection" version = "0.4.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } wheels = [ @@ -3176,8 +3181,8 @@ name = "uvicorn" version = "0.42.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click" }, - { name = "h11" }, + { name = "click", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "h11", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/e3/ad/4a96c425be6fb67e0621e62d86c402b4a17ab2be7f7c055d9bd2f638b9e2/uvicorn-0.42.0.tar.gz", hash = "sha256:9b1f190ce15a2dd22e7758651d9b6d12df09a13d51ba5bf4fc33c383a48e1775", size = 85393, upload-time = "2026-03-16T06:19:50.077Z" } wheels = [ @@ -3228,7 +3233,7 @@ name = "watchfiles" version = "1.1.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio" }, + { name = "anyio", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c2/c9/8869df9b2a2d6c59d79220a4db37679e74f807c559ffe5265e08b227a210/watchfiles-1.1.1.tar.gz", hash = "sha256:a173cb5c16c4f40ab19cecf48a534c409f7ea983ab8fed0741304a1c0a31b3f2", size = 94440, upload-time = "2025-10-14T15:06:21.08Z" } wheels = [ @@ -3248,7 +3253,7 @@ name = "wcmatch" version = "10.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "bracex" }, + { name = "bracex", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/79/3e/c0bdc27cf06f4e47680bd5803a07cb3dfd17de84cde92dd217dcb9e05253/wcmatch-10.1.tar.gz", hash = "sha256:f11f94208c8c8484a16f4f48638a85d771d9513f4ab3f37595978801cb9465af", size = 117421, upload-time = "2025-06-22T19:14:02.49Z" } wheels = [ @@ -3303,9 +3308,9 @@ name = "yarl" version = "1.23.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "idna" }, - { name = "multidict" }, - { name = "propcache" }, + { name = "idna", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "multidict", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "propcache", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/23/6e/beb1beec874a72f23815c1434518bfc4ed2175065173fb138c3705f658d4/yarl-1.23.0.tar.gz", hash = "sha256:53b1ea6ca88ebd4420379c330aea57e258408dd0df9af0992e5de2078dc9f5d5", size = 194676, upload-time = "2026-03-01T22:07:53.373Z" } wheels = [ diff --git a/libs/talon/pyproject.toml b/libs/talon/pyproject.toml index 54c980aca1b..8f4742cb21b 100644 --- a/libs/talon/pyproject.toml +++ b/libs/talon/pyproject.toml @@ -23,9 +23,8 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Python Modules", ] dependencies = [ - # Keep Talon aligned with deepagents-code's prerelease deepagents pin. - "deepagents>=0.6.12,<0.9.0", - "deepagents-code>=0.1.28,<1.0.0", + "deepagents>=0.7.0a3", + "deepagents-code>=0.1.30,<1.0.0", "fleet-deepagents-export>=0.0.1,<1.0.0", "langchain-mcp-adapters>=0.3.0,<1.0.0", ] @@ -40,7 +39,7 @@ speech = [ "transformers>=5.12.1,<6.0.0", ] video = [ - "deepagents[video]>=0.6.12,<0.7.0", + "deepagents[video]>=0.7.0a3", ] [project.scripts] From bac688429dbc8fd156a1a30fb28160eaa6b15bc7 Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Thu, 2 Jul 2026 00:10:48 -0400 Subject: [PATCH 24/36] nit --- libs/code/pyproject.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libs/code/pyproject.toml b/libs/code/pyproject.toml index 37a1a5189d9..70ae02894af 100644 --- a/libs/code/pyproject.toml +++ b/libs/code/pyproject.toml @@ -122,9 +122,10 @@ all-sandboxes = [ "deepagents-code[agentcore,daytona,modal,runloop,vercel]", ] -# Standalone integrations (kept for backwards-compatible install commands) -quickjs = [] +# Standalone integrations video = ["deepagents[video]==0.7.0a3"] +# (quickjs kept for backwards-compatible install commands) +quickjs = [] [project.scripts] From 1a6d323e1d977c2f40b3c63f986f4f721e34bdf3 Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Thu, 2 Jul 2026 00:12:49 -0400 Subject: [PATCH 25/36] cr --- libs/code/deepagents_code/extras_info.py | 5 +++-- libs/deepagents/deepagents/middleware/filesystem.py | 8 ++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/libs/code/deepagents_code/extras_info.py b/libs/code/deepagents_code/extras_info.py index b94553fdf95..5aa88ddd7b7 100644 --- a/libs/code/deepagents_code/extras_info.py +++ b/libs/code/deepagents_code/extras_info.py @@ -207,8 +207,9 @@ class ExtrasIntrospectionError(RuntimeError): STANDALONE_EXTRAS: frozenset[str] = frozenset({"quickjs", "video"}) """Optional extras that don't fit the provider/sandbox taxonomy. -`quickjs` is a core dependency now, but the empty extra remains installable so -older `deepagents-code[quickjs]` and `/install quickjs` workflows stay harmless. +`quickjs` is a core dependency as of 0.1.24, but the empty extra remains +installable so older `deepagents-code[quickjs]` and `/install quickjs` workflows +stay harmless. """ KNOWN_EXTRAS: frozenset[str] = ( diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index d9b0eae3e1c..fe808264c2a 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -87,9 +87,13 @@ ) _FS_WCMATCH_FLAGS = wcglob.BRACE | wcglob.GLOBSTAR +"""wcmatch flags enabling brace expansion and `**` globstar recursion.""" + _SYNC_GLOB_WORKERS = 4 +"""Thread-pool size for synchronous glob operations.""" FilesystemOperation = Literal["read", "write"] +"""Classification of filesystem tools as read-only or mutating.""" _DEFAULT_FS_TOOL_OPS: dict[str, FilesystemOperation] = { "ls": "read", @@ -100,9 +104,13 @@ "edit_file": "write", "delete": "write", } +"""Default mapping from filesystem tool name to its operation category.""" _READ_FILE_MEDIA_RESULT = "read_file_media_result" +"""`additional_kwargs` key marking synthetic `HumanMessage` media from `read_file`.""" + _VIDEO_SAMPLING_RATE = 0.5 +"""Seconds between sampled frames when extracting stills from a video.""" def _tool_error(name: str, tool_call_id: str | None, content: str) -> ToolMessage: From 51e292d116c20cf51c3ca1ba3a1be89779c2f61c Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Thu, 2 Jul 2026 00:14:44 -0400 Subject: [PATCH 26/36] cr --- libs/deepagents/deepagents/middleware/_video.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/libs/deepagents/deepagents/middleware/_video.py b/libs/deepagents/deepagents/middleware/_video.py index 0c7e0ca7af1..35c7782b61c 100644 --- a/libs/deepagents/deepagents/middleware/_video.py +++ b/libs/deepagents/deepagents/middleware/_video.py @@ -29,14 +29,31 @@ MISSING_VIDEO_HINT = "Reading video files requires the optional video dependencies. Install them with `uv add 'deepagents[video]'`." +"""User-facing message shown when a video read is attempted without the `[video]` extra installed.""" + MAX_VIDEO_SAMPLED_FRAMES: Final = 64 +"""Upper bound on the number of frames emitted per video read.""" + MAX_VIDEO_FRAME_PIXELS: Final = 1920 * 1080 +"""Maximum pixel count a decoded frame may have before being downscaled.""" + MAX_VIDEO_FRAME_SIDE: Final = 4096 +"""Maximum width or height of a decoded frame before being downscaled.""" + MAX_VIDEO_OUTPUT_WIDTH: Final = 1920 +"""Target width (pixels) for frames emitted to the model.""" + MAX_VIDEO_OUTPUT_HEIGHT: Final = 1080 +"""Target height (pixels) for frames emitted to the model.""" + MAX_VIDEO_EMITTED_BYTES: Final = 4 * 1024 * 1024 +"""Maximum total encoded byte size for the frame set returned from one read.""" + MAX_VIDEO_DECODE_SECONDS: Final = 10.0 +"""Maximum source duration (seconds) decoded in a single read call.""" + _JPEG_QUALITY: Final = 85 +"""JPEG quality (1-100) used when encoding sampled frames.""" class VideoExtractionError(RuntimeError): From 9d5fc28ac4e92863c919660af0da6f7cc2ae8ebf Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Thu, 2 Jul 2026 00:35:10 -0400 Subject: [PATCH 27/36] cr --- .../deepagents/middleware/_video.py | 56 ++-- .../deepagents/middleware/filesystem.py | 10 +- .../middleware/test_filesystem_video.py | 269 ++++++++++++++++++ .../tests/unit_tests/middleware/test_video.py | 117 +++++++- 4 files changed, 427 insertions(+), 25 deletions(-) diff --git a/libs/deepagents/deepagents/middleware/_video.py b/libs/deepagents/deepagents/middleware/_video.py index 35c7782b61c..b1799a2f566 100644 --- a/libs/deepagents/deepagents/middleware/_video.py +++ b/libs/deepagents/deepagents/middleware/_video.py @@ -1,9 +1,9 @@ -"""Video frame extraction for filesystem and request media handling. +"""Video frame extraction for filesystem reads. This module is the boundary between Deep Agents middleware and the optional video backend (PyAV). It imports PyAV lazily so a `deepagents` install without the `[video]` extra stays lightweight; the import only fires when the agent -actually tries to read or transform a video. +actually tries to read a video. For each video read the module decodes a contiguous slice of the source (`offset`-seconds skip, `limit`-seconds window) and emits sampled @@ -38,7 +38,7 @@ """Maximum pixel count a decoded frame may have before being downscaled.""" MAX_VIDEO_FRAME_SIDE: Final = 4096 -"""Maximum width or height of a decoded frame before being downscaled.""" +"""Maximum width or height of a decoded frame before it is rejected.""" MAX_VIDEO_OUTPUT_WIDTH: Final = 1920 """Target width (pixels) for frames emitted to the model.""" @@ -50,7 +50,7 @@ """Maximum total encoded byte size for the frame set returned from one read.""" MAX_VIDEO_DECODE_SECONDS: Final = 10.0 -"""Maximum source duration (seconds) decoded in a single read call.""" +"""Wall-clock deadline (seconds) for decoding frames in a single read call.""" _JPEG_QUALITY: Final = 85 """JPEG quality (1-100) used when encoding sampled frames.""" @@ -60,7 +60,7 @@ class VideoExtractionError(RuntimeError): """Raised when PyAV cannot produce frames for the requested window.""" -def _import_av() -> Any: # noqa: ANN401 +def _import_av() -> Any: # noqa: ANN401 # PyAV types are unavailable without the [video] extra """Import PyAV lazily so the dep stays optional. Returns: @@ -133,7 +133,14 @@ def extract_video_frames( try: try: video_stream = _find_video_stream(container) - time_base = float(video_stream.time_base) + raw_time_base = video_stream.time_base + if raw_time_base is None: + msg = "Video stream has no time_base; cannot determine frame timestamps" + raise VideoExtractionError(msg) + time_base = float(raw_time_base) + if time_base == 0.0: + msg = "Video stream time_base is zero; cannot determine frame timestamps" + raise VideoExtractionError(msg) stream_start_seconds = _stream_start_seconds(video_stream, time_base) if offset_seconds > 0: # `seek` keeps the math correct across containers that already sit @@ -166,7 +173,7 @@ def extract_video_frames( return blocks -def _open_video_container(av: Any, content: bytes) -> Any: # noqa: ANN401 +def _open_video_container(av: Any, content: bytes) -> Any: # noqa: ANN401 # PyAV types are unavailable without the [video] extra """Open a video byte payload, normalizing PyAV's failure modes. PyAV typically raises `av.error.InvalidDataError` for malformed inputs, @@ -181,7 +188,7 @@ def _open_video_container(av: Any, content: bytes) -> Any: # noqa: ANN401 raise VideoExtractionError(msg) from exc -def _video_backend_error_types(av: Any) -> tuple[type[BaseException], ...]: # noqa: ANN401 +def _video_backend_error_types(av: Any) -> tuple[type[BaseException], ...]: # noqa: ANN401 # PyAV types are unavailable without the [video] extra """Return backend failures that should surface as `VideoExtractionError`.""" errors: list[type[BaseException]] = [OSError] av_error = getattr(av, "error", None) @@ -196,7 +203,7 @@ def _video_backend_error_types(av: Any) -> tuple[type[BaseException], ...]: # n return tuple(errors) -def _find_video_stream(container: Any) -> Any: # noqa: ANN401 +def _find_video_stream(container: Any) -> Any: # noqa: ANN401 # PyAV types are unavailable without the [video] extra """Return the first video stream in `container` or raise.""" video_stream = next((s for s in container.streams if s.type == "video"), None) if video_stream is None: @@ -205,18 +212,18 @@ def _find_video_stream(container: Any) -> Any: # noqa: ANN401 return video_stream -def _stream_start_pts(video_stream: Any) -> int: # noqa: ANN401 +def _stream_start_pts(video_stream: Any) -> int: # noqa: ANN401 # PyAV types are unavailable without the [video] extra """Return the stream start timestamp in stream time-base units.""" start_time = getattr(video_stream, "start_time", None) return int(start_time) if start_time is not None else 0 -def _stream_start_seconds(video_stream: Any, time_base: float) -> float: # noqa: ANN401 +def _stream_start_seconds(video_stream: Any, time_base: float) -> float: # noqa: ANN401 # PyAV types are unavailable without the [video] extra """Return the stream start timestamp in seconds.""" return _stream_start_pts(video_stream) * time_base -def _frame_seconds(frame: Any, *, time_base: float, stream_start_seconds: float) -> float | None: # noqa: ANN401 +def _frame_seconds(frame: Any, *, time_base: float, stream_start_seconds: float) -> float | None: # noqa: ANN401 # PyAV types are unavailable without the [video] extra """Return a frame timestamp normalized to seconds from the video start.""" pts = getattr(frame, "pts", None) if pts is not None: @@ -227,7 +234,7 @@ def _frame_seconds(frame: Any, *, time_base: float, stream_start_seconds: float) return None -def _frame_dimensions(frame: Any) -> tuple[int, int] | None: # noqa: ANN401 +def _frame_dimensions(frame: Any) -> tuple[int, int] | None: # noqa: ANN401 # PyAV types are unavailable without the [video] extra """Return frame dimensions when the decoder exposes them.""" width = getattr(frame, "width", None) height = getattr(frame, "height", None) @@ -253,7 +260,7 @@ def _check_decode_deadline(deadline_seconds: float | None) -> None: def _sample_frames_in_window( - decoded_frames: Any, # noqa: ANN401 + decoded_frames: Any, # noqa: ANN401 # PyAV types are unavailable without the [video] extra *, offset_seconds: float, duration_seconds: float, @@ -270,6 +277,8 @@ def _sample_frames_in_window( blocks: list[ContentBlock] = [] emitted_frames = 0 emitted_bytes = 0 + last_emitted_seconds: float | None = None + truncated = False try: for frame in decoded_frames: _check_decode_deadline(deadline_seconds) @@ -290,9 +299,10 @@ def _sample_frames_in_window( if emitted_frames == 0: msg = f"Video frame output exceeded the {MAX_VIDEO_EMITTED_BYTES} byte safety budget before emitting a frame" raise VideoExtractionError(msg) + truncated = True break - blocks.append({"type": "text", "text": f"Frame at t={ts}"}) + blocks.append({"type": "text", "text": text}) blocks.append( { "type": "image", @@ -302,7 +312,9 @@ def _sample_frames_in_window( ) emitted_frames += 1 emitted_bytes += next_block_bytes + last_emitted_seconds = frame_seconds if emitted_frames >= MAX_VIDEO_SAMPLED_FRAMES: + truncated = True break emitted_index = math.floor((frame_seconds - offset_seconds) / frame_interval_seconds) + 1 next_emit_seconds = max( @@ -312,10 +324,22 @@ def _sample_frames_in_window( except decode_error_types as exc: msg = f"Failed to decode video frames: {exc}" raise VideoExtractionError(msg) from exc + if truncated and last_emitted_seconds is not None: + last_ts = _format_timestamp(last_emitted_seconds) + blocks.append( + { + "type": "text", + "text": ( + f"Coverage truncated at t={last_ts}: the output or frame cap was reached " + f"before the full window was decoded. Re-read with a narrower window " + f"(e.g. offset={last_emitted_seconds:.3f}) to see the remaining frames." + ), + } + ) return blocks -def _encode_jpeg(frame: Any) -> bytes: # noqa: ANN401 +def _encode_jpeg(frame: Any) -> bytes: # noqa: ANN401 # PyAV types are unavailable without the [video] extra """Encode a decoded PyAV frame as JPEG bytes via Pillow. Pillow is part of the `video` extra, and the import stays lazy so module diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index fe808264c2a..a27f3e55395 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -13,7 +13,7 @@ from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass from pathlib import Path, PurePosixPath -from typing import TYPE_CHECKING, Annotated, Any, Literal, NotRequired, cast +from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, NotRequired, cast if TYPE_CHECKING: from langchain_core.runnables.config import RunnableConfig @@ -106,10 +106,10 @@ } """Default mapping from filesystem tool name to its operation category.""" -_READ_FILE_MEDIA_RESULT = "read_file_media_result" +_READ_FILE_MEDIA_RESULT: Final = "read_file_media_result" """`additional_kwargs` key marking synthetic `HumanMessage` media from `read_file`.""" -_VIDEO_SAMPLING_RATE = 0.5 +_VIDEO_SAMPLING_RATE: Final = 0.5 """Seconds between sampled frames when extracting stills from a video.""" @@ -197,7 +197,7 @@ def _err(msg: str) -> ToolMessage: duration_seconds=duration_seconds, sampling_rate=rate, ) - except VideoExtractionError as exc: + except (VideoExtractionError, ValueError) as exc: return _err(str(exc)) blocks.insert(0, {"type": "text", "text": header}) frame_count = sum(1 for block in blocks if isinstance(block, dict) and block.get("type") == "image") @@ -1263,7 +1263,7 @@ def _truncate(content: str, file_path: str, *, line_limit: int | None = None) -> return content - def _handle_read_result( # noqa: PLR0911 + def _handle_read_result( # noqa: PLR0911 # one branch per distinct read-result disposition read_result: ReadResult | str, validated_path: str, tool_call_id: str | None, diff --git a/libs/deepagents/tests/unit_tests/middleware/test_filesystem_video.py b/libs/deepagents/tests/unit_tests/middleware/test_filesystem_video.py index 12f6e53221a..52adc35d831 100644 --- a/libs/deepagents/tests/unit_tests/middleware/test_filesystem_video.py +++ b/libs/deepagents/tests/unit_tests/middleware/test_filesystem_video.py @@ -1,6 +1,8 @@ import base64 +from typing import cast import pytest +from langchain.agents.middleware.types import ModelRequest, ModelResponse from langchain.tools import ToolRuntime from langchain_core.messages import AIMessage, HumanMessage, ToolMessage from langchain_core.messages.utils import convert_to_openai_messages @@ -256,3 +258,270 @@ def read(self, file_path: str, offset: int = 0, limit: int = 100) -> ReadResult: return ReadResult(file_data={"content": payload, "encoding": "base64"}) return VideoBackend() + + +def test_wrap_model_call_reorders_media_after_parallel_tool_results() -> None: + """`wrap_model_call` moves video media behind the full tool-result batch. + + With a parallel batch (video read_file + ls), the provider requires all + `ToolMessage`s before any non-tool message. Deleting the reorder call + from `wrap_model_call` would fail this test because the media + `HumanMessage` would arrive between the two tool results. + """ + mw = FilesystemMiddleware(backend=StateBackend()) + + ai_msg = AIMessage( + content="", + tool_calls=[ + {"id": "call_video", "name": "read_file", "args": {"file_path": "/c.mp4"}}, + {"id": "call_ls", "name": "ls", "args": {"path": "/"}}, + ], + ) + video_tool = ToolMessage(content="sampled frames", name="read_file", tool_call_id="call_video") + video_media = HumanMessage( + content=[{"type": "text", "text": "Reading first 100s of /c.mp4 at 0.5 fps."}], + additional_kwargs={"read_file_media_result": True}, + ) + ls_tool = ToolMessage(content="[]", name="ls", tool_call_id="call_ls") + user_msg = HumanMessage(content="read and list") + + captured: list[ModelRequest] = [] + + def handler(request: ModelRequest) -> ModelResponse: + captured.append(request) + return ModelResponse(result=[AIMessage(content="ok")]) + + request = ModelRequest( + model=None, + messages=[user_msg, ai_msg, video_tool, video_media, ls_tool], + tools=[], + ) + + mw.wrap_model_call(request, handler) + + assert len(captured) == 1 + reordered = captured[0].messages + # All ToolMessages must come before the media HumanMessage. + tool_indices = [i for i, m in enumerate(reordered) if isinstance(m, ToolMessage)] + media_index = next( + i for i, m in enumerate(reordered) if isinstance(m, HumanMessage) and m.additional_kwargs.get("read_file_media_result") is True + ) + assert all(i < media_index for i in tool_indices) + # Specifically: video_tool, ls_tool, then video_media. + assert reordered[-3] == video_tool + assert reordered[-2] == ls_tool + assert reordered[-1] == video_media + + +async def test_awrap_model_call_reorders_media_after_parallel_tool_results() -> None: + """`awrap_model_call` moves video media behind the full tool-result batch (async path).""" + mw = FilesystemMiddleware(backend=StateBackend()) + + ai_msg = AIMessage( + content="", + tool_calls=[ + {"id": "call_video", "name": "read_file", "args": {"file_path": "/c.mp4"}}, + {"id": "call_ls", "name": "ls", "args": {"path": "/"}}, + ], + ) + video_tool = ToolMessage(content="sampled frames", name="read_file", tool_call_id="call_video") + video_media = HumanMessage( + content=[{"type": "text", "text": "Reading first 100s of /c.mp4 at 0.5 fps."}], + additional_kwargs={"read_file_media_result": True}, + ) + ls_tool = ToolMessage(content="[]", name="ls", tool_call_id="call_ls") + user_msg = HumanMessage(content="read and list") + + captured: list[ModelRequest] = [] + + async def handler(request: ModelRequest) -> ModelResponse: + captured.append(request) + return ModelResponse(result=[AIMessage(content="ok")]) + + request = ModelRequest( + model=None, + messages=[user_msg, ai_msg, video_tool, video_media, ls_tool], + tools=[], + ) + + await mw.awrap_model_call(request, handler) + + assert len(captured) == 1 + reordered = captured[0].messages + tool_indices = [i for i, m in enumerate(reordered) if isinstance(m, ToolMessage)] + media_index = next( + i for i, m in enumerate(reordered) if isinstance(m, HumanMessage) and m.additional_kwargs.get("read_file_media_result") is True + ) + assert all(i < media_index for i in tool_indices) + assert reordered[-3] == video_tool + assert reordered[-2] == ls_tool + assert reordered[-1] == video_media + + +# --------------------------------------------------------------------------- +# Async read_file video path (issue #5b) +# --------------------------------------------------------------------------- + + +async def test_async_read_file_video_routes_to_frame_extractor(monkeypatch: pytest.MonkeyPatch) -> None: + """The async `read_file` coroutine produces the same Command as the sync path.""" + sentinel = [ + {"type": "text", "text": "Frame at t=00:00:00.000"}, + {"type": "image", "base64": "AAAA", "mime_type": "image/jpeg"}, + ] + calls: list[dict[str, float | int]] = [] + + def fake_extract( + _content: bytes, + *, + offset_seconds: float, + duration_seconds: float, + sampling_rate: float, + ) -> list[dict[str, str]]: + calls.append( + { + "offset_seconds": offset_seconds, + "duration_seconds": duration_seconds, + "sampling_rate": sampling_rate, + } + ) + return list(sentinel) + + monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) + + middleware = FilesystemMiddleware(backend=_video_backend(b"fake video")) + state = FilesystemState(messages=[], files={}) + runtime = _build_runtime(state, "video-async-1") + read_file_tool = next(t for t in middleware.tools if t.name == "read_file") + + result = await read_file_tool.ainvoke( + { + "file_path": "/clips/intro.mkv", + "offset": 0, + "limit": 30, + "runtime": runtime, + } + ) + + assert isinstance(result, Command) + messages = result.update["messages"] + assert len(messages) == 2 + tool_message, media_message = messages + assert isinstance(tool_message, ToolMessage) + assert tool_message.status == "success" + assert tool_message.additional_kwargs["read_file_frame_count"] == 1 + assert isinstance(media_message, HumanMessage) + assert media_message.additional_kwargs["read_file_media_result"] is True + assert calls == [{"offset_seconds": 0.0, "duration_seconds": 30.0, "sampling_rate": 0.5}] + + +def test_read_file_video_base64_decode_failure(monkeypatch: pytest.MonkeyPatch) -> None: + """Invalid base64 content surfaces as a tool error, not an exception.""" + called = {"count": 0} + + def fake_extract(*_args: object, **_kwargs: object) -> list[dict[str, str]]: + called["count"] += 1 + return [{"type": "text", "text": "ok"}] + + monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) + + class BadBase64Backend(StateBackend): + def read(self, file_path: str, offset: int = 0, limit: int = 100) -> ReadResult: + # Not valid base64 — contains characters outside the base64 alphabet. + return ReadResult(file_data={"content": "!!!not-base64!!!", "encoding": "base64"}) + + middleware = FilesystemMiddleware(backend=BadBase64Backend()) + state = FilesystemState(messages=[], files={}) + runtime = _build_runtime(state, "video-bad-b64") + read_file_tool = next(t for t in middleware.tools if t.name == "read_file") + + result = read_file_tool.invoke({"file_path": "/c.mp4", "runtime": runtime}) + + assert isinstance(result, ToolMessage) + assert result.status == "error" + assert "not valid base64" in result.content + assert called["count"] == 0 + + +def test_read_file_video_offset_header_format(monkeypatch: pytest.MonkeyPatch) -> None: + """A non-zero offset produces the `[x.xxxs, y.yyys)` header format.""" + monkeypatch.setattr( + filesystem_middleware, + "extract_video_frames", + lambda *_a, **_k: [{"type": "text", "text": "ok"}], + ) + middleware = FilesystemMiddleware(backend=_video_backend()) + state = FilesystemState(messages=[], files={}) + runtime = _build_runtime(state, "video-offset") + read_file_tool = next(t for t in middleware.tools if t.name == "read_file") + + result = cast("Command", read_file_tool.invoke({"file_path": "/c.mp4", "offset": 12, "limit": 90, "runtime": runtime})) + messages = result.update["messages"] + media_message = messages[1] + assert isinstance(media_message, HumanMessage) + header_block = media_message.content[0] + assert isinstance(header_block, dict) + text = header_block["text"] + assert text.startswith("Reading [12.000s, 102.000s)") + assert "0.5 fps" in text + + +def test_read_file_video_singular_frame_label(monkeypatch: pytest.MonkeyPatch) -> None: + """A single frame uses '1 frame' (singular), not '1 frames'.""" + sentinel = [ + {"type": "text", "text": "Frame at t=00:00:00.000"}, + {"type": "image", "base64": "AAAA", "mime_type": "image/jpeg"}, + ] + monkeypatch.setattr( + filesystem_middleware, + "extract_video_frames", + lambda *_a, **_k: list(sentinel), + ) + middleware = FilesystemMiddleware(backend=_video_backend()) + state = FilesystemState(messages=[], files={}) + runtime = _build_runtime(state, "video-singular") + read_file_tool = next(t for t in middleware.tools if t.name == "read_file") + + result = cast("Command", read_file_tool.invoke({"file_path": "/c.mp4", "runtime": runtime})) + tool_message = result.update["messages"][0] + assert isinstance(tool_message, ToolMessage) + assert tool_message.content == "Read video /c.mp4: sampled 1 frame. The sampled frames are attached in the following message." + + +def test_read_file_video_truncation_hint_appended_on_byte_cap(monkeypatch: pytest.MonkeyPatch) -> None: + """When the byte budget truncates output, a coverage hint is appended.""" + # Two frames that together exceed a tiny byte budget. + sentinel = [ + {"type": "text", "text": "Frame at t=00:00:00.000"}, + {"type": "image", "base64": "A" * 100, "mime_type": "image/jpeg"}, + {"type": "text", "text": "Frame at t=00:00:02.000"}, + {"type": "image", "base64": "B" * 100, "mime_type": "image/jpeg"}, + { + "type": "text", + "text": ( + "Coverage truncated at t=00:00:00.000: the output or frame cap was reached " + "before the full window was decoded. Re-read with a narrower window " + "(e.g. offset=0.000) to see the remaining frames." + ), + }, + ] + monkeypatch.setattr( + filesystem_middleware, + "extract_video_frames", + lambda *_a, **_k: list(sentinel), + ) + middleware = FilesystemMiddleware(backend=_video_backend()) + state = FilesystemState(messages=[], files={}) + runtime = _build_runtime(state, "video-trunc") + read_file_tool = next(t for t in middleware.tools if t.name == "read_file") + + result = cast("Command", read_file_tool.invoke({"file_path": "/c.mp4", "runtime": runtime})) + media_message = result.update["messages"][1] + assert isinstance(media_message, HumanMessage) + blocks = media_message.content + # The last block should be a truncation hint. + last_block = blocks[-1] + assert isinstance(last_block, dict) + assert last_block["type"] == "text" + assert "Coverage truncated" in last_block["text"] + assert "Re-read with a narrower window" in last_block["text"] diff --git a/libs/deepagents/tests/unit_tests/middleware/test_video.py b/libs/deepagents/tests/unit_tests/middleware/test_video.py index 40cfaaea10f..205b1c717fc 100644 --- a/libs/deepagents/tests/unit_tests/middleware/test_video.py +++ b/libs/deepagents/tests/unit_tests/middleware/test_video.py @@ -98,6 +98,113 @@ def test_extract_returns_interleaved_text_and_image_blocks(synthetic_video_bytes assert jpeg[:3] == b"\xff\xd8\xff" +def test_extract_truncation_hint_on_byte_cap(synthetic_video_bytes: bytes) -> None: + """When the emitted-byte cap truncates output, a coverage hint is appended.""" + # First, measure the size of one frame so we can set a cap that fits + # exactly one frame but not two. + one_frame_blocks = cast( + "list[dict[str, str]]", + video_module.extract_video_frames( + synthetic_video_bytes, + offset_seconds=0, + duration_seconds=1, + sampling_rate=0.5, + ), + ) + one_frame_bytes = sum(len(b.get("text", "").encode()) + len(b.get("base64", "").encode("ascii")) for b in one_frame_blocks) + original_cap = video_module.MAX_VIDEO_EMITTED_BYTES + try: + # Cap fits one frame but not two. + video_module.MAX_VIDEO_EMITTED_BYTES = one_frame_bytes + 10 + blocks = cast( + "list[dict[str, str]]", + video_module.extract_video_frames( + synthetic_video_bytes, + offset_seconds=0, + duration_seconds=30, + sampling_rate=0.5, + ), + ) + finally: + video_module.MAX_VIDEO_EMITTED_BYTES = original_cap + + # The last block should be a truncation hint, not a frame. + last = blocks[-1] + assert last["type"] == "text" + assert "Coverage truncated" in last["text"] + assert "Re-read with a narrower window" in last["text"] + + +def test_extract_truncation_hint_on_frame_count_cap(synthetic_video_bytes: bytes) -> None: + """When the frame-count cap truncates output, a coverage hint is appended.""" + original_cap = video_module.MAX_VIDEO_SAMPLED_FRAMES + try: + video_module.MAX_VIDEO_SAMPLED_FRAMES = 1 + blocks = cast( + "list[dict[str, str]]", + video_module.extract_video_frames( + synthetic_video_bytes, + offset_seconds=0, + duration_seconds=30, + sampling_rate=0.5, + ), + ) + finally: + video_module.MAX_VIDEO_SAMPLED_FRAMES = original_cap + + # Exactly one image block, then the truncation hint. + image_count = sum(1 for b in blocks if b["type"] == "image") + assert image_count == 1 + last = blocks[-1] + assert last["type"] == "text" + assert "Coverage truncated" in last["text"] + + +def test_extract_raises_on_none_time_base(synthetic_video_bytes: bytes) -> None: + """A stream with `time_base=None` raises `VideoExtractionError`, not `TypeError`.""" + real_open = video_module._open_video_container + av_module = video_module._import_av() + + class _FakeStream: + type = "video" + time_base = None + start_time = None + + def __iter__(self) -> Iterator[object]: + return iter([]) + + class _FakeContainer: + def streams(self) -> list[_FakeStream]: + return [_FakeStream()] + + streams = property(streams) + + def decode(self, _stream: object) -> Iterator[object]: + return iter([]) + + def close(self) -> None: + pass + + real_container_open = av_module.open + + def fake_open(_file_or_path: object, **_kwargs: object) -> _FakeContainer: + return _FakeContainer() + + try: + av_module.open = fake_open + with pytest.raises(video_module.VideoExtractionError, match="no time_base"): + video_module.extract_video_frames( + synthetic_video_bytes, + offset_seconds=0, + duration_seconds=1, + sampling_rate=0.5, + ) + finally: + av_module.open = real_container_open + # Restore the module-level reference too. + _ = real_open + + def test_extract_offset_skips_into_source(synthetic_video_bytes: bytes) -> None: """`offset_seconds` seeks into the clip; every emitted frame is at or after that point.""" blocks = cast( @@ -262,9 +369,10 @@ def test_sample_frames_does_not_cap_requested_duration(monkeypatch: pytest.Monke ) headers = [block["text"] for block in blocks if block["type"] == "text"] - # Three frames fit inside the 999 s window without being silently clamped; - # only the frame-count cap (`MAX_VIDEO_SAMPLED_FRAMES = 2`) trims the output. - assert headers == ["Frame at t=00:00:00.000", "Frame at t=00:00:01.000"] + # Two frames emitted before the frame-count cap triggers; the truncation + # hint follows so the model knows the window was not fully decoded. + assert headers[:2] == ["Frame at t=00:00:00.000", "Frame at t=00:00:01.000"] + assert "Coverage truncated" in headers[-1] def test_sample_frames_caps_frame_count(monkeypatch: pytest.MonkeyPatch) -> None: @@ -284,7 +392,8 @@ def test_sample_frames_caps_frame_count(monkeypatch: pytest.MonkeyPatch) -> None ) headers = [block["text"] for block in blocks if block["type"] == "text"] - assert headers == ["Frame at t=00:00:00.000", "Frame at t=00:00:01.000"] + assert headers[:2] == ["Frame at t=00:00:00.000", "Frame at t=00:00:01.000"] + assert "Coverage truncated" in headers[-1] def test_sample_frames_rejects_oversized_output(monkeypatch: pytest.MonkeyPatch) -> None: From 00eeb22179f1b37a141cd97e8ebbd82c2e6bd700 Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Thu, 2 Jul 2026 00:43:21 -0400 Subject: [PATCH 28/36] safely remove annotations --- .../deepagents/middleware/filesystem.py | 96 ++++++++----------- 1 file changed, 38 insertions(+), 58 deletions(-) diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index a27f3e55395..6d4eb761500 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -1160,7 +1160,7 @@ def _create_ls_tool(self) -> BaseTool: def sync_ls( runtime: ToolRuntime[None, FilesystemState], - path: Annotated[str, "Absolute path to the directory to list. Must be absolute, not relative."], + path: str, ) -> ToolMessage: """Synchronous wrapper for ls tool.""" resolved_backend = self._get_backend(runtime) @@ -1199,7 +1199,7 @@ def sync_ls( async def async_ls( runtime: ToolRuntime[None, FilesystemState], - path: Annotated[str, "Absolute path to the directory to list. Must be absolute, not relative."], + path: str, ) -> ToolMessage: """Asynchronous wrapper for ls tool.""" resolved_backend = self._get_backend(runtime) @@ -1361,13 +1361,10 @@ def _handle_read_result( # noqa: PLR0911 # one branch per distinct read-result ) def sync_read_file( - file_path: Annotated[str, "Absolute path to the file to read. Must be absolute, not relative."], + file_path: str, runtime: ToolRuntime[None, FilesystemState], - offset: Annotated[ - int, - "Line number to start reading from for text files (0-indexed). For videos, seconds into the source to start sampling.", - ] = DEFAULT_READ_OFFSET, - limit: Annotated[int, "Maximum number of lines to read for text files. For videos, seconds of source to sample."] = DEFAULT_READ_LIMIT, + offset: int = DEFAULT_READ_OFFSET, + limit: int = DEFAULT_READ_LIMIT, ) -> ToolMessage | Command: """Synchronous wrapper for read_file tool.""" resolved_backend = self._get_backend(runtime) @@ -1391,13 +1388,10 @@ def sync_read_file( return _handle_read_result(read_result, validated_path, runtime.tool_call_id, offset, limit) async def async_read_file( - file_path: Annotated[str, "Absolute path to the file to read. Must be absolute, not relative."], + file_path: str, runtime: ToolRuntime[None, FilesystemState], - offset: Annotated[ - int, - "Line number to start reading from for text files (0-indexed). For videos, seconds into the source to start sampling.", - ] = DEFAULT_READ_OFFSET, - limit: Annotated[int, "Maximum number of lines to read for text files. For videos, seconds of source to sample."] = DEFAULT_READ_LIMIT, + offset: int = DEFAULT_READ_OFFSET, + limit: int = DEFAULT_READ_LIMIT, ) -> ToolMessage | Command: """Asynchronous wrapper for read_file tool.""" resolved_backend = self._get_backend(runtime) @@ -1434,8 +1428,8 @@ def _create_write_file_tool(self) -> BaseTool: tool_description = self._custom_tool_descriptions.get("write_file") or WRITE_FILE_TOOL_DESCRIPTION def sync_write_file( - file_path: Annotated[str, "Absolute path where the file should be written. Must be absolute, not relative."], - content: Annotated[str, "The text content to write to the file. This parameter is required."], + file_path: str, + content: str, runtime: ToolRuntime[None, FilesystemState], ) -> ToolMessage: """Synchronous wrapper for write_file tool.""" @@ -1473,8 +1467,8 @@ def sync_write_file( ) async def async_write_file( - file_path: Annotated[str, "Absolute path where the file should be written. Must be absolute, not relative."], - content: Annotated[str, "The text content to write to the file. This parameter is required."], + file_path: str, + content: str, runtime: ToolRuntime[None, FilesystemState], ) -> ToolMessage: """Asynchronous wrapper for write_file tool.""" @@ -1525,12 +1519,12 @@ def _create_edit_file_tool(self) -> BaseTool: tool_description = self._custom_tool_descriptions.get("edit_file") or EDIT_FILE_TOOL_DESCRIPTION def sync_edit_file( - file_path: Annotated[str, "Absolute path to the file to edit. Must be absolute, not relative."], - old_string: Annotated[str, "The exact text to find and replace. Must be unique in the file unless replace_all is True."], - new_string: Annotated[str, "The text to replace old_string with. Must be different from old_string."], + file_path: str, + old_string: str, + new_string: str, runtime: ToolRuntime[None, FilesystemState], *, - replace_all: Annotated[bool, "If True, replace all occurrences of old_string. If False (default), old_string must be unique."] = False, + replace_all: bool = False, ) -> ToolMessage: """Synchronous wrapper for edit_file tool.""" resolved_backend = self._get_backend(runtime) @@ -1567,12 +1561,12 @@ def sync_edit_file( ) async def async_edit_file( - file_path: Annotated[str, "Absolute path to the file to edit. Must be absolute, not relative."], - old_string: Annotated[str, "The exact text to find and replace. Must be unique in the file unless replace_all is True."], - new_string: Annotated[str, "The text to replace old_string with. Must be different from old_string."], + file_path: str, + old_string: str, + new_string: str, runtime: ToolRuntime[None, FilesystemState], *, - replace_all: Annotated[bool, "If True, replace all occurrences of old_string. If False (default), old_string must be unique."] = False, + replace_all: bool = False, ) -> ToolMessage: """Asynchronous wrapper for edit_file tool.""" resolved_backend = self._get_backend(runtime) @@ -1622,7 +1616,7 @@ def _create_delete_tool(self) -> BaseTool: # Tool wiring + permission/support h tool_description = self._custom_tool_descriptions.get("delete") or DELETE_TOOL_DESCRIPTION def sync_delete( - file_path: Annotated[str, "Absolute path to the file to delete. Must be absolute, not relative."], + file_path: str, runtime: ToolRuntime[None, FilesystemState], ) -> ToolMessage: """Synchronous wrapper for delete tool.""" @@ -1661,7 +1655,7 @@ def sync_delete( ) async def async_delete( - file_path: Annotated[str, "Absolute path to the file to delete. Must be absolute, not relative."], + file_path: str, runtime: ToolRuntime[None, FilesystemState], ) -> ToolMessage: """Asynchronous wrapper for delete tool.""" @@ -1713,9 +1707,9 @@ def _create_glob_tool(self) -> BaseTool: # noqa: C901, PLR0915 # Tool wiring + tool_description = self._custom_tool_descriptions.get("glob") or GLOB_TOOL_DESCRIPTION def sync_glob( # noqa: PLR0911 - early returns for distinct error conditions - pattern: Annotated[str, "Glob pattern to match files (e.g., '**/*.py', '*.txt', '/subdir/**/*.md')."], + pattern: str, runtime: ToolRuntime[None, FilesystemState], - path: Annotated[str | None, "Base directory to search from. Defaults to the backend's default root."] = None, + path: str | None = None, ) -> ToolMessage: """Synchronous wrapper for glob tool.""" resolved_backend = self._get_backend(runtime) @@ -1809,9 +1803,9 @@ def run_glob() -> GlobResult: ) async def async_glob( - pattern: Annotated[str, "Glob pattern to match files (e.g., '**/*.py', '*.txt', '/subdir/**/*.md')."], + pattern: str, runtime: ToolRuntime[None, FilesystemState], - path: Annotated[str | None, "Base directory to search from. Defaults to the backend's default root."] = None, + path: str | None = None, ) -> ToolMessage: """Asynchronous wrapper for glob tool.""" resolved_backend = self._get_backend(runtime) @@ -1886,14 +1880,11 @@ def _create_grep_tool(self) -> BaseTool: tool_description = self._custom_tool_descriptions.get("grep") or GREP_TOOL_DESCRIPTION def sync_grep( - pattern: Annotated[str, "Text pattern to search for (literal string, not regex)."], + pattern: str, runtime: ToolRuntime[None, FilesystemState], - path: Annotated[str | None, "Directory to search in. Defaults to current working directory."] = None, - glob: Annotated[str | None, "Glob pattern to filter which files to search (e.g., '*.py')."] = None, - output_mode: Annotated[ - Literal["files_with_matches", "content", "count"], - "Output format: 'files_with_matches' (file paths only, default), 'content' (matching lines with context), 'count' (match counts per file).", - ] = "files_with_matches", + path: str | None = None, + glob: str | None = None, + output_mode: Literal["files_with_matches", "content", "count"] = "files_with_matches", ) -> ToolMessage: """Synchronous wrapper for grep tool.""" if path is not None: @@ -1929,14 +1920,11 @@ def sync_grep( ) async def async_grep( - pattern: Annotated[str, "Text pattern to search for (literal string, not regex)."], + pattern: str, runtime: ToolRuntime[None, FilesystemState], - path: Annotated[str | None, "Directory to search in. Defaults to current working directory."] = None, - glob: Annotated[str | None, "Glob pattern to filter which files to search (e.g., '*.py')."] = None, - output_mode: Annotated[ - Literal["files_with_matches", "content", "count"], - "Output format: 'files_with_matches' (file paths only, default), 'content' (matching lines with context), 'count' (match counts per file).", - ] = "files_with_matches", + path: str | None = None, + glob: str | None = None, + output_mode: Literal["files_with_matches", "content", "count"] = "files_with_matches", ) -> ToolMessage: """Asynchronous wrapper for grep tool.""" if path is not None: @@ -2053,12 +2041,9 @@ def _create_execute_tool(self) -> BaseTool: # noqa: C901 tool_description = self._custom_tool_descriptions.get("execute") or EXECUTE_TOOL_DESCRIPTION def sync_execute( # noqa: PLR0911 - early returns for distinct error conditions - command: Annotated[str, "Shell command to execute in the sandbox environment."], + command: str, runtime: ToolRuntime[None, FilesystemState], - timeout: Annotated[ - int | None, - "Optional timeout in seconds for this command. Overrides the default timeout. Use 0 for no-timeout execution on backends that support it.", - ] = None, + timeout: int | None = None, ) -> ToolMessage: """Synchronous wrapper for execute tool.""" if timeout is not None: @@ -2143,14 +2128,9 @@ def sync_execute( # noqa: PLR0911 - early returns for distinct error conditions ) async def async_execute( # noqa: PLR0911 - early returns for distinct error conditions - command: Annotated[str, "Shell command to execute in the sandbox environment."], + command: str, runtime: ToolRuntime[None, FilesystemState], - # ASYNC109 - timeout is a semantic parameter forwarded to the - # backend's implementation, not an asyncio.timeout() contract. - timeout: Annotated[ # noqa: ASYNC109 - int | None, - "Optional timeout in seconds for this command. Overrides the default timeout. Use 0 for no-timeout execution on backends that support it.", - ] = None, + timeout: int | None = None, # noqa: ASYNC109 # forwarded to backend, not an asyncio contract ) -> ToolMessage: """Asynchronous wrapper for execute tool.""" if timeout is not None: From 80a92e1d778fd48b543f66eaa99873a53caa22bb Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Thu, 2 Jul 2026 00:45:23 -0400 Subject: [PATCH 29/36] fix false pos --- .../deepagents/middleware/_video.py | 6 ++--- .../tests/unit_tests/middleware/test_video.py | 22 +++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/libs/deepagents/deepagents/middleware/_video.py b/libs/deepagents/deepagents/middleware/_video.py index b1799a2f566..e8c454a74a9 100644 --- a/libs/deepagents/deepagents/middleware/_video.py +++ b/libs/deepagents/deepagents/middleware/_video.py @@ -289,6 +289,9 @@ def _sample_frames_in_window( break if frame_seconds + 1e-6 < next_emit_seconds: continue + if emitted_frames >= MAX_VIDEO_SAMPLED_FRAMES: + truncated = True + break jpeg_bytes = _encode_jpeg(frame) image_base64 = base64.b64encode(jpeg_bytes) @@ -313,9 +316,6 @@ def _sample_frames_in_window( emitted_frames += 1 emitted_bytes += next_block_bytes last_emitted_seconds = frame_seconds - if emitted_frames >= MAX_VIDEO_SAMPLED_FRAMES: - truncated = True - break emitted_index = math.floor((frame_seconds - offset_seconds) / frame_interval_seconds) + 1 next_emit_seconds = max( next_emit_seconds + frame_interval_seconds, diff --git a/libs/deepagents/tests/unit_tests/middleware/test_video.py b/libs/deepagents/tests/unit_tests/middleware/test_video.py index 205b1c717fc..7fb7497d267 100644 --- a/libs/deepagents/tests/unit_tests/middleware/test_video.py +++ b/libs/deepagents/tests/unit_tests/middleware/test_video.py @@ -396,6 +396,28 @@ def test_sample_frames_caps_frame_count(monkeypatch: pytest.MonkeyPatch) -> None assert "Coverage truncated" in headers[-1] +def test_sample_frames_no_truncation_hint_when_cap_exactly_covers_window( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A full window with exactly the frame-count cap is not reported as truncated.""" + monkeypatch.setattr(video_module, "_encode_jpeg", lambda _frame: b"jpeg") + monkeypatch.setattr(video_module, "MAX_VIDEO_SAMPLED_FRAMES", 2) + + blocks = cast( + "list[dict[str, str]]", + video_module._sample_frames_in_window( + [_FakeFrame(0), _FakeFrame(1), _FakeFrame(2)], + offset_seconds=0, + duration_seconds=2, + sampling_rate=1, + time_base=1, + ), + ) + + headers = [block["text"] for block in blocks if block["type"] == "text"] + assert headers == ["Frame at t=00:00:00.000", "Frame at t=00:00:01.000"] + + def test_sample_frames_rejects_oversized_output(monkeypatch: pytest.MonkeyPatch) -> None: """Extraction fails before retaining an oversized first image block.""" monkeypatch.setattr(video_module, "_encode_jpeg", lambda _frame: b"jpeg") From e15895b31733901b320abe31c0b42a9639a5001c Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Thu, 2 Jul 2026 00:47:27 -0400 Subject: [PATCH 30/36] cr --- .../deepagents/middleware/_video.py | 38 ++++++---- .../deepagents/middleware/filesystem.py | 2 +- .../middleware/test_filesystem_video.py | 23 +++++- .../tests/unit_tests/middleware/test_video.py | 71 +++++++++---------- 4 files changed, 79 insertions(+), 55 deletions(-) diff --git a/libs/deepagents/deepagents/middleware/_video.py b/libs/deepagents/deepagents/middleware/_video.py index e8c454a74a9..b29402d8391 100644 --- a/libs/deepagents/deepagents/middleware/_video.py +++ b/libs/deepagents/deepagents/middleware/_video.py @@ -112,18 +112,17 @@ def extract_video_frames( Raises: VideoExtractionError: If PyAV cannot open the payload or the - requested window yields no decodable frames. - ValueError: If argument validation fails before opening the file. + requested window yields no decodable frames, or if argument + validation fails before opening the file. """ - if offset_seconds < 0: - msg = f"offset_seconds must be >= 0, got {offset_seconds!r}" - raise ValueError(msg) - if sampling_rate <= 0: - msg = f"sampling_rate must be > 0, got {sampling_rate!r}" - raise ValueError(msg) - if duration_seconds <= 0: - msg = f"duration_seconds must be > 0, got {duration_seconds!r}" - raise ValueError(msg) + try: + _validate_video_window( + offset_seconds=offset_seconds, + duration_seconds=duration_seconds, + sampling_rate=sampling_rate, + ) + except ValueError as exc: + raise VideoExtractionError(str(exc)) from exc rate = float(sampling_rate) duration = float(duration_seconds) @@ -173,6 +172,19 @@ def extract_video_frames( return blocks +def _validate_video_window(*, offset_seconds: float, duration_seconds: float, sampling_rate: float) -> None: + """Validate the requested sampling window before opening the video.""" + if offset_seconds < 0: + msg = f"offset_seconds must be >= 0, got {offset_seconds!r}" + raise ValueError(msg) + if sampling_rate <= 0: + msg = f"sampling_rate must be > 0, got {sampling_rate!r}" + raise ValueError(msg) + if duration_seconds <= 0: + msg = f"duration_seconds must be > 0, got {duration_seconds!r}" + raise ValueError(msg) + + def _open_video_container(av: Any, content: bytes) -> Any: # noqa: ANN401 # PyAV types are unavailable without the [video] extra """Open a video byte payload, normalizing PyAV's failure modes. @@ -331,8 +343,8 @@ def _sample_frames_in_window( "type": "text", "text": ( f"Coverage truncated at t={last_ts}: the output or frame cap was reached " - f"before the full window was decoded. Re-read with a narrower window " - f"(e.g. offset={last_emitted_seconds:.3f}) to see the remaining frames." + f"before the full window was decoded. Continue from " + f"offset={last_emitted_seconds:.3f} to see the remaining frames." ), } ) diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index 6d4eb761500..4059422c83f 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -197,7 +197,7 @@ def _err(msg: str) -> ToolMessage: duration_seconds=duration_seconds, sampling_rate=rate, ) - except (VideoExtractionError, ValueError) as exc: + except VideoExtractionError as exc: return _err(str(exc)) blocks.insert(0, {"type": "text", "text": header}) frame_count = sum(1 for block in blocks if isinstance(block, dict) and block.get("type") == "image") diff --git a/libs/deepagents/tests/unit_tests/middleware/test_filesystem_video.py b/libs/deepagents/tests/unit_tests/middleware/test_filesystem_video.py index 52adc35d831..d54d63e99f5 100644 --- a/libs/deepagents/tests/unit_tests/middleware/test_filesystem_video.py +++ b/libs/deepagents/tests/unit_tests/middleware/test_filesystem_video.py @@ -192,6 +192,23 @@ def fake_extract(*_args: object, **_kwargs: object) -> list[dict[str, str]]: assert "av not installed" in result.content +def test_read_file_video_unexpected_value_error_is_not_swallowed(monkeypatch: pytest.MonkeyPatch) -> None: + """Unexpected extractor bugs should remain visible to operators.""" + + def fake_extract(*_args: object, **_kwargs: object) -> list[dict[str, str]]: + msg = "unexpected formatter failure" + raise ValueError(msg) + + monkeypatch.setattr(filesystem_middleware, "extract_video_frames", fake_extract) + middleware = FilesystemMiddleware(backend=_video_backend()) + state = FilesystemState(messages=[], files={}) + runtime = _build_runtime(state, "video-read-unexpected-value-error") + read_file_tool = next(t for t in middleware.tools if t.name == "read_file") + + with pytest.raises(ValueError, match="unexpected formatter failure"): + read_file_tool.invoke({"file_path": "/c.mp4", "runtime": runtime}) + + def test_read_file_video_non_positive_limit_rejected(monkeypatch: pytest.MonkeyPatch) -> None: """A non-positive `limit` is rejected as a tool error, not silently clamped.""" called = {"count": 0} @@ -500,8 +517,8 @@ def test_read_file_video_truncation_hint_appended_on_byte_cap(monkeypatch: pytes "type": "text", "text": ( "Coverage truncated at t=00:00:00.000: the output or frame cap was reached " - "before the full window was decoded. Re-read with a narrower window " - "(e.g. offset=0.000) to see the remaining frames." + "before the full window was decoded. Continue from " + "offset=0.000 to see the remaining frames." ), }, ] @@ -524,4 +541,4 @@ def test_read_file_video_truncation_hint_appended_on_byte_cap(monkeypatch: pytes assert isinstance(last_block, dict) assert last_block["type"] == "text" assert "Coverage truncated" in last_block["text"] - assert "Re-read with a narrower window" in last_block["text"] + assert "Continue from offset=" in last_block["text"] diff --git a/libs/deepagents/tests/unit_tests/middleware/test_video.py b/libs/deepagents/tests/unit_tests/middleware/test_video.py index 7fb7497d267..9cccb60c658 100644 --- a/libs/deepagents/tests/unit_tests/middleware/test_video.py +++ b/libs/deepagents/tests/unit_tests/middleware/test_video.py @@ -48,7 +48,7 @@ def synthetic_video_bytes(tmp_path_factory: pytest.TempPathFactory) -> bytes: def test_extract_rejects_negative_offset(synthetic_video_bytes: bytes) -> None: - with pytest.raises(ValueError, match="offset_seconds must be >= 0"): + with pytest.raises(video_module.VideoExtractionError, match="offset_seconds must be >= 0"): video_module.extract_video_frames( synthetic_video_bytes, offset_seconds=-0.1, @@ -58,7 +58,7 @@ def test_extract_rejects_negative_offset(synthetic_video_bytes: bytes) -> None: def test_extract_rejects_zero_or_negative_duration(synthetic_video_bytes: bytes) -> None: - with pytest.raises(ValueError, match="duration_seconds must be > 0"): + with pytest.raises(video_module.VideoExtractionError, match="duration_seconds must be > 0"): video_module.extract_video_frames( synthetic_video_bytes, offset_seconds=0, @@ -68,7 +68,7 @@ def test_extract_rejects_zero_or_negative_duration(synthetic_video_bytes: bytes) def test_extract_rejects_zero_or_negative_sampling_rate(synthetic_video_bytes: bytes) -> None: - with pytest.raises(ValueError, match="sampling_rate must be > 0"): + with pytest.raises(video_module.VideoExtractionError, match="sampling_rate must be > 0"): video_module.extract_video_frames( synthetic_video_bytes, offset_seconds=0, @@ -98,7 +98,10 @@ def test_extract_returns_interleaved_text_and_image_blocks(synthetic_video_bytes assert jpeg[:3] == b"\xff\xd8\xff" -def test_extract_truncation_hint_on_byte_cap(synthetic_video_bytes: bytes) -> None: +def test_extract_truncation_hint_on_byte_cap( + synthetic_video_bytes: bytes, + monkeypatch: pytest.MonkeyPatch, +) -> None: """When the emitted-byte cap truncates output, a coverage hint is appended.""" # First, measure the size of one frame so we can set a cap that fits # exactly one frame but not two. @@ -112,45 +115,40 @@ def test_extract_truncation_hint_on_byte_cap(synthetic_video_bytes: bytes) -> No ), ) one_frame_bytes = sum(len(b.get("text", "").encode()) + len(b.get("base64", "").encode("ascii")) for b in one_frame_blocks) - original_cap = video_module.MAX_VIDEO_EMITTED_BYTES - try: - # Cap fits one frame but not two. - video_module.MAX_VIDEO_EMITTED_BYTES = one_frame_bytes + 10 - blocks = cast( - "list[dict[str, str]]", - video_module.extract_video_frames( - synthetic_video_bytes, - offset_seconds=0, - duration_seconds=30, - sampling_rate=0.5, - ), - ) - finally: - video_module.MAX_VIDEO_EMITTED_BYTES = original_cap + # Cap fits one frame but not two. + monkeypatch.setattr(video_module, "MAX_VIDEO_EMITTED_BYTES", one_frame_bytes + 10) + blocks = cast( + "list[dict[str, str]]", + video_module.extract_video_frames( + synthetic_video_bytes, + offset_seconds=0, + duration_seconds=30, + sampling_rate=0.5, + ), + ) # The last block should be a truncation hint, not a frame. last = blocks[-1] assert last["type"] == "text" assert "Coverage truncated" in last["text"] - assert "Re-read with a narrower window" in last["text"] + assert "Continue from offset=" in last["text"] -def test_extract_truncation_hint_on_frame_count_cap(synthetic_video_bytes: bytes) -> None: +def test_extract_truncation_hint_on_frame_count_cap( + synthetic_video_bytes: bytes, + monkeypatch: pytest.MonkeyPatch, +) -> None: """When the frame-count cap truncates output, a coverage hint is appended.""" - original_cap = video_module.MAX_VIDEO_SAMPLED_FRAMES - try: - video_module.MAX_VIDEO_SAMPLED_FRAMES = 1 - blocks = cast( - "list[dict[str, str]]", - video_module.extract_video_frames( - synthetic_video_bytes, - offset_seconds=0, - duration_seconds=30, - sampling_rate=0.5, - ), - ) - finally: - video_module.MAX_VIDEO_SAMPLED_FRAMES = original_cap + monkeypatch.setattr(video_module, "MAX_VIDEO_SAMPLED_FRAMES", 1) + blocks = cast( + "list[dict[str, str]]", + video_module.extract_video_frames( + synthetic_video_bytes, + offset_seconds=0, + duration_seconds=30, + sampling_rate=0.5, + ), + ) # Exactly one image block, then the truncation hint. image_count = sum(1 for b in blocks if b["type"] == "image") @@ -162,7 +160,6 @@ def test_extract_truncation_hint_on_frame_count_cap(synthetic_video_bytes: bytes def test_extract_raises_on_none_time_base(synthetic_video_bytes: bytes) -> None: """A stream with `time_base=None` raises `VideoExtractionError`, not `TypeError`.""" - real_open = video_module._open_video_container av_module = video_module._import_av() class _FakeStream: @@ -201,8 +198,6 @@ def fake_open(_file_or_path: object, **_kwargs: object) -> _FakeContainer: ) finally: av_module.open = real_container_open - # Restore the module-level reference too. - _ = real_open def test_extract_offset_skips_into_source(synthetic_video_bytes: bytes) -> None: From 9a0b8615106c940905b94979491c63e34be824ce Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Thu, 2 Jul 2026 01:33:57 -0400 Subject: [PATCH 31/36] gate on dep presence --- .../deepagents/backends/filesystem.py | 4 +- .../deepagents/backends/langsmith.py | 4 +- .../deepagents/deepagents/backends/sandbox.py | 4 +- libs/deepagents/deepagents/backends/state.py | 4 +- libs/deepagents/deepagents/backends/store.py | 6 +- libs/deepagents/deepagents/backends/utils.py | 37 +++++++++- .../deepagents/middleware/_video.py | 25 +++++++ .../deepagents/middleware/filesystem.py | 69 ++++++++++++++++--- .../backends/test_filesystem_backend.py | 31 +++++++++ .../backends/test_sandbox_backend.py | 8 +++ .../unit_tests/backends/test_store_backend.py | 21 ++++++ .../tests/unit_tests/backends/test_utils.py | 17 ++++- .../middleware/test_filesystem_video.py | 69 +++++++++++++++++++ .../unit_tests/middleware/test_video_deps.py | 41 +++++++++++ 14 files changed, 318 insertions(+), 22 deletions(-) create mode 100644 libs/deepagents/tests/unit_tests/middleware/test_video_deps.py diff --git a/libs/deepagents/deepagents/backends/filesystem.py b/libs/deepagents/deepagents/backends/filesystem.py index 1fbef959d5f..e1e6d9cff99 100644 --- a/libs/deepagents/deepagents/backends/filesystem.py +++ b/libs/deepagents/deepagents/backends/filesystem.py @@ -38,7 +38,7 @@ ) from deepagents.backends.utils import ( MAX_VIDEO_INPUT_BYTES, - _get_file_type, + _get_backend_read_file_type, check_empty_content, perform_string_replacement, ) @@ -436,7 +436,7 @@ def read( fd = os.open(resolved_path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) try: - file_type = _get_file_type(file_path) + file_type = _get_backend_read_file_type(file_path) if file_type != "text": if file_type == "video" and os.fstat(fd).st_size > MAX_VIDEO_INPUT_BYTES: return ReadResult(error=f"Video file exceeds maximum input size of {MAX_VIDEO_INPUT_BYTES} bytes") diff --git a/libs/deepagents/deepagents/backends/langsmith.py b/libs/deepagents/deepagents/backends/langsmith.py index db9ae587567..7f879fc66ab 100644 --- a/libs/deepagents/deepagents/backends/langsmith.py +++ b/libs/deepagents/deepagents/backends/langsmith.py @@ -20,7 +20,7 @@ TRUNCATION_MSG, BaseSandbox, ) -from deepagents.backends.utils import _get_file_type +from deepagents.backends.utils import _get_backend_read_file_type if TYPE_CHECKING: from langsmith.sandbox import Sandbox @@ -172,7 +172,7 @@ def read( # noqa: PLR0911 - early returns for distinct error conditions # Route by extension first, mirroring _READ_COMMAND_TEMPLATE: anything # not classified as text goes straight to base64 without a decode # attempt. - if _get_file_type(file_path) != "text": + if _get_backend_read_file_type(file_path) != "text": return _binary_read_result(file_path, raw) try: diff --git a/libs/deepagents/deepagents/backends/sandbox.py b/libs/deepagents/deepagents/backends/sandbox.py index b080a7c7f4c..276306a6712 100644 --- a/libs/deepagents/deepagents/backends/sandbox.py +++ b/libs/deepagents/deepagents/backends/sandbox.py @@ -42,7 +42,7 @@ WriteResult, execute_accepts_timeout, ) -from deepagents.backends.utils import _get_file_type +from deepagents.backends.utils import _get_backend_read_file_type logger = logging.getLogger(__name__) @@ -438,7 +438,7 @@ def _parse_ls_output(output: str, path: str) -> LsResult: def _build_read_cmd(file_path: str, offset: int, limit: int) -> str: - file_type = _get_file_type(file_path) + file_type = _get_backend_read_file_type(file_path) path_b64 = base64.b64encode(file_path.encode("utf-8")).decode("ascii") # Defensive int coercion in case callers bypass type checking. return _READ_COMMAND_TEMPLATE.format( diff --git a/libs/deepagents/deepagents/backends/state.py b/libs/deepagents/deepagents/backends/state.py index 87956dd2bd0..00b7f6aa724 100644 --- a/libs/deepagents/deepagents/backends/state.py +++ b/libs/deepagents/deepagents/backends/state.py @@ -24,7 +24,7 @@ WriteResult, ) from deepagents.backends.utils import ( - _get_file_type, + _get_backend_read_file_type, _glob_search_files, _to_legacy_file_data, create_file_data, @@ -231,7 +231,7 @@ def read( if file_data is None: return ReadResult(error=f"File '{file_path}' not found") - if _get_file_type(file_path) != "text": + if _get_backend_read_file_type(file_path) != "text": return ReadResult(file_data=file_data) sliced = slice_read_response(file_data, offset, limit) diff --git a/libs/deepagents/deepagents/backends/store.py b/libs/deepagents/deepagents/backends/store.py index 2a30107980b..ad37a529e07 100644 --- a/libs/deepagents/deepagents/backends/store.py +++ b/libs/deepagents/deepagents/backends/store.py @@ -28,7 +28,7 @@ WriteResult, ) from deepagents.backends.utils import ( - _get_file_type, + _get_backend_read_file_type, _glob_search_files, _to_legacy_file_data, create_file_data, @@ -544,7 +544,7 @@ def read( except ValueError as e: return ReadResult(error=str(e)) - if _get_file_type(file_path) != "text": + if _get_backend_read_file_type(file_path) != "text": return ReadResult(file_data=file_data) sliced = slice_read_response(file_data, offset, limit) @@ -582,7 +582,7 @@ async def aread( except ValueError as e: return ReadResult(error=str(e)) - if _get_file_type(file_path) != "text": + if _get_backend_read_file_type(file_path) != "text": return ReadResult(file_data=file_data) sliced = slice_read_response(file_data, offset, limit) diff --git a/libs/deepagents/deepagents/backends/utils.py b/libs/deepagents/deepagents/backends/utils.py index 27722c2f0bf..f59da0120b8 100644 --- a/libs/deepagents/deepagents/backends/utils.py +++ b/libs/deepagents/deepagents/backends/utils.py @@ -42,7 +42,6 @@ ".flv": "video", ".mpg": "video", ".webm": "video", - ".mkv": "video", ".wmv": "video", ".3gpp": "video", # Audio (https://ai.google.dev/gemini-api/docs/audio) @@ -59,6 +58,10 @@ } """Extension-to-type mapping for non-text files. +Optional features may layer on additional classifications at the use site. For +example, `read_file` treats `.mkv` as video only when the optional video +dependencies are installed. + Derived from Google's multimodal API supported formats: - Images: https://ai.google.dev/gemini-api/docs/image-understanding @@ -194,6 +197,38 @@ def _get_file_type(path: str) -> FileType: return _EXTENSION_TO_FILE_TYPE.get(PurePosixPath(path).suffix.lower(), "text") +_VIDEO_EXTRA_EXTENSIONS: frozenset[str] = frozenset({".mkv"}) +"""Video container extensions handled outside the Google-derived multimodal map. + +These are intentionally absent from `_EXTENSION_TO_FILE_TYPE`, so a `read_file` +without the optional `[video]` extra returns them as a generic file block rather +than a native video block. Backends must still read them as binary — never +text-decode them — and `read_file` layers frame extraction on top only when the +`[video]` dependencies are installed. +""" + + +def _get_backend_read_file_type(path: str) -> FileType: + """Classify a file for backend reads, forcing known video containers to binary. + + Backends decide binary-vs-text on `_get_file_type(...) != "text"`. Extensions + in `_VIDEO_EXTRA_EXTENSIONS` are absent from `_EXTENSION_TO_FILE_TYPE`, so + `_get_file_type` alone would treat them as text and corrupt the bytes (a raw + UTF-8 decode of a video, or line-slicing a base64 blob). Classify them as + `"video"` here so the binary read path runs on every backend. + + Args: + path: File path to classify. + + Returns: + `"video"` for `_VIDEO_EXTRA_EXTENSIONS`; otherwise the shared + `_get_file_type` classification. + """ + if PurePosixPath(path).suffix.lower() in _VIDEO_EXTRA_EXTENSIONS: + return "video" + return _get_file_type(path) + + def _to_legacy_file_data(file_data: FileData) -> dict[str, Any]: r"""Convert a `FileData` dict to the legacy (v1) storage format. diff --git a/libs/deepagents/deepagents/middleware/_video.py b/libs/deepagents/deepagents/middleware/_video.py index b29402d8391..5027684e995 100644 --- a/libs/deepagents/deepagents/middleware/_video.py +++ b/libs/deepagents/deepagents/middleware/_video.py @@ -17,7 +17,9 @@ """ import base64 +import importlib.util import io +import logging import math import time from typing import TYPE_CHECKING, Any, Final @@ -27,10 +29,33 @@ else: ContentBlock = dict # only used at runtime by the agent runtime +logger = logging.getLogger(__name__) + MISSING_VIDEO_HINT = "Reading video files requires the optional video dependencies. Install them with `uv add 'deepagents[video]'`." """User-facing message shown when a video read is attempted without the `[video]` extra installed.""" + +def video_dependencies_available() -> bool: + """Return whether the optional video dependencies appear to be installed. + + Uses `importlib.util.find_spec`, which checks that `av` and Pillow are + *discoverable* rather than performing a full import. A discoverable but + broken install (e.g. a compiled extension that fails to load) is reported as + available here and surfaces later, at actual extraction time, as a + `VideoExtractionError` carrying `MISSING_VIDEO_HINT`. + """ + try: + return importlib.util.find_spec("av") is not None and importlib.util.find_spec("PIL.Image") is not None + except (ImportError, ValueError): + # `find_spec` returns None (not raises) for a simply-absent top-level + # module, so a raised error means the deps are present but unimportable. + # Treat the extra as unavailable, but log it so a broken [video] install + # is debuggable rather than silently disabled. + logger.warning("Video dependency probe failed; treating the [video] extra as unavailable.", exc_info=True) + return False + + MAX_VIDEO_SAMPLED_FRAMES: Final = 64 """Upper bound on the number of frames emitted per video read.""" diff --git a/libs/deepagents/deepagents/middleware/filesystem.py b/libs/deepagents/deepagents/middleware/filesystem.py index 4059422c83f..17307c22a29 100644 --- a/libs/deepagents/deepagents/middleware/filesystem.py +++ b/libs/deepagents/deepagents/middleware/filesystem.py @@ -62,7 +62,9 @@ ) from deepagents.backends.sandbox import BaseSandbox from deepagents.backends.utils import ( + _VIDEO_EXTRA_EXTENSIONS, MAX_VIDEO_INPUT_BYTES, + FileType, _get_file_type, _glob_anchor, _paths_overlap, @@ -84,6 +86,7 @@ from deepagents.middleware._video import ( VideoExtractionError, extract_video_frames, + video_dependencies_available, ) _FS_WCMATCH_FLAGS = wcglob.BRACE | wcglob.GLOBSTAR @@ -235,6 +238,14 @@ def _video_window_header(path: str, offset_seconds: float, duration_seconds: flo return f"Reading [{offset_seconds:.3f}s, {end:.3f}s) of {path} at {rate} fps." +def _get_read_file_type(path: str, *, video_enabled: bool) -> FileType: + """Classify a file for `read_file`, gating optional video extensions.""" + file_type = _get_file_type(path) + if video_enabled and PurePosixPath(path).suffix.lower() in _VIDEO_EXTRA_EXTENSIONS: + return "video" + return file_type + + @dataclass class FilesystemPermission: """A single access rule for filesystem operations.""" @@ -528,6 +539,24 @@ class ReadFileSchema(BaseModel): file_path: str = Field(description="Absolute path to the file to read. Must be absolute, not relative.") + offset: int = Field( + default=DEFAULT_READ_OFFSET, + description="Line number to start reading from (0-indexed). Use for pagination of large files.", + ) + + limit: int = Field( + default=DEFAULT_READ_LIMIT, + description="Maximum number of lines to read. Use for pagination of large files.", + ) + + +class ReadVideoFileSchema(ReadFileSchema): + """Input schema for `read_file` when the optional video frame extraction is available. + + Identical to `ReadFileSchema`; only the `offset`/`limit` descriptions differ + to document their video semantics (interpreted as seconds for video reads). + """ + offset: int = Field( default=DEFAULT_READ_OFFSET, description="Line number to start reading from for text files (0-indexed). For videos, seconds into the source to start sampling.", @@ -607,12 +636,12 @@ class ExecuteSchema(BaseModel): This is useful for exploring the filesystem and finding the right file to read or edit. You should almost ALWAYS use this tool before using the read_file or edit_file tools.""" -READ_FILE_TOOL_DESCRIPTION = """Reads a file from the filesystem. +_READ_FILE_TOOL_DESCRIPTION_TEMPLATE = """Reads a file from the filesystem. Assume this tool is able to read all files. If the User provides a path to a file assume that path is valid. It is okay to read a file that does not exist; an error will be returned. Usage: -- For text files, by default it reads up to 100 lines starting from the beginning of the file +- {first_line} - **IMPORTANT for large files and codebase exploration**: Use pagination with offset and limit parameters to avoid context overflow - First scan: read_file(file_path="...", limit=100) to see file structure - Read more sections: read_file(file_path="...", offset=100, limit=200) for next 200 lines @@ -626,11 +655,32 @@ class ExecuteSchema(BaseModel): For multimodal reads (image, audio, video, PDF, etc.): - Use `read_file(file_path=...)` -- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only -- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s; sampled at a fixed rate). Use smaller windows when you need more temporal detail. +{multimodal_bullets} - If file details were compacted from history, call `read_file` again on the same path - You should ALWAYS make sure a file has been read before editing it.""" +"""Shared `read_file` description body for the text-only and video-aware variants. + +The two variants differ only in the `{first_line}` and `{multimodal_bullets}` +fields, kept in a single template so the common guidance cannot drift between +them. +""" + +_IMAGE_PDF_PAGINATION_BULLET = "- For images and PDFs, pagination via `offset`/`limit` is text-only - supply `file_path` only" +"""Multimodal bullet shared by both `read_file` descriptions (images/PDFs are not paginated).""" + +READ_FILE_TOOL_DESCRIPTION = _READ_FILE_TOOL_DESCRIPTION_TEMPLATE.format( + first_line="By default, it reads up to 100 lines starting from the beginning of the file", + multimodal_bullets=_IMAGE_PDF_PAGINATION_BULLET, +) + +READ_FILE_VIDEO_TOOL_DESCRIPTION = _READ_FILE_TOOL_DESCRIPTION_TEMPLATE.format( + first_line="For text files, by default it reads up to 100 lines starting from the beginning of the file", + multimodal_bullets=( + f"{_IMAGE_PDF_PAGINATION_BULLET}\n" + "- For videos, `offset`/`limit` are interpreted as seconds (default window 100 s; sampled at a fixed rate). Use smaller windows when you need more temporal detail." + ), +) EDIT_FILE_TOOL_DESCRIPTION = """Performs exact string replacements in files. @@ -1247,7 +1297,10 @@ async def async_ls( def _create_read_file_tool(self) -> BaseTool: # noqa: C901 """Create the read_file tool.""" - tool_description = self._custom_tool_descriptions.get("read_file") or READ_FILE_TOOL_DESCRIPTION + video_enabled = video_dependencies_available() + default_description = READ_FILE_VIDEO_TOOL_DESCRIPTION if video_enabled else READ_FILE_TOOL_DESCRIPTION + tool_description = self._custom_tool_descriptions.get("read_file") or default_description + args_schema = ReadVideoFileSchema if video_enabled else ReadFileSchema token_limit = self._tool_token_limit_before_evict def _truncate(content: str, file_path: str, *, line_limit: int | None = None) -> str: @@ -1305,7 +1358,7 @@ def _handle_read_result( # noqa: PLR0911 # one branch per distinct read-result status="error", ) - file_type = _get_file_type(validated_path) + file_type = _get_read_file_type(validated_path, video_enabled=video_enabled) encoding = read_result.file_data.get("encoding", "utf-8") content = read_result.file_data["content"] @@ -1324,7 +1377,7 @@ def _handle_read_result( # noqa: PLR0911 # one branch per distinct read-result # Video reads must be sliced into a sampled frame window before the # generic base64 branch runs; otherwise raw video bytes would reach # the model. - if file_type == "video": + if video_enabled and file_type == "video": return _handle_video_read( content, validated_path, @@ -1420,7 +1473,7 @@ async def async_read_file( func=sync_read_file, coroutine=async_read_file, infer_schema=False, - args_schema=ReadFileSchema, + args_schema=args_schema, ) def _create_write_file_tool(self) -> BaseTool: diff --git a/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py b/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py index 8e9af57e0aa..65215602b7d 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py +++ b/libs/deepagents/tests/unit_tests/backends/test_filesystem_backend.py @@ -1,3 +1,4 @@ +import base64 import logging import shutil import subprocess @@ -257,6 +258,23 @@ def test_filesystem_backend_read_non_utf8_file(tmp_path: Path): assert "chinese.txt" in result.error +def test_filesystem_backend_reads_mkv_as_binary(tmp_path: Path) -> None: + """Local `.mkv` reads must be routed as binary before UTF-8 decoding.""" + target = tmp_path / "clip.mkv" + raw = b"\x80\x81mkv bytes" + target.write_bytes(raw) + + be = FilesystemBackend(root_dir=str(tmp_path), virtual_mode=False) + result = be.read(str(target)) + + assert isinstance(result, ReadResult) + assert result.error is None + assert result.file_data == { + "content": base64.standard_b64encode(raw).decode("ascii"), + "encoding": "base64", + } + + def test_filesystem_backend_rejects_oversized_video_before_read(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Local video reads fail before loading oversized files into memory.""" target = tmp_path / "clip.mp4" @@ -270,6 +288,19 @@ def test_filesystem_backend_rejects_oversized_video_before_read(tmp_path: Path, assert result.error == "Video file exceeds maximum input size of 3 bytes" +def test_filesystem_backend_rejects_oversized_mkv_before_read(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Local `.mkv` reads use the video size guard before loading bytes.""" + target = tmp_path / "clip.mkv" + target.write_bytes(b"abcd") + monkeypatch.setattr(fs_module, "MAX_VIDEO_INPUT_BYTES", 3) + + be = FilesystemBackend(root_dir=str(tmp_path), virtual_mode=False) + result = be.read(str(target)) + + assert isinstance(result, ReadResult) + assert result.error == "Video file exceeds maximum input size of 3 bytes" + + def test_filesystem_backend_intercept_large_tool_result(tmp_path: Path): """Test that FilesystemBackend properly handles large tool result interception.""" root = tmp_path diff --git a/libs/deepagents/tests/unit_tests/backends/test_sandbox_backend.py b/libs/deepagents/tests/unit_tests/backends/test_sandbox_backend.py index 7a062722ed0..9267706b7c7 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_sandbox_backend.py +++ b/libs/deepagents/tests/unit_tests/backends/test_sandbox_backend.py @@ -30,6 +30,7 @@ _READ_COMMAND_TEMPLATE, _WRITE_CHECK_TEMPLATE, BaseSandbox, + _build_read_cmd, _check_preflight_result, _map_edit_error, _parse_grep_output, @@ -1143,6 +1144,13 @@ def test_read_script_genuine_binary_returns_base64(tmp_path: Path) -> None: assert result["encoding"] == "base64" +def test_build_read_cmd_classifies_mkv_as_video() -> None: + """`.mkv` reads must run the binary path, not the text path, in the sandbox.""" + assert "file_type = 'video'" in _build_read_cmd("/clips/a.mkv", 0, 100) + # Sanity: a plain text file still classifies as text. + assert "file_type = 'text'" in _build_read_cmd("/notes.txt", 0, 100) + + def test_read_script_mid_buffer_invalid_utf8_returns_base64(tmp_path: Path) -> None: """Corruption inside the prefix must still route to base64 (not swallowed).""" target = tmp_path / "midbad.dat" diff --git a/libs/deepagents/tests/unit_tests/backends/test_store_backend.py b/libs/deepagents/tests/unit_tests/backends/test_store_backend.py index 9d1cb69a229..5039905409f 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_store_backend.py +++ b/libs/deepagents/tests/unit_tests/backends/test_store_backend.py @@ -51,6 +51,27 @@ def test_store_backend_crud_and_search(): assert any(i["path"] == "/docs/readme.md" for i in g2) +def test_store_backend_reads_mkv_as_binary_without_slicing(): + """`.mkv` reads bypass text line-slicing so binary bytes are returned intact. + + Regression: `.mkv` is not in the shared multimodal map, so a text + classification here would run `slice_read_response` and raise a spurious + "line offset exceeds file length" error for any offset past the content. + """ + mem_store = InMemoryStore() + be = StoreBackend(store=mem_store, namespace=lambda _rt: ("filesystem",)) + be.write("/clip.mkv", "single line of bytes") + + # An offset past the 1-line content would raise a text-slice error if `.mkv` + # were misclassified as text; the binary path returns the file unsliced. + read_result = be.read("/clip.mkv", offset=5, limit=10) + + assert isinstance(read_result, ReadResult) + assert read_result.error is None + assert read_result.file_data is not None + assert read_result.file_data["content"] == "single line of bytes" + + def test_store_backend_write_overwrites_existing_file(): mem_store = InMemoryStore() be = StoreBackend(store=mem_store, namespace=lambda _ctx: ("filesystem",)) diff --git a/libs/deepagents/tests/unit_tests/backends/test_utils.py b/libs/deepagents/tests/unit_tests/backends/test_utils.py index 63fee062e45..9e7f9198419 100644 --- a/libs/deepagents/tests/unit_tests/backends/test_utils.py +++ b/libs/deepagents/tests/unit_tests/backends/test_utils.py @@ -9,6 +9,7 @@ from deepagents.backends.protocol import FileData, ReadResult from deepagents.backends.utils import ( _EXTENSION_TO_FILE_TYPE, + _get_backend_read_file_type, _get_file_type, _glob_search_files, perform_string_replacement, @@ -204,8 +205,20 @@ def test_get_file_type_returns_text_for_unknown_extensions() -> None: assert _get_file_type("/foo/bar") == "text" -def test_get_file_type_recognizes_mkv_as_video() -> None: - assert _get_file_type("/foo/bar.mkv") == "video" +def test_get_file_type_does_not_recognize_mkv() -> None: + """`.mkv` is intentionally absent from the shared multimodal map.""" + assert _get_file_type("/foo/bar.mkv") == "text" + + +def test_get_backend_read_file_type_forces_mkv_to_binary() -> None: + """Backends must read `.mkv` as binary even though it is not in the map.""" + assert _get_backend_read_file_type("/foo/bar.mkv") == "video" + + +def test_get_backend_read_file_type_matches_get_file_type_for_other_extensions() -> None: + """The backend classifier only adds `.mkv`; everything else is unchanged.""" + for path in ("/foo/bar.mp4", "/foo/bar.png", "/foo/bar.txt", "/foo/bar", "/foo/bar.pdf"): + assert _get_backend_read_file_type(path) == _get_file_type(path) def test_get_file_type_non_text_values_are_valid_content_block_types() -> None: diff --git a/libs/deepagents/tests/unit_tests/middleware/test_filesystem_video.py b/libs/deepagents/tests/unit_tests/middleware/test_filesystem_video.py index d54d63e99f5..28271df0f25 100644 --- a/libs/deepagents/tests/unit_tests/middleware/test_filesystem_video.py +++ b/libs/deepagents/tests/unit_tests/middleware/test_filesystem_video.py @@ -14,6 +14,12 @@ from deepagents.middleware.filesystem import FilesystemMiddleware, FilesystemState +@pytest.fixture(autouse=True) +def _enable_video_extra(monkeypatch: pytest.MonkeyPatch) -> None: + """Exercise frame-extraction routing independently of the local test env.""" + monkeypatch.setattr(filesystem_middleware, "video_dependencies_available", lambda: True) + + def test_read_file_video_routes_to_frame_extractor(monkeypatch: pytest.MonkeyPatch) -> None: """Video reads route through the frame extractor, not generic base64 media.""" raw_bytes = b"\x00\x01\x02 fake video bytes" @@ -192,6 +198,69 @@ def fake_extract(*_args: object, **_kwargs: object) -> list[dict[str, str]]: assert "av not installed" in result.content +def test_read_file_without_video_extra_keeps_generic_video_block(monkeypatch: pytest.MonkeyPatch) -> None: + """Missing optional deps preserve the previous generic video read behavior.""" + monkeypatch.setattr(filesystem_middleware, "video_dependencies_available", lambda: False) + monkeypatch.setattr( + filesystem_middleware, + "extract_video_frames", + lambda *_args, **_kwargs: pytest.fail("video extraction should be gated"), + ) + middleware = FilesystemMiddleware(backend=_video_backend(b"video bytes")) + state = FilesystemState(messages=[], files={}) + runtime = _build_runtime(state, "video-read-no-extra") + read_file_tool = next(t for t in middleware.tools if t.name == "read_file") + + result = read_file_tool.invoke({"file_path": "/c.mp4", "runtime": runtime}) + + assert isinstance(result, ToolMessage) + assert result.status == "success" + assert isinstance(result.content, list) + assert result.content[0]["type"] == "video" + assert result.content[0]["base64"] == base64.b64encode(b"video bytes").decode("ascii") + + +def test_read_file_without_video_extra_keeps_mkv_generic(monkeypatch: pytest.MonkeyPatch) -> None: + """The new `.mkv` classification only applies with the video extra.""" + monkeypatch.setattr(filesystem_middleware, "video_dependencies_available", lambda: False) + middleware = FilesystemMiddleware(backend=_video_backend(b"mkv bytes")) + state = FilesystemState(messages=[], files={}) + runtime = _build_runtime(state, "mkv-read-no-extra") + read_file_tool = next(t for t in middleware.tools if t.name == "read_file") + + result = read_file_tool.invoke({"file_path": "/c.mkv", "runtime": runtime}) + + assert isinstance(result, ToolMessage) + assert result.status == "success" + assert isinstance(result.content, list) + assert result.content[0]["type"] == "file" + assert result.content[0]["base64"] == base64.b64encode(b"mkv bytes").decode("ascii") + + +def test_read_file_without_video_extra_uses_text_schema_and_description(monkeypatch: pytest.MonkeyPatch) -> None: + """Video-specific read guidance is hidden when frame extraction is unavailable.""" + monkeypatch.setattr(filesystem_middleware, "video_dependencies_available", lambda: False) + middleware = FilesystemMiddleware(backend=StateBackend()) + read_file_tool = next(t for t in middleware.tools if t.name == "read_file") + + assert "For videos" not in read_file_tool.description + schema = read_file_tool.args_schema.model_json_schema() + assert "For videos" not in schema["properties"]["offset"]["description"] + assert "For videos" not in schema["properties"]["limit"]["description"] + + +def test_read_file_with_video_extra_uses_video_schema_and_description() -> None: + """Video-specific read guidance is exposed when frame extraction is available.""" + # The autouse `_enable_video_extra` fixture forces the extra on for this test. + middleware = FilesystemMiddleware(backend=StateBackend()) + read_file_tool = next(t for t in middleware.tools if t.name == "read_file") + + assert "For videos" in read_file_tool.description + schema = read_file_tool.args_schema.model_json_schema() + assert "For videos" in schema["properties"]["offset"]["description"] + assert "For videos" in schema["properties"]["limit"]["description"] + + def test_read_file_video_unexpected_value_error_is_not_swallowed(monkeypatch: pytest.MonkeyPatch) -> None: """Unexpected extractor bugs should remain visible to operators.""" diff --git a/libs/deepagents/tests/unit_tests/middleware/test_video_deps.py b/libs/deepagents/tests/unit_tests/middleware/test_video_deps.py new file mode 100644 index 00000000000..782a0406f43 --- /dev/null +++ b/libs/deepagents/tests/unit_tests/middleware/test_video_deps.py @@ -0,0 +1,41 @@ +"""Unit tests for `video_dependencies_available`, the optional-extra gate. + +These tests monkeypatch `importlib.util.find_spec` so they run without the +`[video]` extra actually being installed — unlike `test_video.py`, which +`importorskip`s the real PyAV decoder. +""" + +import importlib.util + +import pytest + +from deepagents.middleware._video import video_dependencies_available + + +def test_video_dependencies_available_true_when_both_present(monkeypatch: pytest.MonkeyPatch) -> None: + """Both specs discoverable -> the extra is reported available.""" + monkeypatch.setattr(importlib.util, "find_spec", lambda _name: object()) + assert video_dependencies_available() is True + + +def test_video_dependencies_available_false_when_av_missing(monkeypatch: pytest.MonkeyPatch) -> None: + """A missing `av` short-circuits to unavailable (exercises the `and`).""" + monkeypatch.setattr(importlib.util, "find_spec", lambda name: None if name == "av" else object()) + assert video_dependencies_available() is False + + +def test_video_dependencies_available_false_when_pillow_missing(monkeypatch: pytest.MonkeyPatch) -> None: + """A missing `PIL.Image` reports unavailable even when `av` is present.""" + monkeypatch.setattr(importlib.util, "find_spec", lambda name: None if name == "PIL.Image" else object()) + assert video_dependencies_available() is False + + +def test_video_dependencies_available_false_on_probe_error(monkeypatch: pytest.MonkeyPatch) -> None: + """A raised probe error is caught and treated as unavailable (except branch).""" + + def _raise(_name: str) -> object: + msg = "corrupt spec" + raise ValueError(msg) + + monkeypatch.setattr(importlib.util, "find_spec", _raise) + assert video_dependencies_available() is False From c452870e7ce2c5760405341bfe37477b68124596 Mon Sep 17 00:00:00 2001 From: Mason Daugherty Date: Thu, 2 Jul 2026 01:44:15 -0400 Subject: [PATCH 32/36] perf --- libs/deepagents/deepagents/middleware/_video.py | 2 ++ libs/deepagents/tests/unit_tests/conftest.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/libs/deepagents/deepagents/middleware/_video.py b/libs/deepagents/deepagents/middleware/_video.py index 5027684e995..960d2fc96bd 100644 --- a/libs/deepagents/deepagents/middleware/_video.py +++ b/libs/deepagents/deepagents/middleware/_video.py @@ -22,6 +22,7 @@ import logging import math import time +from functools import lru_cache from typing import TYPE_CHECKING, Any, Final if TYPE_CHECKING: @@ -36,6 +37,7 @@ """User-facing message shown when a video read is attempted without the `[video]` extra installed.""" +@lru_cache(maxsize=1) def video_dependencies_available() -> bool: """Return whether the optional video dependencies appear to be installed. diff --git a/libs/deepagents/tests/unit_tests/conftest.py b/libs/deepagents/tests/unit_tests/conftest.py index 498ac3124f8..35949dc5ad7 100644 --- a/libs/deepagents/tests/unit_tests/conftest.py +++ b/libs/deepagents/tests/unit_tests/conftest.py @@ -62,6 +62,21 @@ def _reset_deprecation_dedupe() -> None: reset_deprecation_dedupe(*_DEDUPED_TARGETS) +@pytest.fixture(autouse=True) +def _reset_video_dep_cache() -> None: + """Clear the `lru_cache` on `video_dependencies_available` before each test. + + The function caches its result for the process lifetime so repeated + `FilesystemMiddleware` construction stays cheap. Tests that monkeypatch + `importlib.util.find_spec` (or rely on the real environment) need the cache + cleared so they observe their own probe result rather than a stale value + from an earlier test. + """ + from deepagents.middleware._video import video_dependencies_available # noqa: PLC0415 + + video_dependencies_available.cache_clear() + + @pytest.fixture(autouse=True, scope="session") def _bootstrap_profile_registries() -> None: """Force the lazy profile bootstrap before any test snapshots the registries. From 1a58061e49277d249d420f607702b08d687ee22b Mon Sep 17 00:00:00 2001 From: Deep Agent Date: Wed, 1 Jul 2026 23:11:00 -0700 Subject: [PATCH 33/36] fix(talon): collapse media extras --- examples/talon/Dockerfile | 2 +- examples/talon/README.md | 4 +- libs/code/pyproject.toml | 2 +- libs/code/uv.lock | 76 +++++++++++++-------------- libs/talon/deepagents_talon/speech.py | 2 +- libs/talon/pyproject.toml | 6 +-- libs/talon/uv.lock | 57 ++++++++++---------- 7 files changed, 75 insertions(+), 74 deletions(-) diff --git a/examples/talon/Dockerfile b/examples/talon/Dockerfile index 1f4a319809d..d78a82c43e1 100644 --- a/examples/talon/Dockerfile +++ b/examples/talon/Dockerfile @@ -57,7 +57,7 @@ RUN mkdir -p /opt/whatsapp-bridge && \ cd /opt/whatsapp-bridge && \ npm install --omit=dev -RUN cd libs/talon && uv sync --frozen --extra speech --extra video +RUN cd libs/talon && uv sync --frozen --extra media WORKDIR /workspace diff --git a/examples/talon/README.md b/examples/talon/README.md index 793677b21ac..611fc6a3d9f 100644 --- a/examples/talon/README.md +++ b/examples/talon/README.md @@ -25,7 +25,7 @@ Uncomment the Telegram env vars in `.env` and set `DEEPAGENTS_TALON_TELEGRAM_BOT ## Voice Transcription -Voice transcription is enabled by default in `.env.example`. The Docker example installs `ffmpeg` plus the Talon `speech` extra, so inbound voice notes are transcribed locally with NVIDIA Parakeet through Transformers before reaching the agent. The first voice message can be slow because the ASR model is downloaded lazily. Set `DEEPAGENTS_TALON_VOICE_TRANSCRIPTION_DEVICE=cuda` when running on a GPU-enabled host. +Voice transcription is enabled by default in `.env.example`. The Docker example installs `ffmpeg` plus the Talon `media` extra, so inbound voice notes are transcribed locally with NVIDIA Parakeet through Transformers before reaching the agent. The first voice message can be slow because the ASR model is downloaded lazily. Set `DEEPAGENTS_TALON_VOICE_TRANSCRIPTION_DEVICE=cuda` when running on a GPU-enabled host. Cron records, downloaded inbound media, and channel session state persist under `~/talon-workspace/.deepagents/`. The agent's default working directory is `/workspace`, so files it creates are written into `~/talon-workspace/` on the host. @@ -43,7 +43,7 @@ cd ../../libs/talon/deepagents_talon/channels/whatsapp_bridge npm install cd ../../../.. -uv sync --directory libs/talon --extra speech +uv sync --directory libs/talon --extra media cp examples/talon/AGENTS.md ~/.deepagents/talon-local/agent/AGENTS.md export DEEPAGENTS_TALON_WORKSPACE=~/talon-workspace uv run --directory libs/talon deepagents-talon --whatsapp diff --git a/libs/code/pyproject.toml b/libs/code/pyproject.toml index 70ae02894af..9daa0a04acc 100644 --- a/libs/code/pyproject.toml +++ b/libs/code/pyproject.toml @@ -123,7 +123,7 @@ all-sandboxes = [ ] # Standalone integrations -video = ["deepagents[video]==0.7.0a3"] +media = ["deepagents[video]==0.7.0a3"] # (quickjs kept for backwards-compatible install commands) quickjs = [] diff --git a/libs/code/uv.lock b/libs/code/uv.lock index dd4decabb06..1b8af5e9511 100644 --- a/libs/code/uv.lock +++ b/libs/code/uv.lock @@ -1224,6 +1224,9 @@ ibm = [ litellm = [ { name = "langchain-litellm" }, ] +media = [ + { name = "deepagents", extra = ["video"] }, +] mistralai = [ { name = "langchain-mistralai" }, ] @@ -1258,9 +1261,6 @@ vercel = [ vertex = [ { name = "langchain-google-vertexai" }, ] -video = [ - { name = "deepagents", extra = ["video"] }, -] xai = [ { name = "langchain-xai" }, ] @@ -1291,7 +1291,7 @@ requires-dist = [ { name = "aiohttp", marker = "extra == 'nvidia'", specifier = ">=3.14.1,<3.15.0" }, { name = "aiosqlite", specifier = ">=0.22.1,<1.0.0" }, { name = "deepagents", editable = "../deepagents" }, - { name = "deepagents", extras = ["video"], marker = "extra == 'video'", editable = "../deepagents" }, + { name = "deepagents", extras = ["video"], marker = "extra == 'media'", editable = "../deepagents" }, { name = "deepagents-acp", specifier = ">=0.0.8,<1.0.0" }, { name = "deepagents-code", extras = ["agentcore", "daytona", "modal", "runloop", "vercel"], marker = "extra == 'all-sandboxes'" }, { name = "deepagents-code", extras = ["anthropic", "baseten", "bedrock", "cohere", "deepseek", "fireworks", "google-genai", "groq", "huggingface", "ibm", "litellm", "mistralai", "nvidia", "ollama", "openai", "openrouter", "perplexity", "together", "vertex", "xai"], marker = "extra == 'all-providers'" }, @@ -1350,7 +1350,7 @@ requires-dist = [ { name = "tomli-w", specifier = ">=1.2.0,<2.0.0" }, { name = "uuid-utils", specifier = ">=0.16.2,<1.0.0" }, ] -provides-extras = ["anthropic", "baseten", "bedrock", "cohere", "deepseek", "fireworks", "google-genai", "groq", "huggingface", "ibm", "litellm", "mistralai", "nvidia", "ollama", "openai", "openrouter", "perplexity", "together", "vertex", "xai", "all-providers", "agentcore", "daytona", "modal", "runloop", "vercel", "all-sandboxes", "quickjs", "video"] +provides-extras = ["anthropic", "baseten", "bedrock", "cohere", "deepseek", "fireworks", "google-genai", "groq", "huggingface", "ibm", "litellm", "mistralai", "nvidia", "ollama", "openai", "openrouter", "perplexity", "together", "vertex", "xai", "all-providers", "agentcore", "daytona", "modal", "runloop", "vercel", "all-sandboxes", "media", "quickjs"] [package.metadata.requires-dev] test = [ @@ -2298,9 +2298,9 @@ name = "ibm-cos-sdk" version = "2.14.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ibm-cos-sdk-core" }, - { name = "ibm-cos-sdk-s3transfer" }, - { name = "jmespath", version = "1.0.1", source = { registry = "https://pypi.org/simple" } }, + { name = "ibm-cos-sdk-core", marker = "python_full_version < '3.15'" }, + { name = "ibm-cos-sdk-s3transfer", marker = "python_full_version < '3.15'" }, + { name = "jmespath", version = "1.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/98/b8/b99f17ece72d4bccd7e75539b9a294d0f73ace5c6c475d8f2631afd6f65b/ibm_cos_sdk-2.14.3.tar.gz", hash = "sha256:643b6f2aa1683adad7f432df23407d11ae5adb9d9ad01214115bee77dc64364a", size = 58831, upload-time = "2025-08-01T06:35:51.722Z" } @@ -2309,10 +2309,10 @@ name = "ibm-cos-sdk-core" version = "2.14.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "jmespath", version = "1.0.1", source = { registry = "https://pypi.org/simple" } }, - { name = "python-dateutil" }, - { name = "requests" }, - { name = "urllib3" }, + { name = "jmespath", version = "1.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.15'" }, + { name = "python-dateutil", marker = "python_full_version < '3.15'" }, + { name = "requests", marker = "python_full_version < '3.15'" }, + { name = "urllib3", marker = "python_full_version < '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7e/45/80c23aa1e13175a9deefe43cbf8e853a3d3bfc8dfa8b6d6fe83e5785fe21/ibm_cos_sdk_core-2.14.3.tar.gz", hash = "sha256:85dee7790c92e8db69bf39dae4c02cac211e3c1d81bb86e64fa2d1e929674623", size = 1103637, upload-time = "2025-08-01T06:35:41.645Z" } @@ -2321,7 +2321,7 @@ name = "ibm-cos-sdk-s3transfer" version = "2.14.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ibm-cos-sdk-core" }, + { name = "ibm-cos-sdk-core", marker = "python_full_version < '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f3/ff/c9baf0997266d398ae08347951a2970e5e96ed6232ed0252f649f2b9a7eb/ibm_cos_sdk_s3transfer-2.14.3.tar.gz", hash = "sha256:2251ebfc4a46144401e431f4a5d9f04c262a0d6f95c88a8e71071da056e55f72", size = 139594, upload-time = "2025-08-01T06:35:46.403Z" } @@ -2330,17 +2330,17 @@ name = "ibm-watsonx-ai" version = "1.5.14" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cachetools" }, - { name = "certifi" }, - { name = "httpx" }, - { name = "ibm-cos-sdk" }, - { name = "lomond" }, - { name = "packaging" }, - { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version != '3.14.*'" }, + { name = "cachetools", marker = "python_full_version < '3.15'" }, + { name = "certifi", marker = "python_full_version < '3.15'" }, + { name = "httpx", marker = "python_full_version < '3.15'" }, + { name = "ibm-cos-sdk", marker = "python_full_version < '3.15'" }, + { name = "lomond", marker = "python_full_version < '3.15'" }, + { name = "packaging", marker = "python_full_version < '3.15'" }, + { name = "pandas", version = "2.3.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.14'" }, { name = "pandas", version = "3.0.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.14.*'" }, - { name = "requests" }, - { name = "tabulate" }, - { name = "urllib3" }, + { name = "requests", marker = "python_full_version < '3.15'" }, + { name = "tabulate", marker = "python_full_version < '3.15'" }, + { name = "urllib3", marker = "python_full_version < '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/29/a3/c756b534696ab2f3f29882fdb7ca7198b7a5c94e10c0a3a327853d6d6b79/ibm_watsonx_ai-1.5.14.tar.gz", hash = "sha256:a756488bd57e87c0fc51be42dcba871143cfe0ac1e805c497c5047e1e4f13e9d", size = 735804, upload-time = "2026-06-22T12:32:43.85Z" } wheels = [ @@ -3438,7 +3438,7 @@ name = "lomond" version = "0.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "six" }, + { name = "six", marker = "python_full_version < '3.15'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c0/9e/ef7813c910d4a893f2bc763ce9246269f55cc68db21dc1327e376d6a2d02/lomond-0.3.3.tar.gz", hash = "sha256:427936596b144b4ec387ead99aac1560b77c8a78107d3d49415d3abbe79acbd3", size = 28789, upload-time = "2018-09-21T15:17:43.297Z" } wheels = [ @@ -4387,11 +4387,11 @@ resolution-markers = [ "python_full_version < '3.12'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12' or python_full_version >= '3.14'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "numpy", version = "2.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12' and python_full_version < '3.14'" }, - { name = "python-dateutil" }, - { name = "pytz" }, - { name = "tzdata" }, + { name = "python-dateutil", marker = "python_full_version < '3.14'" }, + { name = "pytz", marker = "python_full_version < '3.14'" }, + { name = "tzdata", marker = "python_full_version < '3.14'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } wheels = [ @@ -4447,9 +4447,9 @@ resolution-markers = [ "python_full_version == '3.14.*' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.5.0", source = { registry = "https://pypi.org/simple" } }, - { name = "python-dateutil" }, - { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, + { name = "numpy", version = "2.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.14.*'" }, + { name = "python-dateutil", marker = "python_full_version == '3.14.*'" }, + { name = "tzdata", marker = "(python_full_version == '3.14.*' and sys_platform == 'emscripten') or (python_full_version == '3.14.*' and sys_platform == 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f8/87/4341c6252d1c47b08768c3d25ac487362bf403f0313ddae4a2a26c9b1b4c/pandas-3.0.3.tar.gz", hash = "sha256:696a4a00a2a2a35d4e5deb3fc946641b96c944f02230e4f76137fe35d806c4fc", size = 4651414, upload-time = "2026-05-11T18:54:29.21Z" } wheels = [ @@ -5732,8 +5732,8 @@ name = "secretstorage" version = "3.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography" }, - { name = "jeepney" }, + { name = "cryptography", marker = "(python_full_version != '3.14.*' and sys_platform == 'emscripten') or (python_full_version != '3.14.*' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "jeepney", marker = "(python_full_version != '3.14.*' and sys_platform == 'emscripten') or (python_full_version != '3.14.*' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } wheels = [ @@ -6525,11 +6525,11 @@ name = "vercel-workers" version = "0.0.25" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio" }, - { name = "httpx" }, - { name = "pydantic" }, - { name = "python-dotenv" }, - { name = "vercel" }, + { name = "anyio", marker = "python_full_version >= '3.12'" }, + { name = "httpx", marker = "python_full_version >= '3.12'" }, + { name = "pydantic", marker = "python_full_version >= '3.12'" }, + { name = "python-dotenv", marker = "python_full_version >= '3.12'" }, + { name = "vercel", marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/30/df/04d37021ad7ca53b7599c313e411d91623c7a005c741f491d1eefb7a9f0c/vercel_workers-0.0.25.tar.gz", hash = "sha256:212ded01400b524be51d251df49f801caf115ad7d48cca7eb168cbeceda3def3", size = 64149, upload-time = "2026-06-20T19:26:27.177Z" } wheels = [ diff --git a/libs/talon/deepagents_talon/speech.py b/libs/talon/deepagents_talon/speech.py index 5ba25d5297b..9d2bc5190b3 100644 --- a/libs/talon/deepagents_talon/speech.py +++ b/libs/talon/deepagents_talon/speech.py @@ -247,7 +247,7 @@ def _load_local_pipeline(model: str, device: str) -> _LocalSpeechPipeline: module = importlib.import_module("transformers") except ImportError as exc: msg = ( - "Local voice transcription dependencies are missing. Install the `speech` " + "Local voice transcription dependencies are missing. Install the `media` " "extra and ensure ffmpeg is on PATH." ) raise ImportError(msg) from exc diff --git a/libs/talon/pyproject.toml b/libs/talon/pyproject.toml index 8f4742cb21b..59a23964c75 100644 --- a/libs/talon/pyproject.toml +++ b/libs/talon/pyproject.toml @@ -30,7 +30,8 @@ dependencies = [ ] [project.optional-dependencies] -speech = [ +media = [ + "deepagents-code[media]>=0.1.29,<1.0.0", "librosa>=0.11.0,<1.0.0", "numpy>=2.4.6,<3.0.0", "phonemizer>=3.3.0,<4.0.0", @@ -38,9 +39,6 @@ speech = [ "torchaudio>=2.11.0,<2.12.0", "transformers>=5.12.1,<6.0.0", ] -video = [ - "deepagents[video]>=0.7.0a3", -] [project.scripts] deepagents-talon = "deepagents_talon.__main__:main" diff --git a/libs/talon/uv.lock b/libs/talon/uv.lock index 2fea95c3c70..88dd2cd72c7 100644 --- a/libs/talon/uv.lock +++ b/libs/talon/uv.lock @@ -708,34 +708,34 @@ wheels = [ [package.optional-dependencies] cudart = [ - { name = "nvidia-cuda-runtime" }, + { name = "nvidia-cuda-runtime", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, ] cufft = [ - { name = "nvidia-cufft" }, + { name = "nvidia-cufft", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, ] cufile = [ - { name = "nvidia-cufile" }, + { name = "nvidia-cufile", marker = "sys_platform == 'linux'" }, ] cupti = [ - { name = "nvidia-cuda-cupti" }, + { name = "nvidia-cuda-cupti", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, ] curand = [ - { name = "nvidia-curand" }, + { name = "nvidia-curand", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, ] cusolver = [ - { name = "nvidia-cusolver" }, + { name = "nvidia-cusolver", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, ] cusparse = [ - { name = "nvidia-cusparse" }, + { name = "nvidia-cusparse", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc" }, + { name = "nvidia-cuda-nvrtc", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, ] nvtx = [ - { name = "nvidia-nvtx" }, + { name = "nvidia-nvtx", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, ] [[package]] @@ -855,12 +855,17 @@ dependencies = [ { name = "uuid-utils" }, ] +[package.optional-dependencies] +media = [ + { name = "deepagents", extra = ["video"] }, +] + [package.metadata] requires-dist = [ { name = "aiohttp", marker = "extra == 'nvidia'", specifier = ">=3.14.1,<3.15.0" }, { name = "aiosqlite", specifier = ">=0.22.1,<1.0.0" }, { name = "deepagents", editable = "../deepagents" }, - { name = "deepagents", extras = ["video"], marker = "extra == 'video'", editable = "../deepagents" }, + { name = "deepagents", extras = ["video"], marker = "extra == 'media'", editable = "../deepagents" }, { name = "deepagents-acp", specifier = ">=0.0.8,<1.0.0" }, { name = "deepagents-code", extras = ["agentcore", "daytona", "modal", "runloop", "vercel"], marker = "extra == 'all-sandboxes'" }, { name = "deepagents-code", extras = ["anthropic", "baseten", "bedrock", "cohere", "deepseek", "fireworks", "google-genai", "groq", "huggingface", "ibm", "litellm", "mistralai", "nvidia", "ollama", "openai", "openrouter", "perplexity", "together", "vertex", "xai"], marker = "extra == 'all-providers'" }, @@ -919,7 +924,7 @@ requires-dist = [ { name = "tomli-w", specifier = ">=1.2.0,<2.0.0" }, { name = "uuid-utils", specifier = ">=0.16.2,<1.0.0" }, ] -provides-extras = ["anthropic", "baseten", "bedrock", "cohere", "deepseek", "fireworks", "google-genai", "groq", "huggingface", "ibm", "litellm", "mistralai", "nvidia", "ollama", "openai", "openrouter", "perplexity", "together", "vertex", "xai", "all-providers", "agentcore", "daytona", "modal", "runloop", "vercel", "all-sandboxes", "quickjs", "video"] +provides-extras = ["anthropic", "baseten", "bedrock", "cohere", "deepseek", "fireworks", "google-genai", "groq", "huggingface", "ibm", "litellm", "mistralai", "nvidia", "ollama", "openai", "openrouter", "perplexity", "together", "vertex", "xai", "all-providers", "agentcore", "daytona", "modal", "runloop", "vercel", "all-sandboxes", "media", "quickjs"] [package.metadata.requires-dev] test = [ @@ -954,7 +959,8 @@ dependencies = [ ] [package.optional-dependencies] -speech = [ +media = [ + { name = "deepagents-code", extra = ["media"] }, { name = "librosa" }, { name = "numpy" }, { name = "phonemizer" }, @@ -962,9 +968,6 @@ speech = [ { name = "torchaudio" }, { name = "transformers" }, ] -video = [ - { name = "deepagents", extra = ["video"] }, -] [package.dev-dependencies] test = [ @@ -981,18 +984,18 @@ test = [ [package.metadata] requires-dist = [ { name = "deepagents", editable = "../deepagents" }, - { name = "deepagents", extras = ["video"], marker = "extra == 'video'", editable = "../deepagents" }, { name = "deepagents-code", editable = "../code" }, + { name = "deepagents-code", extras = ["media"], marker = "extra == 'media'", editable = "../code" }, { name = "fleet-deepagents-export", specifier = ">=0.0.1,<1.0.0" }, { name = "langchain-mcp-adapters", specifier = ">=0.3.0,<1.0.0" }, - { name = "librosa", marker = "extra == 'speech'", specifier = ">=0.11.0,<1.0.0" }, - { name = "numpy", marker = "extra == 'speech'", specifier = ">=2.4.6,<3.0.0" }, - { name = "phonemizer", marker = "extra == 'speech'", specifier = ">=3.3.0,<4.0.0" }, - { name = "torch", marker = "extra == 'speech'", specifier = ">=2.12.1,<2.13.0" }, - { name = "torchaudio", marker = "extra == 'speech'", specifier = ">=2.11.0,<2.12.0" }, - { name = "transformers", marker = "extra == 'speech'", specifier = ">=5.12.1,<6.0.0" }, + { name = "librosa", marker = "extra == 'media'", specifier = ">=0.11.0,<1.0.0" }, + { name = "numpy", marker = "extra == 'media'", specifier = ">=2.4.6,<3.0.0" }, + { name = "phonemizer", marker = "extra == 'media'", specifier = ">=3.3.0,<4.0.0" }, + { name = "torch", marker = "extra == 'media'", specifier = ">=2.12.1,<2.13.0" }, + { name = "torchaudio", marker = "extra == 'media'", specifier = ">=2.11.0,<2.12.0" }, + { name = "transformers", marker = "extra == 'media'", specifier = ">=5.12.1,<6.0.0" }, ] -provides-extras = ["speech", "video"] +provides-extras = ["media"] [package.metadata.requires-dev] test = [ @@ -3936,8 +3939,8 @@ name = "standard-aifc" version = "3.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "audioop-lts" }, - { name = "standard-chunk" }, + { name = "audioop-lts", marker = "python_full_version >= '3.13'" }, + { name = "standard-chunk", marker = "python_full_version >= '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c4/53/6050dc3dde1671eb3db592c13b55a8005e5040131f7509cef0215212cb84/standard_aifc-3.13.0.tar.gz", hash = "sha256:64e249c7cb4b3daf2fdba4e95721f811bde8bdfc43ad9f936589b7bb2fae2e43", size = 15240, upload-time = "2024-10-30T16:01:31.772Z" } wheels = [ @@ -3958,7 +3961,7 @@ name = "standard-sunau" version = "3.13.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "audioop-lts" }, + { name = "audioop-lts", marker = "python_full_version >= '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/66/e3/ce8d38cb2d70e05ffeddc28bb09bad77cfef979eb0a299c9117f7ed4e6a9/standard_sunau-3.13.0.tar.gz", hash = "sha256:b319a1ac95a09a2378a8442f403c66f4fd4b36616d6df6ae82b8e536ee790908", size = 9368, upload-time = "2024-10-30T16:01:41.626Z" } wheels = [ From e0638b578e692234fa66fed650e2d1bbf25f9357 Mon Sep 17 00:00:00 2001 From: Deep Agent Date: Wed, 1 Jul 2026 23:14:27 -0700 Subject: [PATCH 34/36] fix(talon): sync media extra dependencies --- libs/code/pyproject.toml | 2 +- libs/code/uv.lock | 12 ++++-------- libs/talon/pyproject.toml | 2 +- libs/talon/uv.lock | 12 ++++-------- 4 files changed, 10 insertions(+), 18 deletions(-) diff --git a/libs/code/pyproject.toml b/libs/code/pyproject.toml index 9daa0a04acc..4f245635cbc 100644 --- a/libs/code/pyproject.toml +++ b/libs/code/pyproject.toml @@ -123,7 +123,7 @@ all-sandboxes = [ ] # Standalone integrations -media = ["deepagents[video]==0.7.0a3"] +media = ["av>=17.0.0,<18.0.0", "pillow>=12.2.0,<13.0.0"] # (quickjs kept for backwards-compatible install commands) quickjs = [] diff --git a/libs/code/uv.lock b/libs/code/uv.lock index 1b8af5e9511..70f1df0103e 100644 --- a/libs/code/uv.lock +++ b/libs/code/uv.lock @@ -1059,12 +1059,6 @@ dependencies = [ { name = "wcmatch" }, ] -[package.optional-dependencies] -video = [ - { name = "av" }, - { name = "pillow" }, -] - [package.metadata] requires-dist = [ { name = "av", marker = "extra == 'video'", specifier = ">=17.0.0,<18.0.0" }, @@ -1225,7 +1219,8 @@ litellm = [ { name = "langchain-litellm" }, ] media = [ - { name = "deepagents", extra = ["video"] }, + { name = "av" }, + { name = "pillow" }, ] mistralai = [ { name = "langchain-mistralai" }, @@ -1290,8 +1285,8 @@ test = [ requires-dist = [ { name = "aiohttp", marker = "extra == 'nvidia'", specifier = ">=3.14.1,<3.15.0" }, { name = "aiosqlite", specifier = ">=0.22.1,<1.0.0" }, + { name = "av", marker = "extra == 'media'", specifier = ">=17.0.0,<18.0.0" }, { name = "deepagents", editable = "../deepagents" }, - { name = "deepagents", extras = ["video"], marker = "extra == 'media'", editable = "../deepagents" }, { name = "deepagents-acp", specifier = ">=0.0.8,<1.0.0" }, { name = "deepagents-code", extras = ["agentcore", "daytona", "modal", "runloop", "vercel"], marker = "extra == 'all-sandboxes'" }, { name = "deepagents-code", extras = ["anthropic", "baseten", "bedrock", "cohere", "deepseek", "fireworks", "google-genai", "groq", "huggingface", "ibm", "litellm", "mistralai", "nvidia", "ollama", "openai", "openrouter", "perplexity", "together", "vertex", "xai"], marker = "extra == 'all-providers'" }, @@ -1336,6 +1331,7 @@ requires-dist = [ { name = "mcp", specifier = ">=1.9.2,<2.0.0" }, { name = "packaging", specifier = ">=26.2" }, { name = "pillow", specifier = ">=12.2.0,<13.0.0" }, + { name = "pillow", marker = "extra == 'media'", specifier = ">=12.2.0,<13.0.0" }, { name = "prompt-toolkit", specifier = ">=3.0.52,<4.0.0" }, { name = "pydantic", specifier = ">=2.12.5,<2.14.0a0" }, { name = "pyperclip", specifier = ">=1.11.0,<2.0.0" }, diff --git a/libs/talon/pyproject.toml b/libs/talon/pyproject.toml index 59a23964c75..1c86bbb61cf 100644 --- a/libs/talon/pyproject.toml +++ b/libs/talon/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ [project.optional-dependencies] media = [ - "deepagents-code[media]>=0.1.29,<1.0.0", + "deepagents-code[media]>=0.1.30,<1.0.0", "librosa>=0.11.0,<1.0.0", "numpy>=2.4.6,<3.0.0", "phonemizer>=3.3.0,<4.0.0", diff --git a/libs/talon/uv.lock b/libs/talon/uv.lock index 88dd2cd72c7..2bdeb8e5a81 100644 --- a/libs/talon/uv.lock +++ b/libs/talon/uv.lock @@ -760,12 +760,6 @@ dependencies = [ { name = "wcmatch" }, ] -[package.optional-dependencies] -video = [ - { name = "av" }, - { name = "pillow" }, -] - [package.metadata] requires-dist = [ { name = "av", marker = "extra == 'video'", specifier = ">=17.0.0,<18.0.0" }, @@ -857,15 +851,16 @@ dependencies = [ [package.optional-dependencies] media = [ - { name = "deepagents", extra = ["video"] }, + { name = "av" }, + { name = "pillow" }, ] [package.metadata] requires-dist = [ { name = "aiohttp", marker = "extra == 'nvidia'", specifier = ">=3.14.1,<3.15.0" }, { name = "aiosqlite", specifier = ">=0.22.1,<1.0.0" }, + { name = "av", marker = "extra == 'media'", specifier = ">=17.0.0,<18.0.0" }, { name = "deepagents", editable = "../deepagents" }, - { name = "deepagents", extras = ["video"], marker = "extra == 'media'", editable = "../deepagents" }, { name = "deepagents-acp", specifier = ">=0.0.8,<1.0.0" }, { name = "deepagents-code", extras = ["agentcore", "daytona", "modal", "runloop", "vercel"], marker = "extra == 'all-sandboxes'" }, { name = "deepagents-code", extras = ["anthropic", "baseten", "bedrock", "cohere", "deepseek", "fireworks", "google-genai", "groq", "huggingface", "ibm", "litellm", "mistralai", "nvidia", "ollama", "openai", "openrouter", "perplexity", "together", "vertex", "xai"], marker = "extra == 'all-providers'" }, @@ -910,6 +905,7 @@ requires-dist = [ { name = "mcp", specifier = ">=1.9.2,<2.0.0" }, { name = "packaging", specifier = ">=26.2" }, { name = "pillow", specifier = ">=12.2.0,<13.0.0" }, + { name = "pillow", marker = "extra == 'media'", specifier = ">=12.2.0,<13.0.0" }, { name = "prompt-toolkit", specifier = ">=3.0.52,<4.0.0" }, { name = "pydantic", specifier = ">=2.12.5,<2.14.0a0" }, { name = "pyperclip", specifier = ">=1.11.0,<2.0.0" }, From f22321386e200b8eb8bcc4a25be95acea2f3397b Mon Sep 17 00:00:00 2001 From: Deep Agent Date: Wed, 1 Jul 2026 23:16:55 -0700 Subject: [PATCH 35/36] chore(evals): update lockfile for media extra --- libs/evals/uv.lock | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libs/evals/uv.lock b/libs/evals/uv.lock index ac9a7d8be1b..69cd9393bf8 100644 --- a/libs/evals/uv.lock +++ b/libs/evals/uv.lock @@ -499,8 +499,8 @@ dependencies = [ requires-dist = [ { name = "aiohttp", marker = "extra == 'nvidia'", specifier = ">=3.14.1,<3.15.0" }, { name = "aiosqlite", specifier = ">=0.22.1,<1.0.0" }, + { name = "av", marker = "extra == 'media'", specifier = ">=17.0.0,<18.0.0" }, { name = "deepagents", editable = "../deepagents" }, - { name = "deepagents", extras = ["video"], marker = "extra == 'video'", editable = "../deepagents" }, { name = "deepagents-acp", specifier = ">=0.0.8,<1.0.0" }, { name = "deepagents-code", extras = ["agentcore", "daytona", "modal", "runloop", "vercel"], marker = "extra == 'all-sandboxes'" }, { name = "deepagents-code", extras = ["anthropic", "baseten", "bedrock", "cohere", "deepseek", "fireworks", "google-genai", "groq", "huggingface", "ibm", "litellm", "mistralai", "nvidia", "ollama", "openai", "openrouter", "perplexity", "together", "vertex", "xai"], marker = "extra == 'all-providers'" }, @@ -545,6 +545,7 @@ requires-dist = [ { name = "mcp", specifier = ">=1.9.2,<2.0.0" }, { name = "packaging", specifier = ">=26.2" }, { name = "pillow", specifier = ">=12.2.0,<13.0.0" }, + { name = "pillow", marker = "extra == 'media'", specifier = ">=12.2.0,<13.0.0" }, { name = "prompt-toolkit", specifier = ">=3.0.52,<4.0.0" }, { name = "pydantic", specifier = ">=2.12.5,<2.14.0a0" }, { name = "pyperclip", specifier = ">=1.11.0,<2.0.0" }, @@ -559,7 +560,7 @@ requires-dist = [ { name = "tomli-w", specifier = ">=1.2.0,<2.0.0" }, { name = "uuid-utils", specifier = ">=0.16.2,<1.0.0" }, ] -provides-extras = ["anthropic", "baseten", "bedrock", "cohere", "deepseek", "fireworks", "google-genai", "groq", "huggingface", "ibm", "litellm", "mistralai", "nvidia", "ollama", "openai", "openrouter", "perplexity", "together", "vertex", "xai", "all-providers", "agentcore", "daytona", "modal", "runloop", "vercel", "all-sandboxes", "quickjs", "video"] +provides-extras = ["anthropic", "baseten", "bedrock", "cohere", "deepseek", "fireworks", "google-genai", "groq", "huggingface", "ibm", "litellm", "mistralai", "nvidia", "ollama", "openai", "openrouter", "perplexity", "together", "vertex", "xai", "all-providers", "agentcore", "daytona", "modal", "runloop", "vercel", "all-sandboxes", "media", "quickjs"] [package.metadata.requires-dev] test = [ From e2c4867a2d860aed899630e629a3ebd432e8924d Mon Sep 17 00:00:00 2001 From: Deep Agent Date: Wed, 1 Jul 2026 23:21:59 -0700 Subject: [PATCH 36/36] fix(code): classify media extra --- libs/code/deepagents_code/extras_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/code/deepagents_code/extras_info.py b/libs/code/deepagents_code/extras_info.py index 5aa88ddd7b7..d97ed896102 100644 --- a/libs/code/deepagents_code/extras_info.py +++ b/libs/code/deepagents_code/extras_info.py @@ -204,7 +204,7 @@ class ExtrasIntrospectionError(RuntimeError): ) """Optional extras that add sandbox integrations.""" -STANDALONE_EXTRAS: frozenset[str] = frozenset({"quickjs", "video"}) +STANDALONE_EXTRAS: frozenset[str] = frozenset({"media", "quickjs"}) """Optional extras that don't fit the provider/sandbox taxonomy. `quickjs` is a core dependency as of 0.1.24, but the empty extra remains