fix(agent): retry on JSONDecodeError instead of aborting

model_tools.py

@@ -158,6 +158,7 @@ def _discover_tools():
         "tools.send_message_tool",
         "tools.honcho_tools",
         "tools.homeassistant_tool",
+        "tools.user_notes_tool",
     ]
     import importlib
     for mod_name in _modules:

run_agent.py

@@ -66,6 +66,7 @@
 from tools.browser_tool import cleanup_browser
 
 import requests
+import httpx
 
 from hermes_constants import OPENROUTER_BASE_URL
 
@@ -4153,7 +4154,12 @@ def _build_api_kwargs(self, api_messages: list) -> dict:
             "model": self.model,
             "messages": sanitized_messages,
             "tools": self.tools if self.tools else None,
-            "timeout": float(os.getenv("HERMES_API_TIMEOUT", 900.0)),
+            "timeout": httpx.Timeout(
+                connect=15.0,
+                read=float(os.getenv("HERMES_API_TIMEOUT", 300.0)),
+                write=60.0,
+                pool=15.0,
+            ),
         }
 
         if self.max_tokens is not None:
@@ -6185,6 +6191,15 @@ def _stop_spinner():
                     # Enhanced error logging
                     error_type = type(api_error).__name__
                     error_msg = str(api_error).lower()
+
+                    # Log timeout errors explicitly so retries are visible in Modal logs
+                    is_timeout = isinstance(api_error, (httpx.TimeoutException,)) or "timeout" in error_msg
+                    if is_timeout:
+                        logger.warning(
+                            "API timeout (attempt %s/%s): %s after %.1fs — will retry",
+                            retry_count, max_retries, error_type, elapsed_time,
+                        )
+
                     logger.warning(
                         "API call failed (attempt %s/%s) error_type=%s %s error=%s",
                         retry_count,
@@ -6385,8 +6400,15 @@ def _stop_spinner():
                     # 529 (Anthropic overloaded) is also transient.
                     # Also catch local validation errors (ValueError, TypeError) — these
                     # are programming bugs, not transient failures.
+                    # EXCEPT json.JSONDecodeError (a ValueError subclass): an unparseable
+                    # provider response (truncated body, gateway HTML, SSE mismatch on a
+                    # large/slow request) is transient, not a local bug — let it retry with
+                    # backoff instead of aborting on attempt 1.
                     _RETRYABLE_STATUS_CODES = {413, 429, 529}
-                    is_local_validation_error = isinstance(api_error, (ValueError, TypeError))
+                    is_local_validation_error = (
+                        isinstance(api_error, (ValueError, TypeError))
+                        and not isinstance(api_error, json.JSONDecodeError)
+                    )
                     # Detect generic 400s from Anthropic OAuth (transient server-side failures).
                     # Real invalid_request_error responses include a descriptive message;
                     # transient ones contain only "Error" or are empty. (ref: issue #1608)
@@ -6738,7 +6760,7 @@ def _stop_spinner():
                         # Hard cap: after 6 total retries across all cycles, stop retrying
                         # and instruct the model to use an alternative approach.
                         if self._invalid_json_total_retries >= 6:
-                            self._vprint(f"{self.log_prefix}❌ Hit total invalid JSON retry cap (6). Injecting fallback guidance...")
+                            self._vprint(f"{self.log_prefix}❌ Hit total invalid JSON retry cap (6). Injecting fallback guidance...", force=True)
                             self._invalid_json_retries = 0
 
                             recovery_assistant = self._build_assistant_message(assistant_message, finish_reason)

tests/tools/test_delegate.py

@@ -510,9 +510,14 @@ def test_exit_reason_max_iterations(self):
 
 class TestBlockedTools(unittest.TestCase):
     def test_blocked_tools_constant(self):
-        for tool in ["delegate_task", "clarify", "memory", "send_message", "execute_code"]:
+        for tool in ["delegate_task", "clarify", "memory", "send_message"]:
             self.assertIn(tool, DELEGATE_BLOCKED_TOOLS)
 
+    def test_execute_code_allowed_in_subagents(self):
+        # Deliberately unblocked: subagents need execute_code to write large
+        # files without JSON serialization issues in write_file arguments.
+        self.assertNotIn("execute_code", DELEGATE_BLOCKED_TOOLS)
+
     def test_constants(self):
         self.assertEqual(MAX_CONCURRENT_CHILDREN, 3)
         self.assertEqual(MAX_DEPTH, 2)

tools/code_execution_tool.py

@@ -38,6 +38,26 @@
 
 SANDBOX_AVAILABLE = sys.platform != "win32"
 
+# Thread-local storage for per-execution sandbox environment variables.
+# Each executor thread gets its own copy so concurrent requests in the
+# same Modal container don't interfere with each other.
+_thread_local = threading.local()
+
+
+def set_sandbox_env(env: dict) -> None:
+    """Store per-execution env vars in thread-local storage.
+
+    Must be called inside the executor thread (threading.local is per-thread).
+    These vars are merged into the child subprocess environment by execute_code().
+    """
+    _thread_local.sandbox_env = env
+
+
+def get_sandbox_env() -> dict:
+    """Retrieve per-execution env vars from thread-local storage."""
+    return getattr(_thread_local, "sandbox_env", {})
+
+
 # The 7 tools allowed inside the sandbox. The intersection of this list
 # and the session's enabled tools determines which stubs are generated.
 SANDBOX_ALLOWED_TOOLS = frozenset([
@@ -480,11 +500,15 @@ def execute_code(
         _supa_key = os.getenv("SUPABASE_SERVICE_ROLE_KEY", "")
         if _supa_key:
             child_env["SUPABASE_SERVICE_ROLE_KEY"] = _supa_key
+        # Snowflake programmatic access token is needed for data queries
+        # inside code execution. It's blocked by _SECRET_SUBSTRINGS so we
+        # inject it explicitly.
+        _sf_token = os.getenv("SNOWFLAKE_PROGRAMMATIC_ACCESS_TOKEN", "")
+        if _sf_token:
+            child_env["SNOWFLAKE_PROGRAMMATIC_ACCESS_TOKEN"] = _sf_token
         # Ensure the hermes-agent root is importable in the sandbox so
         # repo-root modules are available to child scripts.
         _hermes_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-        _existing_pp = child_env.get("PYTHONPATH", "")
-        child_env["PYTHONPATH"] = _hermes_root + (os.pathsep + _existing_pp if _existing_pp else "")
         # Inject user's configured timezone so datetime.now() in sandboxed
         # code reflects the correct wall-clock time.
         _tz_name = os.getenv("HERMES_TIMEZONE", "").strip()
@@ -494,6 +518,26 @@ def execute_code(
         # Applied last so it can override any of the above.
         child_env.update(get_sandbox_env())
 
+        # Merge per-execution sandbox env vars (user ID, temp dir, Snowflake
+        # creds) set via set_sandbox_env() in the executor thread.
+        child_env.update(get_sandbox_env())
+
+        # Build PYTHONPATH AFTER merging sandbox env, so:
+        #  - hermes-agent root is always importable (repo modules)
+        #  - the per-execution HERMES_TMP_DIR is also importable, so the
+        #    agent can `import sf_connect` directly without manually doing
+        #    `sys.path.insert(0, os.environ["HERMES_TMP_DIR"])`. The
+        #    setup_sandbox helper drops sf_connect.py into that dir.
+        # This eliminates the "agent burns 5 turns on import retries before
+        # finally finding sf_connect" failure mode.
+        _existing_pp = child_env.get("PYTHONPATH", "")
+        _path_parts = [p for p in _existing_pp.split(os.pathsep) if p]
+        _user_tmp = child_env.get("HERMES_TMP_DIR", "")
+        for _entry in (_user_tmp, _hermes_root):
+            if _entry and _entry not in _path_parts:
+                _path_parts.insert(0, _entry)
+        child_env["PYTHONPATH"] = os.pathsep.join(_path_parts)
+
         proc = subprocess.Popen(
             [sys.executable, "script.py"],
             cwd=tmpdir,

tools/file_operations.py

@@ -314,6 +314,41 @@ def search(self, pattern: str, path: str = ".", target: str = "content",
 MAX_LINE_LENGTH = 2000
 MAX_FILE_SIZE = 50 * 1024  # 50KB
 
+# Matches the LINE_NUM|CONTENT gutter that read_file prepends (see
+# _add_line_numbers). Used to detect read output pasted back into a write.
+_GUTTER_LINE = re.compile(r'^[ \t]*(\d+)\|')
+
+
+def strip_pasted_line_numbers(content: str) -> Tuple[str, int]:
+    """
+    Detect and strip a read_file line-number gutter pasted into write content.
+
+    read_file returns 'LINE_NUM|CONTENT' lines; models sometimes copy blocks
+    from a read result into write_file content verbatim, persisting the gutter
+    into the file (it then breaks CSS/HTML/code). The signature is unmistakable:
+    many lines prefixed 'NNN|' with sequentially increasing numbers. Plain
+    content with pipes (tables, regexes) does not produce sequential runs.
+
+    Returns (cleaned_content, lines_stripped). lines_stripped is 0 when no
+    gutter was detected and content is returned unchanged.
+    """
+    lines = content.split('\n')
+    matched = [_GUTTER_LINE.match(line) for line in lines]
+    nums = [int(m.group(1)) for m in matched if m]
+    nonempty = sum(1 for line in lines if line.strip())
+
+    if len(nums) < 4 or nonempty == 0 or len(nums) / nonempty < 0.5:
+        return content, 0
+    consecutive = sum(1 for a, b in zip(nums, nums[1:]) if b == a + 1)
+    if consecutive / (len(nums) - 1) < 0.9:
+        return content, 0
+
+    cleaned = [
+        line[m.end():] if m else line
+        for line, m in zip(lines, matched)
+    ]
+    return '\n'.join(cleaned), len(nums)
+
 
 class ShellFileOperations(FileOperations):
     """

tools/file_tools.py

@@ -7,7 +7,7 @@
 import os
 import threading
 from typing import Optional
-from tools.file_operations import ShellFileOperations
+from tools.file_operations import ShellFileOperations, strip_pasted_line_numbers
 from agent.redact import redact_sensitive_text
 
 logger = logging.getLogger(__name__)
@@ -289,9 +289,23 @@ def notify_other_tool_call(task_id: str = "default"):
 def write_file_tool(path: str, content: str, task_id: str = "default") -> str:
     """Write content to a file."""
     try:
+        content, gutter_lines = strip_pasted_line_numbers(content)
+        if gutter_lines:
+            logger.warning(
+                "write_file: stripped %d pasted read_file line-number prefixes ('NNN|') from %s",
+                gutter_lines, path,
+            )
         file_ops = _get_file_ops(task_id)
         result = file_ops.write_file(path, content)
-        return json.dumps(result.to_dict(), ensure_ascii=False)
+        result_dict = result.to_dict()
+        if gutter_lines:
+            result_dict["line_number_gutter_stripped"] = gutter_lines
+            result_dict["warning"] = (
+                f"Content contained {gutter_lines} 'LINE_NUM|' prefixes copied from a "
+                "read_file result; they were stripped before writing. Do not include "
+                "the line-number gutter when copying file content."
+            )
+        return json.dumps(result_dict, ensure_ascii=False)
     except Exception as e:
         if _is_expected_write_exception(e):
             logger.debug("write_file expected denial: %s: %s", type(e).__name__, e)