From 151967b04ff475e27a85d17b851d499c217329be Mon Sep 17 00:00:00 2001
From: Emre <125399834+EMREYSLCY@users.noreply.github.com>
Date: Tue, 21 Apr 2026 21:39:28 +0000
Subject: [PATCH] perf: optimize disk I/O by batching writes in
 _process_batch_worker

---
 batch_runner.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/batch_runner.py b/batch_runner.py
index 195452c0ae0ce..49a6d5dfa6c34 100644
--- a/batch_runner.py
+++ b/batch_runner.py
@@ -425,6 +425,7 @@ def _process_batch_worker(args: Tuple) -> Dict[str, Any]:
     batch_tool_stats = {}
     batch_reasoning_stats = {"total_assistant_turns": 0, "turns_with_reasoning": 0, "turns_without_reasoning": 0}
     completed_in_batch = []
+    batch_results_to_write = []
     discarded_no_reasoning = 0
     
     # Process each prompt sequentially in this batch
@@ -469,9 +470,8 @@ def _process_batch_worker(args: Tuple) -> Dict[str, Any]:
                 "tool_error_counts": tool_error_counts  # Simple: {tool: failure_count} - normalized
             }
             
-            # Append to batch output file
-            with open(batch_output_file, 'a', encoding='utf-8') as f:
-                f.write(json.dumps(trajectory_entry, ensure_ascii=False) + "\n")
+            # Accumulate results in memory instead of writing to disk every time
+        batch_results_to_write.append(trajectory_entry)
         
         # Aggregate tool statistics
         for tool_name, stats in result.get("tool_stats", {}).items():
@@ -498,6 +498,11 @@ def _process_batch_worker(args: Tuple) -> Dict[str, Any]:
         else:
             print(f"   ❌ Prompt {prompt_index} failed (will retry on resume)")
     
+    # Write all accumulated results to disk in a single I/O operation
+    if batch_results_to_write:
+        with open(batch_output_file, 'a', encoding='utf-8') as f:
+            for entry in batch_results_to_write:
+                f.write(json.dumps(entry, ensure_ascii=False) + "\n")
     print(f"✅ Batch {batch_num}: Completed ({len(prompts_to_process)} prompts processed)")
     
     return {