agent: Fix a UTF-8 boundary panic in the agent streaming parser

crates/agent/Cargo.toml

@@ -99,6 +99,7 @@ language_model = { workspace = true, "features" = ["test-support"] }
 lsp = { workspace = true, "features" = ["test-support"] }
 pretty_assertions.workspace = true
 project = { workspace = true, "features" = ["test-support"] }
+proptest.workspace = true
 rand.workspace = true
 reqwest_client.workspace = true
 settings = { workspace = true, "features" = ["test-support"] }

crates/agent/src/tools/edit_session/streaming_parser.rs

@@ -169,8 +169,9 @@ impl StreamingParser {
         let mut events = SmallVec::new();
 
         let safe_end = safe_emit_end(content);
-        if safe_end > self.content_emitted_len {
-            let chunk = content[self.content_emitted_len..safe_end].to_string();
+        let safe_start = find_char_boundary(content, self.content_emitted_len);
+        if safe_end > safe_start {
+            let chunk = content[safe_start..safe_end].to_string();
             self.content_emitted_len = safe_end;
             events.push(WriteEvent::ContentChunk { chunk });
         }
@@ -228,7 +229,7 @@ impl StreamingParser {
             }
 
             if !state.old_text_done {
-                let start = state.old_text_emitted_len.min(edit.old_text.len());
+                let start = find_char_boundary(&edit.old_text, state.old_text_emitted_len);
                 let chunk = normalize_done_chunk(edit.old_text[start..].to_string());
                 state.old_text_done = true;
                 state.old_text_emitted_len = edit.old_text.len();
@@ -240,7 +241,7 @@ impl StreamingParser {
             }
 
             if !state.new_text_done {
-                let start = state.new_text_emitted_len.min(edit.new_text.len());
+                let start = find_char_boundary(&edit.new_text, state.new_text_emitted_len);
                 let chunk = normalize_done_chunk(edit.new_text[start..].to_string());
                 state.new_text_done = true;
                 state.new_text_emitted_len = edit.new_text.len();
@@ -259,7 +260,7 @@ impl StreamingParser {
     pub fn finalize_content(&mut self, content: &str) -> SmallVec<[WriteEvent; 1]> {
         let mut events = SmallVec::new();
 
-        let start = self.content_emitted_len.min(content.len());
+        let start = find_char_boundary(content, self.content_emitted_len);
         if content.len() > start {
             let chunk = content[start..].to_string();
             self.content_emitted_len = content.len();
@@ -313,7 +314,7 @@ impl StreamingParser {
 
         if !state.old_text_done {
             let old_text = old_text.unwrap_or_default();
-            let start = state.old_text_emitted_len.min(old_text.len());
+            let start = find_char_boundary(old_text, state.old_text_emitted_len);
             state.old_text_done = true;
             state.old_text_emitted_len = old_text.len();
             events.push(EditEvent::OldTextChunk {
@@ -325,7 +326,7 @@ impl StreamingParser {
 
         if !state.new_text_done {
             let new_text = new_text.unwrap_or_default();
-            let start = state.new_text_emitted_len.min(new_text.len());
+            let start = find_char_boundary(new_text, state.new_text_emitted_len);
             state.new_text_done = true;
             state.new_text_emitted_len = new_text.len();
             state.buffer_new_text_until_old_text_done = false;
@@ -396,6 +397,44 @@ fn normalize_done_chunk(mut chunk: String) -> String {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use proptest::prelude::*;
+
+    fn emitted_len_inside_multibyte_char() -> impl Strategy<Value = (String, String)> {
+        (1usize..8, prop::sample::select(&["。", "—", "é", "🦀"])).prop_map(
+            |(emitted_len, multibyte_char)| {
+                let first = "a".repeat(emitted_len);
+                let second = format!("{}{}", "a".repeat(emitted_len - 1), multibyte_char);
+                (first, second)
+            },
+        )
+    }
+
+    fn boundary_sensitive_text() -> impl Strategy<Value = String> {
+        prop_oneof![
+            emitted_len_inside_multibyte_char().prop_map(|(first, _)| first),
+            emitted_len_inside_multibyte_char().prop_map(|(_, second)| second),
+            prop::sample::select(&[
+                "",
+                "a",
+                "ab",
+                "ab\\",
+                "a。",
+                "a—",
+                "hello,\\",
+                "hello,\n",
+                "hello,\nworld",
+            ])
+            .prop_map(ToString::to_string),
+        ]
+    }
+
+    fn partial_edit() -> impl Strategy<Value = PartialEdit> {
+        (
+            prop::option::of(boundary_sensitive_text()),
+            prop::option::of(boundary_sensitive_text()),
+        )
+            .prop_map(|(old_text, new_text)| PartialEdit { old_text, new_text })
+    }
 
     #[test]
     fn test_first_edit_with_new_text_in_first_chunk_is_held_until_finalize() {
@@ -761,6 +800,30 @@ mod tests {
         );
     }
 
+    proptest! {
+        #[test]
+        fn test_content_finalize_does_not_panic_when_emitted_len_lands_inside_multibyte_char(
+            pair in emitted_len_inside_multibyte_char()
+        ) {
+            let (first, second) = pair;
+            let mut parser = StreamingParser::default();
+
+            parser.push_content(&first);
+            parser.finalize_content(&second);
+        }
+
+        #[test]
+        fn test_push_edits_does_not_panic_on_boundary_sensitive_sequences(
+            partials in prop::collection::vec(prop::collection::vec(partial_edit(), 0..4), 1..12)
+        ) {
+            let mut parser = StreamingParser::default();
+
+            for edits in partials {
+                parser.push_edits(&edits);
+            }
+        }
+    }
+
     #[test]
     fn test_no_partials_direct_finalize() {
         let mut parser = StreamingParser::default();