.NET: Preserve UTF-8 order in HeadTailBuffer

dotnet/src/Microsoft.Agents.AI.Tools.Shell/HeadTailBuffer.cs

@@ -21,9 +21,10 @@ namespace Microsoft.Agents.AI.Tools.Shell;
 /// <para>
 /// The buffer counts UTF-8 bytes (matching the public <c>maxOutputBytes</c> contract
 /// and <see cref="ShellSession.TruncateHeadTail"/>). Append happens one rune at a time
-/// — when the head fills, the next rune's UTF-8 bytes go to the tail as an indivisible
-/// unit, and the oldest rune is dropped from the tail. This guarantees the final
-/// string never contains a split rune (no orphan surrogates, no invalid UTF-8).
+/// — once a complete rune no longer fits in the head, it and all later runes go to
+/// the tail as indivisible units. After the total exceeds the cap, the oldest tail
+/// runes are dropped. This guarantees the final string never contains a split rune
+/// (no orphan surrogates, no invalid UTF-8).
 /// </para>
 /// </remarks>
 internal sealed class HeadTailBuffer
@@ -37,6 +38,7 @@ internal sealed class HeadTailBuffer
     private readonly Queue<byte[]> _tail = new();
     private int _tailBytes;
     private long _totalBytes;
+    private bool _headSealed;
 
     public HeadTailBuffer(int cap)
     {
@@ -63,19 +65,22 @@ private void AppendInternal(string s)
             var n = rune.EncodeToUtf8(scratch);
             this._totalBytes += n;
 
-            if (this._head.Count + n <= this._headCap)
+            if (!this._headSealed && this._head.Count + n <= this._headCap)
             {
                 for (var i = 0; i < n; i++) { this._head.Add(scratch[i]); }
                 continue;
             }
 
-            // Head is full — append to tail as a single rune-sized chunk.
+            // Once a complete rune cannot fit in the head, seal it and keep all later runes in the tail.
+            this._headSealed = true;
             var bytes = scratch[..n].ToArray();
             this._tail.Enqueue(bytes);
             this._tailBytes += n;
 
             // Evict whole runes from the front of the tail until we fit.
-            while (this._tailBytes > this._tailCap && this._tail.Count > 0)
+            while (this._totalBytes > this._cap &&
+                   this._tailBytes > this._tailCap &&
+                   this._tail.Count > 0)
             {
                 var dropped = this._tail.Dequeue();
                 this._tailBytes -= dropped.Length;

dotnet/tests/Microsoft.Agents.AI.Tools.Shell.UnitTests/HeadTailBufferTests.cs

@@ -116,4 +116,39 @@ public void Append_OddCap_AtCap_NoSilentDataDrop()
         Assert.False(truncated);
         Assert.Equal("ABCD\n", text);
     }
+
+    [Fact]
+    public void Append_MultiByteUtf8_ExactlyAtCap_PreservesOrderAndAllContent()
+    {
+        // Arrange
+        const string Input = "aaaaaaa🔥🔥🔥"; // 7 ASCII + 3 * 4-byte runes + newline = 20 bytes.
+        var buf = new HeadTailBuffer(cap: 20);
+
+        // Act
+        buf.AppendLine(Input);
+        var (text, truncated) = buf.ToFinalString();
+
+        // Assert
+        Assert.False(truncated);
+        Assert.Equal(Input + "\n", text);
+    }
+
+    [Fact]
+    public void Append_MultiByteUtf8_Overflow_PreservesHeadAndTailOrder()
+    {
+        // Arrange
+        const string Input = "aaaaaaa🔥🔥🔥x"; // AppendLine makes this one byte over cap.
+        var buf = new HeadTailBuffer(cap: 20);
+
+        // Act
+        buf.AppendLine(Input);
+        var (text, truncated) = buf.ToFinalString();
+
+        // Assert
+        Assert.True(truncated);
+        Assert.StartsWith("aaaaaaa\n", text, System.StringComparison.Ordinal);
+        Assert.Contains("[... truncated 4 bytes ...]", text, System.StringComparison.Ordinal);
+        Assert.EndsWith("🔥🔥x\n", text, System.StringComparison.Ordinal);
+        Assert.DoesNotContain("\uFFFD", text);
+    }
 }