fix(chat): reasoning_content parsing, image passthrough, model-change race

changelogs/v2.5.17.md

@@ -0,0 +1,6 @@
+## Fixes
+
+- **Image attachments now reach every chat model.** The vision gate previously guessed vision support from a hardcoded model-name allow-list (`gpt-4o`, `claude-3`, `gemini`, …), so images were silently stripped for any model whose id didn't match — including custom OpenAI-compatible endpoints (e.g. a local proxy serving `playground-claude-opus-4-8`) and vision-capable on-device models like Gemma. Cairn now always forwards attached images to the model and lets it decide — a non-vision model simply says it can't see them. Images also now flow through the subagent dispatch path, which previously dropped them.
+- **Changing the chat model now applies to the very next message.** A background re-hydrate (triggered after write-tool turns and on every `db:changed` event) could read a stale backend settings cache and silently revert a model you'd just picked — the "have to clear chat twice for the new model to take effect" bug. Refresh hydrates now prefer the live in-memory config over the cache, so a model change sticks immediately.
+- **Qwen/DeepSeek/GLM-style reasoning now shows in the thinking panel on desktop.** The desktop agent stream parsers only read `delta.reasoning`, but these models emit their thinking on `delta.reasoning_content` (and `message.reasoning_content` on non-streaming responses). Cairn now reads `reasoning_content ?? reasoning` in the chat loop, the coding-agent loop, and the subagent dispatch loop — matching mobile — and strips `reasoning_content` before re-sending assistant messages (re-sending it triggers 400s on DeepSeek).
+- **Token counting no longer inflates the context ring for image attachments.** The prompt-token breakdown stringified the whole multimodal message — including the image's base64 data URL — so a single screenshot could add tens of thousands of phantom tokens to the "conversation" segment and spike the context gauge. Images now contribute a flat per-image estimate; only the text parts are tokenised (matching mobile).

electron/ipc/chat.ts

@@ -135,20 +135,11 @@ export function registerChatHandler(db: Database.Database, workspacePath: string
       return;
     }
 
-    // Check if the model supports vision before including images
-    const modelLc = (model ?? "").toLowerCase();
-    const supportsVision = provider !== "localllm" && (
-      modelLc.includes("vision") ||
-      modelLc.includes("gpt-4o") ||
-      modelLc.startsWith("claude-3") ||
-      modelLc.startsWith("claude-sonnet-4") ||
-      modelLc.includes("gemini")
-    );
-
-    if (req.images?.length && !supportsVision) {
-      req.message = "[Images omitted — model does not support vision]\n\n" + req.message;
-      req.images = undefined;
-    }
+    // Images are always passed through to the model. We can't reliably guess
+    // vision support from a model id — cloud APIs, custom OpenAI-compatible
+    // endpoints, and on-device models (some, like Gemma, DO support vision)
+    // all expose arbitrary names. So we forward the images and let the model /
+    // endpoint decide; a non-vision model will simply say it can't see them.
 
     const userMessage: OpenAIMessage = req.images?.length
       ? ({

electron/lib/chat-loop.ts

@@ -86,14 +86,17 @@ export async function runToolLoop(
             res.usage.completion_tokens_details?.reasoning_tokens ?? 0,
           );
         }
-        const rawMsg = choice.message as OpenAIMessage & { reasoning?: string };
-        if (rawMsg.reasoning) {
-          accumulatedReasoning += rawMsg.reasoning;
-          if (onThought) onThought(rawMsg.reasoning);
+        const rawMsg = choice.message as OpenAIMessage & { reasoning?: string; reasoning_content?: string };
+        const rawThought = rawMsg.reasoning_content ?? rawMsg.reasoning;
+        if (rawThought) {
+          accumulatedReasoning += rawThought;
+          if (onThought) onThought(rawThought);
         }
         // Strip reasoning before assigning — it must not enter the messages
-        // array that gets re-sent to the API on subsequent rounds.
-        const { reasoning: _r, ...msgWithoutReasoning } = rawMsg;
+        // array that gets re-sent to the API on subsequent rounds. Both
+        // `reasoning` and `reasoning_content` must be stripped (DeepSeek docs:
+        // re-sending reasoning_content on assistant messages causes 400s).
+        const { reasoning: _r, reasoning_content: _rc, ...msgWithoutReasoning } = rawMsg;
         assistantMsg = msgWithoutReasoning;
 
         // Self-Healing Parser for On-Device XML-style tool calls and tokenizers
@@ -185,12 +188,14 @@ export async function runToolLoop(
             if (onToken) onToken(delta.content);
           }
 
-          // Reasoning / thinking stream (Claude thinking_delta, OpenAI delta.reasoning).
+          // Reasoning / thinking stream (Claude thinking_delta, OpenAI delta.reasoning,
+          // DeepSeek/Qwen-style delta.reasoning_content).
           // Models that don't expose reasoning text simply never emit this field —
           // the panel stays hidden. Reasoning is NOT merged into content/tool JSON.
-          if (delta.reasoning) {
-            accumulatedReasoning += delta.reasoning;
-            if (onThought) onThought(delta.reasoning);
+          const thought = delta.reasoning_content ?? delta.reasoning;
+          if (thought) {
+            accumulatedReasoning += thought;
+            if (onThought) onThought(thought);
           }
 
           if (delta.tool_calls) {

electron/lib/chat-subagent-loop.ts

@@ -418,7 +418,15 @@ export async function runDispatchLoop(
       if (m.name) out.name = m.name;
       return out;
     }),
-    { role: "user", content: req.message },
+    req.images?.length
+      ? ({
+          role: "user",
+          content: [
+            { type: "text", text: req.message },
+            ...req.images.map((img) => ({ type: "image_url", image_url: { url: img.dataUrl } })),
+          ],
+        } as unknown as OpenAIMessage)
+      : { role: "user", content: req.message },
   ];
 
   const maxSteps = req.config?.maxSteps ?? 12;
@@ -482,14 +490,16 @@ export async function runDispatchLoop(
     }
 
     const choice = data.choices?.[0];
-    const assistantMsg = choice?.message as OpenAIMessage & { reasoning?: string } | undefined;
+    const assistantMsg = choice?.message as OpenAIMessage & { reasoning?: string; reasoning_content?: string } | undefined;
     if (!assistantMsg) {
       return { content: "No response from dispatch model.", reasoning: finalReasoning, metrics };
     }
-    if (assistantMsg.reasoning) finalReasoning += assistantMsg.reasoning;
+    const dispatchThought = assistantMsg.reasoning_content ?? assistantMsg.reasoning;
+    if (dispatchThought) finalReasoning += dispatchThought;
 
-    // Strip reasoning before pushing back into history.
-    const { reasoning: _r, ...msgClean } = assistantMsg;
+    // Strip reasoning before pushing back into history. Both `reasoning` and
+    // `reasoning_content` must go (re-sending reasoning_content causes 400s).
+    const { reasoning: _r, reasoning_content: _rc, ...msgClean } = assistantMsg;
 
     if (!msgClean.tool_calls?.length) {
       finalContent = (msgClean.content ?? "").trim();

electron/lib/llm.ts

@@ -203,6 +203,38 @@ function tok(s: string): number {
   return encode(s).length;
 }
 
+/**
+ * Flat per-image token estimate. Vision models bill images by tiles, not by the
+ * length of the base64 payload — so counting the data URL characters (which we
+ * used to do implicitly when the multimodal `content` array got stringified)
+ * over-counts by orders of magnitude. ~1.1k tokens is a reasonable single-image
+ * approximation across OpenAI/Anthropic/Gemini for a typical attachment.
+ */
+const IMAGE_TOKEN_ESTIMATE = 1100;
+
+/**
+ * Count tokens for a message's `content`, which may be a plain string or an
+ * OpenAI multimodal parts array (`[{type:"text"...}, {type:"image_url"...}]`).
+ * Text parts are tokenised; image parts contribute a flat estimate instead of
+ * the base64 data URL length.
+ */
+function tokContent(content: unknown): number {
+  if (typeof content === "string") return tok(content);
+  if (Array.isArray(content)) {
+    let total = 0;
+    for (const part of content) {
+      const p = part as Record<string, unknown>;
+      if (p?.type === "image_url" || p?.image_url) {
+        total += IMAGE_TOKEN_ESTIMATE;
+      } else if (typeof p?.text === "string") {
+        total += tok(p.text);
+      }
+    }
+    return total;
+  }
+  return 0;
+}
+
 export function calculatePromptBreakdown(
   systemPrompt: string | undefined,
   messages: OpenAIMessage[],
@@ -281,8 +313,7 @@ export function calculatePromptBreakdown(
       continue;
     }
 
-    const textContent = msg.content ?? "";
-    const textTokens = tok(textContent);
+    const textTokens = tokContent(msg.content);
     const toolCallsTokens = msg.tool_calls ? tok(JSON.stringify(msg.tool_calls)) : 0;
 
     if (msg.role === "tool") {

electron/lib/pi-agent-loop.ts

@@ -675,11 +675,13 @@ export async function runAgentLoop(
           callbacks.onToken(delta.content);
         }
 
-        // Reasoning / thinking stream (Claude thinking_delta, OpenAI delta.reasoning).
+        // Reasoning / thinking stream (Claude thinking_delta, OpenAI delta.reasoning,
+        // DeepSeek/Qwen-style delta.reasoning_content).
         // Not merged into content or tool-call JSON; surfaced as a separate panel.
-        if (delta.reasoning) {
-          reasoningBuffer += delta.reasoning;
-          callbacks.onThought?.(delta.reasoning);
+        const thought = delta.reasoning_content ?? delta.reasoning;
+        if (thought) {
+          reasoningBuffer += thought;
+          callbacks.onThought?.(thought);
         }
 
         if (delta.tool_calls) {

electron/lib/token-breakdown-images.test.ts

@@ -0,0 +1,76 @@
+/**
+ * Token-breakdown image-content guard.
+ *
+ * A multimodal user message carries an OpenAI parts array
+ * (`[{type:"text"}, {type:"image_url", image_url:{url:"data:...base64..."}}]`)
+ * rather than a plain string. `calculatePromptBreakdown` used to stringify that
+ * whole array, so the giant base64 data URL was tokenised verbatim — inflating
+ * the conversation count (and the context ring) by orders of magnitude.
+ *
+ * These tests lock in that image parts contribute a small flat estimate, NOT
+ * the length of their base64 payload, and that text parts are still counted.
+ */
+import { describe, it, expect } from "vitest";
+import { calculatePromptBreakdown, type OpenAIMessage } from "./llm";
+
+// A ~40KB base64 data URL — realistic for a small screenshot. If this were
+// tokenised, it would be tens of thousands of tokens.
+const BIG_DATA_URL = "data:image/png;base64," + "A".repeat(40_000);
+
+function multimodalUser(text: string): OpenAIMessage {
+  return {
+    role: "user",
+    content: [
+      { type: "text", text },
+      { type: "image_url", image_url: { url: BIG_DATA_URL } },
+    ],
+  } as unknown as OpenAIMessage;
+}
+
+describe("calculatePromptBreakdown image handling", () => {
+  it("does not tokenise the base64 data URL of an image part", () => {
+    const b = calculatePromptBreakdown(undefined, [multimodalUser("what is this?")]);
+    // The 40KB base64 blob alone would be well over 10k tokens if counted.
+    expect(b.conversation).toBeLessThan(2_000);
+  });
+
+  it("still counts the text part of a multimodal message", () => {
+    const withText = calculatePromptBreakdown(undefined, [multimodalUser("describe the architecture diagram in detail please")]);
+    const noText = calculatePromptBreakdown(undefined, [multimodalUser("")]);
+    expect(withText.conversation).toBeGreaterThan(noText.conversation);
+  });
+
+  it("counts a plain-string message the same as before", () => {
+    const b = calculatePromptBreakdown(undefined, [
+      { role: "user", content: "hello world" },
+    ]);
+    expect(b.conversation).toBeGreaterThan(0);
+    expect(b.conversation).toBeLessThan(20);
+  });
+
+  it("adds a nonzero flat image cost independent of base64 payload length", () => {
+    // Same text, no image — the baseline the image cost is measured against.
+    const textOnly = calculatePromptBreakdown(undefined, [
+      { role: "user", content: "hi" },
+    ]);
+    // Same text + a tiny image.
+    const smallImg = calculatePromptBreakdown(undefined, [
+      {
+        role: "user",
+        content: [
+          { type: "text", text: "hi" },
+          { type: "image_url", image_url: { url: "data:image/png;base64," + "A".repeat(100) } },
+        ],
+      } as unknown as OpenAIMessage,
+    ]);
+    // Same text + a 400x larger image payload.
+    const bigImg = calculatePromptBreakdown(undefined, [multimodalUser("hi")]);
+
+    // The image adds a nonzero flat cost on top of the text-only baseline...
+    expect(smallImg.conversation).toBeGreaterThan(textOnly.conversation);
+    // ...and that cost is exactly the flat per-image estimate.
+    expect(smallImg.conversation - textOnly.conversation).toBe(1100);
+    // ...and it does NOT scale with the base64 payload length.
+    expect(bigImg.conversation).toBe(smallImg.conversation);
+  });
+});

src/store/index.ts

@@ -338,9 +338,18 @@ export const useCairnStore = create<CairnStore>()(
       // letting a field-poor backend object reset them to DEFAULT_AI_CONFIG on
       // every hydrate (the "maxSteps reverts to 30" bug).
       const localAiConfig = storage.get<AIConfig>(AI_CONFIG_KEY);
+      // On a REFRESH hydrate (fired after write-tool turns and every db:changed
+      // event) the in-memory aiConfig is the source of truth for the live
+      // session — the user may have just changed the model, and the async
+      // saveAiSettings write may not have landed in the backend cache yet.
+      // Layering the (possibly stale) backend cache on top would silently revert
+      // that selection, which is the "have to clear chat twice for the new model
+      // to apply" bug. So on refresh we prefer the current in-memory config over
+      // the backend cache; on the initial hydrate the backend cache wins.
+      const currentAiConfig = isRefresh ? get().aiConfig : undefined;
       const savedConfig = backendAiConfig
-        ? { ...localAiConfig, ...backendAiConfig }
-        : localAiConfig;
+        ? { ...localAiConfig, ...backendAiConfig, ...currentAiConfig }
+        : (currentAiConfig ?? localAiConfig);
       if (savedConfig) {
         if (savedConfig.provider === ("apple-fm" as unknown as "openai" | "localllm")) {
           savedConfig.provider = "localllm";