Skip to content

fix(ollama): preserve reasoning and tool-call context #6605

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Calcium-Ion merged 1 commit into from
Aug 10, 2026
Merged

fix(ollama): preserve reasoning and tool-call context #6605

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 10 additions & 8 deletions relay/channel/ollama/dto.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,13 @@ import (
)

type OllamaChatMessage struct {
Role string `json:"role"`
Content string `json:"content,omitempty"`
Images []string `json:"images,omitempty"`
ToolCalls []OllamaToolCall `json:"tool_calls,omitempty"`
ToolName string `json:"tool_name,omitempty"`
Thinking json.RawMessage `json:"thinking,omitempty"`
Role string `json:"role"`
Content string `json:"content,omitempty"`
Images []string `json:"images,omitempty"`
ToolCalls []OllamaToolCall `json:"tool_calls,omitempty"`
ToolName string `json:"tool_name,omitempty"`
ToolCallID string `json:"tool_call_id,omitempty"`
Thinking json.RawMessage `json:"thinking,omitempty"`
}

type OllamaToolFunction struct {
Expand All @@ -25,6 +26,7 @@ type OllamaTool struct {
}

type OllamaToolCall struct {
ID string `json:"id,omitempty"`
Function struct {
Name string `json:"name"`
Arguments interface{} `json:"arguments"`
Expand All @@ -36,7 +38,7 @@ type OllamaChatRequest struct {
Messages []OllamaChatMessage `json:"messages"`
Tools interface{} `json:"tools,omitempty"`
Format interface{} `json:"format,omitempty"`
Stream bool `json:"stream,omitempty"`
Stream bool `json:"stream"`
Options map[string]any `json:"options,omitempty"`
KeepAlive interface{} `json:"keep_alive,omitempty"`
Think json.RawMessage `json:"think,omitempty"`
Expand All @@ -48,7 +50,7 @@ type OllamaGenerateRequest struct {
Suffix string `json:"suffix,omitempty"`
Images []string `json:"images,omitempty"`
Format interface{} `json:"format,omitempty"`
Stream bool `json:"stream,omitempty"`
Stream bool `json:"stream"`
Options map[string]any `json:"options,omitempty"`
KeepAlive interface{} `json:"keep_alive,omitempty"`
Think json.RawMessage `json:"think,omitempty"`
Expand Down
137 changes: 95 additions & 42 deletions relay/channel/ollama/relay-ollama.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package ollama

import (
"encoding/json"
"fmt"
"io"
"net/http"
Expand All @@ -19,24 +18,67 @@ import (
"github.com/samber/lo"
)

func toOllamaResponseFormat(responseFormat *dto.ResponseFormat) (any, error) {
if responseFormat == nil {
return nil, nil
}
switch responseFormat.Type {
case "json", "json_object":
return "json", nil
case "json_schema":
if len(responseFormat.JsonSchema) == 0 {
return nil, nil
}
var jsonSchema dto.FormatJsonSchema
if err := common.Unmarshal(responseFormat.JsonSchema, &jsonSchema); err != nil {
return nil, fmt.Errorf("invalid ollama response format: %w", err)
}
return jsonSchema.Schema, nil
default:
return nil, nil
}
}

func openAIChatToOllamaChat(c *gin.Context, r *dto.GeneralOpenAIRequest) (*OllamaChatRequest, error) {
think := r.Think
if len(think) == 0 {
effort := r.ReasoningEffort
if len(r.Reasoning) > 0 {
var reasoning dto.Reasoning
if err := common.Unmarshal(r.Reasoning, &reasoning); err != nil {
return nil, fmt.Errorf("invalid ollama reasoning: %w", err)
}
effort = lo.CoalesceOrEmpty(reasoning.Effort, effort)
}
if effort != "" {
var thinkValue any
switch effort {
case "none":
thinkValue = false
case "low", "medium", "high", "max":
thinkValue = effort
default:
return nil, fmt.Errorf("unsupported ollama reasoning effort %q", effort)
}
var err error
think, err = common.Marshal(thinkValue)
if err != nil {
return nil, fmt.Errorf("marshal ollama think: %w", err)
}
}
}
Comment on lines 42 to +69

@coderabbitai coderabbitai Bot Aug 2, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟠 Major | ⚡ Quick win

🧩 Analysis chain

🌐 Web query:

What values does Ollama's think API parameter currently accept, and what reasoning_effort values does OpenAI currently support across models?

💡 Result:

Ollama's think API parameter accepts boolean values (true/false) and reasoning level strings: "low", "medium", "high", and "max" [1][2]. While these values are supported at the top level for most models, the "max" level is sometimes specifically designated for use within the options object (e.g., options: {"think": "max"}) depending on the specific model and API implementation [3]. Some models, such as GPT-OSS, strictly require one of the reasoning levels ("low", "medium", or "high") and do not accept boolean values [1][4][5]. OpenAI supports several reasoning_effort values across its reasoning-capable models: "none", "minimal", "low", "medium", "high", "xhigh", and "max" [6][7][8]. The availability of these values is model-dependent [6][7]. For instance, certain models may default to "medium" and not support "none", while others might support "xhigh" or "max" only on specific advanced versions [8][9]. Users should consult model-specific documentation to verify which effort levels are compatible with a chosen model [6][7].

Citations:

🌐 Web query:

site:developers.openai.com/api/docs/guides/reasoning reasoning_effort minimal xhigh

💡 Result:

The reasoning_effort parameter is a configuration setting in OpenAI's reasoning models (such as the o-series) that controls the amount of computational processing, or thinking, the model performs before generating a final response [1]. By adjusting this parameter, you can influence the trade-off between latency, cost, and output quality [1]. The available settings typically range from minimal to max, with higher settings instructing the model to spend more time planning, analyzing, and reasoning through complex problems [1]. Key characteristics of these effort levels include: minimal: Prioritizes speed and lower token usage, making it suitable for simpler tasks where immediate, direct answers are preferred [1]. xhigh and max: Direct the model to engage in deeper, more comprehensive reasoning, which is recommended for complex debugging, long-horizon research, high-stakes decision-making, and other intellectually demanding workflows where accuracy and quality are more critical than low latency [1]. In practice, the models are designed to be adaptive, meaning that even with a specific effort setting, they may use fewer tokens for simple tasks and "think harder" only when the complexity of the input requires it [1]. The default effort setting, such as medium, is generally considered the best starting point for balancing performance, quality, and reliability [1].

Citations:

🌐 Web query:

site:docs.ollama.com/capabilities/thinking Ollama think low medium high max false true

💡 Result:

In Ollama, the think parameter is used to control the reasoning trace for supported models [1]. Users can specify the level of reasoning or toggle the feature on and off using the following options: Boolean values: - true: Enables the thinking process (default for supported models) [1]. - false: Disables the reasoning trace [1]. Thinking levels: - low, medium, high: These levels allow users to tune the length and depth of the model's reasoning trace [1]. - max: This option requests the highest level of thinking available for the model [1]. Note that some specific models, such as GPT-OSS, only accept the levels (low, medium, or high) rather than boolean values [1]. In addition to the API parameters, users can manage thinking in the CLI using commands like /set think, /set nothink, or the --hidethinking flag to perform reasoning without displaying the trace in the final output [1].

Citations:

🏁 Script executed:

#!/bin/bash
set -euo pipefail

# Inspect the converter and nearby stream/channel support without executing repository code.
wc -l relay/channel/ollama/relay-ollama.go
sed -n '1,140p' relay/channel/ollama/relay-ollama.go
printf '\n--- streamSupportedChannels references ---\n'
rg -n "streamSupportedChannels|OpenAIChatToOllamaChat|openAIChatToOllamaChat" relay/channel/ollama relay -g '*.go' || true

Repository: QuantumNous/new-api

Length of output: 4646

Allow unsupported OpenAI reasoning-effort values to pass through.

OpenAI accepts minimal and xhigh, but this converter only maps none, low, medium, high, and max. Those values hit default, return an error, and abort the Ollama request conversion. Map minimal to low and xhigh to max, and let Ollama reject any other think value it does not support.

🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@relay/channel/ollama/relay-ollama.go` around lines 42 - 69, Update the effort
switch in openAIChatToOllamaChat to map OpenAI’s “minimal” value to Ollama’s
“low” and “xhigh” to “max”; remove the converter-level error for other
unsupported values so they pass through for Ollama to validate, while preserving
existing mappings and error handling for reasoning unmarshalling and think
marshaling.


chatReq := &OllamaChatRequest{
Model: r.Model,
Stream: lo.FromPtrOr(r.Stream, false),
Options: map[string]any{},
Think: r.Think,
Think: think,
}
if r.ResponseFormat != nil {
if r.ResponseFormat.Type == "json" {
chatReq.Format = "json"
} else if r.ResponseFormat.Type == "json_schema" {
if len(r.ResponseFormat.JsonSchema) > 0 {
var schema any
_ = json.Unmarshal(r.ResponseFormat.JsonSchema, &schema)
chatReq.Format = schema
}
}
format, err := toOllamaResponseFormat(r.ResponseFormat)
if err != nil {
return nil, err
}
chatReq.Format = format

// options mapping
if r.Temperature != nil {
Expand Down Expand Up @@ -68,27 +110,31 @@ func openAIChatToOllamaChat(c *gin.Context, r *dto.GeneralOpenAIRequest) (*Ollam
case []string:
chatReq.Options["stop"] = v
case []any:
arr := make([]string, 0, len(v))
for _, i := range v {
if s, ok := i.(string); ok {
arr = append(arr, s)
}
}
arr := lo.FilterMap(v, func(item any, _ int) (string, bool) {
value, ok := item.(string)
return value, ok
})
if len(arr) > 0 {
chatReq.Options["stop"] = arr
}
}
}

if len(r.Tools) > 0 {
tools := make([]OllamaTool, 0, len(r.Tools))
for _, t := range r.Tools {
tools = append(tools, OllamaTool{Type: "function", Function: OllamaToolFunction{Name: t.Function.Name, Description: t.Function.Description, Parameters: t.Function.Parameters}})
}
chatReq.Tools = tools
chatReq.Tools = lo.Map(r.Tools, func(tool dto.ToolCallRequest, _ int) OllamaTool {
return OllamaTool{
Type: "function",
Function: OllamaToolFunction{
Name: tool.Function.Name,
Description: tool.Function.Description,
Parameters: tool.Function.Parameters,
},
}
})
}

chatReq.Messages = make([]OllamaChatMessage, 0, len(r.Messages))
toolNamesByCallID := make(map[string]string)
for _, m := range r.Messages {
var textBuilder strings.Builder
var images []string
Expand Down Expand Up @@ -117,8 +163,18 @@ func openAIChatToOllamaChat(c *gin.Context, r *dto.GeneralOpenAIRequest) (*Ollam
if len(images) > 0 {
cm.Images = images
}
if m.Role == "tool" && m.Name != nil {
cm.ToolName = *m.Name
if m.Role == "assistant" {
if reasoning, ok := lo.Coalesce(m.ReasoningContent, m.Reasoning); ok {
thinking, err := common.Marshal(*reasoning)
if err != nil {
return nil, fmt.Errorf("marshal ollama thinking: %w", err)
}
cm.Thinking = thinking
}
}
if m.Role == "tool" {
cm.ToolCallID = m.ToolCallId
cm.ToolName = lo.CoalesceOrEmpty(lo.FromPtr(m.Name), toolNamesByCallID[m.ToolCallId])
}
if m.ToolCalls != nil && len(m.ToolCalls) > 0 {
parsed := m.ParseToolCalls()
Expand All @@ -127,15 +183,18 @@ func openAIChatToOllamaChat(c *gin.Context, r *dto.GeneralOpenAIRequest) (*Ollam
for _, tc := range parsed {
var args interface{}
if tc.Function.Arguments != "" {
_ = json.Unmarshal([]byte(tc.Function.Arguments), &args)
_ = common.Unmarshal([]byte(tc.Function.Arguments), &args)
}
if args == nil {
args = map[string]any{}
}
oc := OllamaToolCall{}
oc := OllamaToolCall{ID: tc.ID}
oc.Function.Name = tc.Function.Name
oc.Function.Arguments = args
calls = append(calls, oc)
if tc.ID != "" {
toolNamesByCallID[tc.ID] = tc.Function.Name
}
}
Comment on lines 183 to 198

@coderabbitai coderabbitai Bot Aug 2, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟡 Minor | ⚡ Quick win

Malformed tool-call arguments fail silently instead of surfacing an error.

common.Unmarshal errors on line 186 are discarded with _ =. If tc.Function.Arguments is malformed JSON, args silently falls back to map[string]any{}, and the tool call is forwarded to Ollama with empty arguments instead of the original (possibly still partially useful) payload or a visible error. This can silently drop tool-call context in multi-turn flows, which is the exact failure mode this PR aims to fix for reasoning content.

Log the unmarshal error so malformed payloads are diagnosable, instead of dropping them without a trace.

🐛 Proposed fix to surface unmarshal failures 
 					var args interface{}
 					if tc.Function.Arguments != "" {
-						_ = common.Unmarshal([]byte(tc.Function.Arguments), &args)
+						if err := common.Unmarshal([]byte(tc.Function.Arguments), &args); err != nil {
+							logger.LogError(c, "ollama tool call arguments decode error: "+err.Error())
+						}
 					}
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
for _, tc := range parsed {
var args interface{}
if tc.Function.Arguments != "" {
_ = json.Unmarshal([]byte(tc.Function.Arguments), &args)
_ = common.Unmarshal([]byte(tc.Function.Arguments), &args)
}
if args == nil {
args = map[string]any{}
}
oc := OllamaToolCall{}
oc := OllamaToolCall{ID: tc.ID}
oc.Function.Name = tc.Function.Name
oc.Function.Arguments = args
calls = append(calls, oc)
if tc.ID != "" {
toolNamesByCallID[tc.ID] = tc.Function.Name
}
}
for _, tc := range parsed {
var args interface{}
if tc.Function.Arguments != "" {
if err := common.Unmarshal([]byte(tc.Function.Arguments), &args); err != nil {
logger.LogError(c, "ollama tool call arguments decode error: "+err.Error())
}
}
if args == nil {
args = map[string]any{}
}
oc := OllamaToolCall{ID: tc.ID}
oc.Function.Name = tc.Function.Name
oc.Function.Arguments = args
calls = append(calls, oc)
if tc.ID != "" {
toolNamesByCallID[tc.ID] = tc.Function.Name
}
}
🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@relay/channel/ollama/relay-ollama.go` around lines 183 - 198, In the
tool-call parsing loop around common.Unmarshal, stop discarding JSON unmarshal
errors and log the error with sufficient context, including the affected tool
call or function name. Preserve the existing fallback behavior for nil arguments
while ensuring malformed tc.Function.Arguments produces a visible diagnostic.

cm.ToolCalls = calls
}
Expand Down Expand Up @@ -175,15 +234,11 @@ func openAIToGenerate(c *gin.Context, r *dto.GeneralOpenAIRequest) (*OllamaGener
gen.Suffix = s
}
}
if r.ResponseFormat != nil {
if r.ResponseFormat.Type == "json" {
gen.Format = "json"
} else if r.ResponseFormat.Type == "json_schema" {
var schema any
_ = json.Unmarshal(r.ResponseFormat.JsonSchema, &schema)
gen.Format = schema
}
format, err := toOllamaResponseFormat(r.ResponseFormat)
if err != nil {
return nil, err
}
gen.Format = format
if r.Temperature != nil {
gen.Options["temperature"] = r.Temperature
}
Expand Down Expand Up @@ -212,12 +267,10 @@ func openAIToGenerate(c *gin.Context, r *dto.GeneralOpenAIRequest) (*OllamaGener
case []string:
gen.Options["stop"] = v
case []any:
arr := make([]string, 0, len(v))
for _, i := range v {
if s, ok := i.(string); ok {
arr = append(arr, s)
}
}
arr := lo.FilterMap(v, func(item any, _ int) (string, bool) {
value, ok := item.(string)
return value, ok
})
if len(arr) > 0 {
gen.Options["stop"] = arr
}
Expand Down Expand Up @@ -510,7 +563,7 @@ func FetchOllamaVersion(baseURL, apiKey string) (string, error) {
Version string `json:"version"`
}

if err := json.Unmarshal(body, &versionResp); err != nil {
if err := common.Unmarshal(body, &versionResp); err != nil {
return "", fmt.Errorf("解析响应失败: %v", err)
}

Expand Down
6 changes: 5 additions & 1 deletion relay/channel/ollama/stream.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,12 @@ func ollamaToolCallsToOpenAI(toolCalls []OllamaToolCall, startIndex int, include
argBytes = []byte("{}")
}
}
toolCallID := tc.ID
if toolCallID == "" {
toolCallID = fmt.Sprintf("call_%d", startIndex)
}
tr := dto.ToolCallResponse{
ID: fmt.Sprintf("call_%d", startIndex),
ID: toolCallID,
Type: "function",
Function: dto.FunctionResponse{
Name: tc.Function.Name,
Expand Down
13 changes: 8 additions & 5 deletions relay/channel/ollama/stream_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,14 @@ func TestOllamaChatHandlerNonStreamToolCalls(t *testing.T) {
gin.SetMode(gin.TestMode)

tests := []struct {
name string
raw string
name string
raw string
wantID string
}{
{
name: "compact json per-line parse path",
raw: `{"model":"llama3.1","created_at":"2026-05-27T12:00:00Z","message":{"role":"assistant","content":"","tool_calls":[{"function":{"name":"get_weather","arguments":{"city":"Paris","days":0}}}]},"done":true,"done_reason":"stop","prompt_eval_count":5,"eval_count":7}`,
name: "compact json per-line parse path",
raw: `{"model":"llama3.1","created_at":"2026-05-27T12:00:00Z","message":{"role":"assistant","content":"","tool_calls":[{"id":"call_upstream","function":{"name":"get_weather","arguments":{"city":"Paris","days":0}}}]},"done":true,"done_reason":"stop","prompt_eval_count":5,"eval_count":7}`,
wantID: "call_upstream",
},
{
name: "pretty json fallback parse path",
Expand All @@ -53,6 +55,7 @@ func TestOllamaChatHandlerNonStreamToolCalls(t *testing.T) {
"prompt_eval_count": 5,
"eval_count": 7
}`,
wantID: "call_0",
},
}

Expand Down Expand Up @@ -82,7 +85,7 @@ func TestOllamaChatHandlerNonStreamToolCalls(t *testing.T) {
var toolCalls []dto.ToolCallResponse
require.NoError(t, common.Unmarshal(out.Choices[0].Message.ToolCalls, &toolCalls))
require.Len(t, toolCalls, 1)
assert.NotEmpty(t, toolCalls[0].ID)
assert.Equal(t, tt.wantID, toolCalls[0].ID)
assert.Equal(t, "function", toolCalls[0].Type)
assert.Equal(t, "get_weather", toolCalls[0].Function.Name)
assert.Nil(t, toolCalls[0].Index)
Expand Down
Loading