diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index fa4525389976..158707cf50b3 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -2043,6 +2043,90 @@ static void test_convert_responses_to_chatcmpl() { assert_equals(false, result.contains("tools")); } + + // Test namespaced Responses tools are flattened to ns_{namespace}___{tool} + { + json input = json::parse(R"({ + "input": "Hello", + "model": "test-model", + "tools": [ + { + "type": "namespace", + "name": "browser", + "tools": [ + { + "type": "function", + "name": "click", + "description": "Click an element", + "parameters": {"type": "object", "properties": {}} + }, + { + "type": "web_search" + }, + { + "type": "function", + "name": "navigate", + "description": "Navigate to a URL", + "parameters": {"type": "object", "properties": {}} + } + ] + }, + { + "type": "function", + "name": "get_weather", + "description": "Get weather for a location", + "parameters": {"type": "object", "properties": {}} + } + ] + })"); + + json result = server_chat_convert_responses_to_chatcmpl(input); + + assert_equals(true, result.contains("tools")); + assert_equals((size_t)3, result.at("tools").size()); + assert_equals(std::string("ns_browser___click"), result.at("tools")[0].at("function").at("name").get()); + assert_equals(std::string("ns_browser___navigate"), result.at("tools")[1].at("function").at("name").get()); + assert_equals(std::string("get_weather"), result.at("tools")[2].at("function").at("name").get()); + } + + // Test a function_call input item with a namespace flattens the same way + { + json input = json::parse(R"({ + "input": [ + { + "type": "function_call", + "namespace": "browser", + "name": "click", + "arguments": "{}", + "call_id": "call_1" + } + ], + "model": "test-model" + })"); + + json result = server_chat_convert_responses_to_chatcmpl(input); + + const auto & calls = result.at("messages")[0].at("tool_calls"); + assert_equals(std::string("ns_browser___click"), calls[0].at("function").at("name").get()); + } + + // Test tool name flatten/split round-trip + { + assert_equals(std::string("read_file"), server_chat_responses_tool_name("", "read_file")); + assert_equals(std::string("ns_muse___read_file"), server_chat_responses_tool_name("muse", "read_file")); + + const auto plain = server_chat_responses_split_tool_name("read_file"); + assert_equals(std::string(""), plain.first); + assert_equals(std::string("read_file"), plain.second); + + const auto flat = server_chat_responses_split_tool_name("ns_muse___read_file"); + assert_equals(std::string("muse"), flat.first); + assert_equals(std::string("read_file"), flat.second); + + const auto no_sep = server_chat_responses_split_tool_name("ns_muse"); + assert_equals(std::string(""), no_sep.first); + assert_equals(std::string("ns_muse"), no_sep.second); + } } // Shared LFM2 parser cases - all variants use one output format and parser diff --git a/tools/server/server-chat.cpp b/tools/server/server-chat.cpp index a7431800b40b..21a7b00097ef 100644 --- a/tools/server/server-chat.cpp +++ b/tools/server/server-chat.cpp @@ -3,28 +3,78 @@ #include +// Flattened namespaced tools as "ns_{namespace}___{tool_name}". +static constexpr const char * RESPONSES_NS_PREFIX = "ns_"; +static constexpr const char * RESPONSES_NS_SEP = "___"; + +std::string server_chat_responses_tool_name(const std::string & ns, const std::string & name) { + if (ns.empty()) { + return name; + } + return std::string(RESPONSES_NS_PREFIX) + ns + RESPONSES_NS_SEP + name; +} + +std::pair server_chat_responses_split_tool_name(const std::string & name) { + const std::string prefix = RESPONSES_NS_PREFIX; + const std::string sep = RESPONSES_NS_SEP; + if (name.compare(0, prefix.size(), prefix) != 0) { + return { std::string(), name }; + } + const size_t pos = name.find(sep, prefix.size()); + if (pos == std::string::npos) { + return { std::string(), name }; + } + return { name.substr(prefix.size(), pos - prefix.size()), name.substr(pos + sep.size()) }; +} + json server_chat_convert_responses_tools(const json & tools) { if (!tools.is_array()) { throw std::invalid_argument("'tools' must be an array of objects"); } json chatcmpl_tools = json::array(); - for (json resp_tool : tools) { - json chatcmpl_tool; + auto add_tool = [&](json fn_tool, const std::string & ns) { + fn_tool.erase("type"); + if (!fn_tool.contains("strict")) { + fn_tool["strict"] = true; + } + if (!ns.empty()) { + if (!fn_tool.contains("name") || !fn_tool.at("name").is_string()) { + SRV_WRN("namespaced tool in namespace '%s' missing 'name' skipped\n", ns.c_str()); + return; + } + fn_tool["name"] = server_chat_responses_tool_name(ns, fn_tool.at("name").get()); + } + chatcmpl_tools.push_back(json { + {"type", "function"}, + {"function", fn_tool}, + }); + }; + + for (json resp_tool : tools) { const std::string type = json_value(resp_tool, "type", std::string()); - if (type != "function") { + if (type == "function") { + add_tool(std::move(resp_tool), ""); + } else if (type == "namespace") { + const std::string ns = json_value(resp_tool, "name", std::string()); + if (ns.empty() || !resp_tool.contains("tools") || !resp_tool.at("tools").is_array()) { + // Namespace without a name or tools array carries no functions; + // skip it like other non-function Responses tools. + SRV_WRN("unsupported Responses tool type 'namespace' (namespace '%s') without 'name' or 'tools' array skipped\n", ns.c_str()); + continue; + } + for (json inner_tool : resp_tool.at("tools")) { + const std::string inner_type = json_value(inner_tool, "type", std::string()); + if (inner_type != "function") { + SRV_WRN("unsupported Responses tool type '%s' in namespace '%s' skipped\n", inner_type.c_str(), ns.c_str()); + continue; + } + add_tool(std::move(inner_tool), ns); + } + } else { // Non-function Responses tools have no Chat Completions equivalent. SRV_WRN("unsupported Responses tool type '%s' skipped\n", type.c_str()); - continue; - } - resp_tool.erase("type"); - chatcmpl_tool["type"] = "function"; - - if (!resp_tool.contains("strict")) { - resp_tool["strict"] = true; } - chatcmpl_tool["function"] = resp_tool; - chatcmpl_tools.push_back(chatcmpl_tool); } return chatcmpl_tools; } @@ -37,6 +87,13 @@ json server_chat_convert_responses_to_chatcmpl(const json & response_body) { throw std::invalid_argument("llama.cpp does not support 'previous_response_id'."); } + static auto exists_and_is_array = [](const json & j, const char * key) -> bool { + return j.contains(key) && j.at(key).is_array(); + }; + static auto exists_and_is_string = [](const json & j, const char * key) -> bool { + return j.contains(key) && j.at(key).is_string(); + }; + const json input_value = response_body.at("input"); json chatcmpl_body = response_body; chatcmpl_body.erase("input"); @@ -59,13 +116,6 @@ json server_chat_convert_responses_to_chatcmpl(const json & response_body) { } else if (input_value.is_array()) { // #responses_create-input-input_item_list - static auto exists_and_is_array = [](const json & j, const char * key) -> bool { - return j.contains(key) && j.at(key).is_array(); - }; - static auto exists_and_is_string = [](const json & j, const char * key) -> bool { - return j.contains(key) && j.at(key).is_string(); - }; - for (json item : input_value) { bool merge_prev = !chatcmpl_messages.empty() && chatcmpl_messages.back().value("role", "") == "assistant"; @@ -196,7 +246,7 @@ json server_chat_convert_responses_to_chatcmpl(const json & response_body) { json tool_call = { {"function", json { {"arguments", item.at("arguments")}, - {"name", item.at("name")}, + {"name", server_chat_responses_tool_name(item.value("namespace", ""), item.at("name").get())}, }}, {"id", item.at("call_id")}, {"type", "function"}, diff --git a/tools/server/server-chat.h b/tools/server/server-chat.h index b4b6f1ec6582..581364fde109 100644 --- a/tools/server/server-chat.h +++ b/tools/server/server-chat.h @@ -14,6 +14,12 @@ json server_chat_convert_responses_to_chatcmpl(const json & body); // Convert OpenAI Responses API tools to OpenAI Chat Completions API tools, non-function tools are skipped json server_chat_convert_responses_tools(const json & tools); +// Flatten a Responses-API namespaced tool name. Returns name unchanged when ns is empty. +std::string server_chat_responses_tool_name(const std::string & ns, const std::string & name); + +// Split flattened tool name into (namespace, tool_name) +std::pair server_chat_responses_split_tool_name(const std::string & name); + // Convert Anthropic Messages API format to OpenAI Chat Completions API format json server_chat_convert_anthropic_to_oai(const json & body); diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 48f1e93a21b3..604ba0a87f10 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -5459,6 +5459,24 @@ std::unique_ptr server_routes::handle_completions_impl( dynamic_cast (first_result.get()) != nullptr ); + // Responses SSE events carry a stream-wide sequence number; assign + // them here, where every streamed chunk of this response passes. + auto resp_seq = std::make_shared(0); + auto number_resp_events = [resp_seq](json & events) { + auto number_one = [resp_seq](json & item) { + if (item.is_object() && item.contains("data") && item.at("data").is_object()) { + item["data"]["sequence_number"] = ++(*resp_seq); + } + }; + if (events.is_array()) { + for (auto & item : events) { + number_one(item); + } + } else { + number_one(events); + } + }; + // next responses are streamed // to be sent immediately json first_result_json = first_result->to_json(); @@ -5467,13 +5485,14 @@ std::unique_ptr server_routes::handle_completions_impl( } else if (res_type == TASK_RESPONSE_TYPE_ANTHROPIC) { res->data = format_anthropic_sse(first_result_json); } else if (res_type == TASK_RESPONSE_TYPE_OAI_RESP) { + number_resp_events(first_result_json); res->data = format_oai_resp_sse(first_result_json); } else { res->data = format_oai_sse(first_result_json); } res->status = 200; res->content_type = "text/event-stream"; - res->set_next([res_this = res.get(), res_type, sse_ping_interval](std::string & output) -> bool { + res->set_next([res_this = res.get(), res_type, sse_ping_interval, resp_seq, number_resp_events](std::string & output) mutable -> bool { static auto format_error = [](task_response_type res_type, const json & res_json) { if (res_type == TASK_RESPONSE_TYPE_ANTHROPIC) { return format_anthropic_sse({ @@ -5562,6 +5581,7 @@ std::unique_ptr server_routes::handle_completions_impl( if (res_type == TASK_RESPONSE_TYPE_ANTHROPIC) { output = format_anthropic_sse(res_json); } else if (res_type == TASK_RESPONSE_TYPE_OAI_RESP) { + number_resp_events(res_json); output = format_oai_resp_sse(res_json); } else { output = format_oai_sse(res_json); diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index 1a0d434a1f31..d31ec84b48d4 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -558,14 +558,19 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp() { } for (const common_chat_tool_call & tool_call : oaicompat_msg.tool_calls) { - output.push_back(json { + auto [ns, tool_name] = server_chat_responses_split_tool_name(tool_call.name); + json fc_item = { {"id", "fc_" + tool_call.id}, {"type", "function_call"}, {"status", "completed"}, {"arguments", tool_call.arguments}, {"call_id", "call_" + tool_call.id}, - {"name", tool_call.name}, - }); + {"name", tool_name}, + }; + if (!ns.empty()) { + fc_item["namespace"] = ns; + } + output.push_back(std::move(fc_item)); } std::time_t t = std::time(0); @@ -668,14 +673,18 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp_stream() { for (const common_chat_tool_call & tool_call : oaicompat_msg.tool_calls) { const size_t item_output_index = output_index++; - const json output_item = { + auto [ns, tool_name] = server_chat_responses_split_tool_name(tool_call.name); + json output_item = { {"id", "fc_" + tool_call.id}, {"type", "function_call"}, {"status", "completed"}, {"arguments", tool_call.arguments}, {"call_id", "call_" + tool_call.id}, - {"name", tool_call.name} + {"name", tool_name} }; + if (!ns.empty()) { + output_item["namespace"] = ns; + } server_sent_events.push_back(json { {"event", "response.output_item.done"}, {"data", json { @@ -1282,19 +1291,28 @@ json server_task_result_cmpl_partial::to_json_oaicompat_resp() { } if (!diff.tool_call_delta.name.empty()) { + auto [ns, tool_name] = server_chat_responses_split_tool_name(diff.tool_call_delta.name); + json item = { + {"arguments", ""}, + {"call_id", "fc_" + diff.tool_call_delta.id}, + {"name", tool_name}, + {"type", "function_call"}, + {"status", "in_progress"}, + }; + if (!ns.empty()) { + item["namespace"] = ns; + } events.push_back(json { {"event", "response.output_item.added"}, {"data", json { {"type", "response.output_item.added"}, {"output_index", (anthropic_has_reasoning ? 1 : 0) + (text_block_started ? 1 : 0) + diff.tool_call_index}, - {"item", json { - {"id", "fc_" + diff.tool_call_delta.id}, - {"arguments", ""}, - {"call_id", "call_" + diff.tool_call_delta.id}, - {"name", diff.tool_call_delta.name}, - {"type", "function_call"}, - {"status", "in_progress"}, - }}, + {"item", [&]() { + json it = item; + it["id"] = "fc_" + diff.tool_call_delta.id; + it["call_id"] = "call_" + diff.tool_call_delta.id; + return it; + }()}, }}, }); oai_resp_fc_id = diff.tool_call_delta.id;