Skip to content

server : implement namespaced tools in responses API #23235

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
aldehir wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

server : implement namespaced tools in responses API #23235

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 69 additions & 20 deletions tools/server/server-chat.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,30 @@

#include <sstream>

// Flattened namespaced tools as "ns_{namespace}___{tool_name}".
static constexpr const char * RESPONSES_NS_PREFIX = "ns_";
static constexpr const char * RESPONSES_NS_SEP = "___";

std::string server_chat_responses_tool_name(const std::string & ns, const std::string & name) {
if (ns.empty()) {
return name;
}
return std::string(RESPONSES_NS_PREFIX) + ns + RESPONSES_NS_SEP + name;
}

std::pair<std::string, std::string> server_chat_responses_split_tool_name(const std::string & name) {
const std::string prefix = RESPONSES_NS_PREFIX;
const std::string sep = RESPONSES_NS_SEP;
if (name.compare(0, prefix.size(), prefix) != 0) {
return { std::string(), name };
}
const size_t pos = name.find(sep, prefix.size());
if (pos == std::string::npos) {
return { std::string(), name };
}
return { name.substr(prefix.size(), pos - prefix.size()), name.substr(pos + sep.size()) };
}

json server_chat_convert_responses_to_chatcmpl(const json & response_body) {
if (!response_body.contains("input")) {
throw std::invalid_argument("'input' is required");
Expand All @@ -11,6 +35,13 @@ json server_chat_convert_responses_to_chatcmpl(const json & response_body) {
throw std::invalid_argument("llama.cpp does not support 'previous_response_id'.");
}

static auto exists_and_is_array = [](const json & j, const char * key) -> bool {
return j.contains(key) && j.at(key).is_array();
};
static auto exists_and_is_string = [](const json & j, const char * key) -> bool {
return j.contains(key) && j.at(key).is_string();
};

const json input_value = response_body.at("input");
json chatcmpl_body = response_body;
chatcmpl_body.erase("input");
Expand All @@ -33,13 +64,6 @@ json server_chat_convert_responses_to_chatcmpl(const json & response_body) {
} else if (input_value.is_array()) {
// #responses_create-input-input_item_list

static auto exists_and_is_array = [](const json & j, const char * key) -> bool {
return j.contains(key) && j.at(key).is_array();
};
static auto exists_and_is_string = [](const json & j, const char * key) -> bool {
return j.contains(key) && j.at(key).is_string();
};

for (json item : input_value) {
bool merge_prev = !chatcmpl_messages.empty() && chatcmpl_messages.back().value("role", "") == "assistant";

Expand Down Expand Up @@ -170,7 +194,7 @@ json server_chat_convert_responses_to_chatcmpl(const json & response_body) {
json tool_call = {
{"function", json {
{"arguments", item.at("arguments")},
{"name", item.at("name")},
{"name", server_chat_responses_tool_name(item.value("namespace", ""), item.at("name").get<std::string>())},
}},
{"id", item.at("call_id")},
{"type", "function"},
Expand Down Expand Up @@ -254,23 +278,48 @@ json server_chat_convert_responses_to_chatcmpl(const json & response_body) {
throw std::invalid_argument("'tools' must be an array of objects");
}
std::vector<json> chatcmpl_tools;
for (json resp_tool : response_body.at("tools")) {
json chatcmpl_tool;

auto add_tool = [&](json fn_tool, const std::string & ns) {
fn_tool.erase("type");
if (!fn_tool.contains("strict")) {
fn_tool["strict"] = true;
}
if (!ns.empty()) {
if (!exists_and_is_string(fn_tool, "name")) {
throw std::invalid_argument("namespaced tool requires 'name'");
}
fn_tool["name"] = server_chat_responses_tool_name(ns, fn_tool.at("name").get<std::string>());
}
chatcmpl_tools.push_back(json {
{"type", "function"},
{"function", fn_tool},
});
};

for (json resp_tool : response_body.at("tools")) {
const std::string type = json_value(resp_tool, "type", std::string());
if (type != "function") {
if (type == "function") {
add_tool(std::move(resp_tool), "");
} else if (type == "namespace") {
const std::string ns = json_value(resp_tool, "name", std::string());
if (ns.empty()) {
throw std::invalid_argument("'namespace' tool requires 'name'");
}
if (!exists_and_is_array(resp_tool, "tools")) {
throw std::invalid_argument("'namespace' tool requires 'tools' array");
}
for (json inner_tool : resp_tool.at("tools")) {
const std::string inner_type = json_value(inner_tool, "type", std::string());
if (inner_type != "function") {
SRV_WRN("unsupported Responses tool type '%s' in namespace '%s' skipped\n", inner_type.c_str(), ns.c_str());
continue;
}
add_tool(std::move(inner_tool), ns);
}
} else {
// Non-function Responses tools have no Chat Completions equivalent.
SRV_WRN("unsupported Responses tool type '%s' skipped\n", type.c_str());
continue;
}
resp_tool.erase("type");
chatcmpl_tool["type"] = "function";

if (!resp_tool.contains("strict")) {
resp_tool["strict"] = true;
}
chatcmpl_tool["function"] = resp_tool;
chatcmpl_tools.push_back(chatcmpl_tool);
}
chatcmpl_body.erase("tools");
if (!chatcmpl_tools.empty()) {
Expand Down
6 changes: 6 additions & 0 deletions tools/server/server-chat.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,12 @@ using json = nlohmann::ordered_json;
// Convert OpenAI Responses API format to OpenAI Chat Completions API format
json server_chat_convert_responses_to_chatcmpl(const json & body);

// Flatten a Responses-API namespaced tool name. Returns name unchanged when ns is empty.
std::string server_chat_responses_tool_name(const std::string & ns, const std::string & name);

// Split flattened tool name into (namespace, tool_name)
std::pair<std::string, std::string> server_chat_responses_split_tool_name(const std::string & name);

// Convert Anthropic Messages API format to OpenAI Chat Completions API format
json server_chat_convert_anthropic_to_oai(const json & body);

Expand Down
38 changes: 26 additions & 12 deletions tools/server/server-task.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -973,13 +973,18 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp() {
}

for (const common_chat_tool_call & tool_call : oaicompat_msg.tool_calls) {
output.push_back(json {
auto [ns, tool_name] = server_chat_responses_split_tool_name(tool_call.name);
json fc_item = {
{"type", "function_call"},
{"status", "completed"},
{"arguments", tool_call.arguments},
{"call_id", "fc_" + tool_call.id},
{"name", tool_call.name},
});
{"name", tool_name},
};
if (!ns.empty()) {
fc_item["namespace"] = ns;
}
output.push_back(std::move(fc_item));
}

std::time_t t = std::time(0);
Expand Down Expand Up @@ -1072,13 +1077,17 @@ json server_task_result_cmpl_final::to_json_oaicompat_resp_stream() {
}

for (const common_chat_tool_call & tool_call : oaicompat_msg.tool_calls) {
const json output_item = {
auto [ns, tool_name] = server_chat_responses_split_tool_name(tool_call.name);
json output_item = {
{"type", "function_call"},
{"status", "completed"},
{"arguments", tool_call.arguments},
{"call_id", "fc_" + tool_call.id},
{"name", tool_call.name}
{"name", tool_name}
};
if (!ns.empty()) {
output_item["namespace"] = ns;
}
server_sent_events.push_back(json {
{"event", "response.output_item.done"},
{"data", json {
Expand Down Expand Up @@ -1649,17 +1658,22 @@ json server_task_result_cmpl_partial::to_json_oaicompat_resp() {
}

if (!diff.tool_call_delta.name.empty()) {
auto [ns, tool_name] = server_chat_responses_split_tool_name(diff.tool_call_delta.name);
json item = {
{"arguments", ""},
{"call_id", "fc_" + diff.tool_call_delta.id},
{"name", tool_name},
{"type", "function_call"},
{"status", "in_progress"},
};
if (!ns.empty()) {
item["namespace"] = ns;
}
events.push_back(json {
{"event", "response.output_item.added"},
{"data", json {
{"type", "response.output_item.added"},
{"item", json {
{"arguments", ""},
{"call_id", "fc_" + diff.tool_call_delta.id},
{"name", diff.tool_call_delta.name},
{"type", "function_call"},
{"status", "in_progress"},
}},
{"item", item},
}},
});
oai_resp_fc_id = diff.tool_call_delta.id;
Expand Down
Loading