Skip to content

server : accept data: URLs for input_video and input_audio #27735

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ngxson merged 3 commits into from
Sep 2, 2026
Merged

server : accept data: URLs for input_video and input_audio #27735

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8f6dbc0
server : accept data: URLs for input_video and input_audio
geckguy Aug 26, 2026
7c2ffd1
server : simplify handle_media and drop unused accept_base64_uri flag
geckguy Aug 27, 2026
4d0cf86
server : update comment and add unit test for invalid data URI MIME
geckguy Sep 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 12 additions & 11 deletions tools/server/server-common.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1057,8 +1057,7 @@ json oaicompat_completion_params_parse(const json & body) {
static void handle_media(
std::vector<raw_buffer> & out_files,
const std::string & url,
const std::string & media_path,
bool accept_base64_uri) {
const std::string & media_path) {
if (!media_path.empty()) {
// should already be enforced by arg.cpp, but checking just in case
GGML_ASSERT(media_path.back() == DIRECTORY_SEPARATOR);
Expand Down Expand Up @@ -1099,15 +1098,17 @@ static void handle_media(
data.assign((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
out_files.push_back(data);

} else if (accept_base64_uri && string_starts_with(url, "data:")) {
// try to decode base64 image
} else if (string_starts_with(url, "data:")) {
// try to decode base64 image, video, or audio
std::vector<std::string> parts = string_split<std::string>(url, /*separator*/ ',');
if (parts.size() != 2) {
throw std::runtime_error("Invalid uri-encoded base64 value");
} else if (!string_starts_with(parts[0], "data:image/")) {
throw std::runtime_error("Invalid uri format: " + parts[0]);
throw std::invalid_argument("Invalid uri-encoded base64 value");
} else if (!string_starts_with(parts[0], "data:image/")
&& !string_starts_with(parts[0], "data:video/")
&& !string_starts_with(parts[0], "data:audio/")) {
throw std::invalid_argument("Invalid uri format: " + parts[0]);
} else if (!string_ends_with(parts[0], "base64")) {
throw std::runtime_error("uri must be base64 encoded");
throw std::invalid_argument("uri must be base64 encoded");
} else {
auto base64_data = parts[1];
auto decoded_data = base64_decode(base64_data);
Expand Down Expand Up @@ -1214,7 +1215,7 @@ json oaicompat_chat_params_parse(

json image_url = json_value(p, "image_url", json::object());
std::string url = json_value(image_url, "url", std::string());
handle_media(out_files, url, opt.media_path, true);
handle_media(out_files, url, opt.media_path);

p["type"] = "media_marker";
p["text"] = get_media_marker();
Expand All @@ -1229,7 +1230,7 @@ json oaicompat_chat_params_parse(
json input_audio = json_value(p, "input_audio", json::object());
std::string url = json_value(input_audio, "data",
json_value(input_audio, "url", std::string()));
handle_media(out_files, url, opt.media_path, false);
handle_media(out_files, url, opt.media_path);

p["type"] = "media_marker";
p["text"] = get_media_marker();
Expand All @@ -1243,7 +1244,7 @@ json oaicompat_chat_params_parse(
json input_video = json_value(p, "input_video", json::object());
std::string url = json_value(input_video, "data",
json_value(input_video, "url", std::string()));
handle_media(out_files, url, opt.media_path, false);
handle_media(out_files, url, opt.media_path);

p["type"] = "media_marker";
p["text"] = get_media_marker();
Expand Down
1 change: 1 addition & 0 deletions tools/server/tests/unit/test_vision_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ def test_v1_models_supports_multimodal_capability():
("What is this:\n", "malformed", False, None),
("What is this:\n", "https://google.com/404", False, None), # non-existent image
("What is this:\n", "https://ggml.ai", False, None), # non-image data
("What is this:\n", "data:text/html;base64,aGVsbG8=", False, None), # unsupported data uri mime
# TODO @ngxson : test with multiple images, no images and with audio
]
)
Expand Down
Loading