mikhail/llama.cpp-mtp-turboquant
1
0
Fork 0
Code Issues Pull Requests Actions 30 Packages Projects Releases Wiki Activity

server: add --media-path for local media files (#17697)

Browse Source 
* server: add --media-path for local media files

* remove unused fn
This commit is contained in:
Xuan-Son Nguyen
2025-12-02 22:49:20 +01:00
committed by GitHub
parent a96283adc4
commit 13628d8bdb
9 changed files with 133 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/server/server-common.cpp
+64 -35
View File
@@ -11,6 +11,7 @@
#include <random>
#include <sstream>
#include <fstream>
json format_error_response(const std::string & message, const enum error_type type) {
std::string type_str;
@@ -774,6 +775,65 @@ json oaicompat_completion_params_parse(const json & body) {
return llama_params;
}
// media_path always end with '/', see arg.cpp
static void handle_media(
std::vector<raw_buffer> & out_files,
json & media_obj,
const std::string & media_path) {
std::string url = json_value(media_obj, "url", std::string());
if (string_starts_with(url, "http")) {
// download remote image
// TODO @ngxson : maybe make these params configurable
common_remote_params params;
params.headers.push_back("User-Agent: llama.cpp/" + build_info);
params.max_size = 1024 * 1024 * 10; // 10MB
params.timeout = 10; // seconds
SRV_INF("downloading image from '%s'\n", url.c_str());
auto res = common_remote_get_content(url, params);
if (200 <= res.first && res.first < 300) {
SRV_INF("downloaded %ld bytes\n", res.second.size());
raw_buffer data;
data.insert(data.end(), res.second.begin(), res.second.end());
out_files.push_back(data);
} else {
throw std::runtime_error("Failed to download image");
}
} else if (string_starts_with(url, "file://")) {
if (media_path.empty()) {
throw std::invalid_argument("file:// URLs are not allowed unless --media-path is specified");
}
// load local image file
std::string file_path = url.substr(7); // remove "file://"
raw_buffer data;
if (!fs_validate_filename(file_path, true)) {
throw std::invalid_argument("file path is not allowed: " + file_path);
}
SRV_INF("loading image from local file '%s'\n", (media_path + file_path).c_str());
std::ifstream file(media_path + file_path, std::ios::binary);
if (!file) {
throw std::invalid_argument("file does not exist or cannot be opened: " + file_path);
}
data.assign((std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>());
out_files.push_back(data);
} else {
// try to decode base64 image
std::vector<std::string> parts = string_split<std::string>(url, /*separator*/ ',');
if (parts.size() != 2) {
throw std::runtime_error("Invalid url value");
} else if (!string_starts_with(parts[0], "data:image/")) {
throw std::runtime_error("Invalid url format: " + parts[0]);
} else if (!string_ends_with(parts[0], "base64")) {
throw std::runtime_error("url must be base64 encoded");
} else {
auto base64_data = parts[1];
auto decoded_data = base64_decode(base64_data);
out_files.push_back(decoded_data);
}
}
}
// used by /chat/completions endpoint
json oaicompat_chat_params_parse(
json & body, /* openai api json semantics */
@@ -860,41 +920,8 @@ json oaicompat_chat_params_parse(
throw std::runtime_error("image input is not supported - hint: if this is unexpected, you may need to provide the mmproj");
}
json image_url = json_value(p, "image_url", json::object());
std::string url = json_value(image_url, "url", std::string());
if (string_starts_with(url, "http")) {
// download remote image
// TODO @ngxson : maybe make these params configurable
common_remote_params params;
params.headers.push_back("User-Agent: llama.cpp/" + build_info);
params.max_size = 1024 * 1024 * 10; // 10MB
params.timeout = 10; // seconds
SRV_INF("downloading image from '%s'\n", url.c_str());
auto res = common_remote_get_content(url, params);
if (200 <= res.first && res.first < 300) {
SRV_INF("downloaded %ld bytes\n", res.second.size());
raw_buffer data;
data.insert(data.end(), res.second.begin(), res.second.end());
out_files.push_back(data);
} else {
throw std::runtime_error("Failed to download image");
}
} else {
// try to decode base64 image
std::vector<std::string> parts = string_split<std::string>(url, /*separator*/ ',');
if (parts.size() != 2) {
throw std::invalid_argument("Invalid image_url.url value");
} else if (!string_starts_with(parts[0], "data:image/")) {
throw std::invalid_argument("Invalid image_url.url format: " + parts[0]);
} else if (!string_ends_with(parts[0], "base64")) {
throw std::invalid_argument("image_url.url must be base64 encoded");
} else {
auto base64_data = parts[1];
auto decoded_data = base64_decode(base64_data);
out_files.push_back(decoded_data);
}
}
json image_url = json_value(p, "image_url", json::object());
handle_media(out_files, image_url, opt.media_path);
// replace this chunk with a marker
p["type"] = "text";
@@ -916,6 +943,8 @@ json oaicompat_chat_params_parse(
auto decoded_data = base64_decode(data); // expected to be base64 encoded
out_files.push_back(decoded_data);
// TODO: add audio_url support by reusing handle_media()
// replace this chunk with a marker
p["type"] = "text";
p["text"] = mtmd_default_marker();
tools/server/server-common.h
+1
View File
@@ -284,6 +284,7 @@ struct oaicompat_parser_options {
bool allow_image;
bool allow_audio;
bool enable_thinking = true;
std::string media_path;
};
// used by /chat/completions endpoint
tools/server/server-context.cpp
+1
View File
@@ -788,6 +788,7 @@ struct server_context_impl {
/* allow_image */ mctx ? mtmd_support_vision(mctx) : false,
/* allow_audio */ mctx ? mtmd_support_audio (mctx) : false,
/* enable_thinking */ enable_thinking,
/* media_path */ params_base.media_path,
};
// print sample chat example to make it clear which template is used
tools/server/server.cpp
+2
View File
@@ -38,9 +38,11 @@ static server_http_context::handler_t ex_wrapper(server_http_context::handler_t
try {
return func(req);
} catch (const std::invalid_argument & e) {
// treat invalid_argument as invalid request (400)
error = ERROR_TYPE_INVALID_REQUEST;
message = e.what();
} catch (const std::exception & e) {
// treat other exceptions as server error (500)
error = ERROR_TYPE_SERVER;
message = e.what();
} catch (...) {
tools/server/tests/unit/test_security.py
+31
View File
@@ -94,3 +94,34 @@ def test_cors_options(origin: str, cors_header: str, cors_header_value: str):
assert res.status_code == 200
assert cors_header in res.headers
assert res.headers[cors_header] == cors_header_value
@pytest.mark.parametrize(
"media_path, image_url, success",
[
(None, "file://mtmd/test-1.jpeg", False), # disabled media path, should fail
("../../../tools", "file://mtmd/test-1.jpeg", True),
("../../../tools", "file:////mtmd//test-1.jpeg", True), # should be the same file as above
("../../../tools", "file://mtmd/notfound.jpeg", False), # non-existent file
("../../../tools", "file://../mtmd/test-1.jpeg", False), # no directory traversal
]
)
def test_local_media_file(media_path, image_url, success,):
server = ServerPreset.tinygemma3()
server.media_path = media_path
server.start()
res = server.make_request("POST", "/chat/completions", data={
"max_tokens": 1,
"messages": [
{"role": "user", "content": [
{"type": "text", "text": "test"},
{"type": "image_url", "image_url": {
"url": image_url,
}},
]},
],
})
if success:
assert res.status_code == 200
else:
assert res.status_code == 400
tools/server/tests/utils.py
+3
View File
@@ -95,6 +95,7 @@ class ServerProcess:
chat_template_file: str | None = None
server_path: str | None = None
mmproj_url: str | None = None
media_path: str | None = None
# session variables
process: subprocess.Popen | None = None
@@ -217,6 +218,8 @@ class ServerProcess:
server_args.extend(["--chat-template-file", self.chat_template_file])
if self.mmproj_url:
server_args.extend(["--mmproj-url", self.mmproj_url])
if self.media_path:
server_args.extend(["--media-path", self.media_path])
args = [str(arg) for arg in [server_path, *server_args]]
print(f"tests: starting server with: {' '.join(args)}")