mtmd, server, common: expose modalities to /v1/models (#22952)
* mtmd, server, common: expose modalities to /v1/models * fix build * rename to mtmd_caps
This commit is contained in:
Xuan-Son Nguyen
GitHub
+10
-2
@@ -994,7 +994,7 @@ struct clip_model_loader {
|
||||
bool has_audio = false;
|
||||
|
||||
// TODO @ngxson : we should not pass clip_ctx here, it should be clip_model
|
||||
clip_model_loader(const char * fname) : fname(fname) {
|
||||
clip_model_loader(const char * fname, bool skip_tensors = false) : fname(fname) {
|
||||
struct ggml_context * meta = nullptr;
|
||||
|
||||
struct gguf_init_params params = {
|
||||
@@ -1040,7 +1040,7 @@ struct clip_model_loader {
|
||||
}
|
||||
|
||||
// tensors
|
||||
{
|
||||
if (!skip_tensors) {
|
||||
for (int i = 0; i < n_tensors; ++i) {
|
||||
const char * name = gguf_get_tensor_name(ctx_gguf.get(), i);
|
||||
const size_t offset = gguf_get_tensor_offset(ctx_gguf.get(), i);
|
||||
@@ -2927,6 +2927,14 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params
|
||||
return {ctx_vision, ctx_audio};
|
||||
}
|
||||
|
||||
struct clip_cap clip_get_cap(const char * fname) {
|
||||
clip_cap res;
|
||||
clip_model_loader loader(fname, /* skip_tensors= */ true);
|
||||
res.has_vision = loader.has_vision;
|
||||
res.has_audio = loader.has_audio;
|
||||
return res;
|
||||
}
|
||||
|
||||
struct clip_image_size * clip_image_size_init() {
|
||||
struct clip_image_size * load_image_size = new struct clip_image_size();
|
||||
load_image_size->width = 448;
|
||||
|
||||
@@ -116,3 +116,9 @@ void clip_image_f32_batch_add_mel(struct clip_image_f32_batch * batch, int n_mel
|
||||
bool clip_has_vision_encoder(const struct clip_ctx * ctx);
|
||||
bool clip_has_audio_encoder(const struct clip_ctx * ctx);
|
||||
bool clip_has_whisper_encoder(const struct clip_ctx * ctx);
|
||||
|
||||
struct clip_cap {
|
||||
bool has_vision;
|
||||
bool has_audio;
|
||||
};
|
||||
struct clip_cap clip_get_cap(const char * fname);
|
||||
|
||||
@@ -1423,6 +1423,19 @@ void mtmd_log_set(ggml_log_callback log_callback, void * user_data) {
|
||||
g_logger_state.log_callback_user_data = user_data;
|
||||
}
|
||||
|
||||
struct mtmd_caps mtmd_get_cap_from_file(const char * fname) {
|
||||
try {
|
||||
auto tmp = clip_get_cap(fname);
|
||||
mtmd_caps cap;
|
||||
cap.inp_audio = tmp.has_audio;
|
||||
cap.inp_vision = tmp.has_vision;
|
||||
return cap;
|
||||
} catch (const std::exception & e) {
|
||||
LOG_ERR("%s: failed to get capabilities from file '%s': %s\n", __func__, fname, e.what());
|
||||
return mtmd_caps{ false, false };
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Debugging API (NOT intended for public use)
|
||||
//
|
||||
|
||||
@@ -244,6 +244,14 @@ MTMD_API float * mtmd_get_output_embd(mtmd_context * ctx);
|
||||
// If this is not called, or NULL is supplied, everything is output on stderr.
|
||||
MTMD_API void mtmd_log_set(ggml_log_callback log_callback, void * user_data);
|
||||
|
||||
// EXPERIMENTAL API to get mmproj's capabilities without initializing the full context
|
||||
// This is only intended to be used by llama-server, breaking changes is expected
|
||||
struct mtmd_caps {
|
||||
bool inp_vision;
|
||||
bool inp_audio;
|
||||
};
|
||||
MTMD_API struct mtmd_caps mtmd_get_cap_from_file(const char * mmproj_fname);
|
||||
|
||||
/////////////////////////////////////////
|
||||
|
||||
// test function, to be used in test-mtmd-c-api.c
|
||||
|
||||
@@ -161,6 +161,30 @@ void server_model_meta::update_args(common_preset_context & ctx_preset, std::str
|
||||
args = preset.to_args(bin_path);
|
||||
}
|
||||
|
||||
void server_model_meta::update_caps() {
|
||||
try {
|
||||
common_params params;
|
||||
preset.apply_to_params(params, {
|
||||
"LLAMA_ARG_MODEL",
|
||||
"LLAMA_ARG_MODEL_URL",
|
||||
"LLAMA_ARG_MMPROJ",
|
||||
"LLAMA_ARG_MMPROJ_URL",
|
||||
"LLAMA_ARG_HF_REPO",
|
||||
"LLAMA_ARG_HF_REPO_FILE",
|
||||
});
|
||||
params.offline = true; // avoid any unwanted network call during capability detection
|
||||
common_params_handle_models(params, LLAMA_EXAMPLE_SERVER);
|
||||
if (params.mmproj.path.empty()) {
|
||||
multimodal = { false, false };
|
||||
} else {
|
||||
multimodal = mtmd_get_cap_from_file(params.mmproj.path.c_str());
|
||||
}
|
||||
} catch (const std::exception & e) {
|
||||
LOG_WRN("failed to initialize common_params for multimodal capability detection: %s\n", e.what());
|
||||
multimodal = { false, false };
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// server_models
|
||||
//
|
||||
@@ -236,6 +260,7 @@ void server_models::add_model(server_model_meta && meta) {
|
||||
}
|
||||
|
||||
meta.update_args(ctx_preset, bin_path); // render args
|
||||
meta.update_caps();
|
||||
std::string name = meta.name;
|
||||
mapping[name] = instance_t{
|
||||
/* subproc */ std::make_shared<subprocess_s>(),
|
||||
@@ -346,8 +371,10 @@ void server_models::load_models() {
|
||||
/* status */ SERVER_MODEL_STATUS_UNLOADED,
|
||||
/* last_used */ 0,
|
||||
/* args */ std::vector<std::string>(),
|
||||
/* loaded_info */ {},
|
||||
/* exit_code */ 0,
|
||||
/* stop_timeout */ DEFAULT_STOP_TIMEOUT,
|
||||
/* multimodal */ mtmd_caps{false, false},
|
||||
};
|
||||
add_model(std::move(meta));
|
||||
}
|
||||
@@ -481,6 +508,7 @@ void server_models::load_models() {
|
||||
|
||||
inst.meta.exit_code = 0; // clear failed state so the model can be reloaded
|
||||
inst.meta.update_args(ctx_preset, bin_path);
|
||||
inst.meta.update_caps();
|
||||
}
|
||||
|
||||
// add models that are new in this reload
|
||||
@@ -496,8 +524,10 @@ void server_models::load_models() {
|
||||
/* status */ SERVER_MODEL_STATUS_UNLOADED,
|
||||
/* last_used */ 0,
|
||||
/* args */ std::vector<std::string>(),
|
||||
/* loaded_info */ {},
|
||||
/* exit_code */ 0,
|
||||
/* stop_timeout */ DEFAULT_STOP_TIMEOUT,
|
||||
/* multimodal */ mtmd_caps{false, false},
|
||||
};
|
||||
add_model(std::move(meta));
|
||||
newly_added.push_back(name);
|
||||
@@ -1206,14 +1236,28 @@ void server_models_routes::init_routes() {
|
||||
status["failed"] = true;
|
||||
}
|
||||
|
||||
// pi coding agent multimodal compatibility
|
||||
json input_modalities = json::array({"text"});
|
||||
if (meta.multimodal.inp_vision) {
|
||||
input_modalities.push_back("image");
|
||||
}
|
||||
if (meta.multimodal.inp_audio) {
|
||||
input_modalities.push_back("audio");
|
||||
}
|
||||
json architecture {
|
||||
{"input_modalities", input_modalities},
|
||||
{"output_modalities", json::array({"text"})},
|
||||
};
|
||||
|
||||
json model_info = json {
|
||||
{"id", meta.name},
|
||||
{"aliases", meta.aliases},
|
||||
{"tags", meta.tags},
|
||||
{"object", "model"}, // for OAI-compat
|
||||
{"owned_by", "llamacpp"}, // for OAI-compat
|
||||
{"created", t}, // for OAI-compat
|
||||
{"status", status},
|
||||
{"id", meta.name},
|
||||
{"aliases", meta.aliases},
|
||||
{"tags", meta.tags},
|
||||
{"object", "model"}, // for OAI-compat
|
||||
{"owned_by", "llamacpp"}, // for OAI-compat
|
||||
{"created", t}, // for OAI-compat
|
||||
{"status", status},
|
||||
{"architecture", architecture},
|
||||
// TODO: add other fields, may require reading GGUF metadata
|
||||
};
|
||||
|
||||
|
||||
@@ -66,6 +66,7 @@ struct server_model_meta {
|
||||
json loaded_info; // info to be reflected via /v1/models endpoint
|
||||
int exit_code = 0; // exit code of the model instance process (only valid if status == FAILED)
|
||||
int stop_timeout = 0; // seconds to wait before force-killing the model instance during shutdown
|
||||
mtmd_caps multimodal; // multimodal capabilities
|
||||
|
||||
bool is_ready() const {
|
||||
return status == SERVER_MODEL_STATUS_LOADED;
|
||||
@@ -80,6 +81,7 @@ struct server_model_meta {
|
||||
}
|
||||
|
||||
void update_args(common_preset_context & ctx_presets, std::string bin_path);
|
||||
void update_caps();
|
||||
};
|
||||
|
||||
struct subprocess_s;
|
||||
|
||||
Reference in New Issue
Block a user