mikhail/llama.cpp-mtp-turboquant
1
0
Fork 0
Code Issues Pull Requests Actions 34 Packages Projects Releases Wiki Activity

mtmd: add MiMo v2.5 vision (#22883)

Browse Source 
* mimo-v2.5: vision support

* mimo-v2.5: use fused qkv for vision

* mimi-v2.5: fix f16 vision overflow

* mimo-v2.5: comment cleanups

* mimo-v2.5: Flash doesn't have mmproj
more cleanup
remember to use filter_tensors

* mimo-v2.5: fix trailing whitespace
This commit is contained in:
AesSedai
2026-05-12 02:11:14 -07:00
committed by GitHub
parent 78fbbc2c07
commit 4178259130
12 changed files with 460 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/mtmd/clip.cpp
+125 -1
View File
@@ -642,7 +642,8 @@ ggml_tensor * clip_graph::build_attn(
ggml_tensor * v_cur,
ggml_tensor * kq_mask,
float kq_scale,
int il) const {
int il,
ggml_tensor * sinks) const {
// these nodes are added to the graph together so that they are not reordered
// by doing so, the number of splits in the graph is reduced
ggml_build_forward_expand(gf, q_cur);
@@ -665,6 +666,9 @@ ggml_tensor * clip_graph::build_attn(
cur = ggml_flash_attn_ext(ctx0, q, k, v, kq_mask, kq_scale, 0.0f, 0.0f);
ggml_flash_attn_ext_set_prec(cur, GGML_PREC_F32);
if (sinks != nullptr) {
ggml_flash_attn_ext_add_sinks(cur, sinks);
}
cur = ggml_reshape_2d(ctx0, cur, cur->ne[0]*cur->ne[1], cur->ne[2]*cur->ne[3]);
@@ -677,6 +681,9 @@ ggml_tensor * clip_graph::build_attn(
// ggml_mul_mat_set_prec(kq, GGML_PREC_F32);
kq = ggml_soft_max_ext(ctx0, kq, kq_mask, kq_scale, 0.0f);
if (sinks != nullptr) {
ggml_soft_max_add_sinks(kq, sinks);
}
ggml_tensor * kqv = ggml_mul_mat(ctx0, v, kq);
cur = ggml_permute(ctx0, kqv, 0, 2, 1, 3);
@@ -866,6 +873,10 @@ static ggml_cgraph * clip_image_build_graph(clip_ctx * ctx, const clip_image_f32
{
builder = std::make_unique<clip_graph_qwen3vl>(ctx, img);
} break;
case PROJECTOR_TYPE_MIMOVL:
{
builder = std::make_unique<clip_graph_mimovl>(ctx, img);
} break;
case PROJECTOR_TYPE_STEP3VL:
{
builder = std::make_unique<clip_graph_step3vl>(ctx, img);
@@ -1389,6 +1400,22 @@ struct clip_model_loader {
LOG_WRN("%s: more info: https://github.com/ggml-org/llama.cpp/issues/16842\n\n", __func__);
}
} break;
case PROJECTOR_TYPE_MIMOVL:
{
hparams.n_merge = 2; // spatial_merge_size
hparams.image_resize_algo = RESIZE_ALGO_BICUBIC_PILLOW;
get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge, false);
get_u32(string_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv);
// 1D banded sliding-window radius (visual_token_window_size); required
get_u32(KEY_ATTN_WINDOW_SIZE, hparams.attn_window_size);
std::vector<int> pat;
get_arr_int(KEY_WA_PATTERN_MODE, pat, true);
GGML_ASSERT((int) pat.size() == hparams.n_layer && "mimovl wa_pattern_mode length must equal n_layer");
hparams.wa_pattern_mode.assign(pat.begin(), pat.end());
get_u32(KEY_IMAGE_MIN_PIXELS, hparams.image_min_pixels);
get_u32(KEY_IMAGE_MAX_PIXELS, hparams.image_max_pixels);
hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup
} break;
case PROJECTOR_TYPE_STEP3VL:
{
hparams.n_merge = 4; // two stride-2 downsamplers after patching
@@ -1729,6 +1756,8 @@ struct clip_model_loader {
layer.ff_down_w = get_tensor(string_format(TN_FFN_DOWN, prefix, il, "weight"));
layer.ff_down_b = get_tensor(string_format(TN_FFN_DOWN, prefix, il, "bias"), false);
// mimovl per-head attention sink bias
layer.attn_sinks = get_tensor(string_format(TN_ATTN_SINKS, prefix, il), false);
// qwen3vl deepstack layer
layer.deepstack_norm_w = get_tensor(string_format(TN_DEEPSTACK_NORM, il, "weight"), false);
@@ -1913,6 +1942,13 @@ struct clip_model_loader {
model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
} break;
case PROJECTOR_TYPE_MIMOVL:
{
model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"), false);
model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"), false);
} break;
case PROJECTOR_TYPE_STEP3VL:
{
model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
@@ -3011,6 +3047,7 @@ int clip_n_output_tokens_x(const struct clip_ctx * ctx, struct clip_image_f32 *
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_QWEN3VL:
case PROJECTOR_TYPE_MIMOVL:
case PROJECTOR_TYPE_GLM4V:
case PROJECTOR_TYPE_PADDLEOCR:
case PROJECTOR_TYPE_HUNYUANOCR:
@@ -3032,6 +3069,7 @@ int clip_n_output_tokens_y(const struct clip_ctx * ctx, struct clip_image_f32 *
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_QWEN3VL:
case PROJECTOR_TYPE_MIMOVL:
case PROJECTOR_TYPE_GLM4V:
case PROJECTOR_TYPE_PADDLEOCR:
case PROJECTOR_TYPE_HUNYUANVL:
@@ -3110,6 +3148,7 @@ int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * im
case PROJECTOR_TYPE_QWEN2VL:
case PROJECTOR_TYPE_QWEN25VL:
case PROJECTOR_TYPE_QWEN3VL:
case PROJECTOR_TYPE_MIMOVL:
case PROJECTOR_TYPE_GLM4V:
case PROJECTOR_TYPE_YOUTUVL:
{
@@ -3681,6 +3720,89 @@ bool clip_image_batch_encode(clip_ctx * ctx, const int n_threads, const clip_ima
set_input_i32("positions", positions);
} break;
case PROJECTOR_TYPE_MIMOVL:
{
const int merge = hparams.n_merge; // 2
const int merge_unit = merge * merge; // 4
const int patch = hparams.patch_size; // 16
const int H = image_size_height / patch;
const int W = image_size_width / patch;
const int n_pos_full = H * W;
const int llm_h = H / merge;
const int llm_w = W / merge;
const int n_units = llm_h * llm_w; // n_pos / merge_unit
// Row-major merge-tile-ordered (h, w) positions
std::vector<int32_t> pos_h_row(n_pos_full);
std::vector<int32_t> pos_w_row(n_pos_full);
{
int idx = 0;
for (int ty = 0; ty < llm_h; ty++) {
for (int tx = 0; tx < llm_w; tx++) {
for (int dy = 0; dy < merge; dy++) {
for (int dx = 0; dx < merge; dx++) {
pos_h_row[idx] = ty * merge + dy;
pos_w_row[idx] = tx * merge + dx;
idx++;
}
}
}
}
}
// Col-major merge-unit permutation
std::vector<float> idx_col(n_units);
for (int r = 0; r < llm_h; r++) {
for (int c = 0; c < llm_w; c++) {
int u_row = r * llm_w + c;
int u_col = c * llm_h + r;
idx_col[u_col] = (float) u_row;
}
}
// Col-mode positions: permute pos_*_row by idx_col
std::vector<int32_t> pos_h_col(n_pos_full);
std::vector<int32_t> pos_w_col(n_pos_full);
for (int u = 0; u < n_units; u++) {
int src = (int) idx_col[u];
for (int k = 0; k < merge_unit; k++) {
pos_h_col[u * merge_unit + k] = pos_h_row[src * merge_unit + k];
pos_w_col[u * merge_unit + k] = pos_w_row[src * merge_unit + k];
}
}
// Pack into ggml_rope_multi VISION-mode layout. The non-CPU kernels
// only read slots 0 and 1, so pack h in slot 0, w in slot 1:
// positions[0..n_pos) = h
// positions[n_pos..2*n_pos) = w
// positions[2*n_pos..3*n_pos) = 0
// positions[3*n_pos..4*n_pos) = 0
std::vector<int32_t> positions_row(static_cast<size_t>(n_pos_full) * 4, 0);
std::vector<int32_t> positions_col(static_cast<size_t>(n_pos_full) * 4, 0);
for (int i = 0; i < n_pos_full; i++) {
positions_row[0 * n_pos_full + i] = pos_h_row[i];
positions_row[1 * n_pos_full + i] = pos_w_row[i];
positions_col[0 * n_pos_full + i] = pos_h_col[i];
positions_col[1 * n_pos_full + i] = pos_w_col[i];
}
// Banded 1D sliding-window mask
const int window = hparams.attn_window_size;
GGML_ASSERT(window > 0);
std::vector<float> mask(static_cast<size_t>(n_pos_full) * n_pos_full, std::numeric_limits<float>::lowest());
for (int q = 0; q < n_pos_full; q++) {
int lo = std::max(0, q - window);
int hi = std::min(n_pos_full - 1, q + window);
for (int k = lo; k <= hi; k++) {
mask[static_cast<size_t>(q) * n_pos_full + k] = 0.0f;
}
}
set_input_i32("mimovl_positions_row", positions_row);
set_input_i32("mimovl_positions_col", positions_col);
set_input_f32("mimovl_idx_col", idx_col);
set_input_f32("mimovl_window_mask", mask);
} break;
case PROJECTOR_TYPE_PIXTRAL:
case PROJECTOR_TYPE_KIMIVL:
case PROJECTOR_TYPE_KIMIK25:
@@ -4081,6 +4203,8 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
case PROJECTOR_TYPE_QWEN3VL:
// main path + deepstack paths
return ctx->model.mm_1_b->ne[0] * (1 + ctx->model.n_deepstack_layers);
case PROJECTOR_TYPE_MIMOVL:
return ctx->model.mm_1_w->ne[1];
case PROJECTOR_TYPE_STEP3VL:
return ctx->model.mm_model_proj->ne[1];
case PROJECTOR_TYPE_GEMMA3: