mtmd : support MiniCPM-V 4.6 (#22529)
* Support MiniCPM-V 4.6 in new branch Signed-off-by: tc-mb <tianchi_cai@icloud.com> * fix code bug Signed-off-by: tc-mb <tianchi_cai@icloud.com> * fix pre-commit Signed-off-by: tc-mb <tianchi_cai@icloud.com> * fix convert Signed-off-by: tc-mb <tianchi_cai@icloud.com> * rename clip_graph_minicpmv4_6 Signed-off-by: tc-mb <tianchi_cai@icloud.com> * use new TYPE_MINICPMV4_6 Signed-off-by: tc-mb <tianchi_cai@icloud.com> * use build_attn to allow flash attention support Signed-off-by: tc-mb <tianchi_cai@icloud.com> * no use legacy code, restored here. Signed-off-by: tc-mb <tianchi_cai@icloud.com> * use the existing tensors name Signed-off-by: tc-mb <tianchi_cai@icloud.com> * unused ctx->model.hparams.minicpmv_version Signed-off-by: tc-mb <tianchi_cai@icloud.com> * use n_merge for slice alignment Signed-off-by: tc-mb <tianchi_cai@icloud.com> * borrow wa_layer_indexes for vit_merger insertion point Signed-off-by: tc-mb <tianchi_cai@icloud.com> * fix code style Signed-off-by: tc-mb <tianchi_cai@icloud.com> * Update convert_hf_to_gguf.py Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com> * use filter_tensors and add model.vision_tower Signed-off-by: tc-mb <tianchi_cai@icloud.com> * fix chkhsh Signed-off-by: tc-mb <tianchi_cai@icloud.com> * fix type check Signed-off-by: tc-mb <tianchi_cai@icloud.com> --------- Signed-off-by: tc-mb <tianchi_cai@icloud.com> Co-authored-by: Sigbjørn Skjæret <sigbjorn.skjaeret@scala.com>
This commit is contained in:
tc-mb
GitHub
+90
-2
@@ -1360,6 +1360,9 @@ class TextModel(ModelBase):
|
||||
if chkhsh == "d4540891389ea895b53b399da6ac824becc30f2fba0e9ddbb98f92e55ca0e97c":
|
||||
# ref: https://huggingface.co/Qwen/Qwen3-Embedding-0.6B
|
||||
res = "qwen2"
|
||||
if chkhsh == "1444df51289cfa8063b96f0e62b1125440111bc79a52003ea14b6eac7016fd5f":
|
||||
# ref: https://huggingface.co/openbmb/MiniCPM-V-4_6
|
||||
res = "qwen35"
|
||||
if chkhsh == "66b8d4e19ab16c3bfd89bce5d785fb7e0155e8648708a1f42077cb9fe002c273":
|
||||
# ref: https://huggingface.co/alvarobartt/grok-2-tokenizer
|
||||
res = "grok-2"
|
||||
@@ -5499,16 +5502,101 @@ class _LinearAttentionVReorderBase(Qwen3NextModel):
|
||||
yield from super().modify_tensors(data_torch, name, bid)
|
||||
|
||||
|
||||
class _Qwen35MRopeMixin:
|
||||
# Qwen3.5 always applies interleaved MRoPE (see Qwen3_5RotaryEmbedding in transformers);
|
||||
# the upstream default mrope_section is [11, 11, 10] and llama.cpp's QWEN35 / QWEN35MOE
|
||||
# loaders treat qwen35.rope.dimension_sections as required, so make sure it is always
|
||||
# written even when a particular checkpoint omits the field in `rope_parameters`.
|
||||
_QWEN35_DEFAULT_MROPE_SECTION = [11, 11, 10, 0]
|
||||
|
||||
gguf_writer: gguf.GGUFWriter
|
||||
rope_parameters: dict
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters() # ty: ignore[unresolved-attribute]
|
||||
if "mrope_section" not in self.rope_parameters:
|
||||
self.gguf_writer.add_rope_dimension_sections(self._QWEN35_DEFAULT_MROPE_SECTION)
|
||||
|
||||
|
||||
@ModelBase.register("Qwen3_5ForConditionalGeneration", "Qwen3_5ForCausalLM")
|
||||
class Qwen3_5TextModel(_LinearAttentionVReorderBase):
|
||||
class Qwen3_5TextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN35
|
||||
|
||||
|
||||
@ModelBase.register("Qwen3_5MoeForConditionalGeneration", "Qwen3_5MoeForCausalLM")
|
||||
class Qwen3_5MoeTextModel(_LinearAttentionVReorderBase):
|
||||
class Qwen3_5MoeTextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN35MOE
|
||||
|
||||
|
||||
# MiniCPM-V 4.6: text tower is Qwen3.5 (linear+full hybrid attention) wrapped under
|
||||
# `model.language_model.*`; vision tower is SigLIP + a window-attention ViT merger
|
||||
# + a final DownsampleMLP merger. The same HF arch is registered twice below: once as
|
||||
# the LM (text mode) and once as the mmproj (vision mode), mirroring the Qwen3-VL setup.
|
||||
|
||||
@ModelBase.register("MiniCPMV4_6ForConditionalGeneration")
|
||||
class MiniCPMV4_6TextModel(Qwen3_5TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN35
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
|
||||
if name.startswith("model.merger."):
|
||||
return None
|
||||
# MTP tensors are not used at inference yet; align with Qwen3Next behaviour
|
||||
if name.startswith("mtp"):
|
||||
return None
|
||||
|
||||
return super().filter_tensors(item)
|
||||
|
||||
|
||||
@ModelBase.register("MiniCPMV4_6ForConditionalGeneration")
|
||||
class MiniCPMV4_6VisionModel(MmprojModel):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
if self.hparams_vision is not None:
|
||||
# In MiniCPM-V 4.6 `vision_config.image_size` (980) describes the SigLIP
|
||||
# positional embedding bucket grid (70 x 70), while the per-slice processing
|
||||
# resolution is the preprocessor's `scale_resolution` (typically 448).
|
||||
# The CLIP loader in tools/mtmd/clip.cpp consumes `clip.vision.image_size`
|
||||
# as the slice size and warmup resolution, so report `scale_resolution` there
|
||||
# to match the upstream MiniCPMV4_6ImageProcessorPil slicing rules.
|
||||
scale_resolution = self.preprocessor_config.get("scale_resolution")
|
||||
if scale_resolution is not None:
|
||||
self.hparams_vision["image_size"] = int(scale_resolution)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
assert self.hparams_vision is not None
|
||||
|
||||
# projector type string is consumed by clip_projector_type_from_string() in clip.cpp
|
||||
# (mapped to PROJECTOR_TYPE_MINICPMV4_6).
|
||||
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MINICPMV4_6)
|
||||
|
||||
# ViT merger 2x2 + final merger 2x2 = 4x spatial merge per dimension; used for slice alignment
|
||||
self.gguf_writer.add_vision_projector_scale_factor(4)
|
||||
|
||||
# borrow wa_layer_indexes for vit_merger insertion point
|
||||
insert_layer_id = int(self.global_config.get(
|
||||
"insert_layer_id", self.hparams_vision.get("insert_layer_id", 6)))
|
||||
self.gguf_writer.add_vision_wa_layer_indexes([insert_layer_id])
|
||||
|
||||
# SigLIP vision body uses gelu_pytorch_tanh, which matches ggml_gelu (tanh approx).
|
||||
self.gguf_writer.add_vision_use_gelu(True)
|
||||
self.gguf_writer.add_vision_attention_layernorm_eps(
|
||||
self.hparams_vision.get("layer_norm_eps", 1e-6))
|
||||
|
||||
@classmethod
|
||||
def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
|
||||
name, gen = item
|
||||
|
||||
# lm_head / MTP -> belong to the LM file
|
||||
if name.startswith(("lm_head.", "mtp")):
|
||||
return None
|
||||
|
||||
return super().filter_tensors(item)
|
||||
|
||||
|
||||
@ModelBase.register("GPT2LMHeadModel")
|
||||
class GPT2Model(TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.GPT2
|
||||
|
||||
Reference in New Issue
Block a user