model: try to improve Qwen3 Next (#18683)

* qwen3next: simplify qkvz projection

* use ggml_swiglu_split

* revert swiglu_split, but remove redundant repeat()

* fix missing reshape

* rm 2 redundant transposes

* move mul_mat(k,q) to outside of chunking

* rm redundant cont

* improve g_cs_chunk

* add comments about no cont

* use std::pair instead of ggml_concat

* vectorize key_gdiff calculation

* rm unused tensor

* avoid ggml_concat inside loop

* bring back ggml_concat as it may not work on other backend

* nits
This commit is contained in:
Xuan-Son Nguyen
2026-01-11 12:53:33 +01:00
committed by GitHub
parent 79456a690a
commit 506bb6e010
6 changed files with 245 additions and 187 deletions
+9 -2
View File
@@ -466,7 +466,8 @@ private:
ggml_tensor * cur,
int il);
ggml_tensor * build_delta_net_chunking(
// returns pair of output and new state
std::pair<ggml_tensor *, ggml_tensor *> build_delta_net_chunking(
ggml_tensor * q,
ggml_tensor * k,
ggml_tensor * v,
@@ -478,7 +479,8 @@ private:
ggml_tensor * diag_mask,
int il);
ggml_tensor * build_delta_net_autoregressive(
// returns pair of output and new state
std::pair<ggml_tensor *, ggml_tensor *> build_delta_net_autoregressive(
ggml_tensor * q,
ggml_tensor * k,
ggml_tensor * v,
@@ -493,6 +495,11 @@ private:
ggml_tensor * gate,
int layer);
// returns pair of qkv, z
std::pair<ggml_tensor *, ggml_tensor *> build_qkvz(
ggml_tensor * input,
int il);
const llama_model & model;
};