mikhail/llama.cpp-mtp-turboquant
1
0
Fork 0
Code Issues Pull Requests Actions 32 Packages Projects Releases Wiki Activity

gemma : perform per-layer projections in the first layer (#21612)

Browse Source 
* gemma : reduce graph splits by keeping per-layer ops in the input layer

* gemma : put the per-layer proj in the first layer

* cont : move the projection before the layer loop
This commit is contained in:
Georgi Gerganov
2026-04-08 16:06:30 +03:00
committed by GitHub
parent 87f4744a80
commit 5764d7c6a6
6 changed files with 109 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/models/models.h
+9 -6
View File
@@ -256,9 +256,11 @@ struct llm_build_gemma3n_iswa : public llm_graph_context {
llm_build_gemma3n_iswa(const llama_model & model, const llm_graph_params & params);
ggml_tensor * calc_magnitude(ggml_tensor * x);
ggml_tensor * view_2d_slice(ggml_tensor * x, int idx);
ggml_tensor * get_per_layer_inputs();
ggml_tensor * project_per_layer_inputs(ggml_tensor * inputs_embeds, ggml_tensor * inp_per_layer);
// TODO: refactor in common "per-layer" functionality [TAG_PER_LAYER]
ggml_tensor * build_inp_per_layer();
ggml_tensor * project_per_layer_inputs(ggml_tensor * inp_batch, ggml_tensor * inp_per_layer);
ggml_tensor * gaussian_topk(ggml_tensor * x);
ggml_tensor * altup_compute_router_modalities(ggml_tensor * x, int il);
ggml_tensor * altup_predict(ggml_tensor * cur, int il);
@@ -272,9 +274,10 @@ struct llm_build_gemma4_iswa : public llm_graph_context {
const int64_t n_embd_per_layer;
llm_build_gemma4_iswa(const llama_model & model, const llm_graph_params & params);
ggml_tensor * view_2d_slice(ggml_tensor * x, int idx);
ggml_tensor * get_per_layer_inputs();
ggml_tensor * project_per_layer_inputs(ggml_tensor * inputs_embeds, ggml_tensor * inp_per_layer);
// TODO: refactor in common "per-layer" functionality [TAG_PER_LAYER]
ggml_tensor * build_inp_per_layer();
ggml_tensor * project_per_layer_inputs(ggml_tensor * inp_batch, ggml_tensor * inp_per_layer);
};
struct llm_build_gemma_embedding : public llm_graph_context {