mikhail/llama.cpp-mtp-turboquant
1
0
Fork 0
Code Issues Pull Requests Actions 31 Packages Projects Releases Wiki Activity

models : move the token embedding norms to the first layer (#20943)

Browse Source 
* models : move the token embedding norms to the first layer

* cont : fix LLM_TENSOR_CONV1D + fix il indexing
This commit is contained in:
Georgi Gerganov
2026-03-24 17:00:30 +02:00
committed by GitHub
parent 3fc6f1aed1
commit 9f102a1407
8 changed files with 26 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/models/bert.cpp
+2 -2
View File
@@ -28,8 +28,8 @@ llm_build_bert::llm_build_bert(const llama_model & model, const llm_graph_params
cb(inpL, "inp_embd", -1);
// embed layer norm
inpL = build_norm(inpL, model.tok_norm, model.tok_norm_b, LLM_NORM, -1);
cb(inpL, "inp_norm", -1);
inpL = build_norm(inpL, model.tok_norm, model.tok_norm_b, LLM_NORM, 0);
cb(inpL, "inp_norm", 0);
auto * inp_attn = build_attn_inp_no_cache();
src/models/bloom.cpp
+2 -2
View File
@@ -16,8 +16,8 @@ llm_build_bloom::llm_build_bloom(const llama_model & model, const llm_graph_para
inpL = build_norm(inpL,
model.tok_norm,
model.tok_norm_b,
LLM_NORM, -1);
cb(inpL, "inp_norm", -1);
LLM_NORM, 0);
cb(inpL, "inp_norm", 0);
ggml_tensor * inp_out_ids = build_inp_out_ids();
src/models/modern-bert.cpp
+2 -2
View File
@@ -15,8 +15,8 @@ llm_build_modern_bert::llm_build_modern_bert(const llama_model & model, const ll
cb(inpL, "inp_embd", -1);
// embed layer norm
inpL = build_norm(inpL, model.tok_norm, nullptr, LLM_NORM, -1);
cb(inpL, "inp_norm", -1);
inpL = build_norm(inpL, model.tok_norm, nullptr, LLM_NORM, 0);
cb(inpL, "inp_norm", 0);
ggml_tensor * inp_out_ids = build_inp_out_ids();
src/models/rwkv6.cpp
+1 -1
View File
@@ -8,7 +8,7 @@ llm_build_rwkv6::llm_build_rwkv6(const llama_model & model, const llm_graph_para
ggml_tensor * inpL;
inpL = build_inp_embd(model.tok_embd);
inpL = build_norm(inpL, model.tok_norm, model.tok_norm_b, LLM_NORM, -1);
inpL = build_norm(inpL, model.tok_norm, model.tok_norm_b, LLM_NORM, 0);
auto * rs_inp = build_rs_inp();
src/models/rwkv7.cpp
+1 -1
View File
@@ -9,7 +9,7 @@ llm_build_rwkv7::llm_build_rwkv7(const llama_model & model, const llm_graph_para
ggml_tensor * v_first = nullptr;
inpL = build_inp_embd(model.tok_embd);
inpL = build_norm(inpL, model.tok_norm, model.tok_norm_b, LLM_NORM, -1);
inpL = build_norm(inpL, model.tok_norm, model.tok_norm_b, LLM_NORM, 0);
auto * rs_inp = build_rs_inp();
src/models/wavtokenizer-dec.cpp
+1 -1
View File
@@ -93,7 +93,7 @@ llm_build_wavtokenizer_dec::llm_build_wavtokenizer_dec(const llama_model & model
cur = build_norm(cur,
model.tok_norm,
model.tok_norm_b,
LLM_NORM, -1);
LLM_NORM, 0);
cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));