merge: TurboQuant KV cache types onto upstream master (MTP built-in)
This commit is contained in:
root
@@ -0,0 +1,362 @@
|
||||
=== SMEM M5 Benchmark: baseline ===
|
||||
Model: Qwen3.5-35B-A3B-Q8_0.gguf
|
||||
Date: Sat Mar 28 21:45:40 CDT 2026
|
||||
|
||||
--- turbo3 @ short ---
|
||||
ggml_metal_device_init: testing tensor API for f16 support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x105cffcb0 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_device_init: testing tensor API for bfloat support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x105cfeb30 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_library_init: using embedded metal library
|
||||
ggml_metal_library_init: turbo3 sparse V dequant enabled
|
||||
ggml_metal_library_init: loaded in 6.440 sec
|
||||
ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s)
|
||||
ggml_metal_device_init: GPU name: MTL0
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002)
|
||||
ggml_metal_device_init: simdgroup reduction = true
|
||||
ggml_metal_device_init: simdgroup matrix mul. = true
|
||||
ggml_metal_device_init: has unified memory = true
|
||||
ggml_metal_device_init: has bfloat = true
|
||||
ggml_metal_device_init: has tensor = true
|
||||
ggml_metal_device_init: use residency sets = true
|
||||
ggml_metal_device_init: use shared buffers = true
|
||||
ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB
|
||||
| model | size | params | backend | threads | type_k | type_v | fa | test | t/s |
|
||||
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -----: | -----: | -: | --------------: | -------------------: |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo3 | turbo3 | 1 | tg128 | 78.47 ± 0.56 |
|
||||
|
||||
build: 13afec1 (178)
|
||||
|
||||
--- turbo3 @ 8192 ---
|
||||
ggml_metal_device_init: testing tensor API for f16 support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x1040cfae0 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_device_init: testing tensor API for bfloat support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x1040ce960 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_library_init: using embedded metal library
|
||||
ggml_metal_library_init: turbo3 sparse V dequant enabled
|
||||
ggml_metal_library_init: loaded in 0.010 sec
|
||||
ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s)
|
||||
ggml_metal_device_init: GPU name: MTL0
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002)
|
||||
ggml_metal_device_init: simdgroup reduction = true
|
||||
ggml_metal_device_init: simdgroup matrix mul. = true
|
||||
ggml_metal_device_init: has unified memory = true
|
||||
ggml_metal_device_init: has bfloat = true
|
||||
ggml_metal_device_init: has tensor = true
|
||||
ggml_metal_device_init: use residency sets = true
|
||||
ggml_metal_device_init: use shared buffers = true
|
||||
ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB
|
||||
| model | size | params | backend | threads | type_k | type_v | fa | test | t/s |
|
||||
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -----: | -----: | -: | --------------: | -------------------: |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo3 | turbo3 | 1 | pp8192 | 2144.16 ± 30.18 |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo3 | turbo3 | 1 | tg128 | 78.90 ± 0.24 |
|
||||
|
||||
build: 13afec1 (178)
|
||||
|
||||
--- turbo3 @ 16384 ---
|
||||
ggml_metal_device_init: testing tensor API for f16 support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x10500fc00 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_device_init: testing tensor API for bfloat support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x10500ea80 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_library_init: using embedded metal library
|
||||
ggml_metal_library_init: turbo3 sparse V dequant enabled
|
||||
ggml_metal_library_init: loaded in 0.008 sec
|
||||
ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s)
|
||||
ggml_metal_device_init: GPU name: MTL0
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002)
|
||||
ggml_metal_device_init: simdgroup reduction = true
|
||||
ggml_metal_device_init: simdgroup matrix mul. = true
|
||||
ggml_metal_device_init: has unified memory = true
|
||||
ggml_metal_device_init: has bfloat = true
|
||||
ggml_metal_device_init: has tensor = true
|
||||
ggml_metal_device_init: use residency sets = true
|
||||
ggml_metal_device_init: use shared buffers = true
|
||||
ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB
|
||||
| model | size | params | backend | threads | type_k | type_v | fa | test | t/s |
|
||||
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -----: | -----: | -: | --------------: | -------------------: |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo3 | turbo3 | 1 | pp16384 | 1704.41 ± 21.63 |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo3 | turbo3 | 1 | tg128 | 78.64 ± 0.44 |
|
||||
|
||||
build: 13afec1 (178)
|
||||
|
||||
--- turbo3 @ 32768 ---
|
||||
ggml_metal_device_init: testing tensor API for f16 support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x101c8fb00 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_device_init: testing tensor API for bfloat support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x101c8e980 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_library_init: using embedded metal library
|
||||
ggml_metal_library_init: turbo3 sparse V dequant enabled
|
||||
ggml_metal_library_init: loaded in 0.013 sec
|
||||
ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s)
|
||||
ggml_metal_device_init: GPU name: MTL0
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002)
|
||||
ggml_metal_device_init: simdgroup reduction = true
|
||||
ggml_metal_device_init: simdgroup matrix mul. = true
|
||||
ggml_metal_device_init: has unified memory = true
|
||||
ggml_metal_device_init: has bfloat = true
|
||||
ggml_metal_device_init: has tensor = true
|
||||
ggml_metal_device_init: use residency sets = true
|
||||
ggml_metal_device_init: use shared buffers = true
|
||||
ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB
|
||||
| model | size | params | backend | threads | type_k | type_v | fa | test | t/s |
|
||||
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -----: | -----: | -: | --------------: | -------------------: |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo3 | turbo3 | 1 | pp32768 | 1238.85 ± 6.06 |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo3 | turbo3 | 1 | tg128 | 78.17 ± 0.69 |
|
||||
|
||||
build: 13afec1 (178)
|
||||
|
||||
--- turbo4 @ short ---
|
||||
ggml_metal_device_init: testing tensor API for f16 support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x103c17f70 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_device_init: testing tensor API for bfloat support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x103c16df0 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_library_init: using embedded metal library
|
||||
ggml_metal_library_init: turbo3 sparse V dequant enabled
|
||||
ggml_metal_library_init: loaded in 0.008 sec
|
||||
ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s)
|
||||
ggml_metal_device_init: GPU name: MTL0
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002)
|
||||
ggml_metal_device_init: simdgroup reduction = true
|
||||
ggml_metal_device_init: simdgroup matrix mul. = true
|
||||
ggml_metal_device_init: has unified memory = true
|
||||
ggml_metal_device_init: has bfloat = true
|
||||
ggml_metal_device_init: has tensor = true
|
||||
ggml_metal_device_init: use residency sets = true
|
||||
ggml_metal_device_init: use shared buffers = true
|
||||
ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB
|
||||
| model | size | params | backend | threads | type_k | type_v | fa | test | t/s |
|
||||
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -----: | -----: | -: | --------------: | -------------------: |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo4 | turbo4 | 1 | tg128 | 80.40 ± 0.72 |
|
||||
|
||||
build: 13afec1 (178)
|
||||
|
||||
--- turbo4 @ 8192 ---
|
||||
ggml_metal_device_init: testing tensor API for f16 support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x103e57d30 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_device_init: testing tensor API for bfloat support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x103e56bb0 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_library_init: using embedded metal library
|
||||
ggml_metal_library_init: turbo3 sparse V dequant enabled
|
||||
ggml_metal_library_init: loaded in 0.010 sec
|
||||
ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s)
|
||||
ggml_metal_device_init: GPU name: MTL0
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002)
|
||||
ggml_metal_device_init: simdgroup reduction = true
|
||||
ggml_metal_device_init: simdgroup matrix mul. = true
|
||||
ggml_metal_device_init: has unified memory = true
|
||||
ggml_metal_device_init: has bfloat = true
|
||||
ggml_metal_device_init: has tensor = true
|
||||
ggml_metal_device_init: use residency sets = true
|
||||
ggml_metal_device_init: use shared buffers = true
|
||||
ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB
|
||||
| model | size | params | backend | threads | type_k | type_v | fa | test | t/s |
|
||||
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -----: | -----: | -: | --------------: | -------------------: |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo4 | turbo4 | 1 | pp8192 | 2048.90 ± 43.42 |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo4 | turbo4 | 1 | tg128 | 79.84 ± 0.95 |
|
||||
|
||||
build: 13afec1 (178)
|
||||
|
||||
--- turbo4 @ 16384 ---
|
||||
ggml_metal_device_init: testing tensor API for f16 support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x1060bf740 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_device_init: testing tensor API for bfloat support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x1060be5c0 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_library_init: using embedded metal library
|
||||
ggml_metal_library_init: turbo3 sparse V dequant enabled
|
||||
ggml_metal_library_init: loaded in 0.009 sec
|
||||
ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s)
|
||||
ggml_metal_device_init: GPU name: MTL0
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002)
|
||||
ggml_metal_device_init: simdgroup reduction = true
|
||||
ggml_metal_device_init: simdgroup matrix mul. = true
|
||||
ggml_metal_device_init: has unified memory = true
|
||||
ggml_metal_device_init: has bfloat = true
|
||||
ggml_metal_device_init: has tensor = true
|
||||
ggml_metal_device_init: use residency sets = true
|
||||
ggml_metal_device_init: use shared buffers = true
|
||||
ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB
|
||||
| model | size | params | backend | threads | type_k | type_v | fa | test | t/s |
|
||||
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -----: | -----: | -: | --------------: | -------------------: |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo4 | turbo4 | 1 | pp16384 | 1605.18 ± 20.70 |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo4 | turbo4 | 1 | tg128 | 79.45 ± 1.55 |
|
||||
|
||||
build: 13afec1 (178)
|
||||
|
||||
--- turbo4 @ 32768 ---
|
||||
ggml_metal_device_init: testing tensor API for f16 support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x1040ef870 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_device_init: testing tensor API for bfloat support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x1040ee6f0 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_library_init: using embedded metal library
|
||||
ggml_metal_library_init: turbo3 sparse V dequant enabled
|
||||
ggml_metal_library_init: loaded in 0.010 sec
|
||||
ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s)
|
||||
ggml_metal_device_init: GPU name: MTL0
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002)
|
||||
ggml_metal_device_init: simdgroup reduction = true
|
||||
ggml_metal_device_init: simdgroup matrix mul. = true
|
||||
ggml_metal_device_init: has unified memory = true
|
||||
ggml_metal_device_init: has bfloat = true
|
||||
ggml_metal_device_init: has tensor = true
|
||||
ggml_metal_device_init: use residency sets = true
|
||||
ggml_metal_device_init: use shared buffers = true
|
||||
ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB
|
||||
| model | size | params | backend | threads | type_k | type_v | fa | test | t/s |
|
||||
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -----: | -----: | -: | --------------: | -------------------: |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo4 | turbo4 | 1 | pp32768 | 1157.30 ± 8.01 |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo4 | turbo4 | 1 | tg128 | 80.64 ± 0.72 |
|
||||
|
||||
build: 13afec1 (178)
|
||||
|
||||
--- q8_0 @ short ---
|
||||
ggml_metal_device_init: testing tensor API for f16 support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x1055e78c0 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_device_init: testing tensor API for bfloat support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x1055e6740 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_library_init: using embedded metal library
|
||||
ggml_metal_library_init: turbo3 sparse V dequant enabled
|
||||
ggml_metal_library_init: loaded in 0.008 sec
|
||||
ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s)
|
||||
ggml_metal_device_init: GPU name: MTL0
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002)
|
||||
ggml_metal_device_init: simdgroup reduction = true
|
||||
ggml_metal_device_init: simdgroup matrix mul. = true
|
||||
ggml_metal_device_init: has unified memory = true
|
||||
ggml_metal_device_init: has bfloat = true
|
||||
ggml_metal_device_init: has tensor = true
|
||||
ggml_metal_device_init: use residency sets = true
|
||||
ggml_metal_device_init: use shared buffers = true
|
||||
ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB
|
||||
| model | size | params | backend | threads | type_k | type_v | fa | test | t/s |
|
||||
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -----: | -----: | -: | --------------: | -------------------: |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | q8_0 | q8_0 | 1 | tg128 | 85.48 ± 1.34 |
|
||||
|
||||
build: 13afec1 (178)
|
||||
|
||||
--- q8_0 @ 8192 ---
|
||||
ggml_metal_device_init: testing tensor API for f16 support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x105ac8540 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_device_init: testing tensor API for bfloat support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x105ac73c0 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_library_init: using embedded metal library
|
||||
ggml_metal_library_init: turbo3 sparse V dequant enabled
|
||||
ggml_metal_library_init: loaded in 0.010 sec
|
||||
ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s)
|
||||
ggml_metal_device_init: GPU name: MTL0
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002)
|
||||
ggml_metal_device_init: simdgroup reduction = true
|
||||
ggml_metal_device_init: simdgroup matrix mul. = true
|
||||
ggml_metal_device_init: has unified memory = true
|
||||
ggml_metal_device_init: has bfloat = true
|
||||
ggml_metal_device_init: has tensor = true
|
||||
ggml_metal_device_init: use residency sets = true
|
||||
ggml_metal_device_init: use shared buffers = true
|
||||
ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB
|
||||
| model | size | params | backend | threads | type_k | type_v | fa | test | t/s |
|
||||
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -----: | -----: | -: | --------------: | -------------------: |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | q8_0 | q8_0 | 1 | pp8192 | 2106.47 ± 64.66 |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | q8_0 | q8_0 | 1 | tg128 | 76.72 ± 2.13 |
|
||||
|
||||
build: 13afec1 (178)
|
||||
|
||||
--- q8_0 @ 16384 ---
|
||||
ggml_metal_device_init: testing tensor API for f16 support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x103fefa70 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_device_init: testing tensor API for bfloat support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x103fee8f0 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_library_init: using embedded metal library
|
||||
ggml_metal_library_init: turbo3 sparse V dequant enabled
|
||||
ggml_metal_library_init: loaded in 0.008 sec
|
||||
ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s)
|
||||
ggml_metal_device_init: GPU name: MTL0
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002)
|
||||
ggml_metal_device_init: simdgroup reduction = true
|
||||
ggml_metal_device_init: simdgroup matrix mul. = true
|
||||
ggml_metal_device_init: has unified memory = true
|
||||
ggml_metal_device_init: has bfloat = true
|
||||
ggml_metal_device_init: has tensor = true
|
||||
ggml_metal_device_init: use residency sets = true
|
||||
ggml_metal_device_init: use shared buffers = true
|
||||
ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB
|
||||
| model | size | params | backend | threads | type_k | type_v | fa | test | t/s |
|
||||
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -----: | -----: | -: | --------------: | -------------------: |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | q8_0 | q8_0 | 1 | pp16384 | 1723.71 ± 28.56 |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | q8_0 | q8_0 | 1 | tg128 | 78.09 ± 3.70 |
|
||||
|
||||
build: 13afec1 (178)
|
||||
|
||||
--- q8_0 @ 32768 ---
|
||||
ggml_metal_device_init: testing tensor API for f16 support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x1035f7b10 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_device_init: testing tensor API for bfloat support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x1035f6990 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_library_init: using embedded metal library
|
||||
ggml_metal_library_init: turbo3 sparse V dequant enabled
|
||||
ggml_metal_library_init: loaded in 0.008 sec
|
||||
ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s)
|
||||
ggml_metal_device_init: GPU name: MTL0
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002)
|
||||
ggml_metal_device_init: simdgroup reduction = true
|
||||
ggml_metal_device_init: simdgroup matrix mul. = true
|
||||
ggml_metal_device_init: has unified memory = true
|
||||
ggml_metal_device_init: has bfloat = true
|
||||
ggml_metal_device_init: has tensor = true
|
||||
ggml_metal_device_init: use residency sets = true
|
||||
ggml_metal_device_init: use shared buffers = true
|
||||
ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB
|
||||
| model | size | params | backend | threads | type_k | type_v | fa | test | t/s |
|
||||
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -----: | -----: | -: | --------------: | -------------------: |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | q8_0 | q8_0 | 1 | pp32768 | 1216.99 ± 28.64 |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | q8_0 | q8_0 | 1 | tg128 | 86.83 ± 0.34 |
|
||||
|
||||
build: 13afec1 (178)
|
||||
|
||||
=== Done: baseline ===
|
||||
@@ -0,0 +1,413 @@
|
||||
=== SMEM M5 Benchmark: smem ===
|
||||
Model: Qwen3.5-35B-A3B-Q8_0.gguf
|
||||
Date: Sat Mar 28 22:02:19 CDT 2026
|
||||
|
||||
--- turbo3 @ short ---
|
||||
ggml_metal_device_init: testing tensor API for f16 support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x104fbb670 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_device_init: testing tensor API for bfloat support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x104fbb5f0 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_library_init: using embedded metal library
|
||||
ggml_metal_library_init: turbo3 sparse V dequant enabled
|
||||
ggml_metal_library_init: loaded in 7.366 sec
|
||||
ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s)
|
||||
ggml_metal_device_init: GPU name: MTL0
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002)
|
||||
ggml_metal_device_init: simdgroup reduction = true
|
||||
ggml_metal_device_init: simdgroup matrix mul. = true
|
||||
ggml_metal_device_init: has unified memory = true
|
||||
ggml_metal_device_init: has bfloat = true
|
||||
ggml_metal_device_init: has tensor = true
|
||||
ggml_metal_device_init: use residency sets = true
|
||||
ggml_metal_device_init: use shared buffers = true
|
||||
ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB
|
||||
| model | size | params | backend | threads | type_k | type_v | fa | test | t/s |
|
||||
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -----: | -----: | -: | --------------: | -------------------: |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo3 | turbo3 | 1 | tg128 | 18.39 ± 0.76 |
|
||||
|
||||
build: 13afec1 (178)
|
||||
|
||||
--- turbo3 @ 8192 ---
|
||||
ggml_metal_device_init: testing tensor API for f16 support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x101ee3e50 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_device_init: testing tensor API for bfloat support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x101ee3dd0 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_library_init: using embedded metal library
|
||||
ggml_metal_library_init: turbo3 sparse V dequant enabled
|
||||
ggml_metal_library_init: loaded in 0.009 sec
|
||||
ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s)
|
||||
ggml_metal_device_init: GPU name: MTL0
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002)
|
||||
ggml_metal_device_init: simdgroup reduction = true
|
||||
ggml_metal_device_init: simdgroup matrix mul. = true
|
||||
ggml_metal_device_init: has unified memory = true
|
||||
ggml_metal_device_init: has bfloat = true
|
||||
ggml_metal_device_init: has tensor = true
|
||||
ggml_metal_device_init: use residency sets = true
|
||||
ggml_metal_device_init: use shared buffers = true
|
||||
ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB
|
||||
| model | size | params | backend | threads | type_k | type_v | fa | test | t/s |
|
||||
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -----: | -----: | -: | --------------: | -------------------: |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo3 | turbo3 | 1 | pp16384 | 1337.26 ± 261.92 |
|
||||
| model | size | params | backend | threads | type_k | type_v | fa | test | t/s |
|
||||
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -----: | -----: | -: | --------------: | -------------------: |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo3 | turbo3 | 1 | pp8192 | 1442.03 ± 393.22 |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo3 | turbo3 | 1 | tg128 | 40.38 ± 18.10 |
|
||||
|
||||
build: 13afec1 (178)
|
||||
|
||||
--- turbo3 @ 32768 ---
|
||||
ggml_metal_device_init: testing tensor API for f16 support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x105a3f890 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_device_init: testing tensor API for bfloat support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x105a3e710 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_library_init: using embedded metal library
|
||||
ggml_metal_library_init: turbo3 sparse V dequant enabled
|
||||
ggml_metal_library_init: turbo3/4 SMEM pre-dequant enabled
|
||||
ggml_metal_library_init: loaded in 0.010 sec
|
||||
ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s)
|
||||
ggml_metal_device_init: GPU name: MTL0
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002)
|
||||
ggml_metal_device_init: simdgroup reduction = true
|
||||
ggml_metal_device_init: simdgroup matrix mul. = true
|
||||
ggml_metal_device_init: has unified memory = true
|
||||
ggml_metal_device_init: has bfloat = true
|
||||
ggml_metal_device_init: has tensor = true
|
||||
ggml_metal_device_init: use residency sets = true
|
||||
ggml_metal_device_init: use shared buffers = true
|
||||
ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo3 | turbo3 | 1 | tg128 | 58.20 ± 8.75 |
|
||||
|
||||
build: 13afec1 (178)
|
||||
|
||||
--- turbo3 @ 16384 ---
|
||||
ggml_metal_device_init: testing tensor API for f16 support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x103d7b200 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_device_init: testing tensor API for bfloat support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x103d7b180 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_library_init: using embedded metal library
|
||||
ggml_metal_library_init: turbo3 sparse V dequant enabled
|
||||
ggml_metal_library_init: loaded in 0.009 sec
|
||||
ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s)
|
||||
ggml_metal_device_init: GPU name: MTL0
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002)
|
||||
ggml_metal_device_init: simdgroup reduction = true
|
||||
ggml_metal_device_init: simdgroup matrix mul. = true
|
||||
ggml_metal_device_init: has unified memory = true
|
||||
ggml_metal_device_init: has bfloat = true
|
||||
ggml_metal_device_init: has tensor = true
|
||||
ggml_metal_device_init: use residency sets = true
|
||||
ggml_metal_device_init: use shared buffers = true
|
||||
ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB
|
||||
| model | size | params | backend | threads | type_k | type_v | fa | test | t/s |
|
||||
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -----: | -----: | -: | --------------: | -------------------: |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo3 | turbo3 | 1 | pp16384 | 792.76 ± 57.30 |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo3 | turbo3 | 1 | tg128 | 16.47 ± 1.39 |
|
||||
|
||||
build: 13afec1 (178)
|
||||
|
||||
--- turbo3 @ 32768 ---
|
||||
ggml_metal_device_init: testing tensor API for f16 support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x104dc31e0 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_device_init: testing tensor API for bfloat support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x104dc3160 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_library_init: using embedded metal library
|
||||
ggml_metal_library_init: turbo3 sparse V dequant enabled
|
||||
ggml_metal_library_init: loaded in 0.009 sec
|
||||
ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s)
|
||||
ggml_metal_device_init: GPU name: MTL0
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002)
|
||||
ggml_metal_device_init: simdgroup reduction = true
|
||||
ggml_metal_device_init: simdgroup matrix mul. = true
|
||||
ggml_metal_device_init: has unified memory = true
|
||||
ggml_metal_device_init: has bfloat = true
|
||||
ggml_metal_device_init: has tensor = true
|
||||
ggml_metal_device_init: use residency sets = true
|
||||
ggml_metal_device_init: use shared buffers = true
|
||||
ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB
|
||||
| model | size | params | backend | threads | type_k | type_v | fa | test | t/s |
|
||||
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -----: | -----: | -: | --------------: | -------------------: |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo3 | turbo3 | 1 | pp32768 | 806.43 ± 177.53 |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo3 | turbo3 | 1 | tg128 | 16.19 ± 1.11 |
|
||||
|
||||
build: 13afec1 (178)
|
||||
|
||||
--- turbo4 @ short ---
|
||||
ggml_metal_device_init: testing tensor API for f16 support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x105ccfa30 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_device_init: testing tensor API for bfloat support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x105cce8b0 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_library_init: using embedded metal library
|
||||
ggml_metal_library_init: turbo3 sparse V dequant enabled
|
||||
ggml_metal_library_init: turbo3/4 SMEM pre-dequant enabled
|
||||
ggml_metal_library_init: loaded in 0.008 sec
|
||||
ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s)
|
||||
ggml_metal_device_init: GPU name: MTL0
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002)
|
||||
ggml_metal_device_init: simdgroup reduction = true
|
||||
ggml_metal_device_init: simdgroup matrix mul. = true
|
||||
ggml_metal_device_init: has unified memory = true
|
||||
ggml_metal_device_init: has bfloat = true
|
||||
ggml_metal_device_init: has tensor = true
|
||||
ggml_metal_device_init: use residency sets = true
|
||||
ggml_metal_device_init: use shared buffers = true
|
||||
ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB
|
||||
| model | size | params | backend | threads | type_k | type_v | fa | test | t/s |
|
||||
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -----: | -----: | -: | --------------: | -------------------: |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo4 | turbo4 | 1 | tg128 | 16.93 ± 0.97 |
|
||||
|
||||
build: 13afec1 (178)
|
||||
|
||||
--- turbo4 @ 8192 ---
|
||||
ggml_metal_device_init: testing tensor API for f16 support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x10561bc80 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_device_init: testing tensor API for bfloat support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x10561ab00 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_library_init: using embedded metal library
|
||||
ggml_metal_library_init: turbo3 sparse V dequant enabled
|
||||
ggml_metal_library_init: turbo3/4 SMEM pre-dequant enabled
|
||||
ggml_metal_library_init: loaded in 0.008 sec
|
||||
ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s)
|
||||
ggml_metal_device_init: GPU name: MTL0
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002)
|
||||
ggml_metal_device_init: simdgroup reduction = true
|
||||
ggml_metal_device_init: simdgroup matrix mul. = true
|
||||
ggml_metal_device_init: has unified memory = true
|
||||
ggml_metal_device_init: has bfloat = true
|
||||
ggml_metal_device_init: has tensor = true
|
||||
ggml_metal_device_init: use residency sets = true
|
||||
ggml_metal_device_init: use shared buffers = true
|
||||
ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB
|
||||
| model | size | params | backend | threads | type_k | type_v | fa | test | t/s |
|
||||
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -----: | -----: | -: | --------------: | -------------------: |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo4 | turbo4 | 1 | pp8192 | 942.18 ± 77.19 |
|
||||
| model | size | params | backend | threads | type_k | type_v | fa | test | t/s |
|
||||
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -----: | -----: | -: | --------------: | -------------------: |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo3 | turbo3 | 1 | pp32768 | 941.24 ± 180.34 |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo4 | turbo4 | 1 | tg128 | 44.84 ± 18.74 |
|
||||
|
||||
build: 13afec1 (178)
|
||||
|
||||
--- turbo4 @ 16384 ---
|
||||
ggml_metal_device_init: testing tensor API for f16 support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x1038a3d70 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_device_init: testing tensor API for bfloat support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x1038a2bf0 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_library_init: using embedded metal library
|
||||
ggml_metal_library_init: turbo3 sparse V dequant enabled
|
||||
ggml_metal_library_init: turbo3/4 SMEM pre-dequant enabled
|
||||
ggml_metal_library_init: loaded in 0.008 sec
|
||||
ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s)
|
||||
ggml_metal_device_init: GPU name: MTL0
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002)
|
||||
ggml_metal_device_init: simdgroup reduction = true
|
||||
ggml_metal_device_init: simdgroup matrix mul. = true
|
||||
ggml_metal_device_init: has unified memory = true
|
||||
ggml_metal_device_init: has bfloat = true
|
||||
ggml_metal_device_init: has tensor = true
|
||||
ggml_metal_device_init: use residency sets = true
|
||||
ggml_metal_device_init: use shared buffers = true
|
||||
ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo3 | turbo3 | 1 | tg128 | 61.97 ± 9.79 |
|
||||
|
||||
build: 13afec1 (178)
|
||||
|
||||
--- turbo4 @ short ---
|
||||
ggml_metal_device_init: testing tensor API for f16 support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x10170b580 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_device_init: testing tensor API for bfloat support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x10170b500 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_library_init: using embedded metal library
|
||||
ggml_metal_library_init: turbo3 sparse V dequant enabled
|
||||
ggml_metal_library_init: loaded in 0.008 sec
|
||||
ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s)
|
||||
ggml_metal_device_init: GPU name: MTL0
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002)
|
||||
ggml_metal_device_init: simdgroup reduction = true
|
||||
ggml_metal_device_init: simdgroup matrix mul. = true
|
||||
ggml_metal_device_init: has unified memory = true
|
||||
ggml_metal_device_init: has bfloat = true
|
||||
ggml_metal_device_init: has tensor = true
|
||||
ggml_metal_device_init: use residency sets = true
|
||||
ggml_metal_device_init: use shared buffers = true
|
||||
ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB
|
||||
| model | size | params | backend | threads | type_k | type_v | fa | test | t/s |
|
||||
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -----: | -----: | -: | --------------: | -------------------: |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo4 | turbo4 | 1 | tg128 | 17.82 ± 0.64 |
|
||||
|
||||
build: 13afec1 (178)
|
||||
|
||||
--- turbo4 @ 8192 ---
|
||||
ggml_metal_device_init: testing tensor API for f16 support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x103dab490 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_device_init: testing tensor API for bfloat support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x103dab410 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_library_init: using embedded metal library
|
||||
ggml_metal_library_init: turbo3 sparse V dequant enabled
|
||||
ggml_metal_library_init: loaded in 0.009 sec
|
||||
ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s)
|
||||
ggml_metal_device_init: GPU name: MTL0
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002)
|
||||
ggml_metal_device_init: simdgroup reduction = true
|
||||
ggml_metal_device_init: simdgroup matrix mul. = true
|
||||
ggml_metal_device_init: has unified memory = true
|
||||
ggml_metal_device_init: has bfloat = true
|
||||
ggml_metal_device_init: has tensor = true
|
||||
ggml_metal_device_init: use residency sets = true
|
||||
ggml_metal_device_init: use shared buffers = true
|
||||
ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB
|
||||
| model | size | params | backend | threads | type_k | type_v | fa | test | t/s |
|
||||
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -----: | -----: | -: | --------------: | -------------------: |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo4 | turbo4 | 1 | pp16384 | 1187.08 ± 274.35 |
|
||||
| model | size | params | backend | threads | type_k | type_v | fa | test | t/s |
|
||||
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -----: | -----: | -: | --------------: | -------------------: |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo4 | turbo4 | 1 | pp8192 | 1098.56 ± 217.82 |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo4 | turbo4 | 1 | tg128 | 50.13 ± 12.92 |
|
||||
|
||||
build: 13afec1 (178)
|
||||
|
||||
--- turbo4 @ 32768 ---
|
||||
ggml_metal_device_init: testing tensor API for f16 support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x105f20300 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_device_init: testing tensor API for bfloat support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x105f1f180 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_library_init: using embedded metal library
|
||||
ggml_metal_library_init: turbo3 sparse V dequant enabled
|
||||
ggml_metal_library_init: turbo3/4 SMEM pre-dequant enabled
|
||||
ggml_metal_library_init: loaded in 0.008 sec
|
||||
ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s)
|
||||
ggml_metal_device_init: GPU name: MTL0
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002)
|
||||
ggml_metal_device_init: simdgroup reduction = true
|
||||
ggml_metal_device_init: simdgroup matrix mul. = true
|
||||
ggml_metal_device_init: has unified memory = true
|
||||
ggml_metal_device_init: has bfloat = true
|
||||
ggml_metal_device_init: has tensor = true
|
||||
ggml_metal_device_init: use residency sets = true
|
||||
ggml_metal_device_init: use shared buffers = true
|
||||
ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo4 | turbo4 | 1 | tg128 | 58.25 ± 4.07 |
|
||||
|
||||
build: 13afec1 (178)
|
||||
|
||||
--- turbo4 @ 16384 ---
|
||||
ggml_metal_device_init: testing tensor API for f16 support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x10588f220 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_device_init: testing tensor API for bfloat support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x10588f1a0 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_library_init: using embedded metal library
|
||||
ggml_metal_library_init: turbo3 sparse V dequant enabled
|
||||
ggml_metal_library_init: loaded in 0.008 sec
|
||||
ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s)
|
||||
ggml_metal_device_init: GPU name: MTL0
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002)
|
||||
ggml_metal_device_init: simdgroup reduction = true
|
||||
ggml_metal_device_init: simdgroup matrix mul. = true
|
||||
ggml_metal_device_init: has unified memory = true
|
||||
ggml_metal_device_init: has bfloat = true
|
||||
ggml_metal_device_init: has tensor = true
|
||||
ggml_metal_device_init: use residency sets = true
|
||||
ggml_metal_device_init: use shared buffers = true
|
||||
ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB
|
||||
| model | size | params | backend | threads | type_k | type_v | fa | test | t/s |
|
||||
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -----: | -----: | -: | --------------: | -------------------: |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo4 | turbo4 | 1 | pp16384 | 755.20 ± 28.45 |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo4 | turbo4 | 1 | tg128 | 15.58 ± 1.31 |
|
||||
|
||||
build: 13afec1 (178)
|
||||
|
||||
--- turbo4 @ 32768 ---
|
||||
ggml_metal_device_init: testing tensor API for f16 support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x1018533e0 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_device_init: testing tensor API for bfloat support
|
||||
ggml_metal_library_compile_pipeline: compiling pipeline: base = 'dummy_kernel', name = 'dummy_kernel'
|
||||
ggml_metal_library_compile_pipeline: loaded dummy_kernel 0x101853360 | th_max = 1024 | th_width = 32
|
||||
ggml_metal_library_init: using embedded metal library
|
||||
ggml_metal_library_init: turbo3 sparse V dequant enabled
|
||||
ggml_metal_library_init: loaded in 0.009 sec
|
||||
ggml_metal_rsets_init: creating a residency set collection (keep_alive = 180 s)
|
||||
ggml_metal_device_init: GPU name: MTL0
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyApple10 (1010)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyCommon3 (3003)
|
||||
ggml_metal_device_init: GPU family: MTLGPUFamilyMetal4 (5002)
|
||||
ggml_metal_device_init: simdgroup reduction = true
|
||||
ggml_metal_device_init: simdgroup matrix mul. = true
|
||||
ggml_metal_device_init: has unified memory = true
|
||||
ggml_metal_device_init: has bfloat = true
|
||||
ggml_metal_device_init: has tensor = true
|
||||
ggml_metal_device_init: use residency sets = true
|
||||
ggml_metal_device_init: use shared buffers = true
|
||||
ggml_metal_device_init: recommendedMaxWorkingSetSize = 115448.73 MB
|
||||
| model | size | params | backend | threads | type_k | type_v | fa | test | t/s |
|
||||
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -----: | -----: | -: | --------------: | -------------------: |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo4 | turbo4 | 1 | pp32768 | 732.00 ± 172.10 |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo4 | turbo4 | 1 | tg128 | 16.29 ± 1.78 |
|
||||
|
||||
build: 13afec1 (178)
|
||||
|
||||
SKIP: q8_0 + smem (q8_0 unaffected by SMEM)
|
||||
SKIP: q8_0 + smem (q8_0 unaffected by SMEM)
|
||||
SKIP: q8_0 + smem (q8_0 unaffected by SMEM)
|
||||
SKIP: q8_0 + smem (q8_0 unaffected by SMEM)
|
||||
=== Done: smem ===
|
||||
| model | size | params | backend | threads | type_k | type_v | fa | test | t/s |
|
||||
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -----: | -----: | -: | --------------: | -------------------: |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo4 | turbo4 | 1 | pp32768 | 1018.88 ± 235.19 |
|
||||
| qwen35moe 35B.A3B Q8_0 | 34.36 GiB | 34.66 B | MTL,BLAS | 1 | turbo4 | turbo4 | 1 | tg128 | 81.62 ± 0.05 |
|
||||
|
||||
build: 13afec1 (178)
|
||||
|
||||
SKIP: q8_0 + smem (q8_0 unaffected by SMEM)
|
||||
SKIP: q8_0 + smem (q8_0 unaffected by SMEM)
|
||||
SKIP: q8_0 + smem (q8_0 unaffected by SMEM)
|
||||
SKIP: q8_0 + smem (q8_0 unaffected by SMEM)
|
||||
=== Done: smem ===
|
||||
@@ -407,6 +407,9 @@ const std::vector<ggml_type> kv_cache_types = {
|
||||
GGML_TYPE_IQ4_NL,
|
||||
GGML_TYPE_Q5_0,
|
||||
GGML_TYPE_Q5_1,
|
||||
GGML_TYPE_TURBO2_0,
|
||||
GGML_TYPE_TURBO3_0,
|
||||
GGML_TYPE_TURBO4_0,
|
||||
};
|
||||
|
||||
static ggml_type kv_cache_type_from_str(const std::string & s) {
|
||||
|
||||
+15
-1
@@ -429,7 +429,10 @@ extern "C" {
|
||||
GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block)
|
||||
GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale)
|
||||
GGML_TYPE_Q1_0 = 41,
|
||||
GGML_TYPE_COUNT = 42,
|
||||
GGML_TYPE_TURBO3_0 = 42, // TurboQuant 3-bit KV cache: 2-bit PolarQuant + 1-bit QJL
|
||||
GGML_TYPE_TURBO4_0 = 43, // TurboQuant 4-bit KV cache: 3-bit PolarQuant + 1-bit QJL
|
||||
GGML_TYPE_TURBO2_0 = 44, // TurboQuant 2-bit KV cache: 2-bit PolarQuant (no QJL)
|
||||
GGML_TYPE_COUNT = 45,
|
||||
};
|
||||
|
||||
// precision
|
||||
@@ -567,6 +570,7 @@ extern "C" {
|
||||
GGML_OP_RWKV_WKV7,
|
||||
GGML_OP_SOLVE_TRI,
|
||||
GGML_OP_GATED_DELTA_NET,
|
||||
GGML_OP_TURBO_WHT,
|
||||
|
||||
GGML_OP_UNARY,
|
||||
|
||||
@@ -2555,6 +2559,16 @@ extern "C" {
|
||||
struct ggml_tensor * beta,
|
||||
struct ggml_tensor * state);
|
||||
|
||||
// TurboQuant Walsh-Hadamard Transform (O(d log d) rotation for KV cache compression)
|
||||
// Applies WHT rotation to 128-element groups along ne[0]: sign1 → butterfly → sign2 → normalize
|
||||
// direction: 0 = forward (signs1 → WHT → signs2), 1 = inverse (signs2 → WHT → signs1)
|
||||
GGML_API struct ggml_tensor * ggml_turbo_wht(
|
||||
struct ggml_context * ctx,
|
||||
struct ggml_tensor * a,
|
||||
int direction,
|
||||
int group_size, // 0 = auto (64 or 128 from ne[0])
|
||||
struct ggml_tensor * scale); // NULL = no InnerQ scaling
|
||||
|
||||
// custom operators
|
||||
|
||||
typedef void (*ggml_custom1_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, int ith, int nth, void * userdata);
|
||||
|
||||
@@ -206,6 +206,7 @@ add_library(ggml-base
|
||||
ggml-threading.h
|
||||
ggml-quants.c
|
||||
ggml-quants.h
|
||||
ggml-turbo-quant.c
|
||||
gguf.cpp)
|
||||
|
||||
set_target_properties(ggml-base PROPERTIES
|
||||
|
||||
@@ -277,6 +277,73 @@ typedef struct {
|
||||
} block_tq2_0;
|
||||
static_assert(sizeof(block_tq2_0) == sizeof(ggml_half) + QK_K / 4, "wrong tq2_0 block size/padding");
|
||||
|
||||
// TurboQuant 3-bit MSE-only: 3-bit PolarQuant indices (no QJL)
|
||||
// Storage block size = 32 (matches q4_0 for optimal GPU parallelism)
|
||||
// Transform group size = 128 (head_dim, for rotation Gaussianization)
|
||||
// Per block: norm(fp16) + 2-bit indices (8 bytes) + 1-bit extra (4 bytes) = 14 bytes per 32 values
|
||||
// = 3.5 bits/value → 4.6× compression vs fp16
|
||||
// The 3-bit index is split: lower 2 bits in qs[], upper 1 bit in signs[]
|
||||
#define QK_TURBO3 128 // Block size 128: one block per rotation group, eliminates redundant norms
|
||||
#define QK_TURBO3_GROUP 128 // rotation group size = head_dim
|
||||
// Derived: FA template nl parameters (auto-scale with block size)
|
||||
#define NL_TURBO3 (QK_TURBO3 / 16) // non-vec FA iterations per block
|
||||
#define NL_TURBO3_VEC (QK_TURBO3 / 4) // vec FA iterations per block
|
||||
typedef struct {
|
||||
ggml_half norm; // 2 bytes: vector L2 norm (for rescaling)
|
||||
uint8_t qs[QK_TURBO3 / 4]; // 8 bytes: lower 2-bit indices (4 per byte)
|
||||
uint8_t signs[QK_TURBO3 / 8]; // 4 bytes: upper 1-bit of 3-bit index (8 per byte)
|
||||
} block_turbo3_0; // 14 bytes total
|
||||
static_assert(sizeof(block_turbo3_0) == sizeof(ggml_half) + QK_TURBO3/4 + QK_TURBO3/8, "wrong turbo3_0 block size/padding");
|
||||
|
||||
// TurboQuant 4-bit: 3-bit PolarQuant indices + 1-bit QJL signs
|
||||
// TURBO4_USE_4BIT: switch between 4-bit PolarQuant (new) and 3-bit+QJL (legacy)
|
||||
// Default: 4-bit on all backends (Metal + CUDA validated)
|
||||
#ifndef TURBO4_USE_4BIT
|
||||
# define TURBO4_USE_4BIT 1
|
||||
#endif
|
||||
|
||||
#define QK_TURBO4 128
|
||||
|
||||
#if TURBO4_USE_4BIT
|
||||
// 4-bit PolarQuant: 16 optimal centroids, nibble packed, no QJL
|
||||
// Per block: norm(fp16) + rnorm(fp16, reserved) + 4-bit indices (64 bytes)
|
||||
// = 68 bytes per 128 values = 4.25 bits/value → 3.8× compression vs fp16
|
||||
typedef struct {
|
||||
ggml_half norm; // 2 bytes
|
||||
ggml_half rnorm; // 2 bytes (reserved, unused in 4-bit mode)
|
||||
uint8_t qs[QK_TURBO4 / 2]; // 64 bytes: 4-bit PolarQuant indices (nibble packed)
|
||||
} block_turbo4_0; // 68 bytes total
|
||||
static_assert(sizeof(block_turbo4_0) == 68, "wrong turbo4_0 block size");
|
||||
#else
|
||||
// Legacy 3-bit PolarQuant + 1-bit QJL (original paper design)
|
||||
// Per block: norm(fp16) + rnorm(fp16) + 3-bit indices (48 bytes) + 1-bit QJL signs (16 bytes)
|
||||
// = 68 bytes per 128 values = 4.25 bits/value → 3.8× compression vs fp16
|
||||
typedef struct {
|
||||
ggml_half norm; // 2 bytes
|
||||
ggml_half rnorm; // 2 bytes: residual norm for QJL scale
|
||||
uint8_t qs[QK_TURBO4 * 3 / 8]; // 48 bytes: 3-bit PolarQuant indices
|
||||
uint8_t signs[QK_TURBO4 / 8]; // 16 bytes: 1-bit QJL signs
|
||||
} block_turbo4_0; // 68 bytes total
|
||||
static_assert(sizeof(block_turbo4_0) == 2*sizeof(ggml_half) + QK_TURBO4*3/8 + QK_TURBO4/8, "wrong turbo4_0 block size");
|
||||
#endif
|
||||
|
||||
static_assert(QK_TURBO4 == 128, "turbo4 kernels assume QK_TURBO4 == 128");
|
||||
|
||||
// TurboQuant 2-bit: 2-bit PolarQuant indices only (no QJL)
|
||||
// Per block: norm(fp16) + 2-bit indices (8 bytes) = 10 bytes per 32 values
|
||||
// = 2.5 bits/value → 6.4× compression vs fp16
|
||||
// 4 centroids (Lloyd-Max for N(0, 1/128)): {-0.133462, -0.039994, 0.039994, 0.133462}
|
||||
#define QK_TURBO2 128 // Block size 128: one block per rotation group
|
||||
#define QK_TURBO2_GROUP 128 // rotation group size = head_dim
|
||||
// Derived: FA template nl parameters (auto-scale with block size)
|
||||
#define NL_TURBO2 (QK_TURBO2 / 16) // non-vec FA iterations per block
|
||||
#define NL_TURBO2_VEC (QK_TURBO2 / 4) // vec FA iterations per block
|
||||
typedef struct {
|
||||
ggml_half norm; // 2 bytes: corrected L2 norm
|
||||
uint8_t qs[QK_TURBO2 / 4]; // 8 bytes: 2-bit indices (4 per byte)
|
||||
} block_turbo2_0; // 10 bytes total
|
||||
static_assert(sizeof(block_turbo2_0) == sizeof(ggml_half) + QK_TURBO2/4, "wrong turbo2_0 block size/padding");
|
||||
|
||||
//
|
||||
// Super-block quantization structures
|
||||
//
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
#include "ggml-cpu-impl.h"
|
||||
#include "ggml-impl.h"
|
||||
#include "quants.h"
|
||||
#include "ggml-quants.h"
|
||||
#include "ggml-threading.h"
|
||||
#include "unary-ops.h"
|
||||
#include "binary-ops.h"
|
||||
@@ -208,6 +209,17 @@ typedef pthread_t ggml_thread_t;
|
||||
#include <TargetConditionals.h>
|
||||
#endif
|
||||
|
||||
// Forward declarations — defined below, after utility functions
|
||||
static void ggml_vec_dot_turbo3_0_f32(int n, float * GGML_RESTRICT s, size_t bs,
|
||||
const void * GGML_RESTRICT vx, size_t bx,
|
||||
const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||
static void ggml_vec_dot_turbo2_0_f32(int n, float * GGML_RESTRICT s, size_t bs,
|
||||
const void * GGML_RESTRICT vx, size_t bx,
|
||||
const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||
static void ggml_vec_dot_turbo4_0_f32(int n, float * GGML_RESTRICT s, size_t bs,
|
||||
const void * GGML_RESTRICT vx, size_t bx,
|
||||
const void * GGML_RESTRICT vy, size_t by, int nrc);
|
||||
|
||||
static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = {
|
||||
[GGML_TYPE_F32] = {
|
||||
.from_float = (ggml_from_float_t) ggml_cpu_fp32_to_fp32,
|
||||
@@ -403,6 +415,24 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = {
|
||||
[GGML_TYPE_I32] = {
|
||||
.from_float = (ggml_from_float_t) ggml_cpu_fp32_to_i32,
|
||||
},
|
||||
[GGML_TYPE_TURBO3_0] = {
|
||||
.from_float = (ggml_from_float_t) quantize_row_turbo3_0_ref,
|
||||
.vec_dot = (ggml_vec_dot_t) ggml_vec_dot_turbo3_0_f32,
|
||||
.vec_dot_type = GGML_TYPE_F32,
|
||||
.nrows = 1,
|
||||
},
|
||||
[GGML_TYPE_TURBO2_0] = {
|
||||
.from_float = (ggml_from_float_t) quantize_row_turbo2_0_ref,
|
||||
.vec_dot = (ggml_vec_dot_t) ggml_vec_dot_turbo2_0_f32,
|
||||
.vec_dot_type = GGML_TYPE_F32,
|
||||
.nrows = 1,
|
||||
},
|
||||
[GGML_TYPE_TURBO4_0] = {
|
||||
.from_float = (ggml_from_float_t) quantize_row_turbo4_0_ref,
|
||||
.vec_dot = (ggml_vec_dot_t) ggml_vec_dot_turbo4_0_f32,
|
||||
.vec_dot_type = GGML_TYPE_F32,
|
||||
.nrows = 1,
|
||||
},
|
||||
};
|
||||
|
||||
const struct ggml_type_traits_cpu * ggml_get_type_traits_cpu(enum ggml_type type) {
|
||||
@@ -2047,6 +2077,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
|
||||
{
|
||||
ggml_compute_forward_gated_delta_net(params, tensor);
|
||||
} break;
|
||||
case GGML_OP_TURBO_WHT:
|
||||
{
|
||||
ggml_compute_forward_turbo_wht(params, tensor);
|
||||
} break;
|
||||
case GGML_OP_MAP_CUSTOM1:
|
||||
{
|
||||
ggml_compute_forward_map_custom1(params, tensor);
|
||||
@@ -2227,6 +2261,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
|
||||
case GGML_OP_COUNT_EQUAL:
|
||||
case GGML_OP_SOLVE_TRI:
|
||||
case GGML_OP_GATED_DELTA_NET:
|
||||
case GGML_OP_TURBO_WHT:
|
||||
{
|
||||
n_tasks = n_threads;
|
||||
} break;
|
||||
@@ -2947,6 +2982,10 @@ struct ggml_cplan ggml_graph_plan(
|
||||
const int64_t per_thread = S_v + (K > 1 ? S_v * S_v : 0);
|
||||
cur = per_thread * sizeof(float) * n_tasks;
|
||||
} break;
|
||||
case GGML_OP_TURBO_WHT:
|
||||
{
|
||||
cur = 0; // no extra workspace needed
|
||||
} break;
|
||||
case GGML_OP_COUNT:
|
||||
{
|
||||
GGML_ABORT("fatal error");
|
||||
@@ -3385,6 +3424,65 @@ enum ggml_status ggml_graph_compute_with_ctx(struct ggml_context * ctx, struct g
|
||||
return ggml_graph_compute(cgraph, &cplan);
|
||||
}
|
||||
|
||||
// TurboQuant3 vec_dot: dequantize turbo3 block to f32, then dot with f32 operand.
|
||||
// Used by CPU flash attention for models with D not supported by CUDA FA (e.g. D=192).
|
||||
static void ggml_vec_dot_turbo3_0_f32(int n, float * GGML_RESTRICT s, size_t bs,
|
||||
const void * GGML_RESTRICT vx, size_t bx,
|
||||
const void * GGML_RESTRICT vy, size_t by, int nrc) {
|
||||
GGML_ASSERT(nrc == 1);
|
||||
GGML_UNUSED(bs); GGML_UNUSED(bx); GGML_UNUSED(by); GGML_UNUSED(nrc);
|
||||
|
||||
// Dequantize turbo3 to f32 temp buffer, then dot
|
||||
float tmp[4096]; // max head_dim
|
||||
GGML_ASSERT(n <= 4096);
|
||||
ggml_get_type_traits(GGML_TYPE_TURBO3_0)->to_float(vx, tmp, n);
|
||||
|
||||
const float * y = (const float *)vy;
|
||||
float sum = 0.0f;
|
||||
for (int i = 0; i < n; i++) {
|
||||
sum += tmp[i] * y[i];
|
||||
}
|
||||
*s = sum;
|
||||
}
|
||||
|
||||
// TurboQuant2 vec_dot: dequantize turbo2 block to f32, then dot with f32 operand.
|
||||
static void ggml_vec_dot_turbo2_0_f32(int n, float * GGML_RESTRICT s, size_t bs,
|
||||
const void * GGML_RESTRICT vx, size_t bx,
|
||||
const void * GGML_RESTRICT vy, size_t by, int nrc) {
|
||||
GGML_ASSERT(nrc == 1);
|
||||
GGML_UNUSED(bs); GGML_UNUSED(bx); GGML_UNUSED(by); GGML_UNUSED(nrc);
|
||||
|
||||
float tmp[4096];
|
||||
GGML_ASSERT(n <= 4096);
|
||||
ggml_get_type_traits(GGML_TYPE_TURBO2_0)->to_float(vx, tmp, n);
|
||||
|
||||
const float * y = (const float *)vy;
|
||||
float sum = 0.0f;
|
||||
for (int i = 0; i < n; i++) {
|
||||
sum += tmp[i] * y[i];
|
||||
}
|
||||
*s = sum;
|
||||
}
|
||||
|
||||
// TurboQuant4 vec_dot: dequantize turbo4 block to f32, then dot with f32 operand.
|
||||
static void ggml_vec_dot_turbo4_0_f32(int n, float * GGML_RESTRICT s, size_t bs,
|
||||
const void * GGML_RESTRICT vx, size_t bx,
|
||||
const void * GGML_RESTRICT vy, size_t by, int nrc) {
|
||||
GGML_ASSERT(nrc == 1);
|
||||
GGML_UNUSED(bs); GGML_UNUSED(bx); GGML_UNUSED(by); GGML_UNUSED(nrc);
|
||||
|
||||
float tmp[4096];
|
||||
GGML_ASSERT(n <= 4096);
|
||||
ggml_get_type_traits(GGML_TYPE_TURBO4_0)->to_float(vx, tmp, n);
|
||||
|
||||
const float * y = (const float *)vy;
|
||||
float sum = 0.0f;
|
||||
for (int i = 0; i < n; i++) {
|
||||
sum += tmp[i] * y[i];
|
||||
}
|
||||
*s = sum;
|
||||
}
|
||||
|
||||
void ggml_cpu_fp32_to_fp32(const float * x, float * y, int64_t n) {
|
||||
memcpy(y, x, n * sizeof(float));
|
||||
}
|
||||
|
||||
@@ -4978,6 +4978,14 @@ static void ggml_compute_forward_set_rows_f32(
|
||||
|
||||
ggml_from_float_t const from_float = ggml_get_type_traits_cpu(dst->type)->from_float;
|
||||
|
||||
// For turbo types: communicate WHT group size to the quantize function via global
|
||||
if (dst->type == GGML_TYPE_TURBO3_0 || dst->type == GGML_TYPE_TURBO4_0 || dst->type == GGML_TYPE_TURBO2_0) {
|
||||
extern int turbo3_cpu_wht_group_size;
|
||||
int gs = 0;
|
||||
memcpy(&gs, dst->op_params, sizeof(int));
|
||||
turbo3_cpu_wht_group_size = (gs == 64 || gs == 128) ? gs : 0;
|
||||
}
|
||||
|
||||
for (int64_t i03 = 0; i03 < ne03; ++i03) {
|
||||
for (int64_t i02 = 0; i02 < ne02; ++i02) {
|
||||
for (int64_t i = ir0; i < ir1; ++i) {
|
||||
@@ -10686,6 +10694,104 @@ void ggml_compute_forward_gated_delta_net(
|
||||
}
|
||||
}
|
||||
|
||||
// ggml_compute_forward_turbo_wht
|
||||
|
||||
// WHT sign arrays (must match Metal shader turbo_wht_signs1/2)
|
||||
static const float turbo_wht_s1[128] = {-1,1,1,-1,-1,1,-1,1,-1,-1,1,1,1,1,1,1,1,-1,1,-1,1,-1,-1,1,1,1,-1,1,1,-1,-1,-1,-1,1,1,-1,1,1,-1,1,-1,1,1,-1,-1,1,-1,1,1,1,1,-1,-1,-1,-1,-1,1,-1,1,1,1,1,-1,1,-1,-1,1,-1,-1,-1,1,-1,-1,-1,1,-1,-1,-1,1,1,1,-1,-1,1,1,1,-1,-1,1,1,-1,1,1,-1,1,-1,-1,1,1,-1,1,-1,1,-1,1,1,1,1,-1,1,-1,1,1,-1,1,1,-1,-1,-1,-1,-1,1,1,-1,1,1,-1,1};
|
||||
static const float turbo_wht_s2[128] = {1,1,1,1,-1,1,1,-1,1,-1,-1,-1,1,-1,-1,-1,1,1,-1,-1,1,-1,1,-1,1,-1,-1,1,-1,1,1,1,1,1,-1,-1,-1,1,-1,-1,-1,-1,-1,-1,1,1,1,-1,1,-1,1,1,1,-1,-1,1,-1,-1,-1,-1,-1,-1,1,1,1,-1,1,-1,-1,-1,-1,1,-1,1,-1,1,-1,-1,1,1,-1,1,-1,1,1,-1,1,-1,-1,-1,-1,1,-1,-1,1,-1,1,-1,1,1,1,-1,-1,1,-1,1,-1,1,1,-1,-1,1,-1,1,-1,1,1,-1,1,-1,1,-1,-1,-1,-1,-1,1,-1};
|
||||
|
||||
static void ggml_compute_forward_turbo_wht_f32(
|
||||
const ggml_compute_params * params,
|
||||
ggml_tensor * dst) {
|
||||
const ggml_tensor * src = dst->src[0];
|
||||
const ggml_tensor * scale_tensor = dst->src[1]; // InnerQ scale_inv (may be NULL)
|
||||
const float * src_data = (const float *) src->data;
|
||||
float * dst_data = (float *) dst->data;
|
||||
const float * scale_inv = scale_tensor ? (const float *) scale_tensor->data : NULL;
|
||||
|
||||
int direction;
|
||||
int group_size;
|
||||
memcpy(&direction, dst->op_params + 0, sizeof(int));
|
||||
memcpy(&group_size, dst->op_params + sizeof(int), sizeof(int));
|
||||
|
||||
const int64_t head_dim = src->ne[0];
|
||||
const int64_t n_heads = ggml_nelements(src) / head_dim;
|
||||
const int64_t groups_per_head = head_dim / group_size;
|
||||
const int tail_size = (int)(head_dim % group_size);
|
||||
const int64_t n_groups = groups_per_head * n_heads;
|
||||
|
||||
const float inv_sqrt = 1.0f / sqrtf((float)group_size);
|
||||
|
||||
// Parallel over groups
|
||||
const int64_t ith = params->ith;
|
||||
const int64_t nth = params->nth;
|
||||
const int64_t grp_start = (n_groups * ith) / nth;
|
||||
const int64_t grp_end = (n_groups * (ith + 1)) / nth;
|
||||
|
||||
// Select sign arrays: for 64-group, use first 64 elements of the 128-element arrays
|
||||
const float * s_first = (direction == 0) ? turbo_wht_s1 : turbo_wht_s2;
|
||||
const float * s_second = (direction == 0) ? turbo_wht_s2 : turbo_wht_s1;
|
||||
|
||||
for (int64_t g = grp_start; g < grp_end; g++) {
|
||||
const int64_t head_idx = g / groups_per_head;
|
||||
const int64_t grp_in_head = g % groups_per_head;
|
||||
const int64_t base = head_idx * head_dim + grp_in_head * group_size;
|
||||
|
||||
float x[128]; // max group_size
|
||||
const float * in = src_data + base;
|
||||
|
||||
// InnerQ forward: apply scale_inv BEFORE signs+WHT (for Q pre-rotation)
|
||||
if (direction == 0 && scale_inv != NULL) {
|
||||
for (int i = 0; i < group_size; i++) x[i] = in[i] * scale_inv[i % group_size];
|
||||
} else {
|
||||
for (int i = 0; i < group_size; i++) x[i] = in[i];
|
||||
}
|
||||
|
||||
// Apply first signs
|
||||
for (int i = 0; i < group_size; i++) x[i] *= s_first[i];
|
||||
|
||||
// WHT butterfly (log2(group_size) stages)
|
||||
for (int h = 1; h < group_size; h *= 2) {
|
||||
for (int i = 0; i < group_size; i += h * 2) {
|
||||
for (int j = i; j < i + h; j++) {
|
||||
float a = x[j], b = x[j + h];
|
||||
x[j] = a + b;
|
||||
x[j + h] = a - b;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Normalize + second signs
|
||||
float * out = dst_data + base;
|
||||
for (int i = 0; i < group_size; i++) {
|
||||
float val = x[i] * inv_sqrt * s_second[i];
|
||||
// InnerQ inverse: apply scale_inv AFTER WHT+signs (for V un-rotation)
|
||||
if (direction == 1 && scale_inv != NULL) {
|
||||
val *= scale_inv[i % group_size];
|
||||
}
|
||||
out[i] = val;
|
||||
}
|
||||
}
|
||||
|
||||
// Copy tail elements unchanged (identity pass-through)
|
||||
if (tail_size > 0 && ith == 0) {
|
||||
const int64_t tail_offset = groups_per_head * group_size;
|
||||
for (int64_t h = 0; h < n_heads; h++) {
|
||||
const int64_t base = h * head_dim + tail_offset;
|
||||
memcpy(dst_data + base, src_data + base, tail_size * sizeof(float));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void ggml_compute_forward_turbo_wht(
|
||||
const ggml_compute_params * params,
|
||||
ggml_tensor * dst) {
|
||||
switch (dst->src[0]->type) {
|
||||
case GGML_TYPE_F32: ggml_compute_forward_turbo_wht_f32(params, dst); break;
|
||||
default: GGML_ABORT("fatal error");
|
||||
}
|
||||
}
|
||||
|
||||
// ggml_compute_forward_rwkv_wkv7
|
||||
|
||||
static void ggml_compute_forward_rwkv_wkv7_f32(
|
||||
|
||||
@@ -104,6 +104,7 @@ void ggml_compute_forward_rwkv_wkv7(const struct ggml_compute_params * params, s
|
||||
void ggml_compute_forward_solve_tri(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_gla(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_gated_delta_net(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_turbo_wht(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_map_custom1(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_map_custom2(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
void ggml_compute_forward_map_custom3(const struct ggml_compute_params * params, struct ggml_tensor * dst);
|
||||
|
||||
@@ -120,7 +120,22 @@ if (CUDAToolkit_FOUND)
|
||||
template-instances/fattn-vec-instance-f16-f16.cu
|
||||
template-instances/fattn-vec-instance-q4_0-q4_0.cu
|
||||
template-instances/fattn-vec-instance-q8_0-q8_0.cu
|
||||
template-instances/fattn-vec-instance-bf16-bf16.cu)
|
||||
template-instances/fattn-vec-instance-bf16-bf16.cu
|
||||
template-instances/fattn-vec-instance-turbo3_0-turbo3_0.cu
|
||||
template-instances/fattn-vec-instance-turbo3_0-q8_0.cu
|
||||
template-instances/fattn-vec-instance-q8_0-turbo3_0.cu
|
||||
template-instances/fattn-vec-instance-turbo2_0-turbo2_0.cu
|
||||
template-instances/fattn-vec-instance-turbo2_0-q8_0.cu
|
||||
template-instances/fattn-vec-instance-q8_0-turbo2_0.cu
|
||||
template-instances/fattn-vec-instance-turbo3_0-turbo2_0.cu
|
||||
template-instances/fattn-vec-instance-turbo2_0-turbo3_0.cu
|
||||
template-instances/fattn-vec-instance-turbo4_0-turbo4_0.cu
|
||||
template-instances/fattn-vec-instance-turbo4_0-q8_0.cu
|
||||
template-instances/fattn-vec-instance-q8_0-turbo4_0.cu
|
||||
template-instances/fattn-vec-instance-turbo4_0-turbo3_0.cu
|
||||
template-instances/fattn-vec-instance-turbo3_0-turbo4_0.cu
|
||||
template-instances/fattn-vec-instance-turbo4_0-turbo2_0.cu
|
||||
template-instances/fattn-vec-instance-turbo2_0-turbo4_0.cu)
|
||||
endif()
|
||||
|
||||
ggml_add_backend_library(ggml-cuda
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#include "convert.cuh"
|
||||
#include "dequantize.cuh"
|
||||
#include "turbo-quant.cuh"
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
@@ -758,6 +759,12 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) {
|
||||
return dequantize_row_mxfp4_cuda;
|
||||
case GGML_TYPE_NVFP4:
|
||||
return dequantize_row_nvfp4_cuda;
|
||||
case GGML_TYPE_TURBO3_0:
|
||||
return dequantize_block_cont_cuda<QK_TURBO3, QR_TURBO3, dequantize_turbo3_0>;
|
||||
case GGML_TYPE_TURBO2_0:
|
||||
return dequantize_block_cont_cuda<QK_TURBO2, QR_TURBO2, dequantize_turbo2_0>;
|
||||
case GGML_TYPE_TURBO4_0:
|
||||
return dequantize_block_cont_cuda<QK_TURBO4, QR_TURBO4, dequantize_turbo4_0>;
|
||||
case GGML_TYPE_F32:
|
||||
return convert_unary_cont_cuda<float>;
|
||||
case GGML_TYPE_BF16:
|
||||
@@ -813,6 +820,12 @@ to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) {
|
||||
return dequantize_row_mxfp4_cuda;
|
||||
case GGML_TYPE_NVFP4:
|
||||
return dequantize_row_nvfp4_cuda;
|
||||
case GGML_TYPE_TURBO3_0:
|
||||
return dequantize_block_cont_cuda<QK_TURBO3, QR_TURBO3, dequantize_turbo3_0>;
|
||||
case GGML_TYPE_TURBO2_0:
|
||||
return dequantize_block_cont_cuda<QK_TURBO2, QR_TURBO2, dequantize_turbo2_0>;
|
||||
case GGML_TYPE_TURBO4_0:
|
||||
return dequantize_block_cont_cuda<QK_TURBO4, QR_TURBO4, dequantize_turbo4_0>;
|
||||
case GGML_TYPE_F16:
|
||||
return convert_unary_cont_cuda<half>;
|
||||
case GGML_TYPE_BF16:
|
||||
@@ -838,6 +851,12 @@ to_fp16_nc_cuda_t ggml_get_to_fp16_nc_cuda(ggml_type type) {
|
||||
return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;
|
||||
case GGML_TYPE_Q8_0:
|
||||
return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;
|
||||
case GGML_TYPE_TURBO3_0:
|
||||
return dequantize_block_cuda<QK_TURBO3, QR_TURBO3, dequantize_turbo3_0>;
|
||||
case GGML_TYPE_TURBO2_0:
|
||||
return dequantize_block_cuda<QK_TURBO2, QR_TURBO2, dequantize_turbo2_0>;
|
||||
case GGML_TYPE_TURBO4_0:
|
||||
return dequantize_block_cuda<QK_TURBO4, QR_TURBO4, dequantize_turbo4_0>;
|
||||
case GGML_TYPE_BF16:
|
||||
return convert_unary_cuda<nv_bfloat16>;
|
||||
default:
|
||||
@@ -884,6 +903,12 @@ to_fp32_nc_cuda_t ggml_get_to_fp32_nc_cuda(ggml_type type) {
|
||||
return dequantize_block_cuda<QK5_1, QR5_1, dequantize_q5_1>;
|
||||
case GGML_TYPE_Q8_0:
|
||||
return dequantize_block_cuda<QK8_0, QR8_0, dequantize_q8_0>;
|
||||
case GGML_TYPE_TURBO3_0:
|
||||
return dequantize_block_cuda<QK_TURBO3, QR_TURBO3, dequantize_turbo3_0>;
|
||||
case GGML_TYPE_TURBO2_0:
|
||||
return dequantize_block_cuda<QK_TURBO2, QR_TURBO2, dequantize_turbo2_0>;
|
||||
case GGML_TYPE_TURBO4_0:
|
||||
return dequantize_block_cuda<QK_TURBO4, QR_TURBO4, dequantize_turbo4_0>;
|
||||
case GGML_TYPE_BF16:
|
||||
return convert_unary_cuda<nv_bfloat16, float>;
|
||||
default:
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#include "common.cuh"
|
||||
#include "turbo-quant.cuh"
|
||||
|
||||
static __device__ __forceinline__ void dequantize_q1_0(const void * vx, const int64_t ib, const int iqs, float2 & v){
|
||||
const block_q1_0 * x = (const block_q1_0 *) vx;
|
||||
@@ -97,3 +98,29 @@ static __device__ __forceinline__ void dequantize_q8_0(const void * vx, const in
|
||||
v.x *= d;
|
||||
v.y *= d;
|
||||
}
|
||||
|
||||
// Turbo4: 4-bit PolarQuant (nibble packed), block size 128
|
||||
// iqs is the element index within the block (even), produces elements iqs and iqs+1
|
||||
static __device__ __forceinline__ void dequantize_turbo4_0(const void * vx, const int64_t ib, const int iqs, float2 & v){
|
||||
const block_turbo4_0 * x = (const block_turbo4_0 *) vx;
|
||||
const float norm = __half2float(x[ib].norm);
|
||||
v.x = turbo4_dequant_element(&x[ib], iqs + 0, norm);
|
||||
v.y = turbo4_dequant_element(&x[ib], iqs + 1, norm);
|
||||
}
|
||||
|
||||
// Turbo3: 3-bit PolarQuant (2-bit qs + 1-bit sign), block size 32
|
||||
// iqs is the element index within the block (even), produces elements iqs and iqs+1
|
||||
static __device__ __forceinline__ void dequantize_turbo3_0(const void * vx, const int64_t ib, const int iqs, float2 & v){
|
||||
const block_turbo3_0 * x = (const block_turbo3_0 *) vx;
|
||||
const float norm = __half2float(x[ib].norm);
|
||||
v.x = turbo3_dequant_element(&x[ib], iqs + 0, norm);
|
||||
v.y = turbo3_dequant_element(&x[ib], iqs + 1, norm);
|
||||
}
|
||||
|
||||
// Turbo2: 2-bit PolarQuant (2-bit qs only, no sign), block size 32
|
||||
static __device__ __forceinline__ void dequantize_turbo2_0(const void * vx, const int64_t ib, const int iqs, float2 & v){
|
||||
const block_turbo2_0 * x = (const block_turbo2_0 *) vx;
|
||||
const float norm = __half2float(x[ib].norm);
|
||||
v.x = turbo2_dequant_element(&x[ib], iqs + 0, norm);
|
||||
v.y = turbo2_dequant_element(&x[ib], iqs + 1, norm);
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
#include "common.cuh"
|
||||
#include "convert.cuh"
|
||||
#include "vecdotq.cuh"
|
||||
#include "turbo-quant.cuh"
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
@@ -288,6 +289,164 @@ static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_q8_0(
|
||||
return sum;
|
||||
}
|
||||
|
||||
// Turbo3 KQ dot product: dequantize K from turbo3 blocks, dot with Q (float2/half2)
|
||||
// Uses float Q path (like f16), not q8_1 integer path.
|
||||
// Q_v is half2[] or float2[] with D/2 pairs, partitioned nthreads-strided.
|
||||
//
|
||||
// Matches the f16 pattern: outer loop steps by nthreads*cpy_ne, inner loop
|
||||
// processes cpy_ne pairs per thread per iteration so Q_v and K indices stay aligned.
|
||||
// elem0 = 2*k_KQ is always even, so elem0 and elem0+1 always share the same
|
||||
// turbo3 block (ib), qs byte, and signs byte — loaded once per pair.
|
||||
template <int D, int nthreads>
|
||||
static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_turbo3_0(
|
||||
const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) {
|
||||
|
||||
const block_turbo3_0 * K_turbo = (const block_turbo3_0 *) K_c;
|
||||
GGML_UNUSED(Q_q8);
|
||||
GGML_UNUSED(Q_ds_v);
|
||||
|
||||
constexpr int cpy_nb = ggml_cuda_get_max_cpy_bytes();
|
||||
constexpr int cpy_ne = cpy_nb / 4;
|
||||
|
||||
float sum = 0.0f;
|
||||
|
||||
#pragma unroll
|
||||
for (int k_KQ_0 = 0; k_KQ_0 < D/2; k_KQ_0 += nthreads*cpy_ne) {
|
||||
#pragma unroll
|
||||
for (int k_KQ_1 = 0; k_KQ_1 < cpy_ne; ++k_KQ_1) {
|
||||
const int k_KQ = k_KQ_0 + (threadIdx.x % nthreads)*cpy_ne + k_KQ_1;
|
||||
|
||||
// elem0 is always even; elem0 and elem1 are always in the same block,
|
||||
// the same qs byte (j0%4 ∈ {0,2}), and the same signs byte (j0%8 ∈ {0,2,4,6}).
|
||||
const int elem0 = k_KQ * 2; // always even
|
||||
const int ib = elem0 / QK_TURBO3; // shared block index
|
||||
const int j0 = elem0 % QK_TURBO3; // always even, 0..30
|
||||
|
||||
// Single loads for the shared block fields
|
||||
const float norm = __half2float(K_turbo[ib].norm);
|
||||
const uint8_t qs_byte = K_turbo[ib].qs[j0 / 4]; // covers both j0 and j0+1
|
||||
const uint8_t sgn_byte = K_turbo[ib].signs[j0 / 8]; // covers both j0 and j0+1
|
||||
|
||||
// Extract 3-bit indices for elem0 and elem1 from shared bytes
|
||||
const int shift = (j0 % 4) * 2; // 0 or 4
|
||||
const uint8_t idx0 = ((qs_byte >> shift) & 0x3) | (((sgn_byte >> (j0 % 8)) & 0x1) << 2);
|
||||
const uint8_t idx1 = ((qs_byte >> (shift+2)) & 0x3) | (((sgn_byte >> (j0 % 8 + 1)) & 0x1) << 2);
|
||||
|
||||
float2 kv;
|
||||
kv.x = TURBO_CENTROIDS_3BIT[idx0] * norm;
|
||||
kv.y = TURBO_CENTROIDS_3BIT[idx1] * norm;
|
||||
|
||||
#ifdef V_DOT2_F32_F16_AVAILABLE
|
||||
const half2 qv = ((const half2 *) Q_v)[k_KQ_0/nthreads + k_KQ_1];
|
||||
ggml_cuda_mad(sum, make_float2(kv.x, kv.y), __half22float2(qv));
|
||||
#else
|
||||
const float2 qv = ((const float2 *) Q_v)[k_KQ_0/nthreads + k_KQ_1];
|
||||
sum += kv.x * qv.x + kv.y * qv.y;
|
||||
#endif // V_DOT2_F32_F16_AVAILABLE
|
||||
}
|
||||
}
|
||||
|
||||
return sum;
|
||||
}
|
||||
|
||||
// Turbo2 KQ dot product: dequantize K from turbo2 blocks, dot with Q (float2/half2)
|
||||
// Same structure as turbo3 but reads 2-bit indices from qs only (no signs).
|
||||
template <int D, int nthreads>
|
||||
static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_turbo2_0(
|
||||
const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) {
|
||||
|
||||
const block_turbo2_0 * K_turbo = (const block_turbo2_0 *) K_c;
|
||||
GGML_UNUSED(Q_q8);
|
||||
GGML_UNUSED(Q_ds_v);
|
||||
|
||||
constexpr int cpy_nb = ggml_cuda_get_max_cpy_bytes();
|
||||
constexpr int cpy_ne = cpy_nb / 4;
|
||||
|
||||
float sum = 0.0f;
|
||||
|
||||
#pragma unroll
|
||||
for (int k_KQ_0 = 0; k_KQ_0 < D/2; k_KQ_0 += nthreads*cpy_ne) {
|
||||
#pragma unroll
|
||||
for (int k_KQ_1 = 0; k_KQ_1 < cpy_ne; ++k_KQ_1) {
|
||||
const int k_KQ = k_KQ_0 + (threadIdx.x % nthreads)*cpy_ne + k_KQ_1;
|
||||
|
||||
const int elem0 = k_KQ * 2;
|
||||
const int ib = elem0 / QK_TURBO2;
|
||||
const int j0 = elem0 % QK_TURBO2;
|
||||
|
||||
const float norm = __half2float(K_turbo[ib].norm);
|
||||
const uint8_t qs_byte = K_turbo[ib].qs[j0 / 4];
|
||||
|
||||
const int shift = (j0 % 4) * 2;
|
||||
const uint8_t idx0 = (qs_byte >> shift) & 0x3;
|
||||
const uint8_t idx1 = (qs_byte >> (shift+2)) & 0x3;
|
||||
|
||||
float2 kv;
|
||||
kv.x = TURBO_CENTROIDS_2BIT[idx0] * norm;
|
||||
kv.y = TURBO_CENTROIDS_2BIT[idx1] * norm;
|
||||
|
||||
#ifdef V_DOT2_F32_F16_AVAILABLE
|
||||
const half2 qv = ((const half2 *) Q_v)[k_KQ_0/nthreads + k_KQ_1];
|
||||
ggml_cuda_mad(sum, make_float2(kv.x, kv.y), __half22float2(qv));
|
||||
#else
|
||||
const float2 qv = ((const float2 *) Q_v)[k_KQ_0/nthreads + k_KQ_1];
|
||||
sum += kv.x * qv.x + kv.y * qv.y;
|
||||
#endif // V_DOT2_F32_F16_AVAILABLE
|
||||
}
|
||||
}
|
||||
|
||||
return sum;
|
||||
}
|
||||
|
||||
// Turbo4 KQ dot product: dequantize K from turbo4 blocks, dot with Q (float2/half2)
|
||||
// 4-bit nibble packed: qs[j/2] >> ((j%2)*4) & 0xF
|
||||
template <int D, int nthreads>
|
||||
static __device__ __forceinline__ float vec_dot_fattn_vec_KQ_turbo4_0(
|
||||
const char * __restrict__ K_c, const void * __restrict__ Q_v, const int * __restrict__ Q_q8, const void * __restrict__ Q_ds_v) {
|
||||
|
||||
const block_turbo4_0 * K_turbo = (const block_turbo4_0 *) K_c;
|
||||
GGML_UNUSED(Q_q8);
|
||||
GGML_UNUSED(Q_ds_v);
|
||||
|
||||
constexpr int cpy_nb = ggml_cuda_get_max_cpy_bytes();
|
||||
constexpr int cpy_ne = cpy_nb / 4;
|
||||
|
||||
float sum = 0.0f;
|
||||
|
||||
#pragma unroll
|
||||
for (int k_KQ_0 = 0; k_KQ_0 < D/2; k_KQ_0 += nthreads*cpy_ne) {
|
||||
#pragma unroll
|
||||
for (int k_KQ_1 = 0; k_KQ_1 < cpy_ne; ++k_KQ_1) {
|
||||
const int k_KQ = k_KQ_0 + (threadIdx.x % nthreads)*cpy_ne + k_KQ_1;
|
||||
|
||||
const int elem0 = k_KQ * 2; // always even
|
||||
const int ib = elem0 / QK_TURBO4; // block index
|
||||
const int j0 = elem0 % QK_TURBO4; // always even
|
||||
|
||||
const float norm = __half2float(K_turbo[ib].norm);
|
||||
// Both j0 and j0+1 are adjacent nibbles: j0/2 == (j0+1)/2 when j0 is even
|
||||
const uint8_t qs_byte = K_turbo[ib].qs[j0 / 2];
|
||||
|
||||
const uint8_t idx0 = (qs_byte >> 0) & 0xF; // low nibble = j0
|
||||
const uint8_t idx1 = (qs_byte >> 4) & 0xF; // high nibble = j0+1
|
||||
|
||||
float2 kv;
|
||||
kv.x = TURBO_CENTROIDS_4BIT[idx0] * norm;
|
||||
kv.y = TURBO_CENTROIDS_4BIT[idx1] * norm;
|
||||
|
||||
#ifdef V_DOT2_F32_F16_AVAILABLE
|
||||
const half2 qv = ((const half2 *) Q_v)[k_KQ_0/nthreads + k_KQ_1];
|
||||
ggml_cuda_mad(sum, make_float2(kv.x, kv.y), __half22float2(qv));
|
||||
#else
|
||||
const float2 qv = ((const float2 *) Q_v)[k_KQ_0/nthreads + k_KQ_1];
|
||||
sum += kv.x * qv.x + kv.y * qv.y;
|
||||
#endif // V_DOT2_F32_F16_AVAILABLE
|
||||
}
|
||||
}
|
||||
|
||||
return sum;
|
||||
}
|
||||
|
||||
template <typename Tds, int ni>
|
||||
static __device__ __forceinline__ void quantize_q8_1_to_shared(
|
||||
const float * __restrict__ x, const float scale, int * __restrict__ yq32, void * __restrict__ yds) {
|
||||
@@ -577,6 +736,186 @@ static __device__ __forceinline__ void dequantize_V_q8_0(const void * __restrict
|
||||
}
|
||||
}
|
||||
|
||||
// Turbo3 V dequantize: extract `ne` float/half values at position i0.
|
||||
//
|
||||
// Optimised for the ne==4 path (used by the VEC kernel with turbo3 V):
|
||||
// i0 is always a multiple of 4 from the VEC kernel access pattern, so all 4
|
||||
// elements share one qs byte and one signs byte — we load each once.
|
||||
template <typename T, int ne>
|
||||
static __device__ __forceinline__ void dequantize_V_turbo3_0(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) {
|
||||
const block_turbo3_0 * x = (const block_turbo3_0 *) vx;
|
||||
|
||||
const int64_t ib = i0 / QK_TURBO3;
|
||||
const int j0 = i0 % QK_TURBO3;
|
||||
const float norm = __half2float(x[ib].norm);
|
||||
|
||||
static_assert(ne == 2 || ne == 4, "bad ne");
|
||||
|
||||
if constexpr (ne == 4) {
|
||||
// When j0 % 4 == 0 (always true from VEC kernel), all 4 elements share one
|
||||
// qs byte (4 elements per byte) and one signs byte (8 elements per byte).
|
||||
const uint8_t qs_byte = x[ib].qs[j0 / 4];
|
||||
const uint8_t sgn_byte = x[ib].signs[j0 / 8];
|
||||
const int shift_s = j0 % 8; // 0 or 4
|
||||
|
||||
const uint8_t idx0 = ((qs_byte >> 0) & 0x3) | (((sgn_byte >> (shift_s+0)) & 0x1) << 2);
|
||||
const uint8_t idx1 = ((qs_byte >> 2) & 0x3) | (((sgn_byte >> (shift_s+1)) & 0x1) << 2);
|
||||
const uint8_t idx2 = ((qs_byte >> 4) & 0x3) | (((sgn_byte >> (shift_s+2)) & 0x1) << 2);
|
||||
const uint8_t idx3 = ((qs_byte >> 6) & 0x3) | (((sgn_byte >> (shift_s+3)) & 0x1) << 2);
|
||||
|
||||
#ifdef FP16_AVAILABLE
|
||||
if constexpr (std::is_same_v<T, half>) {
|
||||
((half2 *) dst)[0] = make_half2(
|
||||
__float2half(TURBO_CENTROIDS_3BIT[idx0] * norm),
|
||||
__float2half(TURBO_CENTROIDS_3BIT[idx1] * norm));
|
||||
((half2 *) dst)[1] = make_half2(
|
||||
__float2half(TURBO_CENTROIDS_3BIT[idx2] * norm),
|
||||
__float2half(TURBO_CENTROIDS_3BIT[idx3] * norm));
|
||||
} else
|
||||
#endif // FP16_AVAILABLE
|
||||
if constexpr (std::is_same_v<T, float>) {
|
||||
((float2 *) dst)[0] = make_float2(
|
||||
TURBO_CENTROIDS_3BIT[idx0] * norm,
|
||||
TURBO_CENTROIDS_3BIT[idx1] * norm);
|
||||
((float2 *) dst)[1] = make_float2(
|
||||
TURBO_CENTROIDS_3BIT[idx2] * norm,
|
||||
TURBO_CENTROIDS_3BIT[idx3] * norm);
|
||||
} else {
|
||||
static_assert(std::is_same_v<T, void>, "unsupported type");
|
||||
}
|
||||
} else { // ne == 2
|
||||
#ifdef FP16_AVAILABLE
|
||||
if constexpr (std::is_same_v<T, half>) {
|
||||
float v0 = turbo3_dequant_element(&x[ib], j0, norm);
|
||||
float v1 = turbo3_dequant_element(&x[ib], j0+1, norm);
|
||||
((half2 *) dst)[0] = make_half2(__float2half(v0), __float2half(v1));
|
||||
} else
|
||||
#endif // FP16_AVAILABLE
|
||||
if constexpr (std::is_same_v<T, float>) {
|
||||
((float *) dst)[0] = turbo3_dequant_element(&x[ib], j0, norm);
|
||||
((float *) dst)[1] = turbo3_dequant_element(&x[ib], j0+1, norm);
|
||||
} else {
|
||||
static_assert(std::is_same_v<T, void>, "unsupported type");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Turbo2 V dequantize: extract `ne` float/half values at position i0.
|
||||
template <typename T, int ne>
|
||||
static __device__ __forceinline__ void dequantize_V_turbo2_0(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) {
|
||||
const block_turbo2_0 * x = (const block_turbo2_0 *) vx;
|
||||
|
||||
const int64_t ib = i0 / QK_TURBO2;
|
||||
const int j0 = i0 % QK_TURBO2;
|
||||
const float norm = __half2float(x[ib].norm);
|
||||
|
||||
static_assert(ne == 2 || ne == 4, "bad ne");
|
||||
|
||||
if constexpr (ne == 4) {
|
||||
const uint8_t qs_byte = x[ib].qs[j0 / 4];
|
||||
|
||||
const uint8_t idx0 = (qs_byte >> 0) & 0x3;
|
||||
const uint8_t idx1 = (qs_byte >> 2) & 0x3;
|
||||
const uint8_t idx2 = (qs_byte >> 4) & 0x3;
|
||||
const uint8_t idx3 = (qs_byte >> 6) & 0x3;
|
||||
|
||||
#ifdef FP16_AVAILABLE
|
||||
if constexpr (std::is_same_v<T, half>) {
|
||||
((half2 *) dst)[0] = make_half2(
|
||||
__float2half(TURBO_CENTROIDS_2BIT[idx0] * norm),
|
||||
__float2half(TURBO_CENTROIDS_2BIT[idx1] * norm));
|
||||
((half2 *) dst)[1] = make_half2(
|
||||
__float2half(TURBO_CENTROIDS_2BIT[idx2] * norm),
|
||||
__float2half(TURBO_CENTROIDS_2BIT[idx3] * norm));
|
||||
} else
|
||||
#endif // FP16_AVAILABLE
|
||||
if constexpr (std::is_same_v<T, float>) {
|
||||
((float2 *) dst)[0] = make_float2(
|
||||
TURBO_CENTROIDS_2BIT[idx0] * norm,
|
||||
TURBO_CENTROIDS_2BIT[idx1] * norm);
|
||||
((float2 *) dst)[1] = make_float2(
|
||||
TURBO_CENTROIDS_2BIT[idx2] * norm,
|
||||
TURBO_CENTROIDS_2BIT[idx3] * norm);
|
||||
} else {
|
||||
static_assert(std::is_same_v<T, void>, "unsupported type");
|
||||
}
|
||||
} else { // ne == 2
|
||||
#ifdef FP16_AVAILABLE
|
||||
if constexpr (std::is_same_v<T, half>) {
|
||||
float v0 = turbo2_dequant_element(&x[ib], j0, norm);
|
||||
float v1 = turbo2_dequant_element(&x[ib], j0+1, norm);
|
||||
((half2 *) dst)[0] = make_half2(__float2half(v0), __float2half(v1));
|
||||
} else
|
||||
#endif // FP16_AVAILABLE
|
||||
if constexpr (std::is_same_v<T, float>) {
|
||||
((float *) dst)[0] = turbo2_dequant_element(&x[ib], j0, norm);
|
||||
((float *) dst)[1] = turbo2_dequant_element(&x[ib], j0+1, norm);
|
||||
} else {
|
||||
static_assert(std::is_same_v<T, void>, "unsupported type");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Turbo4 V dequantize: extract `ne` float/half values at position i0.
|
||||
// 4-bit nibble packed, block size 128.
|
||||
template <typename T, int ne>
|
||||
static __device__ __forceinline__ void dequantize_V_turbo4_0(const void * __restrict__ vx, void * __restrict__ dst, const int64_t i0) {
|
||||
const block_turbo4_0 * x = (const block_turbo4_0 *) vx;
|
||||
|
||||
const int64_t ib = i0 / QK_TURBO4;
|
||||
const int j0 = i0 % QK_TURBO4;
|
||||
const float norm = __half2float(x[ib].norm);
|
||||
|
||||
static_assert(ne == 2 || ne == 4, "bad ne");
|
||||
|
||||
if constexpr (ne == 4) {
|
||||
// j0 is always a multiple of 4 from the VEC kernel access pattern.
|
||||
// 4 consecutive elements span 2 qs bytes: j0/2 and j0/2+1.
|
||||
const uint8_t qs_byte0 = x[ib].qs[j0 / 2]; // elements j0, j0+1
|
||||
const uint8_t qs_byte1 = x[ib].qs[j0 / 2 + 1]; // elements j0+2, j0+3
|
||||
|
||||
const uint8_t idx0 = (qs_byte0 >> 0) & 0xF;
|
||||
const uint8_t idx1 = (qs_byte0 >> 4) & 0xF;
|
||||
const uint8_t idx2 = (qs_byte1 >> 0) & 0xF;
|
||||
const uint8_t idx3 = (qs_byte1 >> 4) & 0xF;
|
||||
|
||||
#ifdef FP16_AVAILABLE
|
||||
if constexpr (std::is_same_v<T, half>) {
|
||||
((half2 *) dst)[0] = make_half2(
|
||||
__float2half(TURBO_CENTROIDS_4BIT[idx0] * norm),
|
||||
__float2half(TURBO_CENTROIDS_4BIT[idx1] * norm));
|
||||
((half2 *) dst)[1] = make_half2(
|
||||
__float2half(TURBO_CENTROIDS_4BIT[idx2] * norm),
|
||||
__float2half(TURBO_CENTROIDS_4BIT[idx3] * norm));
|
||||
} else
|
||||
#endif // FP16_AVAILABLE
|
||||
if constexpr (std::is_same_v<T, float>) {
|
||||
((float2 *) dst)[0] = make_float2(
|
||||
TURBO_CENTROIDS_4BIT[idx0] * norm,
|
||||
TURBO_CENTROIDS_4BIT[idx1] * norm);
|
||||
((float2 *) dst)[1] = make_float2(
|
||||
TURBO_CENTROIDS_4BIT[idx2] * norm,
|
||||
TURBO_CENTROIDS_4BIT[idx3] * norm);
|
||||
} else {
|
||||
static_assert(std::is_same_v<T, void>, "unsupported type");
|
||||
}
|
||||
} else { // ne == 2
|
||||
#ifdef FP16_AVAILABLE
|
||||
if constexpr (std::is_same_v<T, half>) {
|
||||
float v0 = turbo4_dequant_element(&x[ib], j0, norm);
|
||||
float v1 = turbo4_dequant_element(&x[ib], j0+1, norm);
|
||||
((half2 *) dst)[0] = make_half2(__float2half(v0), __float2half(v1));
|
||||
} else
|
||||
#endif // FP16_AVAILABLE
|
||||
if constexpr (std::is_same_v<T, float>) {
|
||||
((float *) dst)[0] = turbo4_dequant_element(&x[ib], j0, norm);
|
||||
((float *) dst)[1] = turbo4_dequant_element(&x[ib], j0+1, norm);
|
||||
} else {
|
||||
static_assert(std::is_same_v<T, void>, "unsupported type");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
template <ggml_type type_K, int D, int nthreads>
|
||||
constexpr __device__ vec_dot_KQ_t get_vec_dot_KQ() {
|
||||
if constexpr (type_K == GGML_TYPE_F16) {
|
||||
@@ -593,6 +932,12 @@ constexpr __device__ vec_dot_KQ_t get_vec_dot_KQ() {
|
||||
return vec_dot_fattn_vec_KQ_q8_0<D, nthreads>;
|
||||
} else if constexpr (type_K == GGML_TYPE_BF16) {
|
||||
return vec_dot_fattn_vec_KQ_bf16<D, nthreads>;
|
||||
} else if constexpr (type_K == GGML_TYPE_TURBO3_0) {
|
||||
return vec_dot_fattn_vec_KQ_turbo3_0<D, nthreads>;
|
||||
} else if constexpr (type_K == GGML_TYPE_TURBO2_0) {
|
||||
return vec_dot_fattn_vec_KQ_turbo2_0<D, nthreads>;
|
||||
} else if constexpr (type_K == GGML_TYPE_TURBO4_0) {
|
||||
return vec_dot_fattn_vec_KQ_turbo4_0<D, nthreads>;
|
||||
} else {
|
||||
static_assert(type_K == -1, "bad type");
|
||||
return nullptr;
|
||||
@@ -615,6 +960,12 @@ constexpr __device__ dequantize_V_t get_dequantize_V() {
|
||||
return dequantize_V_q8_0<T, ne>;
|
||||
} else if constexpr (type_V == GGML_TYPE_BF16) {
|
||||
return dequantize_V_bf16<float, ne>;
|
||||
} else if constexpr (type_V == GGML_TYPE_TURBO3_0) {
|
||||
return dequantize_V_turbo3_0<T, ne>;
|
||||
} else if constexpr (type_V == GGML_TYPE_TURBO2_0) {
|
||||
return dequantize_V_turbo2_0<T, ne>;
|
||||
} else if constexpr (type_V == GGML_TYPE_TURBO4_0) {
|
||||
return dequantize_V_turbo4_0<T, ne>;
|
||||
} else {
|
||||
static_assert(type_V == -1, "bad type");
|
||||
return nullptr;
|
||||
|
||||
@@ -84,6 +84,11 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 32, 128, 2, 32, 160, 128, 128, 1, false);
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 64, 256, 1, 32, 160, 128, 128, 1, false);
|
||||
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE(640, 512, 8, 64, 4, 32, 288, 256, 128, 1, false);
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE(640, 512, 16, 64, 4, 32, 288, 256, 128, 1, false);
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE(640, 512, 32, 128, 2, 32, 160, 128, 128, 1, false);
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE(640, 512, 64, 256, 1, 32, 160, 128, 128, 1, false);
|
||||
|
||||
return fattn_mma_config(32, 1, 0, 0, 0, 0, 0, false);
|
||||
}
|
||||
|
||||
@@ -106,6 +111,11 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 32, 128, 2, 32, 160, 128, 128, 1, false);
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 64, 256, 1, 32, 160, 128, 128, 1, false);
|
||||
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE(640, 512, 8, 64, 4, 32, 96, 64, 128, 1, false);
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE(640, 512, 16, 64, 4, 32, 96, 64, 128, 1, false);
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE(640, 512, 32, 128, 2, 32, 160, 128, 128, 1, false);
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE(640, 512, 64, 256, 1, 32, 160, 128, 128, 1, false);
|
||||
|
||||
return ggml_cuda_fattn_mma_get_config_ampere(DKQ, DV, ncols);
|
||||
}
|
||||
|
||||
@@ -120,6 +130,11 @@ static constexpr __host__ __device__ fattn_mma_config ggml_cuda_fattn_mma_get_co
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 32, 128, 2, 32, 160, 128, 64, 1, false);
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE(576, 512, 64, 256, 1, 32, 160, 128, 64, 1, false);
|
||||
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE(640, 512, 8, 64, 4, 32, 288, 256, 64, 1, false);
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE(640, 512, 16, 64, 4, 32, 288, 256, 64, 1, false);
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE(640, 512, 32, 128, 2, 32, 160, 128, 64, 1, false);
|
||||
GGML_CUDA_FATTN_MMA_CONFIG_CASE(640, 512, 64, 256, 1, 32, 160, 128, 64, 1, false);
|
||||
|
||||
// TODO tune specifically for Volta
|
||||
return ggml_cuda_fattn_mma_get_config_ampere(DKQ, DV, ncols);
|
||||
}
|
||||
@@ -2018,3 +2033,8 @@ extern DECL_FATTN_MMA_F16_CASE(576, 512, 8, 4);
|
||||
extern DECL_FATTN_MMA_F16_CASE(576, 512, 16, 4);
|
||||
extern DECL_FATTN_MMA_F16_CASE(576, 512, 1, 32);
|
||||
extern DECL_FATTN_MMA_F16_CASE(576, 512, 2, 32);
|
||||
|
||||
// D=640: padded turbo KV cache for GLM-4.7 Flash (ncols2=16 only)
|
||||
extern DECL_FATTN_MMA_F16_CASE(640, 512, 1, 16);
|
||||
extern DECL_FATTN_MMA_F16_CASE(640, 512, 2, 16);
|
||||
extern DECL_FATTN_MMA_F16_CASE(640, 512, 4, 16);
|
||||
|
||||
@@ -50,10 +50,17 @@ void ggml_cuda_flash_attn_ext_tile(ggml_backend_cuda_context & ctx, ggml_tensor
|
||||
GGML_ASSERT(V->ne[0] == K->ne[0]);
|
||||
ggml_cuda_flash_attn_ext_tile_case<512, 512>(ctx, dst);
|
||||
} break;
|
||||
#ifndef GGML_USE_HIP
|
||||
// D>=576 tile kernels exceed HIP local memory limit (67584 > 65536)
|
||||
case 576: {
|
||||
GGML_ASSERT(V->ne[0] == 512);
|
||||
ggml_cuda_flash_attn_ext_tile_case<576, 512>(ctx, dst);
|
||||
} break;
|
||||
case 640: {
|
||||
GGML_ASSERT(V->ne[0] == 512);
|
||||
ggml_cuda_flash_attn_ext_tile_case<640, 512>(ctx, dst);
|
||||
} break;
|
||||
#endif
|
||||
default: {
|
||||
GGML_ABORT("Unsupported head size");
|
||||
} break;
|
||||
|
||||
@@ -84,6 +84,10 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_nv
|
||||
GGML_CUDA_FATTN_TILE_CONFIG_CASE(576, 512, 8, 256, 2, 64, 64)
|
||||
GGML_CUDA_FATTN_TILE_CONFIG_CASE(576, 512, 16, 256, 2, 64, 64)
|
||||
|
||||
GGML_CUDA_FATTN_TILE_CONFIG_CASE(640, 512, 4, 128, 2, 64, 64)
|
||||
GGML_CUDA_FATTN_TILE_CONFIG_CASE(640, 512, 8, 256, 2, 64, 64)
|
||||
GGML_CUDA_FATTN_TILE_CONFIG_CASE(640, 512, 16, 256, 2, 64, 64)
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -152,6 +156,10 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_nv
|
||||
GGML_CUDA_FATTN_TILE_CONFIG_CASE(576, 512, 8, 256, 2, 32, 64)
|
||||
GGML_CUDA_FATTN_TILE_CONFIG_CASE(576, 512, 16, 256, 2, 32, 64)
|
||||
|
||||
GGML_CUDA_FATTN_TILE_CONFIG_CASE(640, 512, 4, 128, 2, 32, 64)
|
||||
GGML_CUDA_FATTN_TILE_CONFIG_CASE(640, 512, 8, 256, 2, 32, 64)
|
||||
GGML_CUDA_FATTN_TILE_CONFIG_CASE(640, 512, 16, 256, 2, 32, 64)
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -229,6 +237,11 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_am
|
||||
GGML_CUDA_FATTN_TILE_CONFIG_CASE(576, 512, 16, 256, 2, 64, 64)
|
||||
GGML_CUDA_FATTN_TILE_CONFIG_CASE(576, 512, 32, 512, 1, 128, 64)
|
||||
|
||||
GGML_CUDA_FATTN_TILE_CONFIG_CASE(640, 512, 4, 128, 2, 64, 64)
|
||||
GGML_CUDA_FATTN_TILE_CONFIG_CASE(640, 512, 8, 256, 2, 64, 64)
|
||||
GGML_CUDA_FATTN_TILE_CONFIG_CASE(640, 512, 16, 256, 2, 64, 64)
|
||||
GGML_CUDA_FATTN_TILE_CONFIG_CASE(640, 512, 32, 512, 1, 128, 64)
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -306,6 +319,11 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_am
|
||||
GGML_CUDA_FATTN_TILE_CONFIG_CASE(576, 512, 16, 256, 4, 64, 64)
|
||||
GGML_CUDA_FATTN_TILE_CONFIG_CASE(576, 512, 32, 256, 2, 128, 64)
|
||||
|
||||
GGML_CUDA_FATTN_TILE_CONFIG_CASE(640, 512, 4, 128, 2, 64, 64)
|
||||
GGML_CUDA_FATTN_TILE_CONFIG_CASE(640, 512, 8, 256, 2, 64, 64)
|
||||
GGML_CUDA_FATTN_TILE_CONFIG_CASE(640, 512, 16, 256, 4, 64, 64)
|
||||
GGML_CUDA_FATTN_TILE_CONFIG_CASE(640, 512, 32, 256, 2, 128, 64)
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -1239,7 +1257,7 @@ static void launch_fattn_tile_switch_ncols2(ggml_backend_cuda_context & ctx, ggm
|
||||
const int gqa_ratio = Q->ne[2] / K->ne[2];
|
||||
|
||||
// On NVIDIA (Pascal and older) the GQA optimizations seem to be detrimental in some cases.
|
||||
// However, for DKQ == 576, DV == 512 only the kernel variant with GQA optimizations is implemented.
|
||||
// However, for DKQ == 576/640, DV == 512 only the kernel variant with GQA optimizations is implemented.
|
||||
const bool nvidia = GGML_CUDA_CC_IS_NVIDIA(ggml_cuda_info().devices[ggml_cuda_get_device()].cc);
|
||||
const int gqa_limit = nvidia && gqa_ratio <= 4 && DV <= 256 ? 16 : INT_MAX;
|
||||
const bool use_gqa_opt = mask && max_bias == 0.0f && Q->ne[1] <= gqa_limit && K->ne[1] % FATTN_KQ_STRIDE == 0;
|
||||
@@ -1345,3 +1363,4 @@ extern DECL_FATTN_TILE_CASE(256, 256);
|
||||
extern DECL_FATTN_TILE_CASE(320, 256);
|
||||
extern DECL_FATTN_TILE_CASE(512, 512);
|
||||
extern DECL_FATTN_TILE_CASE(576, 512);
|
||||
extern DECL_FATTN_TILE_CASE(640, 512);
|
||||
|
||||
@@ -75,17 +75,20 @@ static __global__ void flash_attn_ext_vec(
|
||||
#endif // GGML_USE_HIP
|
||||
|
||||
constexpr int nthreads = ggml_cuda_fattn_vec_get_nthreads_device();
|
||||
constexpr int nthreads_KQ = (type_K == GGML_TYPE_F16 || type_K == GGML_TYPE_BF16) ? 128 / cpy_nb : nthreads_KQ_q;
|
||||
constexpr int nthreads_V = (type_V == GGML_TYPE_F16 || type_V == GGML_TYPE_BF16) ? 128 / cpy_nb : nthreads_V_q;
|
||||
// Turbo3 uses the float Q path (like f16/bf16), not q8_1 integer path
|
||||
constexpr bool K_is_unquantized = (type_K == GGML_TYPE_F16 || type_K == GGML_TYPE_BF16 || type_K == GGML_TYPE_TURBO3_0 || type_K == GGML_TYPE_TURBO2_0 || type_K == GGML_TYPE_TURBO4_0);
|
||||
constexpr bool V_is_unquantized = (type_V == GGML_TYPE_F16 || type_V == GGML_TYPE_BF16 || type_V == GGML_TYPE_TURBO3_0 || type_V == GGML_TYPE_TURBO2_0 || type_V == GGML_TYPE_TURBO4_0);
|
||||
constexpr int nthreads_KQ = K_is_unquantized ? 128 / cpy_nb : nthreads_KQ_q;
|
||||
constexpr int nthreads_V = V_is_unquantized ? ((type_V == GGML_TYPE_TURBO3_0 || type_V == GGML_TYPE_TURBO2_0 || type_V == GGML_TYPE_TURBO4_0) ? nthreads_V_q : 128 / cpy_nb) : nthreads_V_q;
|
||||
|
||||
static_assert(WARP_SIZE % nthreads_KQ == 0, "bad nthreads_K");
|
||||
static_assert(WARP_SIZE % nthreads_V == 0, "bad nthreads_V");
|
||||
|
||||
constexpr int V_rows_per_thread = (type_V == GGML_TYPE_F16 || type_V == GGML_TYPE_BF16) ? 2*cpy_ne : 4;
|
||||
constexpr int V_rows_per_thread = V_is_unquantized ? ((type_V == GGML_TYPE_TURBO3_0 || type_V == GGML_TYPE_TURBO2_0 || type_V == GGML_TYPE_TURBO4_0) ? 4 : 2*cpy_ne) : 4;
|
||||
constexpr int V_cols_per_iter = WARP_SIZE / nthreads_V;
|
||||
|
||||
constexpr vec_dot_KQ_t vec_dot_KQ = get_vec_dot_KQ<type_K, D, nthreads_KQ>();
|
||||
constexpr bool Q_q8_1 = type_K != GGML_TYPE_F16 && type_K != GGML_TYPE_BF16;
|
||||
constexpr bool Q_q8_1 = !K_is_unquantized;
|
||||
#ifdef V_DOT2_F32_F16_AVAILABLE
|
||||
constexpr dequantize_V_t dequantize_V = get_dequantize_V<type_V, half, V_rows_per_thread>();
|
||||
#else
|
||||
@@ -120,6 +123,14 @@ static __global__ void flash_attn_ext_vec(
|
||||
__shared__ float KQ[ne_KQ > ne_combine ? ne_KQ : ne_combine];
|
||||
#endif // V_DOT2_F32_F16_AVAILABLE
|
||||
|
||||
// Sparse V: skip V dequant for positions with negligible attention weights.
|
||||
// At long context, most V positions contribute < 1e-6 to the output — skipping
|
||||
// their dequant saves significant compute (especially for quantized V types).
|
||||
constexpr float sparse_v_threshold_f = 1e-6f;
|
||||
#ifdef V_DOT2_F32_F16_AVAILABLE
|
||||
const half sparse_v_threshold_h = __float2half(sparse_v_threshold_f);
|
||||
#endif
|
||||
|
||||
float KQ_max[ncols];
|
||||
float KQ_sum[ncols];
|
||||
#pragma unroll
|
||||
@@ -320,6 +331,17 @@ static __global__ void flash_attn_ext_vec(
|
||||
for (int j = 0; j < ncols; ++j) {
|
||||
KQ_k[j] = __half2half2(KQ[j*nthreads + k]);
|
||||
}
|
||||
|
||||
// Sparse V: skip V dequant if all attention weights for this position are negligible
|
||||
{
|
||||
bool dominated = true;
|
||||
#pragma unroll
|
||||
for (int j = 0; j < ncols; ++j) {
|
||||
if (__hgt(__low2half(KQ_k[j]), sparse_v_threshold_h)) { dominated = false; break; }
|
||||
}
|
||||
if (dominated) { continue; }
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V*V_rows_per_thread/2) {
|
||||
half2 tmp[V_rows_per_thread/2];
|
||||
@@ -349,6 +371,17 @@ static __global__ void flash_attn_ext_vec(
|
||||
for (int j = 0; j < ncols; ++j) {
|
||||
KQ_k[j] = KQ[j*nthreads + k];
|
||||
}
|
||||
|
||||
// Sparse V: skip V dequant if all attention weights for this position are negligible
|
||||
{
|
||||
bool dominated = true;
|
||||
#pragma unroll
|
||||
for (int j = 0; j < ncols; ++j) {
|
||||
if (KQ_k[j] >= sparse_v_threshold_f) { dominated = false; break; }
|
||||
}
|
||||
if (dominated) { continue; }
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
for (int i_VKQ_0 = 0; i_VKQ_0 < D/2; i_VKQ_0 += nthreads_V*V_rows_per_thread/2) {
|
||||
float2 tmp[V_rows_per_thread/2];
|
||||
@@ -598,3 +631,72 @@ EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q5_0)
|
||||
EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q5_1)
|
||||
EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_Q8_0)
|
||||
EXTERN_DECL_FATTN_VEC_CASES(256, GGML_TYPE_BF16)
|
||||
|
||||
// TurboQuant3 — turbo3 K + turbo3 V (KV cache uses same type)
|
||||
extern DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0);
|
||||
extern DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0);
|
||||
extern DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0);
|
||||
|
||||
// Mixed turbo3/q8_0 KV cache types
|
||||
extern DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO3_0, GGML_TYPE_Q8_0);
|
||||
extern DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO3_0, GGML_TYPE_Q8_0);
|
||||
extern DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO3_0, GGML_TYPE_Q8_0);
|
||||
|
||||
extern DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_0);
|
||||
extern DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_0);
|
||||
extern DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_0);
|
||||
|
||||
// TurboQuant2 -- turbo2 K + turbo2 V
|
||||
extern DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0);
|
||||
extern DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0);
|
||||
extern DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0);
|
||||
|
||||
// Mixed turbo2/q8_0 KV cache types
|
||||
extern DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO2_0, GGML_TYPE_Q8_0);
|
||||
extern DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO2_0, GGML_TYPE_Q8_0);
|
||||
extern DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO2_0, GGML_TYPE_Q8_0);
|
||||
|
||||
extern DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0);
|
||||
extern DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0);
|
||||
extern DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0);
|
||||
|
||||
// Mixed turbo3/turbo2 KV cache types
|
||||
extern DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO2_0);
|
||||
extern DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO2_0);
|
||||
extern DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO2_0);
|
||||
|
||||
extern DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO3_0);
|
||||
extern DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO3_0);
|
||||
extern DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO3_0);
|
||||
|
||||
// TurboQuant4 — turbo4 K + turbo4 V (KV cache uses same type)
|
||||
extern DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO4_0);
|
||||
extern DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO4_0);
|
||||
extern DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO4_0);
|
||||
|
||||
// Mixed turbo4/q8_0 KV cache types
|
||||
extern DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO4_0, GGML_TYPE_Q8_0);
|
||||
extern DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO4_0, GGML_TYPE_Q8_0);
|
||||
extern DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO4_0, GGML_TYPE_Q8_0);
|
||||
|
||||
extern DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_TURBO4_0);
|
||||
extern DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_TURBO4_0);
|
||||
extern DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_TURBO4_0);
|
||||
|
||||
// Mixed turbo4/turbo3 KV cache types
|
||||
extern DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO3_0);
|
||||
extern DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO3_0);
|
||||
extern DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO3_0);
|
||||
|
||||
extern DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO4_0);
|
||||
extern DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO4_0);
|
||||
extern DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO4_0);
|
||||
|
||||
// Mixed turbo4/turbo2 KV cache types
|
||||
extern DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO2_0);
|
||||
extern DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO2_0);
|
||||
extern DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO2_0);
|
||||
|
||||
extern DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO4_0);
|
||||
extern DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO4_0);
|
||||
extern DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO4_0);
|
||||
|
||||
@@ -236,6 +236,17 @@ static void ggml_cuda_flash_attn_ext_mma_f16(ggml_backend_cuda_context & ctx, gg
|
||||
ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1<576, 512, 4>(ctx, dst);
|
||||
}
|
||||
} break;
|
||||
case 640: {
|
||||
// Padded turbo KV cache for GLM-4.7 Flash (K head_dim=576 zero-padded to 640).
|
||||
// D=640 shared memory (Q storage = ncols*(DKQ/2+4)*4) exceeds hardware limit at ncols1>=4.
|
||||
// Cap at ncols1=2 (ncols=32): Q=32*324*4=41KB + KV≈37KB = ~78KB total.
|
||||
GGML_ASSERT(V->ne[0] == 512);
|
||||
if (Q->ne[1] <= 1) {
|
||||
ggml_cuda_flash_attn_ext_mma_f16_case<640, 512, 1, 16>(ctx, dst);
|
||||
} else {
|
||||
ggml_cuda_flash_attn_ext_mma_f16_case<640, 512, 2, 16>(ctx, dst);
|
||||
}
|
||||
} break;
|
||||
default:
|
||||
GGML_ABORT("fatal error");
|
||||
break;
|
||||
@@ -325,6 +336,39 @@ static void ggml_cuda_flash_attn_ext_vec(ggml_backend_cuda_context & ctx, ggml_t
|
||||
FATTN_VEC_CASES_ALL_D(GGML_TYPE_BF16, GGML_TYPE_BF16)
|
||||
#endif // GGML_CUDA_FA_ALL_QUANTS
|
||||
|
||||
// TurboQuant3 KV cache types (always enabled)
|
||||
FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0)
|
||||
|
||||
// Mixed turbo3/q8_0 KV cache types
|
||||
FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO3_0, GGML_TYPE_Q8_0)
|
||||
FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_0)
|
||||
|
||||
// TurboQuant2 KV cache types (always enabled)
|
||||
FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0)
|
||||
|
||||
// Mixed turbo2/q8_0 KV cache types
|
||||
FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO2_0, GGML_TYPE_Q8_0)
|
||||
FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0)
|
||||
|
||||
// Mixed turbo3/turbo2 KV cache types
|
||||
FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO2_0)
|
||||
FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO3_0)
|
||||
|
||||
// TurboQuant4 KV cache types (always enabled)
|
||||
FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO4_0)
|
||||
|
||||
// Mixed turbo4/q8_0 KV cache types
|
||||
FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO4_0, GGML_TYPE_Q8_0)
|
||||
FATTN_VEC_CASES_ALL_D(GGML_TYPE_Q8_0, GGML_TYPE_TURBO4_0)
|
||||
|
||||
// Mixed turbo4/turbo3 KV cache types
|
||||
FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO3_0)
|
||||
FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO4_0)
|
||||
|
||||
// Mixed turbo4/turbo2 KV cache types
|
||||
FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO2_0)
|
||||
FATTN_VEC_CASES_ALL_D(GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO4_0)
|
||||
|
||||
GGML_ABORT("fatal error");
|
||||
}
|
||||
|
||||
@@ -410,6 +454,7 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
|
||||
}
|
||||
break;
|
||||
case 576:
|
||||
case 640:
|
||||
if (V->ne[0] != 512) {
|
||||
return BEST_FATTN_KERNEL_NONE;
|
||||
}
|
||||
@@ -423,8 +468,14 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
|
||||
|
||||
#ifndef GGML_CUDA_FA_ALL_QUANTS
|
||||
if (K->type != V->type) {
|
||||
// Allow mixed turbo KV types (any combination of turbo2, turbo3, q8_0)
|
||||
auto is_turbo = [](ggml_type t) {
|
||||
return t == GGML_TYPE_TURBO2_0 || t == GGML_TYPE_TURBO3_0 || t == GGML_TYPE_TURBO4_0 || t == GGML_TYPE_Q8_0;
|
||||
};
|
||||
if (!is_turbo(K->type) || !is_turbo(V->type)) {
|
||||
return BEST_FATTN_KERNEL_NONE;
|
||||
}
|
||||
}
|
||||
#endif // GGML_CUDA_FA_ALL_QUANTS
|
||||
|
||||
switch (K->type) {
|
||||
@@ -441,6 +492,24 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
|
||||
case GGML_TYPE_Q8_0:
|
||||
case GGML_TYPE_BF16:
|
||||
break;
|
||||
case GGML_TYPE_TURBO3_0:
|
||||
// turbo3 VEC kernel instantiated for D in {64, 128, 256}.
|
||||
if (K->ne[0] % 64 != 0) {
|
||||
return BEST_FATTN_KERNEL_NONE;
|
||||
}
|
||||
break;
|
||||
case GGML_TYPE_TURBO2_0:
|
||||
// turbo2 VEC kernel instantiated for D in {64, 128, 256}.
|
||||
if (K->ne[0] % 64 != 0) {
|
||||
return BEST_FATTN_KERNEL_NONE;
|
||||
}
|
||||
break;
|
||||
case GGML_TYPE_TURBO4_0:
|
||||
// turbo4 VEC kernel instantiated for D in {64, 128, 256}.
|
||||
if (K->ne[0] % 64 != 0) {
|
||||
return BEST_FATTN_KERNEL_NONE;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
return BEST_FATTN_KERNEL_NONE;
|
||||
}
|
||||
@@ -478,7 +547,7 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
|
||||
return BEST_FATTN_KERNEL_MMA_F16;
|
||||
}
|
||||
|
||||
const int ncols2_max = Q->ne[0] == 320 ? 32 : ((Q->ne[0] == 576 || Q->ne[0] == 192) ? 16 : 8);
|
||||
const int ncols2_max = Q->ne[0] == 320 ? 32 : ((Q->ne[0] == 576 || Q->ne[0] == 192 || Q->ne[0] == 640) ? 16 : 8);
|
||||
int gqa_ratio_eff = 1;
|
||||
while (gqa_ratio % (2*gqa_ratio_eff) == 0 && gqa_ratio_eff < ncols2_max) {
|
||||
gqa_ratio_eff *= 2;
|
||||
@@ -495,7 +564,7 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const
|
||||
}
|
||||
|
||||
// Use the WMMA kernel if possible:
|
||||
if (ggml_cuda_should_use_wmma_fattn(cc) && K->ne[1] % FATTN_KQ_STRIDE == 0 && Q->ne[0] != 40 && Q->ne[0] != 72 && Q->ne[0] != 192 && Q->ne[0] != 512 && Q->ne[0] != 576) {
|
||||
if (ggml_cuda_should_use_wmma_fattn(cc) && K->ne[1] % FATTN_KQ_STRIDE == 0 && Q->ne[0] != 40 && Q->ne[0] != 72 && Q->ne[0] != 192 && Q->ne[0] != 512 && Q->ne[0] != 576 && Q->ne[0] != 640) {
|
||||
if (can_use_vector_kernel && Q->ne[1] <= 2) {
|
||||
return BEST_FATTN_KERNEL_VEC;
|
||||
}
|
||||
|
||||
@@ -58,6 +58,7 @@
|
||||
#include "ggml-cuda/gated_delta_net.cuh"
|
||||
#include "ggml-cuda/set.cuh"
|
||||
#include "ggml-cuda/set-rows.cuh"
|
||||
#include "ggml-cuda/turbo-wht.cuh"
|
||||
#include "ggml-cuda/pad_reflect_1d.cuh"
|
||||
#include "ggml-cuda/solve_tri.cuh"
|
||||
#include "ggml-cuda/tri.cuh"
|
||||
@@ -2800,6 +2801,9 @@ static bool ggml_cuda_compute_forward(ggml_backend_cuda_context & ctx, struct gg
|
||||
case GGML_OP_SET_ROWS:
|
||||
ggml_cuda_op_set_rows(ctx, dst);
|
||||
break;
|
||||
case GGML_OP_TURBO_WHT:
|
||||
ggml_cuda_turbo_wht(ctx, dst);
|
||||
break;
|
||||
case GGML_OP_SET:
|
||||
ggml_cuda_op_set(ctx, dst);
|
||||
break;
|
||||
@@ -5191,9 +5195,18 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
} break;
|
||||
case GGML_OP_SET_ROWS:
|
||||
{
|
||||
// turbo types require head_dim divisible by appropriate group size
|
||||
if ((op->type == GGML_TYPE_TURBO3_0 || op->type == GGML_TYPE_TURBO2_0) && op->src[0]->ne[0] % 64 != 0) {
|
||||
return false;
|
||||
}
|
||||
// turbo4 block size is 128, so head_dim must be divisible by 128
|
||||
if (op->type == GGML_TYPE_TURBO4_0 && op->src[0]->ne[0] % 128 != 0) {
|
||||
return false;
|
||||
}
|
||||
return (op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16 || op->type == GGML_TYPE_BF16 ||
|
||||
op->type == GGML_TYPE_Q4_0 || op->type == GGML_TYPE_Q4_1 || op->type == GGML_TYPE_Q5_0 ||
|
||||
op->type == GGML_TYPE_Q5_1 || op->type == GGML_TYPE_Q8_0 || op->type == GGML_TYPE_IQ4_NL) &&
|
||||
op->type == GGML_TYPE_Q5_1 || op->type == GGML_TYPE_Q8_0 || op->type == GGML_TYPE_IQ4_NL ||
|
||||
op->type == GGML_TYPE_TURBO3_0 || op->type == GGML_TYPE_TURBO2_0 || op->type == GGML_TYPE_TURBO4_0) &&
|
||||
op->src[0]->type == GGML_TYPE_F32 &&
|
||||
(op->src[1]->type == GGML_TYPE_I64 || op->src[1]->type == GGML_TYPE_I32);
|
||||
} break;
|
||||
@@ -5323,6 +5336,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g
|
||||
return (op->src[0]->type == GGML_TYPE_F32 || op->src[0]->type == GGML_TYPE_F16) &&
|
||||
(op->src[1]->type == GGML_TYPE_F32 || op->src[1]->type == GGML_TYPE_F16) &&
|
||||
(op->type == GGML_TYPE_F32 || op->type == GGML_TYPE_F16);
|
||||
case GGML_OP_TURBO_WHT:
|
||||
return op->src[0]->type == GGML_TYPE_F32 && op->type == GGML_TYPE_F32 &&
|
||||
op->src[0]->ne[0] % 64 == 0; // head dim must be divisible by 64 (supports 64 and 128 WHT groups)
|
||||
case GGML_OP_SSM_SCAN: {
|
||||
if (op->src[3]->ne[0] == 1) {
|
||||
// Mamba2
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#include "set-rows.cuh"
|
||||
#include "cpy-utils.cuh"
|
||||
#include "turbo-quant.cuh"
|
||||
|
||||
typedef void (*set_rows_kernel_t)(const char * src, char * dst);
|
||||
|
||||
@@ -209,6 +210,928 @@ static void set_rows_cuda(
|
||||
}
|
||||
}
|
||||
|
||||
// ---- TurboQuant3 set_rows: GROUP_SIZE-element groups with WHT rotation + norm correction ----
|
||||
//
|
||||
// Templated on GROUP_SIZE (128 or 64).
|
||||
// Parallel kernel: one CUDA block per group, GROUP_SIZE threads per block.
|
||||
// Thread j handles element j within the group.
|
||||
//
|
||||
// Steps (all parallel):
|
||||
// 1. Load element j from global memory
|
||||
// 2. Parallel L2 norm (warp reduce + inter-warp via shared memory)
|
||||
// 3. Normalize
|
||||
// 4. Forward WHT (log2(GROUP_SIZE) butterfly stages, shared memory)
|
||||
// 5. Quantize element j to 3-bit centroid index
|
||||
// 6. Pack qs (warp shuffle) and signs (__ballot_sync) into turbo3 block, no atomics
|
||||
// 7. Parallel reconstruction norm (same pattern as step 2)
|
||||
// 8. Write corrected norm (one thread per sub-block)
|
||||
|
||||
template <typename idx_t, int GROUP_SIZE>
|
||||
__launch_bounds__(128) // max of 128 or 64
|
||||
static __global__ void k_set_rows_turbo3(
|
||||
const float * __restrict__ src0,
|
||||
const idx_t * __restrict__ src1,
|
||||
block_turbo3_0 * __restrict__ dst,
|
||||
const int64_t ne00,
|
||||
const int64_t ne01,
|
||||
const int64_t ne10,
|
||||
const int64_t ne11,
|
||||
const int64_t ne12,
|
||||
const int64_t ne13,
|
||||
const int64_t s01,
|
||||
const int64_t s02,
|
||||
const int64_t s03,
|
||||
const int64_t s10,
|
||||
const int64_t s11,
|
||||
const int64_t s12,
|
||||
const int64_t s1,
|
||||
const int64_t s2,
|
||||
const int64_t s3) {
|
||||
|
||||
static_assert(GROUP_SIZE == 128 || GROUP_SIZE == 64, "GROUP_SIZE must be 128 or 64");
|
||||
|
||||
// blockIdx.x = flat group index; threadIdx.x = element within group (0..GROUP_SIZE-1)
|
||||
const int j = threadIdx.x;
|
||||
|
||||
// Decode blockIdx.x → (i_grp, i01, i02, i03)
|
||||
constexpr int blocks_per_group = GROUP_SIZE / QK_TURBO3;
|
||||
const int64_t n_groups_per_row = ne00 / GROUP_SIZE;
|
||||
const int64_t g = blockIdx.x;
|
||||
const int64_t i_grp = g % n_groups_per_row;
|
||||
int64_t tmp = g / n_groups_per_row;
|
||||
const int64_t i01 = tmp % ne01;
|
||||
tmp = tmp / ne01;
|
||||
const int64_t i02 = tmp % ne12;
|
||||
const int64_t i03 = tmp / ne12;
|
||||
|
||||
const int64_t i12 = i02;
|
||||
const int64_t i11 = i01 % ne11;
|
||||
const int64_t i10 = i01;
|
||||
|
||||
const int64_t dst_row = *(src1 + i10*s10 + i11*s11 + i12*s12);
|
||||
const float * src_row = src0 + i01*s01 + i02*s02 + i03*s03;
|
||||
block_turbo3_0 * dst_row_ptr = (block_turbo3_0 *)((char *)dst + dst_row*s1 + i02*s2 + i03*s3);
|
||||
block_turbo3_0 * blk_base = dst_row_ptr + i_grp * blocks_per_group;
|
||||
|
||||
// ---- Step 1: Load element j (coalesced) ----
|
||||
__shared__ float x[GROUP_SIZE];
|
||||
x[j] = src_row[i_grp * GROUP_SIZE + j];
|
||||
__syncthreads();
|
||||
|
||||
// ---- InnerQ: calibrate on original (unscaled) values ----
|
||||
if (d_innerq_calibrating) {
|
||||
atomicAdd(&d_innerq_sq_accum[j], x[j] * x[j]);
|
||||
if (j == 0) atomicAdd(&d_innerq_count, 1);
|
||||
}
|
||||
|
||||
// ---- InnerQ: apply channel scale (only when active) ----
|
||||
if (d_innerq_active) {
|
||||
x[j] *= d_innerq_scale[j];
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
// ---- Step 2: Parallel L2 norm ----
|
||||
constexpr int n_warps = GROUP_SIZE / WARP_SIZE;
|
||||
__shared__ float warp_accum[n_warps];
|
||||
float v = x[j];
|
||||
float v2 = v * v;
|
||||
for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1)
|
||||
v2 += __shfl_xor_sync(0xffffffff, v2, offset);
|
||||
if (j % WARP_SIZE == 0)
|
||||
warp_accum[j / WARP_SIZE] = v2;
|
||||
__syncthreads();
|
||||
|
||||
__shared__ float s_norm_sq;
|
||||
if (j == 0) {
|
||||
float total = 0.0f;
|
||||
for (int w = 0; w < n_warps; w++) total += warp_accum[w];
|
||||
s_norm_sq = total;
|
||||
}
|
||||
__syncthreads();
|
||||
const float grp_norm = sqrtf(s_norm_sq);
|
||||
const float inv_norm = (grp_norm > 1e-10f) ? 1.0f / grp_norm : 0.0f;
|
||||
|
||||
// ---- Step 3: Normalize ----
|
||||
x[j] *= inv_norm;
|
||||
__syncthreads();
|
||||
|
||||
// ---- Step 4: Forward WHT (signs1 → butterfly → signs2, normalized) ----
|
||||
if (GROUP_SIZE == 128) {
|
||||
x[j] *= TURBO_WHT_SIGNS1[j];
|
||||
} else {
|
||||
x[j] *= TURBO_WHT_SIGNS1_64[j];
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
#define WHT_STAGE_SHARED(h) \
|
||||
if (j % (2*(h)) < (h)) { float a = x[j], b = x[j+(h)]; x[j] = a+b; x[j+(h)] = a-b; } \
|
||||
__syncthreads();
|
||||
|
||||
// Butterfly stages: loop from h=1 to h<GROUP_SIZE, doubling each time
|
||||
WHT_STAGE_SHARED(1)
|
||||
WHT_STAGE_SHARED(2)
|
||||
WHT_STAGE_SHARED(4)
|
||||
WHT_STAGE_SHARED(8)
|
||||
WHT_STAGE_SHARED(16)
|
||||
WHT_STAGE_SHARED(32)
|
||||
if (GROUP_SIZE == 128) { WHT_STAGE_SHARED(64) }
|
||||
#undef WHT_STAGE_SHARED
|
||||
|
||||
constexpr float inv_sqrt_group = (GROUP_SIZE == 128) ? 0.08838834764831845f : 0.125f;
|
||||
if (GROUP_SIZE == 128) {
|
||||
x[j] = x[j] * inv_sqrt_group * TURBO_WHT_SIGNS2[j];
|
||||
} else {
|
||||
x[j] = x[j] * inv_sqrt_group * TURBO_WHT_SIGNS2_64[j];
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
// ---- Step 5: Quantize element j ----
|
||||
const float rv = x[j];
|
||||
const uint8_t idx = turbo_nearest_centroid_3bit(rv);
|
||||
|
||||
// ---- Step 6: Pack qs and signs (warp-cooperative, no atomics) ----
|
||||
// Each warp handles 32 elements. With QK_TURBO3 > WARP_SIZE, multiple warps
|
||||
// share one block and write to different byte offsets within it.
|
||||
const int warp_id = j / WARP_SIZE;
|
||||
const int lane = j % WARP_SIZE;
|
||||
const int elem_in_block = j % QK_TURBO3;
|
||||
block_turbo3_0 * blk = blk_base + (j / QK_TURBO3);
|
||||
|
||||
// Pack qs: 4 elements per byte, 2 bits each.
|
||||
// All 4 threads in a qs-group gather their low2 bits via shuffle.
|
||||
const int qs_byte_idx = elem_in_block / 4;
|
||||
const uint8_t my_low2 = idx & 0x3;
|
||||
uint8_t qs_byte = 0;
|
||||
#pragma unroll
|
||||
for (int k = 0; k < 4; k++) {
|
||||
uint8_t contrib = __shfl_sync(0xffffffff, my_low2, (lane & ~3) + k);
|
||||
qs_byte |= contrib << (k * 2);
|
||||
}
|
||||
if (lane % 4 == 0) blk->qs[qs_byte_idx] = qs_byte;
|
||||
|
||||
// Pack signs: 8 elements per byte, 1 bit each. __ballot_sync across warp.
|
||||
// Ballot is per-warp (32 bits); extract local byte, write to global position in block.
|
||||
const uint32_t ballot = __ballot_sync(0xffffffff, (idx >> 2) & 1);
|
||||
const int local_signs_byte = lane / 8; // byte within 32-bit ballot (0..3)
|
||||
const int global_signs_byte = elem_in_block / 8; // byte within block's signs array
|
||||
const uint8_t signs_byte = (uint8_t)((ballot >> (local_signs_byte * 8)) & 0xFF);
|
||||
if (lane % 8 == 0) blk->signs[global_signs_byte] = signs_byte;
|
||||
|
||||
// ---- Step 7: Reconstruction norm (parallel, same pattern as step 2) ----
|
||||
const float c = TURBO_CENTROIDS_3BIT[idx];
|
||||
float rc = c * c;
|
||||
for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1)
|
||||
rc += __shfl_xor_sync(0xffffffff, rc, offset);
|
||||
if (j % WARP_SIZE == 0)
|
||||
warp_accum[j / WARP_SIZE] = rc;
|
||||
__syncthreads();
|
||||
|
||||
__shared__ float s_recon_sq;
|
||||
if (j == 0) {
|
||||
float total = 0.0f;
|
||||
for (int w = 0; w < n_warps; w++) total += warp_accum[w];
|
||||
s_recon_sq = total;
|
||||
}
|
||||
__syncthreads();
|
||||
const float recon_norm = sqrtf(s_recon_sq);
|
||||
const float corrected_norm = (recon_norm > 1e-10f) ? grp_norm / recon_norm : grp_norm;
|
||||
|
||||
// ---- Step 8: Write corrected norm (one per turbo3 block) ----
|
||||
if (elem_in_block == 0) blk->norm = __float2half(corrected_norm);
|
||||
|
||||
GGML_UNUSED(ne10);
|
||||
GGML_UNUSED(ne13);
|
||||
}
|
||||
|
||||
// ---- TurboQuant3 tail kernel: straight 3-bit quantize without WHT rotation ----
|
||||
//
|
||||
// For head dims not divisible by 128 (e.g. 576 = 4*128 + 64), the remainder
|
||||
// elements can't use the 128-element WHT. They are quantised directly into
|
||||
// standard turbo3 blocks. Q is also NOT rotated for these positions (the graph
|
||||
// guards on ne[0] % 128), so <Q_tail, K_tail> stays in the original space.
|
||||
//
|
||||
// One CUDA block per row, with tail_size threads (must be multiple of 32).
|
||||
|
||||
template <typename idx_t>
|
||||
static __global__ void k_set_rows_turbo3_tail(
|
||||
const float * __restrict__ src0,
|
||||
const idx_t * __restrict__ src1,
|
||||
block_turbo3_0 * __restrict__ dst,
|
||||
const int64_t ne00,
|
||||
const int64_t ne01,
|
||||
const int64_t ne10,
|
||||
const int64_t ne11,
|
||||
const int64_t ne12,
|
||||
const int64_t ne13,
|
||||
const int64_t s01,
|
||||
const int64_t s02,
|
||||
const int64_t s03,
|
||||
const int64_t s10,
|
||||
const int64_t s11,
|
||||
const int64_t s12,
|
||||
const int64_t s1,
|
||||
const int64_t s2,
|
||||
const int64_t s3,
|
||||
const int tail_size) {
|
||||
|
||||
const int j = threadIdx.x; // 0 .. tail_size-1
|
||||
|
||||
// Decode blockIdx.x → (i01, i02, i03)
|
||||
int64_t tmp = blockIdx.x;
|
||||
const int64_t i01 = tmp % ne01; tmp /= ne01;
|
||||
const int64_t i02 = tmp % ne12;
|
||||
const int64_t i03 = tmp / ne12;
|
||||
|
||||
const int64_t i11 = i01 % ne11;
|
||||
const int64_t i10 = i01;
|
||||
const int64_t i12 = i02;
|
||||
|
||||
const int64_t dst_row = *(src1 + i10*s10 + i11*s11 + i12*s12);
|
||||
const float * src_row = src0 + i01*s01 + i02*s02 + i03*s03;
|
||||
block_turbo3_0 * dst_row_ptr = (block_turbo3_0 *)((char *)dst + dst_row*s1 + i02*s2 + i03*s3);
|
||||
|
||||
// Tail starts after all full 128-element groups
|
||||
const int64_t n_full = ne00 / QK_TURBO3_GROUP;
|
||||
const int64_t tail_start = n_full * QK_TURBO3_GROUP;
|
||||
block_turbo3_0 * blk_base = dst_row_ptr + n_full * (QK_TURBO3_GROUP / QK_TURBO3);
|
||||
|
||||
// ---- Load ----
|
||||
const float val = src_row[tail_start + j];
|
||||
|
||||
// ---- L2 norm over the tail group (warp reduce + inter-warp) ----
|
||||
const int n_warps = tail_size / WARP_SIZE;
|
||||
const int warp_id = j / WARP_SIZE;
|
||||
const int lane = j % WARP_SIZE;
|
||||
|
||||
__shared__ float warp_accum[4]; // max 3 warps (tail ≤ 96)
|
||||
float v2 = val * val;
|
||||
for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1)
|
||||
v2 += __shfl_xor_sync(0xffffffff, v2, offset);
|
||||
if (lane == 0) warp_accum[warp_id] = v2;
|
||||
__syncthreads();
|
||||
|
||||
__shared__ float s_norm_sq;
|
||||
if (j == 0) {
|
||||
float total = 0.0f;
|
||||
for (int w = 0; w < n_warps; w++) total += warp_accum[w];
|
||||
s_norm_sq = total;
|
||||
}
|
||||
__syncthreads();
|
||||
const float grp_norm = sqrtf(s_norm_sq);
|
||||
const float inv_norm = (grp_norm > 1e-10f) ? 1.0f / grp_norm : 0.0f;
|
||||
|
||||
// ---- Normalize (no WHT!) ----
|
||||
const float rv = val * inv_norm;
|
||||
|
||||
// ---- Quantize ----
|
||||
const uint8_t idx = turbo_nearest_centroid_3bit(rv);
|
||||
|
||||
// ---- Pack qs and signs (same warp-cooperative logic) ----
|
||||
block_turbo3_0 * blk = blk_base + warp_id;
|
||||
|
||||
const uint8_t my_low2 = idx & 0x3;
|
||||
uint8_t qs_byte = 0;
|
||||
#pragma unroll
|
||||
for (int k = 0; k < 4; k++) {
|
||||
uint8_t contrib = __shfl_sync(0xffffffff, my_low2, (lane & ~3) + k);
|
||||
qs_byte |= contrib << (k * 2);
|
||||
}
|
||||
if (lane % 4 == 0) blk->qs[lane / 4] = qs_byte;
|
||||
|
||||
const uint32_t ballot = __ballot_sync(0xffffffff, (idx >> 2) & 1);
|
||||
const int signs_byte_idx = lane / 8;
|
||||
const uint8_t signs_byte = (uint8_t)((ballot >> (signs_byte_idx * 8)) & 0xFF);
|
||||
if (lane % 8 == 0) blk->signs[signs_byte_idx] = signs_byte;
|
||||
|
||||
// ---- Reconstruction norm ----
|
||||
const float c = TURBO_CENTROIDS_3BIT[idx];
|
||||
float rc = c * c;
|
||||
for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1)
|
||||
rc += __shfl_xor_sync(0xffffffff, rc, offset);
|
||||
if (lane == 0) warp_accum[warp_id] = rc;
|
||||
__syncthreads();
|
||||
|
||||
__shared__ float s_recon_sq;
|
||||
if (j == 0) {
|
||||
float total = 0.0f;
|
||||
for (int w = 0; w < n_warps; w++) total += warp_accum[w];
|
||||
s_recon_sq = total;
|
||||
}
|
||||
__syncthreads();
|
||||
const float recon_norm = sqrtf(s_recon_sq);
|
||||
const float corrected_norm = (recon_norm > 1e-10f) ? grp_norm / recon_norm : grp_norm;
|
||||
|
||||
if (lane == 0) blk->norm = __float2half(corrected_norm);
|
||||
|
||||
GGML_UNUSED(ne10);
|
||||
GGML_UNUSED(ne13);
|
||||
}
|
||||
|
||||
template<typename idx_t>
|
||||
static void set_rows_cuda_turbo3(
|
||||
ggml_backend_cuda_context & ctx,
|
||||
const ggml_tensor * src0,
|
||||
const ggml_tensor * src1,
|
||||
ggml_tensor * dst) {
|
||||
|
||||
const float * src0_d = (const float *)src0->data;
|
||||
const idx_t * src1_d = (const idx_t *)src1->data;
|
||||
|
||||
GGML_TENSOR_BINARY_OP_LOCALS
|
||||
GGML_ASSERT(ne00 % QK_TURBO3 == 0); // must be block-aligned (32)
|
||||
|
||||
cudaStream_t stream = ctx.stream();
|
||||
|
||||
// Read WHT group size from op_params (set by llama-kv-cache.cpp based on head_dim).
|
||||
// Default to 128 if not set (backward compat with head_dim=128 models).
|
||||
int group_size = 128;
|
||||
memcpy(&group_size, dst->op_params, sizeof(int));
|
||||
if (group_size != 64 && group_size != 128) group_size = 128;
|
||||
GGML_ASSERT(ne00 % group_size == 0);
|
||||
|
||||
const int64_t n_full_groups = ne00 / group_size;
|
||||
const int tail_size = (int)(ne00 % group_size);
|
||||
|
||||
const int64_t s01 = nb01/sizeof(float);
|
||||
const int64_t s02 = nb02/sizeof(float);
|
||||
const int64_t s03 = nb03/sizeof(float);
|
||||
const int64_t s10 = nb10/sizeof(idx_t);
|
||||
const int64_t s11 = nb11/sizeof(idx_t);
|
||||
const int64_t s12 = nb12/sizeof(idx_t);
|
||||
|
||||
// InnerQ: check/finalize calibration before kernel launch
|
||||
turbo_innerq_check_finalize(group_size, ne00);
|
||||
|
||||
// Launch 1: full groups with WHT rotation
|
||||
if (n_full_groups > 0) {
|
||||
const int64_t ne_total = n_full_groups * ne01 * ne02 * ne03;
|
||||
if (group_size == 128) {
|
||||
k_set_rows_turbo3<idx_t, 128><<<(int)ne_total, 128, 0, stream>>>(
|
||||
src0_d, src1_d, (block_turbo3_0 *)dst->data,
|
||||
ne00, ne01, ne10, ne11, ne12, ne13,
|
||||
s01, s02, s03, s10, s11, s12,
|
||||
nb1, nb2, nb3);
|
||||
} else {
|
||||
k_set_rows_turbo3<idx_t, 64><<<(int)ne_total, 64, 0, stream>>>(
|
||||
src0_d, src1_d, (block_turbo3_0 *)dst->data,
|
||||
ne00, ne01, ne10, ne11, ne12, ne13,
|
||||
s01, s02, s03, s10, s11, s12,
|
||||
nb1, nb2, nb3);
|
||||
}
|
||||
}
|
||||
|
||||
// Launch 2: tail elements (no WHT, straight quantize)
|
||||
// Not needed for 64-aligned dims but kept for potential future use
|
||||
if (tail_size > 0) {
|
||||
GGML_ASSERT(tail_size % QK_TURBO3 == 0); // tail must be block-aligned
|
||||
const int64_t n_rows = ne01 * ne02 * ne03;
|
||||
k_set_rows_turbo3_tail<idx_t><<<(int)n_rows, tail_size, 0, stream>>>(
|
||||
src0_d, src1_d, (block_turbo3_0 *)dst->data,
|
||||
ne00, ne01, ne10, ne11, ne12, ne13,
|
||||
s01, s02, s03, s10, s11, s12,
|
||||
nb1, nb2, nb3, tail_size);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- TurboQuant2 set_rows: GROUP_SIZE-element groups with WHT rotation + norm correction ----
|
||||
//
|
||||
// Same structure as turbo3 but 2-bit quantization only (no signs byte).
|
||||
|
||||
template <typename idx_t, int GROUP_SIZE>
|
||||
__launch_bounds__(128)
|
||||
static __global__ void k_set_rows_turbo2(
|
||||
const float * __restrict__ src0,
|
||||
const idx_t * __restrict__ src1,
|
||||
block_turbo2_0 * __restrict__ dst,
|
||||
const int64_t ne00,
|
||||
const int64_t ne01,
|
||||
const int64_t ne10,
|
||||
const int64_t ne11,
|
||||
const int64_t ne12,
|
||||
const int64_t ne13,
|
||||
const int64_t s01,
|
||||
const int64_t s02,
|
||||
const int64_t s03,
|
||||
const int64_t s10,
|
||||
const int64_t s11,
|
||||
const int64_t s12,
|
||||
const int64_t s1,
|
||||
const int64_t s2,
|
||||
const int64_t s3) {
|
||||
|
||||
static_assert(GROUP_SIZE == 128 || GROUP_SIZE == 64, "GROUP_SIZE must be 128 or 64");
|
||||
|
||||
const int j = threadIdx.x;
|
||||
|
||||
constexpr int blocks_per_group = GROUP_SIZE / QK_TURBO2;
|
||||
const int64_t n_groups_per_row = ne00 / GROUP_SIZE;
|
||||
const int64_t g = blockIdx.x;
|
||||
const int64_t i_grp = g % n_groups_per_row;
|
||||
int64_t tmp = g / n_groups_per_row;
|
||||
const int64_t i01 = tmp % ne01;
|
||||
tmp = tmp / ne01;
|
||||
const int64_t i02 = tmp % ne12;
|
||||
const int64_t i03 = tmp / ne12;
|
||||
|
||||
const int64_t i12 = i02;
|
||||
const int64_t i11 = i01 % ne11;
|
||||
const int64_t i10 = i01;
|
||||
|
||||
const int64_t dst_row = *(src1 + i10*s10 + i11*s11 + i12*s12);
|
||||
const float * src_row = src0 + i01*s01 + i02*s02 + i03*s03;
|
||||
block_turbo2_0 * dst_row_ptr = (block_turbo2_0 *)((char *)dst + dst_row*s1 + i02*s2 + i03*s3);
|
||||
block_turbo2_0 * blk_base = dst_row_ptr + i_grp * blocks_per_group;
|
||||
|
||||
// ---- Step 1: Load element j (coalesced) ----
|
||||
__shared__ float x[GROUP_SIZE];
|
||||
x[j] = src_row[i_grp * GROUP_SIZE + j];
|
||||
__syncthreads();
|
||||
|
||||
// ---- InnerQ: calibrate on original (unscaled) values ----
|
||||
if (d_innerq_calibrating) {
|
||||
atomicAdd(&d_innerq_sq_accum[j], x[j] * x[j]);
|
||||
if (j == 0) atomicAdd(&d_innerq_count, 1);
|
||||
}
|
||||
|
||||
// ---- InnerQ: apply channel scale (only when active) ----
|
||||
if (d_innerq_active) {
|
||||
x[j] *= d_innerq_scale[j];
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
// ---- Step 2: Parallel L2 norm ----
|
||||
constexpr int n_warps = GROUP_SIZE / WARP_SIZE;
|
||||
__shared__ float warp_accum[n_warps];
|
||||
float v = x[j];
|
||||
float v2 = v * v;
|
||||
for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1)
|
||||
v2 += __shfl_xor_sync(0xffffffff, v2, offset);
|
||||
if (j % WARP_SIZE == 0)
|
||||
warp_accum[j / WARP_SIZE] = v2;
|
||||
__syncthreads();
|
||||
|
||||
__shared__ float s_norm_sq;
|
||||
if (j == 0) {
|
||||
float total = 0.0f;
|
||||
for (int w = 0; w < n_warps; w++) total += warp_accum[w];
|
||||
s_norm_sq = total;
|
||||
}
|
||||
__syncthreads();
|
||||
const float grp_norm = sqrtf(s_norm_sq);
|
||||
const float inv_norm = (grp_norm > 1e-10f) ? 1.0f / grp_norm : 0.0f;
|
||||
|
||||
// ---- Step 3: Normalize ----
|
||||
x[j] *= inv_norm;
|
||||
__syncthreads();
|
||||
|
||||
// ---- Step 4: Forward WHT ----
|
||||
if (GROUP_SIZE == 128) {
|
||||
x[j] *= TURBO_WHT_SIGNS1[j];
|
||||
} else {
|
||||
x[j] *= TURBO_WHT_SIGNS1_64[j];
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
#define WHT_STAGE_SHARED_T2(h) \
|
||||
if (j % (2*(h)) < (h)) { float a = x[j], b = x[j+(h)]; x[j] = a+b; x[j+(h)] = a-b; } \
|
||||
__syncthreads();
|
||||
|
||||
WHT_STAGE_SHARED_T2(1)
|
||||
WHT_STAGE_SHARED_T2(2)
|
||||
WHT_STAGE_SHARED_T2(4)
|
||||
WHT_STAGE_SHARED_T2(8)
|
||||
WHT_STAGE_SHARED_T2(16)
|
||||
WHT_STAGE_SHARED_T2(32)
|
||||
if (GROUP_SIZE == 128) { WHT_STAGE_SHARED_T2(64) }
|
||||
#undef WHT_STAGE_SHARED_T2
|
||||
|
||||
constexpr float inv_sqrt_group = (GROUP_SIZE == 128) ? 0.08838834764831845f : 0.125f;
|
||||
if (GROUP_SIZE == 128) {
|
||||
x[j] = x[j] * inv_sqrt_group * TURBO_WHT_SIGNS2[j];
|
||||
} else {
|
||||
x[j] = x[j] * inv_sqrt_group * TURBO_WHT_SIGNS2_64[j];
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
// ---- Step 5: Quantize element j to 2-bit centroid ----
|
||||
const float rv = x[j];
|
||||
const uint8_t idx = turbo_nearest_centroid_2bit(rv);
|
||||
|
||||
// ---- Step 6: Pack qs (warp-cooperative, no atomics) ----
|
||||
// Each warp handles 32 elements. With QK_TURBO2 > WARP_SIZE, multiple warps
|
||||
// share one block and write to different byte offsets within it.
|
||||
const int warp_id = j / WARP_SIZE;
|
||||
const int lane = j % WARP_SIZE;
|
||||
const int elem_in_block = j % QK_TURBO2;
|
||||
block_turbo2_0 * blk = blk_base + (j / QK_TURBO2);
|
||||
|
||||
// Pack qs: 4 elements per byte, 2 bits each.
|
||||
const uint8_t my_bits = idx & 0x3;
|
||||
uint8_t qs_byte = 0;
|
||||
#pragma unroll
|
||||
for (int k = 0; k < 4; k++) {
|
||||
uint8_t contrib = __shfl_sync(0xffffffff, my_bits, (lane & ~3) + k);
|
||||
qs_byte |= contrib << (k * 2);
|
||||
}
|
||||
if (lane % 4 == 0) blk->qs[elem_in_block / 4] = qs_byte;
|
||||
|
||||
// No signs packing needed for turbo2
|
||||
|
||||
// ---- Step 7: Reconstruction norm ----
|
||||
const float c = TURBO_CENTROIDS_2BIT[idx];
|
||||
float rc = c * c;
|
||||
for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1)
|
||||
rc += __shfl_xor_sync(0xffffffff, rc, offset);
|
||||
if (j % WARP_SIZE == 0)
|
||||
warp_accum[j / WARP_SIZE] = rc;
|
||||
__syncthreads();
|
||||
|
||||
__shared__ float s_recon_sq;
|
||||
if (j == 0) {
|
||||
float total = 0.0f;
|
||||
for (int w = 0; w < n_warps; w++) total += warp_accum[w];
|
||||
s_recon_sq = total;
|
||||
}
|
||||
__syncthreads();
|
||||
const float recon_norm = sqrtf(s_recon_sq);
|
||||
const float corrected_norm = (recon_norm > 1e-10f) ? grp_norm / recon_norm : grp_norm;
|
||||
|
||||
// ---- Step 8: Write corrected norm (one per turbo2 block) ----
|
||||
if (elem_in_block == 0) blk->norm = __float2half(corrected_norm);
|
||||
|
||||
GGML_UNUSED(ne10);
|
||||
GGML_UNUSED(ne13);
|
||||
}
|
||||
|
||||
// ---- TurboQuant2 tail kernel: straight 2-bit quantize without WHT rotation ----
|
||||
|
||||
template <typename idx_t>
|
||||
static __global__ void k_set_rows_turbo2_tail(
|
||||
const float * __restrict__ src0,
|
||||
const idx_t * __restrict__ src1,
|
||||
block_turbo2_0 * __restrict__ dst,
|
||||
const int64_t ne00,
|
||||
const int64_t ne01,
|
||||
const int64_t ne10,
|
||||
const int64_t ne11,
|
||||
const int64_t ne12,
|
||||
const int64_t ne13,
|
||||
const int64_t s01,
|
||||
const int64_t s02,
|
||||
const int64_t s03,
|
||||
const int64_t s10,
|
||||
const int64_t s11,
|
||||
const int64_t s12,
|
||||
const int64_t s1,
|
||||
const int64_t s2,
|
||||
const int64_t s3,
|
||||
const int tail_size) {
|
||||
|
||||
const int j = threadIdx.x;
|
||||
|
||||
int64_t tmp = blockIdx.x;
|
||||
const int64_t i01 = tmp % ne01; tmp /= ne01;
|
||||
const int64_t i02 = tmp % ne12;
|
||||
const int64_t i03 = tmp / ne12;
|
||||
|
||||
const int64_t i11 = i01 % ne11;
|
||||
const int64_t i10 = i01;
|
||||
const int64_t i12 = i02;
|
||||
|
||||
const int64_t dst_row = *(src1 + i10*s10 + i11*s11 + i12*s12);
|
||||
const float * src_row = src0 + i01*s01 + i02*s02 + i03*s03;
|
||||
block_turbo2_0 * dst_row_ptr = (block_turbo2_0 *)((char *)dst + dst_row*s1 + i02*s2 + i03*s3);
|
||||
|
||||
const int64_t n_full = ne00 / QK_TURBO2_GROUP;
|
||||
const int64_t tail_start = n_full * QK_TURBO2_GROUP;
|
||||
block_turbo2_0 * blk_base = dst_row_ptr + n_full * (QK_TURBO2_GROUP / QK_TURBO2);
|
||||
|
||||
// ---- Load ----
|
||||
const float val = src_row[tail_start + j];
|
||||
|
||||
// ---- L2 norm ----
|
||||
const int n_warps = tail_size / WARP_SIZE;
|
||||
const int warp_id = j / WARP_SIZE;
|
||||
const int lane = j % WARP_SIZE;
|
||||
|
||||
__shared__ float warp_accum[4];
|
||||
float v2 = val * val;
|
||||
for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1)
|
||||
v2 += __shfl_xor_sync(0xffffffff, v2, offset);
|
||||
if (lane == 0) warp_accum[warp_id] = v2;
|
||||
__syncthreads();
|
||||
|
||||
__shared__ float s_norm_sq;
|
||||
if (j == 0) {
|
||||
float total = 0.0f;
|
||||
for (int w = 0; w < n_warps; w++) total += warp_accum[w];
|
||||
s_norm_sq = total;
|
||||
}
|
||||
__syncthreads();
|
||||
const float grp_norm = sqrtf(s_norm_sq);
|
||||
const float inv_norm = (grp_norm > 1e-10f) ? 1.0f / grp_norm : 0.0f;
|
||||
|
||||
// ---- Normalize (no WHT!) ----
|
||||
const float rv = val * inv_norm;
|
||||
|
||||
// ---- Quantize ----
|
||||
const uint8_t idx = turbo_nearest_centroid_2bit(rv);
|
||||
|
||||
// ---- Pack qs ----
|
||||
block_turbo2_0 * blk = blk_base + warp_id;
|
||||
|
||||
const uint8_t my_bits = idx & 0x3;
|
||||
uint8_t qs_byte = 0;
|
||||
#pragma unroll
|
||||
for (int k = 0; k < 4; k++) {
|
||||
uint8_t contrib = __shfl_sync(0xffffffff, my_bits, (lane & ~3) + k);
|
||||
qs_byte |= contrib << (k * 2);
|
||||
}
|
||||
if (lane % 4 == 0) blk->qs[lane / 4] = qs_byte;
|
||||
|
||||
// ---- Reconstruction norm ----
|
||||
const float c = TURBO_CENTROIDS_2BIT[idx];
|
||||
float rc = c * c;
|
||||
for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1)
|
||||
rc += __shfl_xor_sync(0xffffffff, rc, offset);
|
||||
if (lane == 0) warp_accum[warp_id] = rc;
|
||||
__syncthreads();
|
||||
|
||||
__shared__ float s_recon_sq;
|
||||
if (j == 0) {
|
||||
float total = 0.0f;
|
||||
for (int w = 0; w < n_warps; w++) total += warp_accum[w];
|
||||
s_recon_sq = total;
|
||||
}
|
||||
__syncthreads();
|
||||
const float recon_norm = sqrtf(s_recon_sq);
|
||||
const float corrected_norm = (recon_norm > 1e-10f) ? grp_norm / recon_norm : grp_norm;
|
||||
|
||||
if (lane == 0) blk->norm = __float2half(corrected_norm);
|
||||
|
||||
GGML_UNUSED(ne10);
|
||||
GGML_UNUSED(ne13);
|
||||
GGML_UNUSED(ne00);
|
||||
}
|
||||
|
||||
template<typename idx_t>
|
||||
static void set_rows_cuda_turbo2(
|
||||
ggml_backend_cuda_context & ctx,
|
||||
const ggml_tensor * src0,
|
||||
const ggml_tensor * src1,
|
||||
ggml_tensor * dst) {
|
||||
|
||||
const float * src0_d = (const float *)src0->data;
|
||||
const idx_t * src1_d = (const idx_t *)src1->data;
|
||||
|
||||
GGML_TENSOR_BINARY_OP_LOCALS
|
||||
GGML_ASSERT(ne00 % QK_TURBO2 == 0);
|
||||
|
||||
cudaStream_t stream = ctx.stream();
|
||||
|
||||
int group_size = 128;
|
||||
memcpy(&group_size, dst->op_params, sizeof(int));
|
||||
if (group_size != 64 && group_size != 128) group_size = 128;
|
||||
GGML_ASSERT(ne00 % group_size == 0);
|
||||
|
||||
const int64_t n_full_groups = ne00 / group_size;
|
||||
const int tail_size = (int)(ne00 % group_size);
|
||||
|
||||
const int64_t s01 = nb01/sizeof(float);
|
||||
const int64_t s02 = nb02/sizeof(float);
|
||||
const int64_t s03 = nb03/sizeof(float);
|
||||
const int64_t s10 = nb10/sizeof(idx_t);
|
||||
const int64_t s11 = nb11/sizeof(idx_t);
|
||||
const int64_t s12 = nb12/sizeof(idx_t);
|
||||
|
||||
// InnerQ: check/finalize calibration before kernel launch
|
||||
turbo_innerq_check_finalize(group_size, ne00);
|
||||
|
||||
if (n_full_groups > 0) {
|
||||
const int64_t ne_total = n_full_groups * ne01 * ne02 * ne03;
|
||||
if (group_size == 128) {
|
||||
k_set_rows_turbo2<idx_t, 128><<<(int)ne_total, 128, 0, stream>>>(
|
||||
src0_d, src1_d, (block_turbo2_0 *)dst->data,
|
||||
ne00, ne01, ne10, ne11, ne12, ne13,
|
||||
s01, s02, s03, s10, s11, s12,
|
||||
nb1, nb2, nb3);
|
||||
} else {
|
||||
k_set_rows_turbo2<idx_t, 64><<<(int)ne_total, 64, 0, stream>>>(
|
||||
src0_d, src1_d, (block_turbo2_0 *)dst->data,
|
||||
ne00, ne01, ne10, ne11, ne12, ne13,
|
||||
s01, s02, s03, s10, s11, s12,
|
||||
nb1, nb2, nb3);
|
||||
}
|
||||
}
|
||||
|
||||
if (tail_size > 0) {
|
||||
GGML_ASSERT(tail_size % QK_TURBO2 == 0);
|
||||
const int64_t n_rows = ne01 * ne02 * ne03;
|
||||
k_set_rows_turbo2_tail<idx_t><<<(int)n_rows, tail_size, 0, stream>>>(
|
||||
src0_d, src1_d, (block_turbo2_0 *)dst->data,
|
||||
ne00, ne01, ne10, ne11, ne12, ne13,
|
||||
s01, s02, s03, s10, s11, s12,
|
||||
nb1, nb2, nb3, tail_size);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- TurboQuant4 set_rows: 128-element groups with WHT rotation + 4-bit quantization ----
|
||||
//
|
||||
// turbo4 block size IS the WHT group size (128), so 1 CUDA block = 1 turbo4 block.
|
||||
// 128 threads per block, thread j handles element j.
|
||||
// 4-bit centroids (16 values), nibble packed: qs[j/2] |= (idx & 0xF) << ((j%2)*4)
|
||||
|
||||
template <typename idx_t>
|
||||
__launch_bounds__(128)
|
||||
static __global__ void k_set_rows_turbo4(
|
||||
const float * __restrict__ src0,
|
||||
const idx_t * __restrict__ src1,
|
||||
block_turbo4_0 * __restrict__ dst,
|
||||
const int64_t ne00,
|
||||
const int64_t ne01,
|
||||
const int64_t ne10,
|
||||
const int64_t ne11,
|
||||
const int64_t ne12,
|
||||
const int64_t ne13,
|
||||
const int64_t s01,
|
||||
const int64_t s02,
|
||||
const int64_t s03,
|
||||
const int64_t s10,
|
||||
const int64_t s11,
|
||||
const int64_t s12,
|
||||
const int64_t s1,
|
||||
const int64_t s2,
|
||||
const int64_t s3) {
|
||||
|
||||
// blockIdx.x = flat block index; threadIdx.x = element within block (0..127)
|
||||
const int j = threadIdx.x;
|
||||
|
||||
// Decode blockIdx.x → (i_blk, i01, i02, i03)
|
||||
const int64_t n_blocks_per_row = ne00 / QK_TURBO4;
|
||||
const int64_t g = blockIdx.x;
|
||||
const int64_t i_blk = g % n_blocks_per_row;
|
||||
int64_t tmp = g / n_blocks_per_row;
|
||||
const int64_t i01 = tmp % ne01;
|
||||
tmp = tmp / ne01;
|
||||
const int64_t i02 = tmp % ne12;
|
||||
const int64_t i03 = tmp / ne12;
|
||||
|
||||
const int64_t i12 = i02;
|
||||
const int64_t i11 = i01 % ne11;
|
||||
const int64_t i10 = i01;
|
||||
|
||||
const int64_t dst_row = *(src1 + i10*s10 + i11*s11 + i12*s12);
|
||||
const float * src_row = src0 + i01*s01 + i02*s02 + i03*s03;
|
||||
block_turbo4_0 * dst_row_ptr = (block_turbo4_0 *)((char *)dst + dst_row*s1 + i02*s2 + i03*s3);
|
||||
block_turbo4_0 * blk = dst_row_ptr + i_blk;
|
||||
|
||||
// ---- Step 1: Load element j (coalesced) ----
|
||||
__shared__ float x[128];
|
||||
x[j] = src_row[i_blk * QK_TURBO4 + j];
|
||||
__syncthreads();
|
||||
|
||||
// ---- InnerQ: calibrate on original (unscaled) values ----
|
||||
if (d_innerq_calibrating) {
|
||||
atomicAdd(&d_innerq_sq_accum[j], x[j] * x[j]);
|
||||
if (j == 0) atomicAdd(&d_innerq_count, 1);
|
||||
}
|
||||
|
||||
// ---- InnerQ: apply channel scale (only when active) ----
|
||||
if (d_innerq_active) {
|
||||
x[j] *= d_innerq_scale[j];
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
// ---- Step 2: Parallel L2 norm ----
|
||||
constexpr int n_warps = 128 / WARP_SIZE; // = 4
|
||||
__shared__ float warp_accum[n_warps];
|
||||
float v = x[j];
|
||||
float v2 = v * v;
|
||||
for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1)
|
||||
v2 += __shfl_xor_sync(0xffffffff, v2, offset);
|
||||
if (j % WARP_SIZE == 0)
|
||||
warp_accum[j / WARP_SIZE] = v2;
|
||||
__syncthreads();
|
||||
|
||||
__shared__ float s_norm_sq;
|
||||
if (j == 0) {
|
||||
float total = 0.0f;
|
||||
for (int w = 0; w < n_warps; w++) total += warp_accum[w];
|
||||
s_norm_sq = total;
|
||||
}
|
||||
__syncthreads();
|
||||
const float grp_norm = sqrtf(s_norm_sq);
|
||||
const float inv_norm = (grp_norm > 1e-10f) ? 1.0f / grp_norm : 0.0f;
|
||||
|
||||
// ---- Step 3: Normalize ----
|
||||
x[j] *= inv_norm;
|
||||
__syncthreads();
|
||||
|
||||
// ---- Step 4: Forward WHT (signs1 → butterfly → signs2, normalized) ----
|
||||
x[j] *= TURBO_WHT_SIGNS1[j];
|
||||
__syncthreads();
|
||||
|
||||
#define WHT_STAGE_SHARED_T4(h) \
|
||||
if (j % (2*(h)) < (h)) { float a = x[j], b = x[j+(h)]; x[j] = a+b; x[j+(h)] = a-b; } \
|
||||
__syncthreads();
|
||||
|
||||
WHT_STAGE_SHARED_T4(1)
|
||||
WHT_STAGE_SHARED_T4(2)
|
||||
WHT_STAGE_SHARED_T4(4)
|
||||
WHT_STAGE_SHARED_T4(8)
|
||||
WHT_STAGE_SHARED_T4(16)
|
||||
WHT_STAGE_SHARED_T4(32)
|
||||
WHT_STAGE_SHARED_T4(64)
|
||||
#undef WHT_STAGE_SHARED_T4
|
||||
|
||||
constexpr float inv_sqrt_128 = 0.08838834764831845f;
|
||||
x[j] = x[j] * inv_sqrt_128 * TURBO_WHT_SIGNS2[j];
|
||||
__syncthreads();
|
||||
|
||||
// ---- Step 5: Quantize element j to 4-bit centroid ----
|
||||
const float rv = x[j];
|
||||
const uint8_t idx = turbo_nearest_centroid_4bit(rv);
|
||||
|
||||
// ---- Step 6: Pack qs (nibble packed, warp-cooperative) ----
|
||||
// 2 elements per byte, 4 bits each.
|
||||
// Thread pairs (j, j+1) share a qs byte.
|
||||
const int lane = j % WARP_SIZE;
|
||||
const uint8_t my_nibble = idx & 0xF;
|
||||
uint8_t qs_byte = 0;
|
||||
// Gather nibble from partner thread
|
||||
uint8_t partner_nibble = __shfl_sync(0xffffffff, my_nibble, lane ^ 1);
|
||||
if (j % 2 == 0) {
|
||||
qs_byte = my_nibble | (partner_nibble << 4);
|
||||
blk->qs[j / 2] = qs_byte;
|
||||
}
|
||||
|
||||
// ---- Step 7: Reconstruction norm (parallel) ----
|
||||
const float c = TURBO_CENTROIDS_4BIT[idx];
|
||||
float rc = c * c;
|
||||
for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1)
|
||||
rc += __shfl_xor_sync(0xffffffff, rc, offset);
|
||||
if (j % WARP_SIZE == 0)
|
||||
warp_accum[j / WARP_SIZE] = rc;
|
||||
__syncthreads();
|
||||
|
||||
__shared__ float s_recon_sq;
|
||||
if (j == 0) {
|
||||
float total = 0.0f;
|
||||
for (int w = 0; w < n_warps; w++) total += warp_accum[w];
|
||||
s_recon_sq = total;
|
||||
}
|
||||
__syncthreads();
|
||||
const float recon_norm = sqrtf(s_recon_sq);
|
||||
const float corrected_norm = (recon_norm > 1e-10f) ? grp_norm / recon_norm : grp_norm;
|
||||
|
||||
// ---- Step 8: Write corrected norm and zero rnorm (one thread) ----
|
||||
if (j == 0) {
|
||||
blk->norm = __float2half(corrected_norm);
|
||||
blk->rnorm = __float2half(0.0f);
|
||||
}
|
||||
|
||||
GGML_UNUSED(ne10);
|
||||
GGML_UNUSED(ne13);
|
||||
}
|
||||
|
||||
template<typename idx_t>
|
||||
static void set_rows_cuda_turbo4(
|
||||
ggml_backend_cuda_context & ctx,
|
||||
const ggml_tensor * src0,
|
||||
const ggml_tensor * src1,
|
||||
ggml_tensor * dst) {
|
||||
|
||||
const float * src0_d = (const float *)src0->data;
|
||||
const idx_t * src1_d = (const idx_t *)src1->data;
|
||||
|
||||
GGML_TENSOR_BINARY_OP_LOCALS
|
||||
GGML_ASSERT(ne00 % QK_TURBO4 == 0); // must be block-aligned (128)
|
||||
|
||||
cudaStream_t stream = ctx.stream();
|
||||
|
||||
// turbo4 block size = WHT group size = 128, always
|
||||
const int64_t n_blocks = ne00 / QK_TURBO4;
|
||||
|
||||
const int64_t s01 = nb01/sizeof(float);
|
||||
const int64_t s02 = nb02/sizeof(float);
|
||||
const int64_t s03 = nb03/sizeof(float);
|
||||
const int64_t s10 = nb10/sizeof(idx_t);
|
||||
const int64_t s11 = nb11/sizeof(idx_t);
|
||||
const int64_t s12 = nb12/sizeof(idx_t);
|
||||
|
||||
// InnerQ: check/finalize calibration before kernel launch
|
||||
turbo_innerq_check_finalize(QK_TURBO4, ne00);
|
||||
|
||||
if (n_blocks > 0) {
|
||||
const int64_t ne_total = n_blocks * ne01 * ne02 * ne03;
|
||||
k_set_rows_turbo4<idx_t><<<(int)ne_total, 128, 0, stream>>>(
|
||||
src0_d, src1_d, (block_turbo4_0 *)dst->data,
|
||||
ne00, ne01, ne10, ne11, ne12, ne13,
|
||||
s01, s02, s03, s10, s11, s12,
|
||||
nb1, nb2, nb3);
|
||||
}
|
||||
}
|
||||
|
||||
template<typename src_t, typename idx_t>
|
||||
static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
|
||||
const src_t * src0_d = (const src_t *)src0->data;
|
||||
@@ -309,6 +1232,12 @@ static void set_rows_cuda(ggml_backend_cuda_context & ctx, const ggml_tensor * s
|
||||
nb1, nb2, nb3,
|
||||
stream
|
||||
);
|
||||
} else if (dst->type == GGML_TYPE_TURBO3_0) {
|
||||
set_rows_cuda_turbo3<idx_t>(ctx, src0, src1, dst);
|
||||
} else if (dst->type == GGML_TYPE_TURBO2_0) {
|
||||
set_rows_cuda_turbo2<idx_t>(ctx, src0, src1, dst);
|
||||
} else if (dst->type == GGML_TYPE_TURBO4_0) {
|
||||
set_rows_cuda_turbo4<idx_t>(ctx, src0, src1, dst);
|
||||
} else {
|
||||
GGML_ABORT("unsupported type %s", ggml_type_name(dst->type));
|
||||
}
|
||||
|
||||
@@ -4,3 +4,4 @@
|
||||
|
||||
DECL_FATTN_MMA_F16_CASE(192, 128, 1, 16);
|
||||
DECL_FATTN_MMA_F16_CASE(576, 512, 1, 16);
|
||||
DECL_FATTN_MMA_F16_CASE(640, 512, 1, 16);
|
||||
|
||||
@@ -4,3 +4,4 @@
|
||||
|
||||
DECL_FATTN_MMA_F16_CASE(192, 128, 2, 16);
|
||||
DECL_FATTN_MMA_F16_CASE(576, 512, 2, 16);
|
||||
DECL_FATTN_MMA_F16_CASE(640, 512, 2, 16);
|
||||
|
||||
@@ -4,3 +4,4 @@
|
||||
|
||||
DECL_FATTN_MMA_F16_CASE(192, 128, 4, 16);
|
||||
DECL_FATTN_MMA_F16_CASE(576, 512, 4, 16);
|
||||
DECL_FATTN_MMA_F16_CASE(640, 512, 4, 16);
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
||||
|
||||
#include "../fattn-tile.cuh"
|
||||
|
||||
DECL_FATTN_TILE_CASE(640, 512);
|
||||
@@ -0,0 +1,7 @@
|
||||
// Mixed KV: q8_0 K + turbo2 V
|
||||
|
||||
#include "../fattn-vec.cuh"
|
||||
|
||||
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0);
|
||||
DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0);
|
||||
DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_TURBO2_0);
|
||||
@@ -0,0 +1,7 @@
|
||||
// Mixed KV: q8_0 K + turbo3 V
|
||||
|
||||
#include "../fattn-vec.cuh"
|
||||
|
||||
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_0);
|
||||
DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_0);
|
||||
DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_TURBO3_0);
|
||||
@@ -0,0 +1,7 @@
|
||||
// Mixed KV: q8_0 K + turbo4 V
|
||||
|
||||
#include "../fattn-vec.cuh"
|
||||
|
||||
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q8_0, GGML_TYPE_TURBO4_0);
|
||||
DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q8_0, GGML_TYPE_TURBO4_0);
|
||||
DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q8_0, GGML_TYPE_TURBO4_0);
|
||||
@@ -0,0 +1,7 @@
|
||||
// Mixed KV: turbo2 K + q8_0 V
|
||||
|
||||
#include "../fattn-vec.cuh"
|
||||
|
||||
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO2_0, GGML_TYPE_Q8_0);
|
||||
DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO2_0, GGML_TYPE_Q8_0);
|
||||
DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO2_0, GGML_TYPE_Q8_0);
|
||||
@@ -0,0 +1,7 @@
|
||||
// TurboQuant2 CUDA flash attention vec kernel instantiation
|
||||
|
||||
#include "../fattn-vec.cuh"
|
||||
|
||||
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0);
|
||||
DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0);
|
||||
DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO2_0);
|
||||
@@ -0,0 +1,7 @@
|
||||
// Mixed KV: turbo2 K + turbo3 V
|
||||
|
||||
#include "../fattn-vec.cuh"
|
||||
|
||||
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO3_0);
|
||||
DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO3_0);
|
||||
DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO3_0);
|
||||
@@ -0,0 +1,7 @@
|
||||
// Mixed KV: turbo2 K + turbo4 V
|
||||
|
||||
#include "../fattn-vec.cuh"
|
||||
|
||||
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO4_0);
|
||||
DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO4_0);
|
||||
DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO2_0, GGML_TYPE_TURBO4_0);
|
||||
@@ -0,0 +1,7 @@
|
||||
// Mixed KV: turbo3 K + q8_0 V
|
||||
|
||||
#include "../fattn-vec.cuh"
|
||||
|
||||
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO3_0, GGML_TYPE_Q8_0);
|
||||
DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO3_0, GGML_TYPE_Q8_0);
|
||||
DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO3_0, GGML_TYPE_Q8_0);
|
||||
@@ -0,0 +1,7 @@
|
||||
// Mixed KV: turbo3 K + turbo2 V
|
||||
|
||||
#include "../fattn-vec.cuh"
|
||||
|
||||
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO2_0);
|
||||
DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO2_0);
|
||||
DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO2_0);
|
||||
@@ -0,0 +1,7 @@
|
||||
// TurboQuant3 CUDA flash attention vec kernel instantiation
|
||||
|
||||
#include "../fattn-vec.cuh"
|
||||
|
||||
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0);
|
||||
DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0);
|
||||
DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO3_0);
|
||||
@@ -0,0 +1,7 @@
|
||||
// Mixed KV: turbo3 K + turbo4 V
|
||||
|
||||
#include "../fattn-vec.cuh"
|
||||
|
||||
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO4_0);
|
||||
DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO4_0);
|
||||
DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO3_0, GGML_TYPE_TURBO4_0);
|
||||
@@ -0,0 +1,7 @@
|
||||
// Mixed KV: turbo4 K + q8_0 V
|
||||
|
||||
#include "../fattn-vec.cuh"
|
||||
|
||||
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO4_0, GGML_TYPE_Q8_0);
|
||||
DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO4_0, GGML_TYPE_Q8_0);
|
||||
DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO4_0, GGML_TYPE_Q8_0);
|
||||
@@ -0,0 +1,7 @@
|
||||
// Mixed KV: turbo4 K + turbo2 V
|
||||
|
||||
#include "../fattn-vec.cuh"
|
||||
|
||||
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO2_0);
|
||||
DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO2_0);
|
||||
DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO2_0);
|
||||
@@ -0,0 +1,7 @@
|
||||
// Mixed KV: turbo4 K + turbo3 V
|
||||
|
||||
#include "../fattn-vec.cuh"
|
||||
|
||||
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO3_0);
|
||||
DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO3_0);
|
||||
DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO3_0);
|
||||
@@ -0,0 +1,7 @@
|
||||
// TurboQuant4 CUDA flash attention vec kernel instantiation
|
||||
|
||||
#include "../fattn-vec.cuh"
|
||||
|
||||
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO4_0);
|
||||
DECL_FATTN_VEC_CASE(128, GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO4_0);
|
||||
DECL_FATTN_VEC_CASE(256, GGML_TYPE_TURBO4_0, GGML_TYPE_TURBO4_0);
|
||||
@@ -0,0 +1,32 @@
|
||||
#include "turbo-innerq.cuh"
|
||||
#include <cstring>
|
||||
|
||||
// Host-side shared state for InnerQ cross-TU communication
|
||||
bool g_innerq_finalized = false;
|
||||
float g_innerq_scale_inv_host[INNERQ_MAX_CHANNELS] = {
|
||||
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
|
||||
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
|
||||
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
|
||||
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1
|
||||
};
|
||||
|
||||
static bool g_innerq_tensor_needs_update = false;
|
||||
|
||||
void turbo_innerq_publish(const float * scale_inv, int group_size) {
|
||||
for (int i = 0; i < group_size && i < INNERQ_MAX_CHANNELS; i++) {
|
||||
g_innerq_scale_inv_host[i] = scale_inv[i];
|
||||
}
|
||||
for (int i = group_size; i < INNERQ_MAX_CHANNELS; i++) {
|
||||
g_innerq_scale_inv_host[i] = 1.0f;
|
||||
}
|
||||
g_innerq_finalized = true;
|
||||
g_innerq_tensor_needs_update = true;
|
||||
}
|
||||
|
||||
bool turbo_innerq_needs_tensor_update(void) {
|
||||
return g_innerq_tensor_needs_update;
|
||||
}
|
||||
|
||||
void turbo_innerq_mark_tensor_updated(void) {
|
||||
g_innerq_tensor_needs_update = false;
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
#pragma once
|
||||
|
||||
// TurboQuant InnerQ per-channel equalization — cross-TU shared state
|
||||
// The host-side state lives in turbo-innerq.cu; device-side state is per-TU
|
||||
// in turbo-quant.cuh (only set-rows.cu needs device access).
|
||||
|
||||
#define INNERQ_MAX_CHANNELS 128
|
||||
|
||||
// Host-side shared state (defined in turbo-innerq.cu)
|
||||
extern bool g_innerq_finalized;
|
||||
extern float g_innerq_scale_inv_host[INNERQ_MAX_CHANNELS];
|
||||
|
||||
// Called from set-rows.cu after InnerQ finalization to publish scale_inv
|
||||
void turbo_innerq_publish(const float * scale_inv, int group_size);
|
||||
|
||||
// Called from llama-kv-cache.cpp (or equivalent) to check if tensor needs update
|
||||
// Returns true if there are new scale_inv values to upload
|
||||
bool turbo_innerq_needs_tensor_update(void);
|
||||
|
||||
// Called after tensor update to clear the flag
|
||||
void turbo_innerq_mark_tensor_updated(void);
|
||||
@@ -0,0 +1,421 @@
|
||||
/*
|
||||
* TurboQuant CUDA kernels for KV cache compression
|
||||
* Based on: arXiv 2504.19874 (ICLR 2026)
|
||||
*
|
||||
* Implements GGML_TYPE_TURBO3_0 (3-bit PolarQuant, block size 32)
|
||||
* Constants, WHT rotation, quantize/dequantize device functions.
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "common.cuh"
|
||||
#include "turbo-innerq.cuh"
|
||||
#include <cstdlib>
|
||||
#include <cmath>
|
||||
|
||||
// ---- Quantization ratios for dequantize_block template ----
|
||||
#define QR_TURBO3 1 // Each dequantize call produces 2 consecutive elements (like q8_0)
|
||||
#define QR_TURBO2 1 // Each dequantize call produces 2 consecutive elements (like q8_0)
|
||||
#define QR_TURBO4 1 // Each dequantize call produces 2 consecutive elements (like q8_0)
|
||||
|
||||
// ---- 2-bit centroids (Lloyd-Max for N(0, 1/128)) ----
|
||||
|
||||
static __constant__ float TURBO_CENTROIDS_2BIT[4] = {
|
||||
-0.133462f, -0.039994f, 0.039994f, 0.133462f
|
||||
};
|
||||
|
||||
static __constant__ float TURBO_MID_2BIT[3] = {
|
||||
-0.086728f, 0.0f, 0.086728f
|
||||
};
|
||||
|
||||
// ---- 3-bit centroids (Lloyd-Max for N(0, 1/128)) ----
|
||||
|
||||
static __constant__ float TURBO_CENTROIDS_3BIT[8] = {
|
||||
-0.190685f, -0.117832f, -0.065717f, -0.021460f,
|
||||
0.021460f, 0.065717f, 0.117832f, 0.190685f
|
||||
};
|
||||
|
||||
// ---- Midpoints for nearest centroid lookup ----
|
||||
|
||||
static __constant__ float TURBO_MID_3BIT[7] = {
|
||||
-0.154259f, -0.091775f, -0.043589f, 0.0f,
|
||||
0.043589f, 0.091775f, 0.154259f
|
||||
};
|
||||
|
||||
// ---- WHT sign arrays (seed=42) ----
|
||||
|
||||
static __constant__ float TURBO_WHT_SIGNS1[128] = {
|
||||
-1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f,
|
||||
1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, -1.0f,
|
||||
-1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f,
|
||||
1.0f, 1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f,
|
||||
-1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, 1.0f,
|
||||
1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f,
|
||||
-1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f,
|
||||
1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f
|
||||
};
|
||||
|
||||
static __constant__ float TURBO_WHT_SIGNS2[128] = {
|
||||
1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f,
|
||||
1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f,
|
||||
1.0f, 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f,
|
||||
1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, 1.0f,
|
||||
1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f,
|
||||
-1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f,
|
||||
1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f,
|
||||
-1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f
|
||||
};
|
||||
|
||||
// ---- 64-element WHT sign arrays (first 64 of the 128-element arrays) ----
|
||||
|
||||
static __constant__ float TURBO_WHT_SIGNS1_64[64] = {
|
||||
-1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f,
|
||||
1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, -1.0f,
|
||||
-1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f,
|
||||
1.0f, 1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f
|
||||
};
|
||||
|
||||
static __constant__ float TURBO_WHT_SIGNS2_64[64] = {
|
||||
1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f,
|
||||
1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f,
|
||||
1.0f, 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f,
|
||||
1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, 1.0f
|
||||
};
|
||||
|
||||
// ---- Fast Walsh-Hadamard Transform (in-place, normalized) ----
|
||||
// O(n log n) = 896 ops for n=128
|
||||
|
||||
static __device__ __forceinline__ void turbo_fwht_128(float * x) {
|
||||
for (int h = 1; h < 128; h *= 2) {
|
||||
for (int i = 0; i < 128; i += h * 2) {
|
||||
for (int j = i; j < i + h; j++) {
|
||||
float a = x[j];
|
||||
float b = x[j + h];
|
||||
x[j] = a + b;
|
||||
x[j + h] = a - b;
|
||||
}
|
||||
}
|
||||
}
|
||||
const float inv_sqrt_128 = 0.08838834764831845f;
|
||||
for (int i = 0; i < 128; i++) {
|
||||
x[i] *= inv_sqrt_128;
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Fast Walsh-Hadamard Transform for 64-element groups ----
|
||||
// O(n log n) = 384 ops for n=64
|
||||
|
||||
static __device__ __forceinline__ void turbo_fwht_64(float * x) {
|
||||
for (int h = 1; h < 64; h *= 2) {
|
||||
for (int i = 0; i < 64; i += h * 2) {
|
||||
for (int j = i; j < i + h; j++) {
|
||||
float a = x[j];
|
||||
float b = x[j + h];
|
||||
x[j] = a + b;
|
||||
x[j + h] = a - b;
|
||||
}
|
||||
}
|
||||
}
|
||||
const float inv_sqrt_64 = 0.125f;
|
||||
for (int i = 0; i < 64; i++) {
|
||||
x[i] *= inv_sqrt_64;
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Forward rotation: signs1 → FWHT → signs2 ----
|
||||
|
||||
static __device__ __forceinline__ void turbo_rotate_forward(float * x) {
|
||||
for (int i = 0; i < 128; i++) x[i] *= TURBO_WHT_SIGNS1[i];
|
||||
turbo_fwht_128(x);
|
||||
for (int i = 0; i < 128; i++) x[i] *= TURBO_WHT_SIGNS2[i];
|
||||
}
|
||||
|
||||
// ---- Forward rotation for 64-element groups ----
|
||||
|
||||
static __device__ __forceinline__ void turbo_rotate_forward_64(float * x) {
|
||||
for (int i = 0; i < 64; i++) x[i] *= TURBO_WHT_SIGNS1_64[i];
|
||||
turbo_fwht_64(x);
|
||||
for (int i = 0; i < 64; i++) x[i] *= TURBO_WHT_SIGNS2_64[i];
|
||||
}
|
||||
|
||||
// ---- InnerQ per-channel equalization ----
|
||||
// Equalizes K channel variances before WHT rotation to reduce quantization error.
|
||||
// Enabled via TURBO_INNERQ=N env var (N = calibration token count).
|
||||
// Math: <Q/s, s*K> = <Q, K> preserves dot products.
|
||||
// INNERQ_MAX_CHANNELS is defined in turbo-innerq.cuh
|
||||
|
||||
static __device__ float d_innerq_scale[INNERQ_MAX_CHANNELS];
|
||||
static __device__ float d_innerq_scale_inv[INNERQ_MAX_CHANNELS];
|
||||
static __device__ float d_innerq_sq_accum[INNERQ_MAX_CHANNELS];
|
||||
static __device__ int d_innerq_count;
|
||||
static __device__ int d_innerq_active; // 0 = scales are identity, 1 = scales applied
|
||||
static __device__ int d_innerq_calibrating; // 1 = accumulating K² stats
|
||||
|
||||
static int innerq_enabled = 0; // host: 0=off, 1=calibrating, 2=active
|
||||
static int innerq_target_tokens = 0;
|
||||
static float innerq_strength = 0.5f;
|
||||
static bool innerq_initialized = false;
|
||||
|
||||
// Host: read TURBO_INNERQ env, start calibration if enabled
|
||||
static void turbo_innerq_init(void) {
|
||||
if (innerq_initialized) return;
|
||||
innerq_initialized = true;
|
||||
|
||||
const char * env = getenv("TURBO_INNERQ");
|
||||
if (!env || atoi(env) <= 0) {
|
||||
innerq_enabled = 0;
|
||||
return;
|
||||
}
|
||||
innerq_target_tokens = atoi(env);
|
||||
innerq_enabled = 1; // calibrating
|
||||
|
||||
const char * env_str = getenv("TURBO_INNERQ_STRENGTH");
|
||||
if (env_str) innerq_strength = atof(env_str);
|
||||
if (innerq_strength <= 0.0f || innerq_strength > 1.0f) innerq_strength = 0.5f;
|
||||
|
||||
// Zero accumulators and set calibrating flag on device
|
||||
float zeros[INNERQ_MAX_CHANNELS] = {0};
|
||||
int zero = 0, one = 1;
|
||||
cudaMemcpyToSymbol(d_innerq_sq_accum, zeros, sizeof(zeros));
|
||||
cudaMemcpyToSymbol(d_innerq_count, &zero, sizeof(int));
|
||||
cudaMemcpyToSymbol(d_innerq_active, &zero, sizeof(int));
|
||||
cudaMemcpyToSymbol(d_innerq_calibrating, &one, sizeof(int));
|
||||
|
||||
GGML_LOG_INFO("%s: InnerQ calibration started (target=%d tokens, strength=%.2f)\n",
|
||||
__func__, innerq_target_tokens, innerq_strength);
|
||||
}
|
||||
|
||||
// Host: finalize calibration — compute scales, upload, activate
|
||||
static void turbo_innerq_finalize(int group_size) {
|
||||
// Read accumulators from device
|
||||
float sq_accum[INNERQ_MAX_CHANNELS];
|
||||
int count = 0;
|
||||
cudaMemcpyFromSymbol(sq_accum, d_innerq_sq_accum, group_size * sizeof(float));
|
||||
cudaMemcpyFromSymbol(&count, d_innerq_count, sizeof(int));
|
||||
|
||||
if (count <= 0) {
|
||||
GGML_LOG_WARN("%s: InnerQ calibration got 0 tokens, disabling\n", __func__);
|
||||
innerq_enabled = 0;
|
||||
int zero = 0;
|
||||
cudaMemcpyToSymbol(d_innerq_calibrating, &zero, sizeof(int));
|
||||
return;
|
||||
}
|
||||
|
||||
// Compute per-channel RMS
|
||||
float rms[INNERQ_MAX_CHANNELS];
|
||||
float mean_rms = 0.0f;
|
||||
float max_ratio = 0.0f, min_ratio = 1e30f;
|
||||
for (int i = 0; i < group_size; i++) {
|
||||
rms[i] = sqrtf(sq_accum[i] / (float)count);
|
||||
mean_rms += rms[i];
|
||||
}
|
||||
mean_rms /= (float)group_size;
|
||||
|
||||
// Compute scale[i] = (mean_rms / channel_rms[i])^strength, clamp to [0.5, 2.0]
|
||||
float scale[INNERQ_MAX_CHANNELS];
|
||||
float scale_inv[INNERQ_MAX_CHANNELS];
|
||||
for (int i = 0; i < group_size; i++) {
|
||||
float ratio = (rms[i] > 1e-10f) ? (mean_rms / rms[i]) : 1.0f;
|
||||
float s = powf(ratio, innerq_strength);
|
||||
if (s < 0.5f) s = 0.5f;
|
||||
if (s > 2.0f) s = 2.0f;
|
||||
scale[i] = s;
|
||||
scale_inv[i] = 1.0f / s;
|
||||
if (ratio > max_ratio) max_ratio = ratio;
|
||||
if (ratio < min_ratio) min_ratio = ratio;
|
||||
}
|
||||
|
||||
// Auto-skip if max channel ratio < 1.2 (already balanced)
|
||||
if (max_ratio < 1.2f && min_ratio > (1.0f / 1.2f)) {
|
||||
GGML_LOG_INFO("%s: InnerQ auto-disabled (channels already balanced, max_ratio=%.3f)\n",
|
||||
__func__, max_ratio);
|
||||
innerq_enabled = 0;
|
||||
int zero = 0;
|
||||
cudaMemcpyToSymbol(d_innerq_calibrating, &zero, sizeof(int));
|
||||
return;
|
||||
}
|
||||
|
||||
// Stop calibrating, upload scales, activate
|
||||
int zero = 0, one = 1;
|
||||
cudaMemcpyToSymbol(d_innerq_calibrating, &zero, sizeof(int));
|
||||
cudaMemcpyToSymbol(d_innerq_scale, scale, group_size * sizeof(float));
|
||||
cudaMemcpyToSymbol(d_innerq_scale_inv, scale_inv, group_size * sizeof(float));
|
||||
cudaDeviceSynchronize(); // ensure scales are visible before activating
|
||||
cudaMemcpyToSymbol(d_innerq_active, &one, sizeof(int));
|
||||
|
||||
innerq_enabled = 2; // active
|
||||
|
||||
// Publish scale_inv to shared host state for cross-TU tensor update
|
||||
turbo_innerq_publish(scale_inv, group_size);
|
||||
|
||||
GGML_LOG_INFO("%s: InnerQ finalized (%d tokens, max_ratio=%.3f, min_ratio=%.3f)\n",
|
||||
__func__, count, max_ratio, min_ratio);
|
||||
}
|
||||
|
||||
// Host: called before each set_rows kernel launch
|
||||
static void turbo_innerq_check_finalize(int group_size, int64_t ne00) {
|
||||
if (!innerq_initialized) {
|
||||
turbo_innerq_init();
|
||||
}
|
||||
if (innerq_enabled == 0) return;
|
||||
|
||||
// InnerQ only works when each WHT group = one head (group_size == head_dim).
|
||||
// For standard models: ne00 = n_heads * head_dim, group_size = head_dim → ne00 % group_size == 0, fine.
|
||||
// For non-standard models (head_dim > group_size, e.g. GLM 576 → 64-group):
|
||||
// ne00 = head_dim (single head), group_size = 64, ne00/group_size = 9 groups per head → WRONG.
|
||||
// Detect: if ne00 / group_size doesn't divide evenly into standard head counts (1,2,4,8,16,32,64,128),
|
||||
// it's likely multi-group-per-head. Simpler check: group_size < 128 means head_dim > 128.
|
||||
const bool multi_group_per_head = (group_size < 128); // 64-group → head_dim > 128, multi-group
|
||||
if (multi_group_per_head) {
|
||||
if (innerq_enabled == 1) {
|
||||
GGML_LOG_WARN("%s: InnerQ disabled (ne00=%lld != group_size=%d, multi-group heads)\n",
|
||||
__func__, (long long)ne00, group_size);
|
||||
innerq_enabled = 0;
|
||||
int zero = 0;
|
||||
cudaMemcpyToSymbol(d_innerq_calibrating, &zero, sizeof(int));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if calibration is complete
|
||||
if (innerq_enabled == 1) {
|
||||
int count = 0;
|
||||
cudaMemcpyFromSymbol(&count, d_innerq_count, sizeof(int));
|
||||
if (count >= innerq_target_tokens) {
|
||||
turbo_innerq_finalize(group_size);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Host: check if InnerQ is currently active (finalized)
|
||||
static bool turbo_innerq_is_active(void) {
|
||||
return innerq_enabled == 2;
|
||||
}
|
||||
|
||||
// ---- 4-bit centroids (Lloyd-Max for N(0, 1/128)) ----
|
||||
|
||||
static __constant__ float TURBO_CENTROIDS_4BIT[16] = {
|
||||
-0.173926f, -0.117195f, -0.089527f, -0.068756f,
|
||||
-0.051262f, -0.035597f, -0.020989f, -0.006938f,
|
||||
0.006938f, 0.020989f, 0.035597f, 0.051262f,
|
||||
0.068756f, 0.089527f, 0.117195f, 0.173926f
|
||||
};
|
||||
|
||||
// ---- Midpoints for nearest 4-bit centroid lookup ----
|
||||
|
||||
static __constant__ float TURBO_MID_4BIT[15] = {
|
||||
-0.145561f, -0.103361f, -0.079142f, -0.060009f,
|
||||
-0.043430f, -0.028293f, -0.013964f, 0.000000f,
|
||||
0.013964f, 0.028293f, 0.043430f, 0.060009f,
|
||||
0.079142f, 0.103361f, 0.145561f
|
||||
};
|
||||
|
||||
// ---- Nearest 4-bit centroid index ----
|
||||
|
||||
static __device__ __forceinline__ uint8_t turbo_nearest_centroid_4bit(float val) {
|
||||
if (val < TURBO_MID_4BIT[ 0]) return 0;
|
||||
else if (val < TURBO_MID_4BIT[ 1]) return 1;
|
||||
else if (val < TURBO_MID_4BIT[ 2]) return 2;
|
||||
else if (val < TURBO_MID_4BIT[ 3]) return 3;
|
||||
else if (val < TURBO_MID_4BIT[ 4]) return 4;
|
||||
else if (val < TURBO_MID_4BIT[ 5]) return 5;
|
||||
else if (val < TURBO_MID_4BIT[ 6]) return 6;
|
||||
else if (val < TURBO_MID_4BIT[ 7]) return 7;
|
||||
else if (val < TURBO_MID_4BIT[ 8]) return 8;
|
||||
else if (val < TURBO_MID_4BIT[ 9]) return 9;
|
||||
else if (val < TURBO_MID_4BIT[10]) return 10;
|
||||
else if (val < TURBO_MID_4BIT[11]) return 11;
|
||||
else if (val < TURBO_MID_4BIT[12]) return 12;
|
||||
else if (val < TURBO_MID_4BIT[13]) return 13;
|
||||
else if (val < TURBO_MID_4BIT[14]) return 14;
|
||||
else return 15;
|
||||
}
|
||||
|
||||
// ---- Per-block quantize for turbo4 (128 elements, expects already-rotated input) ----
|
||||
|
||||
static __device__ void quantize_f32_turbo4_0_block(const float * __restrict__ src,
|
||||
block_turbo4_0 * __restrict__ dst) {
|
||||
for (int j = 0; j < QK_TURBO4 / 2; j++) dst->qs[j] = 0;
|
||||
|
||||
for (int j = 0; j < QK_TURBO4; j++) {
|
||||
uint8_t idx = turbo_nearest_centroid_4bit(src[j]);
|
||||
dst->qs[j / 2] |= (idx & 0xF) << ((j % 2) * 4);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Inline dequant helper: extract one float from turbo4 block ----
|
||||
|
||||
static __device__ __forceinline__ float turbo4_dequant_element(
|
||||
const block_turbo4_0 * __restrict__ x, int j, float norm) {
|
||||
uint8_t idx = (x->qs[j / 2] >> ((j % 2) * 4)) & 0xF;
|
||||
return TURBO_CENTROIDS_4BIT[idx] * norm;
|
||||
}
|
||||
|
||||
// ---- Nearest 3-bit centroid index ----
|
||||
|
||||
static __device__ __forceinline__ uint8_t turbo_nearest_centroid_3bit(float val) {
|
||||
if (val < TURBO_MID_3BIT[0]) return 0;
|
||||
else if (val < TURBO_MID_3BIT[1]) return 1;
|
||||
else if (val < TURBO_MID_3BIT[2]) return 2;
|
||||
else if (val < TURBO_MID_3BIT[3]) return 3;
|
||||
else if (val < TURBO_MID_3BIT[4]) return 4;
|
||||
else if (val < TURBO_MID_3BIT[5]) return 5;
|
||||
else if (val < TURBO_MID_3BIT[6]) return 6;
|
||||
else return 7;
|
||||
}
|
||||
|
||||
// ---- Per-block quantize (32 elements, expects already-rotated input) ----
|
||||
// Used by set_rows after group-level WHT rotation
|
||||
|
||||
static __device__ void quantize_f32_turbo3_0_block(const float * __restrict__ src,
|
||||
block_turbo3_0 * __restrict__ dst) {
|
||||
for (int j = 0; j < QK_TURBO3 / 4; j++) dst->qs[j] = 0;
|
||||
for (int j = 0; j < QK_TURBO3 / 8; j++) dst->signs[j] = 0;
|
||||
|
||||
for (int j = 0; j < QK_TURBO3; j++) {
|
||||
uint8_t idx = turbo_nearest_centroid_3bit(src[j]);
|
||||
dst->qs[j / 4] |= (idx & 0x3) << ((j % 4) * 2);
|
||||
if (idx & 0x4) {
|
||||
dst->signs[j / 8] |= (1 << (j % 8));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Inline dequant helper: extract one float from turbo3 block ----
|
||||
|
||||
static __device__ __forceinline__ float turbo3_dequant_element(
|
||||
const block_turbo3_0 * __restrict__ x, int j, float norm) {
|
||||
uint8_t low2 = (x->qs[j / 4] >> ((j % 4) * 2)) & 0x3;
|
||||
uint8_t hi1 = (x->signs[j / 8] >> (j % 8)) & 0x1;
|
||||
uint8_t idx = low2 | (hi1 << 2);
|
||||
return TURBO_CENTROIDS_3BIT[idx] * norm;
|
||||
}
|
||||
|
||||
// ---- Nearest 2-bit centroid index ----
|
||||
|
||||
static __device__ __forceinline__ uint8_t turbo_nearest_centroid_2bit(float val) {
|
||||
if (val < TURBO_MID_2BIT[0]) return 0;
|
||||
else if (val < TURBO_MID_2BIT[1]) return 1;
|
||||
else if (val < TURBO_MID_2BIT[2]) return 2;
|
||||
else return 3;
|
||||
}
|
||||
|
||||
// ---- Per-block quantize for turbo2 (32 elements, expects already-rotated input) ----
|
||||
|
||||
static __device__ void quantize_f32_turbo2_0_block(const float * __restrict__ src,
|
||||
block_turbo2_0 * __restrict__ dst) {
|
||||
for (int j = 0; j < QK_TURBO2 / 4; j++) dst->qs[j] = 0;
|
||||
|
||||
for (int j = 0; j < QK_TURBO2; j++) {
|
||||
uint8_t idx = turbo_nearest_centroid_2bit(src[j]);
|
||||
dst->qs[j / 4] |= (idx & 0x3) << ((j % 4) * 2);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Inline dequant helper: extract one float from turbo2 block ----
|
||||
|
||||
static __device__ __forceinline__ float turbo2_dequant_element(
|
||||
const block_turbo2_0 * __restrict__ x, int j, float norm) {
|
||||
uint8_t idx = (x->qs[j / 4] >> ((j % 4) * 2)) & 0x3;
|
||||
return TURBO_CENTROIDS_2BIT[idx] * norm;
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
#include "turbo-quant.cuh"
|
||||
#include "turbo-wht.cuh"
|
||||
|
||||
// ─── CUDA kernel ──────────────────────────────────────────────────────────────
|
||||
//
|
||||
// Templated on direction and group_size (128 or 64).
|
||||
// One block per group, group_size threads per block.
|
||||
// direction: 0 = forward (signs1 → WHT → signs2), 1 = inverse (signs2 → WHT → signs1)
|
||||
//
|
||||
// When head_dim is not a multiple of group_size, only the full groups
|
||||
// within each head are processed. Tail elements are left unchanged (identity).
|
||||
//
|
||||
// Algorithm mirrors the CPU implementation in ggml-cpu/ops.cpp:
|
||||
// 1. Apply s_first elementwise
|
||||
// 2. Radix-2 Hadamard butterfly (log2(group_size) stages, in-place)
|
||||
// 3. Normalize by 1/sqrt(group_size) and apply s_second elementwise
|
||||
//
|
||||
// InnerQ scale_inv: when non-null, applies per-channel inverse scaling for
|
||||
// Q/V equalization. For forward (Q rotation): multiply BEFORE signs+WHT.
|
||||
// For inverse (V un-rotation): multiply AFTER WHT+signs.
|
||||
|
||||
template <int direction, int group_size>
|
||||
static __global__ void k_turbo_wht_f32(const float * __restrict__ src,
|
||||
float * __restrict__ dst,
|
||||
const float * __restrict__ scale_inv,
|
||||
int64_t n_groups,
|
||||
int64_t head_dim,
|
||||
int64_t groups_per_head) {
|
||||
static_assert(group_size == 128 || group_size == 64, "group_size must be 128 or 64");
|
||||
|
||||
const int64_t g = blockIdx.x;
|
||||
if (g >= n_groups) return;
|
||||
|
||||
const int t = threadIdx.x; // 0 .. group_size-1
|
||||
|
||||
// Map group index to position in the tensor:
|
||||
// each head has groups_per_head full groups, then a gap of tail elements.
|
||||
const int64_t head_idx = g / groups_per_head;
|
||||
const int64_t grp_in_head = g % groups_per_head;
|
||||
const int64_t base = head_idx * head_dim + grp_in_head * group_size;
|
||||
|
||||
__shared__ float x[group_size];
|
||||
|
||||
// Load from global memory
|
||||
x[t] = src[base + t];
|
||||
__syncthreads();
|
||||
|
||||
// InnerQ forward: apply scale_inv BEFORE signs+WHT (for Q pre-rotation)
|
||||
if (direction == 0 && scale_inv != nullptr) {
|
||||
x[t] *= scale_inv[t % group_size];
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
// Apply first sign array
|
||||
if (group_size == 128) {
|
||||
x[t] *= (direction == 0) ? TURBO_WHT_SIGNS1[t] : TURBO_WHT_SIGNS2[t];
|
||||
} else {
|
||||
x[t] *= (direction == 0) ? TURBO_WHT_SIGNS1_64[t] : TURBO_WHT_SIGNS2_64[t];
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
// WHT butterfly — log2(group_size) stages.
|
||||
// In stage h, threads where (t % (2h)) < h read x[t] and x[t+h],
|
||||
// then write x[t] = a+b and x[t+h] = a-b. Each active thread
|
||||
// owns a disjoint pair, so no intra-stage conflicts exist.
|
||||
#define WHT_STAGE(h) \
|
||||
if (t % (2*(h)) < (h)) { float a = x[t], b = x[t+(h)]; x[t] = a+b; x[t+(h)] = a-b; } \
|
||||
__syncthreads();
|
||||
|
||||
WHT_STAGE(1)
|
||||
WHT_STAGE(2)
|
||||
WHT_STAGE(4)
|
||||
WHT_STAGE(8)
|
||||
WHT_STAGE(16)
|
||||
WHT_STAGE(32)
|
||||
if (group_size == 128) { WHT_STAGE(64) }
|
||||
#undef WHT_STAGE
|
||||
|
||||
// Normalize and apply second sign array, write to output
|
||||
constexpr float inv_sqrt = (group_size == 128) ? 0.08838834764831845f : 0.125f;
|
||||
float result;
|
||||
if (group_size == 128) {
|
||||
result = x[t] * inv_sqrt *
|
||||
((direction == 0) ? TURBO_WHT_SIGNS2[t] : TURBO_WHT_SIGNS1[t]);
|
||||
} else {
|
||||
result = x[t] * inv_sqrt *
|
||||
((direction == 0) ? TURBO_WHT_SIGNS2_64[t] : TURBO_WHT_SIGNS1_64[t]);
|
||||
}
|
||||
|
||||
// InnerQ inverse: apply scale_inv AFTER WHT+signs (for V un-rotation)
|
||||
if (direction == 1 && scale_inv != nullptr) {
|
||||
result *= scale_inv[t % group_size];
|
||||
}
|
||||
|
||||
dst[base + t] = result;
|
||||
}
|
||||
|
||||
// ─── Simple copy kernel for tail elements (identity pass-through) ────────────
|
||||
|
||||
static __global__ void k_turbo_wht_copy_tail(const float * __restrict__ src,
|
||||
float * __restrict__ dst,
|
||||
int64_t n_heads,
|
||||
int64_t head_dim,
|
||||
int64_t tail_offset,
|
||||
int tail_size) {
|
||||
const int64_t i = (int64_t)blockIdx.x * blockDim.x + threadIdx.x;
|
||||
if (i >= n_heads * tail_size) return;
|
||||
|
||||
const int64_t head_idx = i / tail_size;
|
||||
const int64_t tail_elem = i % tail_size;
|
||||
const int64_t offset = head_idx * head_dim + tail_offset + tail_elem;
|
||||
dst[offset] = src[offset];
|
||||
}
|
||||
|
||||
// ─── Dispatch ─────────────────────────────────────────────────────────────────
|
||||
|
||||
void ggml_cuda_turbo_wht(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
|
||||
const ggml_tensor * src = dst->src[0];
|
||||
const ggml_tensor * scale_tensor = dst->src[1]; // InnerQ scale_inv (may be NULL)
|
||||
|
||||
GGML_ASSERT(src->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(ggml_is_contiguous(src));
|
||||
GGML_ASSERT(ggml_is_contiguous(dst));
|
||||
|
||||
int direction;
|
||||
int group_size;
|
||||
memcpy(&direction, dst->op_params + 0, sizeof(int));
|
||||
memcpy(&group_size, dst->op_params + sizeof(int), sizeof(int));
|
||||
|
||||
const int64_t head_dim = src->ne[0];
|
||||
const int64_t n_heads = ggml_nelements(src) / head_dim;
|
||||
|
||||
GGML_ASSERT(group_size == 64 || group_size == 128);
|
||||
const int64_t groups_per_head = head_dim / group_size;
|
||||
const int tail_size = (int)(head_dim % group_size);
|
||||
const int64_t n_groups = groups_per_head * n_heads;
|
||||
|
||||
const float * src_ptr = (const float *) src->data;
|
||||
float * dst_ptr = (float *) dst->data;
|
||||
const float * scale_inv_ptr = scale_tensor ? (const float *) scale_tensor->data : nullptr;
|
||||
|
||||
cudaStream_t stream = ctx.stream();
|
||||
|
||||
// Process full groups
|
||||
if (n_groups > 0) {
|
||||
dim3 blocks(n_groups);
|
||||
if (group_size == 128) {
|
||||
dim3 threads(128);
|
||||
if (direction == 0) {
|
||||
k_turbo_wht_f32<0, 128><<<blocks, threads, 0, stream>>>(src_ptr, dst_ptr, scale_inv_ptr, n_groups, head_dim, groups_per_head);
|
||||
} else {
|
||||
k_turbo_wht_f32<1, 128><<<blocks, threads, 0, stream>>>(src_ptr, dst_ptr, scale_inv_ptr, n_groups, head_dim, groups_per_head);
|
||||
}
|
||||
} else {
|
||||
dim3 threads(64);
|
||||
if (direction == 0) {
|
||||
k_turbo_wht_f32<0, 64><<<blocks, threads, 0, stream>>>(src_ptr, dst_ptr, scale_inv_ptr, n_groups, head_dim, groups_per_head);
|
||||
} else {
|
||||
k_turbo_wht_f32<1, 64><<<blocks, threads, 0, stream>>>(src_ptr, dst_ptr, scale_inv_ptr, n_groups, head_dim, groups_per_head);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Pass through tail elements unchanged (no rotation)
|
||||
// Not needed for 64-aligned dims but kept for completeness
|
||||
if (tail_size > 0) {
|
||||
const int64_t total_tail = n_heads * tail_size;
|
||||
const int block_sz = 256;
|
||||
const int n_blocks = (int)((total_tail + block_sz - 1) / block_sz);
|
||||
k_turbo_wht_copy_tail<<<n_blocks, block_sz, 0, stream>>>(
|
||||
src_ptr, dst_ptr, n_heads, head_dim, groups_per_head * group_size, tail_size);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
#pragma once
|
||||
|
||||
#include "common.cuh"
|
||||
|
||||
void ggml_cuda_turbo_wht(ggml_backend_cuda_context & ctx, ggml_tensor * dst);
|
||||
Vendored
+28
-3
@@ -33,11 +33,32 @@
|
||||
#define CU_MEM_LOCATION_TYPE_DEVICE hipMemLocationTypeDevice
|
||||
#define CU_MEM_ACCESS_FLAGS_PROT_READWRITE hipMemAccessFlagsProtReadWrite
|
||||
#define CU_CHECK(fn) {hipError_t err = fn; if(err != hipSuccess) { GGML_ABORT("HipVMM Failure: %s\n", hipGetErrorString(err)); }}
|
||||
#define __shfl_sync(mask, var, laneMask, width) __shfl(var, laneMask, width)
|
||||
#define __shfl_up_sync(mask, var, laneMask, width) __shfl_up(var, laneMask, width)
|
||||
#define __shfl_xor_sync(mask, var, laneMask, width) __shfl_xor(var, laneMask, width)
|
||||
// __shfl_sync: support both 3-arg (mask, var, srcLane) and 4-arg (mask, var, srcLane, width) calls
|
||||
// HIP ignores the mask but requires it to be 64-bit, so we cast explicitly.
|
||||
#define __SHFL_SYNC_3(mask, var, srcLane) __shfl(var, srcLane, warpSize)
|
||||
#define __SHFL_SYNC_4(mask, var, srcLane, width) __shfl(var, srcLane, width)
|
||||
#define __SHFL_GET_MACRO(_1, _2, _3, _4, NAME, ...) NAME
|
||||
#define __shfl_sync(...) __SHFL_GET_MACRO(__VA_ARGS__, __SHFL_SYNC_4, __SHFL_SYNC_3)(__VA_ARGS__)
|
||||
// __shfl_up_sync: support 3-arg and 4-arg calls (HIP ignores mask)
|
||||
#define __SHFL_UP_SYNC_3(mask, var, delta) __shfl_up(var, delta, warpSize)
|
||||
#define __SHFL_UP_SYNC_4(mask, var, delta, width) __shfl_up(var, delta, width)
|
||||
#define __SHFL_UP_GET(_1, _2, _3, _4, NAME, ...) NAME
|
||||
#define __shfl_up_sync(...) __SHFL_UP_GET(__VA_ARGS__, __SHFL_UP_SYNC_4, __SHFL_UP_SYNC_3)(__VA_ARGS__)
|
||||
|
||||
// __shfl_xor_sync: support 3-arg and 4-arg calls (HIP ignores mask)
|
||||
#define __SHFL_XOR_SYNC_3(mask, var, laneMask) __shfl_xor(var, laneMask, warpSize)
|
||||
#define __SHFL_XOR_SYNC_4(mask, var, laneMask, width) __shfl_xor(var, laneMask, width)
|
||||
#define __SHFL_XOR_GET(_1, _2, _3, _4, NAME, ...) NAME
|
||||
#define __shfl_xor_sync(...) __SHFL_XOR_GET(__VA_ARGS__, __SHFL_XOR_SYNC_4, __SHFL_XOR_SYNC_3)(__VA_ARGS__)
|
||||
|
||||
// __shfl_down_sync: support 3-arg and 4-arg calls (HIP ignores mask)
|
||||
#define __SHFL_DOWN_SYNC_3(mask, var, delta) __shfl_down(var, delta, warpSize)
|
||||
#define __SHFL_DOWN_SYNC_4(mask, var, delta, width) __shfl_down(var, delta, width)
|
||||
#define __SHFL_DOWN_GET(_1, _2, _3, _4, NAME, ...) NAME
|
||||
#define __shfl_down_sync(...) __SHFL_DOWN_GET(__VA_ARGS__, __SHFL_DOWN_SYNC_4, __SHFL_DOWN_SYNC_3)(__VA_ARGS__)
|
||||
#define __all_sync(mask, var) __all(var)
|
||||
#define __any_sync(mask, var) __any(var)
|
||||
#define __ballot_sync(mask, var) ((uint32_t)__ballot(var))
|
||||
#define cublasStrsmBatched hipblasStrsmBatched
|
||||
#define cublasCreate hipblasCreate
|
||||
#define cublasDestroy hipblasDestroy
|
||||
@@ -121,6 +142,10 @@
|
||||
#define cudaStreamPerThread hipStreamPerThread
|
||||
#define cudaStreamSynchronize hipStreamSynchronize
|
||||
#define cudaStreamWaitEvent hipStreamWaitEvent
|
||||
#define cudaMemcpyToSymbol hipMemcpyToSymbol
|
||||
#define cudaMemcpyFromSymbol hipMemcpyFromSymbol
|
||||
#define cudaMemcpyHostToDevice hipMemcpyHostToDevice
|
||||
#define cudaMemcpyDeviceToHost hipMemcpyDeviceToHost
|
||||
#define cudaGraphExec_t hipGraphExec_t
|
||||
#define cudaGraphNode_t hipGraphNode_t
|
||||
#define cudaKernelNodeParams hipKernelNodeParams
|
||||
|
||||
@@ -62,6 +62,8 @@ list(APPEND GGML_HEADERS_ROCM "../../include/ggml-cuda.h")
|
||||
|
||||
file(GLOB GGML_SOURCES_ROCM "../ggml-cuda/*.cu")
|
||||
file(GLOB SRCS "../ggml-cuda/template-instances/fattn-tile*.cu")
|
||||
# Exclude D>=576 tile kernels: exceed HIP local memory limit (67584 > 65536)
|
||||
list(FILTER SRCS EXCLUDE REGEX "dkq(576|640)")
|
||||
list(APPEND GGML_SOURCES_ROCM ${SRCS})
|
||||
file(GLOB SRCS "../ggml-cuda/template-instances/fattn-mma*.cu")
|
||||
list(APPEND GGML_SOURCES_ROCM ${SRCS})
|
||||
@@ -79,7 +81,22 @@ else()
|
||||
../ggml-cuda/template-instances/fattn-vec-instance-f16-f16.cu
|
||||
../ggml-cuda/template-instances/fattn-vec-instance-q4_0-q4_0.cu
|
||||
../ggml-cuda/template-instances/fattn-vec-instance-q8_0-q8_0.cu
|
||||
../ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu)
|
||||
../ggml-cuda/template-instances/fattn-vec-instance-bf16-bf16.cu
|
||||
../ggml-cuda/template-instances/fattn-vec-instance-turbo3_0-turbo3_0.cu
|
||||
../ggml-cuda/template-instances/fattn-vec-instance-turbo3_0-q8_0.cu
|
||||
../ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo3_0.cu
|
||||
../ggml-cuda/template-instances/fattn-vec-instance-turbo2_0-turbo2_0.cu
|
||||
../ggml-cuda/template-instances/fattn-vec-instance-turbo2_0-q8_0.cu
|
||||
../ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo2_0.cu
|
||||
../ggml-cuda/template-instances/fattn-vec-instance-turbo3_0-turbo2_0.cu
|
||||
../ggml-cuda/template-instances/fattn-vec-instance-turbo2_0-turbo3_0.cu
|
||||
../ggml-cuda/template-instances/fattn-vec-instance-turbo4_0-turbo4_0.cu
|
||||
../ggml-cuda/template-instances/fattn-vec-instance-turbo4_0-q8_0.cu
|
||||
../ggml-cuda/template-instances/fattn-vec-instance-q8_0-turbo4_0.cu
|
||||
../ggml-cuda/template-instances/fattn-vec-instance-turbo4_0-turbo3_0.cu
|
||||
../ggml-cuda/template-instances/fattn-vec-instance-turbo3_0-turbo4_0.cu
|
||||
../ggml-cuda/template-instances/fattn-vec-instance-turbo4_0-turbo2_0.cu
|
||||
../ggml-cuda/template-instances/fattn-vec-instance-turbo2_0-turbo4_0.cu)
|
||||
endif()
|
||||
|
||||
ggml_add_backend_library(ggml-hip
|
||||
|
||||
@@ -650,6 +650,23 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_solve_tri(ggml_m
|
||||
return res;
|
||||
}
|
||||
|
||||
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_turbo_wht(ggml_metal_library_t lib) {
|
||||
const char * name = "kernel_turbo_wht";
|
||||
|
||||
ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
|
||||
if (!res.pipeline) {
|
||||
// No function constants needed — compile with empty cv
|
||||
ggml_metal_cv_t cv = ggml_metal_cv_init();
|
||||
res = ggml_metal_library_compile_pipeline(lib, name, name, cv);
|
||||
ggml_metal_cv_free(cv);
|
||||
}
|
||||
|
||||
res.nsg = 1;
|
||||
res.smem = 0;
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_ext(ggml_metal_library_t lib, const ggml_tensor * op, int nsg, int nxpsg, int r1ptg) {
|
||||
char base[256];
|
||||
char name[256];
|
||||
@@ -1387,12 +1404,17 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext(
|
||||
// do bounds checks for the mask?
|
||||
const bool bc_mask = op->src[3] && (op->src[3]->ne[1] % 8 != 0);
|
||||
|
||||
snprintf(base, 256, "kernel_%s_%s_dk%d_dv%d",
|
||||
// Asymmetric K/V: always encode both K and V types in the pipeline name.
|
||||
// Symmetric case: ktype == vtype, so the name just has the type twice.
|
||||
// This avoids ambiguity if a type name contains underscores (e.g. q4_0).
|
||||
snprintf(base, 256, "kernel_%s_k%s_v%s_dk%d_dv%d",
|
||||
"flash_attn_ext",
|
||||
ggml_type_name(op->src[1]->type),
|
||||
ggml_type_name(op->src[2]->type),
|
||||
dk,
|
||||
dv);
|
||||
|
||||
|
||||
snprintf(name, 256, "%s_mask=%d_sinks=%d_bias=%d_scap=%d_kvpad=%d_bcm=%d_ns10=%d_ns20=%d_nsg=%d",
|
||||
base,
|
||||
has_mask,
|
||||
@@ -1450,12 +1472,16 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_flash_attn_ext_v
|
||||
const int32_t ns10 = op->src[1]->nb[1]/op->src[1]->nb[0];
|
||||
const int32_t ns20 = op->src[2]->nb[1]/op->src[2]->nb[0];
|
||||
|
||||
snprintf(base, 256, "kernel_%s_%s_dk%d_dv%d",
|
||||
// Asymmetric K/V: always encode both K and V types in the pipeline name.
|
||||
// Uses k/v prefix to avoid ambiguity with type names containing underscores.
|
||||
snprintf(base, 256, "kernel_%s_k%s_v%s_dk%d_dv%d",
|
||||
"flash_attn_ext_vec",
|
||||
ggml_type_name(op->src[1]->type),
|
||||
ggml_type_name(op->src[2]->type),
|
||||
dk,
|
||||
dv);
|
||||
|
||||
|
||||
snprintf(name, 256, "%s_mask=%d_sink=%d_bias=%d_scap=%d_kvpad=%d_ns10=%d_ns20=%d_nsg=%d_nwg=%d",
|
||||
base,
|
||||
has_mask,
|
||||
|
||||
@@ -129,6 +129,7 @@ struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_ssm_scan
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_rwkv (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_gated_delta_net (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_solve_tri (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_turbo_wht (ggml_metal_library_t lib);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv_ext (ggml_metal_library_t lib, const struct ggml_tensor * op, int nsg, int nxpsg, int r1ptg);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
struct ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mv (ggml_metal_library_t lib, const struct ggml_tensor * op);
|
||||
|
||||
@@ -225,6 +225,43 @@ ggml_metal_library_t ggml_metal_library_init(ggml_metal_device_t dev) {
|
||||
[prep setObject:@"1" forKey:@"GGML_METAL_EMBED_LIBRARY"];
|
||||
#endif
|
||||
|
||||
// TurboQuant: auto-select dequant path based on hardware
|
||||
// M1/M2/M3/M4 (no tensor API): 4-mag LUT (+38-45% decode at long ctx)
|
||||
// M5+ (has tensor API): 8-entry full LUT (best decode speed)
|
||||
{
|
||||
const char * force_4mag = getenv("TURBO_FORCE_4MAG");
|
||||
// Always compile with 4-mag support. The dispatch code selects
|
||||
// 4-mag vs 8-LUT based on context depth at runtime.
|
||||
// Pre-M5: always 4-mag (constant cache too slow)
|
||||
// M5+: 4-mag for mid-context (8K-20K), 8-LUT otherwise
|
||||
if (!ggml_metal_device_get_props(dev)->has_tensor || (force_4mag && force_4mag[0] == '1')) {
|
||||
[prep setObject:@"1" forKey:@"TURBO_USE_4MAG"];
|
||||
GGML_LOG_INFO("%s: turbo3 using 4-mag LUT%s\n", __func__,
|
||||
force_4mag ? " (forced)" : " (pre-M5 hardware)");
|
||||
}
|
||||
// Sparse V dequant: skip V for negligible attention weights
|
||||
// Enabled by default on all Metal (validated: PPL identical, NIAH 9/9, 30+ testers)
|
||||
// Opt-out via TURBO_SPARSE_V=0
|
||||
const char * sparse_v_env = getenv("TURBO_SPARSE_V");
|
||||
const bool sparse_v_disabled = sparse_v_env && sparse_v_env[0] == '0';
|
||||
if (!sparse_v_disabled) {
|
||||
[prep setObject:@"1" forKey:@"TURBO_SPARSE_V"];
|
||||
GGML_LOG_INFO("%s: turbo3 sparse V dequant enabled (opt-out: TURBO_SPARSE_V=0)\n", __func__);
|
||||
}
|
||||
// TODO: context-adaptive dispatch — compile both 4-mag and 8-LUT
|
||||
// FA kernel instantiations, select based on ne11 (KV cache size)
|
||||
// at dispatch time in ggml_metal_op_flash_attn_ext()
|
||||
}
|
||||
|
||||
// TurboQuant profiling: set TURBO_PROFILE_MODE env var (0-4)
|
||||
{
|
||||
const char * pm = getenv("TURBO_PROFILE_MODE");
|
||||
if (pm && pm[0] >= '0' && pm[0] <= '4') {
|
||||
[prep setObject:[NSString stringWithUTF8String:pm] forKey:@"TURBO_PROFILE_MODE"];
|
||||
GGML_LOG_INFO("%s: TURBO_PROFILE_MODE=%s\n", __func__, pm);
|
||||
}
|
||||
}
|
||||
|
||||
MTLCompileOptions * options = [MTLCompileOptions new];
|
||||
options.preprocessorMacros = prep;
|
||||
|
||||
@@ -1164,8 +1201,24 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
|
||||
return false;
|
||||
}
|
||||
if (op->src[1]->type != op->src[2]->type) {
|
||||
// Allow asymmetric K/V for supported mixed pairs:
|
||||
// - turbo x turbo (any combination)
|
||||
// - q8_0 x turbo (either direction)
|
||||
const bool k_is_turbo = (op->src[1]->type == GGML_TYPE_TURBO2_0 ||
|
||||
op->src[1]->type == GGML_TYPE_TURBO3_0 ||
|
||||
op->src[1]->type == GGML_TYPE_TURBO4_0);
|
||||
const bool v_is_turbo = (op->src[2]->type == GGML_TYPE_TURBO2_0 ||
|
||||
op->src[2]->type == GGML_TYPE_TURBO3_0 ||
|
||||
op->src[2]->type == GGML_TYPE_TURBO4_0);
|
||||
const bool k_is_q8 = (op->src[1]->type == GGML_TYPE_Q8_0);
|
||||
const bool v_is_q8 = (op->src[2]->type == GGML_TYPE_Q8_0);
|
||||
const bool supported = (k_is_turbo && v_is_turbo) ||
|
||||
(k_is_q8 && v_is_turbo) ||
|
||||
(k_is_turbo && v_is_q8);
|
||||
if (!supported) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
switch (op->src[1]->type) {
|
||||
case GGML_TYPE_F32:
|
||||
case GGML_TYPE_F16:
|
||||
@@ -1192,6 +1245,8 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
|
||||
return true;
|
||||
case GGML_OP_GATED_DELTA_NET:
|
||||
return has_simdgroup_reduction && op->src[2]->ne[0] % 32 == 0;
|
||||
case GGML_OP_TURBO_WHT:
|
||||
return op->src[0]->ne[0] % 128 == 0;
|
||||
case GGML_OP_SOLVE_TRI:
|
||||
case GGML_OP_MUL_MAT:
|
||||
case GGML_OP_MUL_MAT_ID:
|
||||
@@ -1272,6 +1327,9 @@ bool ggml_metal_device_supports_op(ggml_metal_device_t dev, const struct ggml_te
|
||||
case GGML_TYPE_Q5_0:
|
||||
case GGML_TYPE_Q5_1:
|
||||
case GGML_TYPE_IQ4_NL:
|
||||
case GGML_TYPE_TURBO2_0:
|
||||
case GGML_TYPE_TURBO3_0:
|
||||
case GGML_TYPE_TURBO4_0:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
|
||||
@@ -101,6 +101,7 @@
|
||||
#define FC_SUM_ROWS 1400
|
||||
#define FC_UPSCALE 1500
|
||||
#define FC_GATED_DELTA_NET 1600
|
||||
#define FC_TURBO_WHT 1700
|
||||
|
||||
// op-specific constants
|
||||
#define OP_FLASH_ATTN_EXT_NQPSG 8
|
||||
@@ -889,6 +890,11 @@ typedef struct {
|
||||
uint64_t nb3;
|
||||
} ggml_metal_kargs_gated_delta_net;
|
||||
|
||||
typedef struct {
|
||||
int64_t n_elements; // total elements in tensor
|
||||
int32_t direction; // 0 = forward, 1 = inverse
|
||||
} ggml_metal_kargs_turbo_wht;
|
||||
|
||||
typedef struct {
|
||||
int32_t ne00;
|
||||
int32_t ne01;
|
||||
|
||||
@@ -337,6 +337,10 @@ static int ggml_metal_op_encode_impl(ggml_metal_op_t ctx, int idx) {
|
||||
{
|
||||
n_fuse = ggml_metal_op_gated_delta_net(ctx, idx);
|
||||
} break;
|
||||
case GGML_OP_TURBO_WHT:
|
||||
{
|
||||
n_fuse = ggml_metal_op_turbo_wht(ctx, idx);
|
||||
} break;
|
||||
case GGML_OP_SOLVE_TRI:
|
||||
{
|
||||
n_fuse = ggml_metal_op_solve_tri(ctx, idx);
|
||||
@@ -1656,6 +1660,39 @@ int ggml_metal_op_gated_delta_net(ggml_metal_op_t ctx, int idx) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
int ggml_metal_op_turbo_wht(ggml_metal_op_t ctx, int idx) {
|
||||
ggml_tensor * op = ctx->node(idx);
|
||||
|
||||
ggml_metal_library_t lib = ctx->lib;
|
||||
ggml_metal_encoder_t enc = ctx->enc;
|
||||
|
||||
int direction;
|
||||
memcpy(&direction, op->op_params, sizeof(int));
|
||||
|
||||
const int64_t n_elements = ggml_nelements(op->src[0]);
|
||||
const int64_t n_groups = n_elements / 128;
|
||||
|
||||
auto pipeline = ggml_metal_library_get_pipeline_turbo_wht(lib);
|
||||
|
||||
ggml_metal_kargs_turbo_wht args = {
|
||||
/*.n_elements =*/ n_elements,
|
||||
/*.direction =*/ direction,
|
||||
};
|
||||
|
||||
int ida = 0;
|
||||
ggml_metal_encoder_set_pipeline(enc, pipeline);
|
||||
ggml_metal_encoder_set_bytes (enc, &args, sizeof(args), ida++);
|
||||
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op->src[0]), ida++);
|
||||
ggml_metal_encoder_set_buffer (enc, ggml_metal_get_buffer_id(op), ida++);
|
||||
|
||||
// One thread per 128-element group, 256 threads per threadgroup
|
||||
const int threads_per_tg = 256;
|
||||
const int n_threadgroups = (n_groups + threads_per_tg - 1) / threads_per_tg;
|
||||
ggml_metal_encoder_dispatch_threadgroups(enc, n_threadgroups, 1, 1, threads_per_tg, 1, 1);
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
int ggml_metal_op_solve_tri(ggml_metal_op_t ctx, int idx) {
|
||||
ggml_tensor * op = ctx->node(idx);
|
||||
|
||||
@@ -2511,6 +2548,17 @@ bool ggml_metal_op_flash_attn_ext_use_vec(const ggml_tensor * op) {
|
||||
const int64_t ne01 = op->src[0]->ne[1]; // batch size
|
||||
|
||||
// use vec kernel if the batch size is small and if the head size is supported
|
||||
// EXPERIMENT: force non-vec for turbo3 on pre-M5 hardware.
|
||||
// The vec kernel uses nl=8 (4 elements per dequant call) which has 4x more
|
||||
// loop iterations than the non-vec nl=2 path. On M2 Pro, this loop overhead
|
||||
// dominates — the non-vec path may be faster even for batch=1.
|
||||
const ggml_type ktype = op->src[1]->type;
|
||||
if (ktype == GGML_TYPE_TURBO2_0 || ktype == GGML_TYPE_TURBO3_0 || ktype == GGML_TYPE_TURBO4_0) {
|
||||
const char * force_nonvec = getenv("TURBO_FORCE_NONVEC");
|
||||
if (force_nonvec && force_nonvec[0] == '1') {
|
||||
return false; // force non-vec path
|
||||
}
|
||||
}
|
||||
return (ne01 < 20) && (ne00 % 32 == 0);
|
||||
}
|
||||
|
||||
@@ -2650,7 +2698,22 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
|
||||
GGML_ASSERT(ne00 % 4 == 0);
|
||||
|
||||
GGML_ASSERT(op->src[0]->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(op->src[1]->type == op->src[2]->type);
|
||||
|
||||
// Allow asymmetric K/V quantization for supported mixed pairs
|
||||
{
|
||||
const ggml_type type_k = op->src[1]->type;
|
||||
const ggml_type type_v = op->src[2]->type;
|
||||
if (type_k != type_v) {
|
||||
const bool k_is_turbo = (type_k == GGML_TYPE_TURBO2_0 || type_k == GGML_TYPE_TURBO3_0 || type_k == GGML_TYPE_TURBO4_0);
|
||||
const bool v_is_turbo = (type_v == GGML_TYPE_TURBO2_0 || type_v == GGML_TYPE_TURBO3_0 || type_v == GGML_TYPE_TURBO4_0);
|
||||
const bool k_is_q8 = (type_k == GGML_TYPE_Q8_0);
|
||||
const bool v_is_q8 = (type_v == GGML_TYPE_Q8_0);
|
||||
const bool supported = (k_is_turbo && v_is_turbo) ||
|
||||
(k_is_q8 && v_is_turbo) ||
|
||||
(k_is_turbo && v_is_q8);
|
||||
GGML_ASSERT(supported && "asymmetric K/V types only supported for turbo and q8_0 mixed pairs");
|
||||
}
|
||||
}
|
||||
|
||||
//GGML_ASSERT(ggml_are_same_shape (src1, src2));
|
||||
GGML_ASSERT(ne11 == ne21);
|
||||
@@ -2935,7 +2998,8 @@ int ggml_metal_op_flash_attn_ext(ggml_metal_op_t ctx, int idx) {
|
||||
// ne20*(nsg)
|
||||
// each simdgroup has a full f32 head vector in shared mem to accumulate results
|
||||
//
|
||||
#define FATTN_SMEM(nsg) (GGML_PAD(((GGML_PAD(ne00, 128) + 4*ncpsg + 2*GGML_PAD(ne20, 128))*(nsg))*(sizeof(float)/2), 16))
|
||||
// Extra 128 floats (512 bytes) for TurboQuant pre-dequantized block cache in threadgroup memory
|
||||
#define FATTN_SMEM(nsg) (GGML_PAD(((GGML_PAD(ne00, 128) + 4*ncpsg + 2*GGML_PAD(ne20, 128))*(nsg))*(sizeof(float)/2) + 128*sizeof(float), 16))
|
||||
|
||||
int64_t nsg = 1;
|
||||
|
||||
|
||||
@@ -59,6 +59,7 @@ int ggml_metal_op_ssm_conv (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_ssm_scan (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_rwkv (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_gated_delta_net (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_turbo_wht (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_solve_tri (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_set (ggml_metal_op_t ctx, int idx);
|
||||
int ggml_metal_op_cpy (ggml_metal_op_t ctx, int idx);
|
||||
|
||||
+1502
-200
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,49 @@
|
||||
// TurboQuant Fast Walsh-Hadamard rotation for Metal
|
||||
// Replaces 256KB dense matrices with 512 bytes of sign arrays + O(d log d) butterfly
|
||||
// Generated with seed=42 (rotation) and seed=1042 (QJL)
|
||||
|
||||
// --- Rotation sign arrays ---
|
||||
constant float turbo_wht_signs1[128] = {
|
||||
-1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f};
|
||||
constant float turbo_wht_signs2[128] = {
|
||||
1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f};
|
||||
|
||||
// --- QJL sign arrays ---
|
||||
constant float turbo_qjl_wht_signs1[128] = {
|
||||
1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f};
|
||||
constant float turbo_qjl_wht_signs2[128] = {
|
||||
1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, -1.0f, 1.0f, -1.0f, 1.0f, -1.0f, 1.0f, 1.0f, 1.0f, 1.0f, -1.0f};
|
||||
|
||||
// --- Fast Walsh-Hadamard Transform (in-place, normalized) ---
|
||||
// O(n log n) = 896 operations for n=128, vs O(n²) = 16384 for dense matvec
|
||||
static void turbo_fwht_128(thread float * x) {
|
||||
for (int h = 1; h < 128; h *= 2) {
|
||||
for (int i = 0; i < 128; i += h * 2) {
|
||||
for (int j = i; j < i + h; j++) {
|
||||
float a = x[j];
|
||||
float b = x[j + h];
|
||||
x[j] = a + b;
|
||||
x[j + h] = a - b;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Normalize by 1/sqrt(128)
|
||||
const float inv_sqrt_128 = 0.08838834764831845f; // 1/sqrt(128)
|
||||
for (int i = 0; i < 128; i++) {
|
||||
x[i] *= inv_sqrt_128;
|
||||
}
|
||||
}
|
||||
|
||||
// --- Forward rotation: signs1 → FWHT → signs2 ---
|
||||
static void turbo_rotate_forward(thread float * x, constant float * s1, constant float * s2) {
|
||||
for (int i = 0; i < 128; i++) x[i] *= s1[i];
|
||||
turbo_fwht_128(x);
|
||||
for (int i = 0; i < 128; i++) x[i] *= s2[i];
|
||||
}
|
||||
|
||||
// --- Inverse rotation: signs2 → FWHT → signs1 (FWHT is its own inverse) ---
|
||||
static void turbo_rotate_inverse(thread float * x, constant float * s1, constant float * s2) {
|
||||
for (int i = 0; i < 128; i++) x[i] *= s2[i];
|
||||
turbo_fwht_128(x);
|
||||
for (int i = 0; i < 128; i++) x[i] *= s1[i];
|
||||
}
|
||||
@@ -102,6 +102,17 @@ GGML_API size_t quantize_q8_0(const float * GGML_RESTRICT src, void * GGML_RESTR
|
||||
GGML_API size_t quantize_mxfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
||||
GGML_API size_t quantize_nvfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
||||
|
||||
// TurboQuant KV cache compression (arXiv 2504.19874)
|
||||
GGML_API void quantize_row_turbo3_0_ref(const float * GGML_RESTRICT x, block_turbo3_0 * GGML_RESTRICT y, int64_t k);
|
||||
GGML_API void quantize_row_turbo4_0_ref(const float * GGML_RESTRICT x, block_turbo4_0 * GGML_RESTRICT y, int64_t k);
|
||||
GGML_API void dequantize_row_turbo3_0(const block_turbo3_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
||||
GGML_API void dequantize_row_turbo4_0(const block_turbo4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
||||
GGML_API size_t quantize_turbo3_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
||||
GGML_API size_t quantize_turbo4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
||||
GGML_API void quantize_row_turbo2_0_ref(const float * GGML_RESTRICT x, block_turbo2_0 * GGML_RESTRICT y, int64_t k);
|
||||
GGML_API void dequantize_row_turbo2_0(const block_turbo2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k);
|
||||
GGML_API size_t quantize_turbo2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix);
|
||||
|
||||
GGML_API void iq2xs_init_impl(enum ggml_type type);
|
||||
GGML_API void iq2xs_free_impl(enum ggml_type type);
|
||||
GGML_API void iq3xs_init_impl(int grid_size);
|
||||
|
||||
@@ -0,0 +1,627 @@
|
||||
/*
|
||||
* TurboQuant: KV cache compression via PolarQuant + QJL
|
||||
* Based on: arXiv 2504.19874 (ICLR 2026)
|
||||
*
|
||||
* Implements GGML_TYPE_TURBO2_0 (2-bit), GGML_TYPE_TURBO3_0 (3-bit) and
|
||||
* GGML_TYPE_TURBO4_0 (4-bit) for use as --cache-type-k turboN in llama-server.
|
||||
*/
|
||||
|
||||
#include "ggml-quants.h"
|
||||
#include "ggml-common.h"
|
||||
#include "ggml-impl.h"
|
||||
|
||||
#include <math.h>
|
||||
#include <string.h>
|
||||
#include <assert.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
/* Global: WHT group size for CPU quantize path (set by CPU SET_ROWS handler) */
|
||||
int turbo3_cpu_wht_group_size = 0;
|
||||
|
||||
/* ---------- constants ---------- */
|
||||
|
||||
#define TURBO_SEED_ROTATION 42
|
||||
#define TURBO_SEED_QJL 1042
|
||||
#define TURBO_D 128 /* rotation group size = head_dim (independent of block size) */
|
||||
#define TURBO_QJL_CONST 1.2533141373155003f /* sqrt(pi/2) */
|
||||
|
||||
/* TURBO_D must match QK_TURBO3_GROUP from ggml-common.h — they represent
|
||||
* the same rotation group size but are defined separately. Guard against
|
||||
* silent divergence so GPU kernels and CPU reference stay in sync. */
|
||||
static_assert(TURBO_D == QK_TURBO3_GROUP,
|
||||
"TURBO_D must equal QK_TURBO3_GROUP (rotation group size)");
|
||||
|
||||
/* Optimal centroids from paper (scaled by 1/sqrt(d)) */
|
||||
/* 2-bit: {±0.453, ±1.51} / sqrt(d) */
|
||||
static const float CENTROIDS_2BIT[4] = { -0.133462f, -0.039994f, 0.039994f, 0.133462f };
|
||||
|
||||
/* 3-bit: Lloyd-Max for N(0, 1/128), pre-computed */
|
||||
static const float CENTROIDS_3BIT[8] = {
|
||||
-0.190685f, -0.117832f, -0.065717f, -0.021460f,
|
||||
0.021460f, 0.065717f, 0.117832f, 0.190685f
|
||||
};
|
||||
|
||||
/* ---------- rotation matrix (lazy init) ---------- */
|
||||
|
||||
static float turbo_rotation[TURBO_D * TURBO_D];
|
||||
static float turbo_rotation_t[TURBO_D * TURBO_D]; /* transpose */
|
||||
static int turbo_rotation_initialized = 0;
|
||||
|
||||
/* Simple LCG PRNG for deterministic rotation generation */
|
||||
static uint64_t turbo_prng_state;
|
||||
|
||||
static void turbo_prng_seed(uint64_t seed) {
|
||||
turbo_prng_state = seed;
|
||||
}
|
||||
|
||||
static double turbo_prng_normal(void) {
|
||||
/* Box-Muller transform from uniform LCG */
|
||||
turbo_prng_state = turbo_prng_state * 6364136223846793005ULL + 1442695040888963407ULL;
|
||||
double u1 = (double)(turbo_prng_state >> 11) / (double)(1ULL << 53);
|
||||
if (u1 < 1e-15) u1 = 1e-15;
|
||||
turbo_prng_state = turbo_prng_state * 6364136223846793005ULL + 1442695040888963407ULL;
|
||||
double u2 = (double)(turbo_prng_state >> 11) / (double)(1ULL << 53);
|
||||
return sqrt(-2.0 * log(u1)) * cos(2.0 * M_PI * u2);
|
||||
}
|
||||
|
||||
static void turbo_init_rotation(void) {
|
||||
if (turbo_rotation_initialized) return;
|
||||
|
||||
const int d = TURBO_D;
|
||||
|
||||
/* Generate random Gaussian matrix directly into turbo_rotation.
|
||||
* Previous code used a 64KB stack-local G[128*128] then memcpy'd —
|
||||
* this segfaults on llama.cpp worker threads with reduced stack
|
||||
* sizes (512KB macOS, 64KB some Linux configs). Writing directly
|
||||
* into the static array avoids the stack allocation entirely. */
|
||||
turbo_prng_seed(TURBO_SEED_ROTATION);
|
||||
for (int i = 0; i < d * d; i++) {
|
||||
turbo_rotation[i] = (float)turbo_prng_normal();
|
||||
}
|
||||
|
||||
/* QR decomposition via modified Gram-Schmidt */
|
||||
/* Q stored column-major in turbo_rotation */
|
||||
|
||||
for (int j = 0; j < d; j++) {
|
||||
/* Normalize column j */
|
||||
float norm = 0.0f;
|
||||
for (int i = 0; i < d; i++) {
|
||||
norm += turbo_rotation[i * d + j] * turbo_rotation[i * d + j];
|
||||
}
|
||||
norm = sqrtf(norm);
|
||||
if (norm > 1e-10f) {
|
||||
for (int i = 0; i < d; i++) {
|
||||
turbo_rotation[i * d + j] /= norm;
|
||||
}
|
||||
}
|
||||
|
||||
/* Orthogonalize remaining columns against j */
|
||||
for (int k = j + 1; k < d; k++) {
|
||||
float dot = 0.0f;
|
||||
for (int i = 0; i < d; i++) {
|
||||
dot += turbo_rotation[i * d + j] * turbo_rotation[i * d + k];
|
||||
}
|
||||
for (int i = 0; i < d; i++) {
|
||||
turbo_rotation[i * d + k] -= dot * turbo_rotation[i * d + j];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Compute transpose */
|
||||
for (int i = 0; i < d; i++) {
|
||||
for (int j = 0; j < d; j++) {
|
||||
turbo_rotation_t[i * d + j] = turbo_rotation[j * d + i];
|
||||
}
|
||||
}
|
||||
|
||||
turbo_rotation_initialized = 1;
|
||||
}
|
||||
|
||||
/* ---------- QJL projection matrix (lazy init, seed-based) ---------- */
|
||||
|
||||
static float turbo_qjl_matrix[TURBO_D * TURBO_D];
|
||||
static float turbo_qjl_matrix_t[TURBO_D * TURBO_D];
|
||||
static int turbo_qjl_initialized = 0;
|
||||
|
||||
static void turbo_init_qjl(void) {
|
||||
if (turbo_qjl_initialized) return;
|
||||
|
||||
const int d = TURBO_D;
|
||||
turbo_prng_seed(TURBO_SEED_QJL);
|
||||
|
||||
for (int i = 0; i < d * d; i++) {
|
||||
turbo_qjl_matrix[i] = (float)turbo_prng_normal();
|
||||
}
|
||||
|
||||
/* Transpose */
|
||||
for (int i = 0; i < d; i++) {
|
||||
for (int j = 0; j < d; j++) {
|
||||
turbo_qjl_matrix_t[i * d + j] = turbo_qjl_matrix[j * d + i];
|
||||
}
|
||||
}
|
||||
|
||||
turbo_qjl_initialized = 1;
|
||||
}
|
||||
|
||||
/* ---------- helper: matrix-vector multiply ---------- */
|
||||
|
||||
static void matvec(const float * M, const float * x, float * y, int d) {
|
||||
/* y = M @ x, M is row-major d×d */
|
||||
for (int i = 0; i < d; i++) {
|
||||
float sum = 0.0f;
|
||||
for (int j = 0; j < d; j++) {
|
||||
sum += M[i * d + j] * x[j];
|
||||
}
|
||||
y[i] = sum;
|
||||
}
|
||||
}
|
||||
|
||||
/* ---------- nearest centroid ---------- */
|
||||
|
||||
static int nearest_centroid_2bit(float val) {
|
||||
/* Binary search on midpoints: {-0.133, -0.040, 0.040, 0.133} */
|
||||
if (val < -0.086728f) return 0; /* midpoint(-0.133, -0.040) */
|
||||
if (val < 0.000000f) return 1; /* midpoint(-0.040, 0.040) */
|
||||
if (val < 0.086728f) return 2; /* midpoint(0.040, 0.133) */
|
||||
return 3;
|
||||
}
|
||||
|
||||
static int nearest_centroid_3bit(float val) {
|
||||
/* 8 centroids, find nearest via midpoints */
|
||||
if (val < -0.154259f) return 0;
|
||||
if (val < -0.091775f) return 1;
|
||||
if (val < -0.043589f) return 2;
|
||||
if (val < 0.000000f) return 3;
|
||||
if (val < 0.043589f) return 4;
|
||||
if (val < 0.091775f) return 5;
|
||||
if (val < 0.154259f) return 6;
|
||||
return 7;
|
||||
}
|
||||
|
||||
static int nearest_centroid_4bit(float val) {
|
||||
/* 16 centroids, optimal for N(0, 1/sqrt(128)), find nearest via midpoints */
|
||||
if (val < -0.145560f) return 0;
|
||||
if (val < -0.103361f) return 1;
|
||||
if (val < -0.079142f) return 2;
|
||||
if (val < -0.060009f) return 3;
|
||||
if (val < -0.043430f) return 4;
|
||||
if (val < -0.028293f) return 5;
|
||||
if (val < -0.013963f) return 6;
|
||||
if (val < 0.000000f) return 7;
|
||||
if (val < 0.013963f) return 8;
|
||||
if (val < 0.028293f) return 9;
|
||||
if (val < 0.043430f) return 10;
|
||||
if (val < 0.060009f) return 11;
|
||||
if (val < 0.079142f) return 12;
|
||||
if (val < 0.103361f) return 13;
|
||||
if (val < 0.145560f) return 14;
|
||||
return 15;
|
||||
}
|
||||
|
||||
/* ---------- WHT sign arrays (must match CUDA/Metal, seed=42) ---------- */
|
||||
|
||||
static const float turbo_cpu_s1[128] = {
|
||||
-1,1,1,-1,-1,1,-1,1,-1,-1,1,1,1,1,1,1,1,-1,1,-1,1,-1,-1,1,1,1,-1,1,1,-1,-1,-1,
|
||||
-1,1,1,-1,1,1,-1,1,-1,1,1,-1,-1,1,-1,1,1,1,1,-1,-1,-1,-1,-1,1,-1,1,1,1,1,-1,1,
|
||||
-1,-1,1,-1,-1,-1,1,-1,-1,-1,1,-1,-1,-1,1,1,1,-1,-1,1,1,1,-1,-1,1,1,-1,1,1,-1,1,-1,
|
||||
-1,1,1,-1,1,-1,1,-1,1,1,1,1,-1,1,-1,1,1,-1,1,1,-1,-1,-1,-1,-1,1,1,-1,1,1,-1,1
|
||||
};
|
||||
|
||||
static const float turbo_cpu_s2[128] = {
|
||||
1,1,1,1,-1,1,1,-1,1,-1,-1,-1,1,-1,-1,-1,1,1,-1,-1,1,-1,1,-1,1,-1,-1,1,-1,1,1,1,
|
||||
1,1,-1,-1,-1,1,-1,-1,-1,-1,-1,-1,1,1,1,-1,1,-1,1,1,1,-1,-1,1,-1,-1,-1,-1,-1,-1,1,1,
|
||||
1,-1,1,-1,-1,-1,-1,1,-1,1,-1,1,-1,-1,1,1,-1,1,-1,1,1,-1,1,-1,-1,-1,-1,1,-1,-1,1,-1,
|
||||
1,-1,1,1,1,-1,-1,1,-1,1,-1,1,1,-1,-1,1,-1,1,-1,1,1,-1,1,-1,1,-1,-1,-1,-1,-1,1,-1
|
||||
};
|
||||
|
||||
/* ---------- CPU forward WHT (in-place, group_size elements) ---------- */
|
||||
|
||||
static void turbo_cpu_fwht(float * x, int group_size) {
|
||||
const float * s1 = turbo_cpu_s1;
|
||||
const float * s2 = turbo_cpu_s2;
|
||||
const float inv_sqrt = (group_size == 128) ? 0.08838834764831845f : 0.125f;
|
||||
|
||||
// signs1
|
||||
for (int i = 0; i < group_size; i++) x[i] *= s1[i];
|
||||
|
||||
// butterfly stages
|
||||
for (int h = 1; h < group_size; h *= 2) {
|
||||
for (int i = 0; i < group_size; i += h * 2) {
|
||||
for (int j = i; j < i + h; j++) {
|
||||
float a = x[j], b = x[j + h];
|
||||
x[j] = a + b;
|
||||
x[j + h] = a - b;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// normalize + signs2
|
||||
for (int i = 0; i < group_size; i++) x[i] *= inv_sqrt * s2[i];
|
||||
}
|
||||
|
||||
/* ---------- TURBO3_0: 3-bit PolarQuant with WHT rotation ---------- */
|
||||
|
||||
void quantize_row_turbo3_0_ref(const float * GGML_RESTRICT x, block_turbo3_0 * GGML_RESTRICT y, int64_t k) {
|
||||
assert(k % QK_TURBO3 == 0);
|
||||
|
||||
// Read WHT group size from global (set by CPU SET_ROWS handler before each call).
|
||||
// Fallback: 128 if row is 128-aligned, else 64.
|
||||
extern int turbo3_cpu_wht_group_size;
|
||||
int group_size = turbo3_cpu_wht_group_size;
|
||||
if (group_size != 64 && group_size != 128) {
|
||||
group_size = (k % 128 == 0) ? 128 : 64;
|
||||
}
|
||||
if (k % group_size != 0) group_size = (group_size == 128) ? 64 : 128;
|
||||
assert(k % group_size == 0);
|
||||
|
||||
const int n_groups = k / group_size;
|
||||
const int blocks_per_group = group_size / QK_TURBO3;
|
||||
|
||||
for (int g = 0; g < n_groups; g++) {
|
||||
const float * grp_src = x + g * group_size;
|
||||
block_turbo3_0 * grp_dst = y + g * blocks_per_group;
|
||||
|
||||
// 1. L2 norm over the group
|
||||
float norm_sq = 0.0f;
|
||||
float buf[128]; // max group_size
|
||||
for (int j = 0; j < group_size; j++) {
|
||||
buf[j] = grp_src[j];
|
||||
norm_sq += buf[j] * buf[j];
|
||||
}
|
||||
float grp_norm = sqrtf(norm_sq);
|
||||
float inv_norm = (grp_norm > 1e-10f) ? 1.0f / grp_norm : 0.0f;
|
||||
|
||||
// 2. Normalize
|
||||
for (int j = 0; j < group_size; j++) buf[j] *= inv_norm;
|
||||
|
||||
// 3. Forward WHT rotation
|
||||
turbo_cpu_fwht(buf, group_size);
|
||||
|
||||
// 4. Quantize + pack into sub-blocks
|
||||
float recon_sq = 0.0f;
|
||||
for (int b = 0; b < blocks_per_group; b++) {
|
||||
block_turbo3_0 * blk = &grp_dst[b];
|
||||
const int off = b * QK_TURBO3;
|
||||
|
||||
memset(blk->qs, 0, QK_TURBO3 / 4);
|
||||
memset(blk->signs, 0, QK_TURBO3 / 8);
|
||||
|
||||
for (int j = 0; j < QK_TURBO3; j++) {
|
||||
int idx = nearest_centroid_3bit(buf[off + j]);
|
||||
blk->qs[j / 4] |= (idx & 0x3) << ((j % 4) * 2);
|
||||
if (idx & 0x4) {
|
||||
blk->signs[j / 8] |= (1 << (j % 8));
|
||||
}
|
||||
recon_sq += CENTROIDS_3BIT[idx] * CENTROIDS_3BIT[idx];
|
||||
}
|
||||
}
|
||||
|
||||
// 5. Corrected norm: grp_norm / recon_norm (matching CUDA kernel)
|
||||
float recon_norm = sqrtf(recon_sq);
|
||||
float corrected = (recon_norm > 1e-10f) ? grp_norm / recon_norm : grp_norm;
|
||||
for (int b = 0; b < blocks_per_group; b++) {
|
||||
grp_dst[b].norm = GGML_FP32_TO_FP16(corrected);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void dequantize_row_turbo3_0(const block_turbo3_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
|
||||
// Stub — Metal shader handles dequant on GPU.
|
||||
assert(k % QK_TURBO3 == 0);
|
||||
const int nb = k / QK_TURBO3;
|
||||
for (int block = 0; block < nb; block++) {
|
||||
float norm = GGML_FP16_TO_FP32(x[block].norm);
|
||||
for (int j = 0; j < QK_TURBO3; j++) {
|
||||
uint8_t low2 = (x[block].qs[j/4] >> ((j%4)*2)) & 0x3;
|
||||
uint8_t hi1 = (x[block].signs[j/8] >> (j%8)) & 0x1;
|
||||
uint8_t idx = low2 | (hi1 << 2);
|
||||
y[block * QK_TURBO3 + j] = CENTROIDS_3BIT[idx] * norm;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
size_t quantize_turbo3_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst,
|
||||
int64_t nrows, int64_t n_per_row, const float * imatrix) {
|
||||
GGML_UNUSED(imatrix);
|
||||
assert(n_per_row % QK_TURBO3 == 0);
|
||||
|
||||
size_t row_size = (n_per_row / QK_TURBO3) * sizeof(block_turbo3_0);
|
||||
for (int64_t row = 0; row < nrows; row++) {
|
||||
quantize_row_turbo3_0_ref(
|
||||
src + row * n_per_row,
|
||||
(block_turbo3_0 *)((char *)dst + row * row_size),
|
||||
n_per_row
|
||||
);
|
||||
}
|
||||
return nrows * row_size;
|
||||
}
|
||||
|
||||
/* ---------- TURBO2_0: 2-bit PolarQuant (no QJL) ---------- */
|
||||
|
||||
void quantize_row_turbo2_0_ref(const float * GGML_RESTRICT x, block_turbo2_0 * GGML_RESTRICT y, int64_t k) {
|
||||
assert(k % QK_TURBO2 == 0);
|
||||
|
||||
extern int turbo3_cpu_wht_group_size;
|
||||
int group_size = turbo3_cpu_wht_group_size;
|
||||
if (group_size != 64 && group_size != 128) {
|
||||
group_size = (k % 128 == 0) ? 128 : 64;
|
||||
}
|
||||
if (k % group_size != 0) group_size = (group_size == 128) ? 64 : 128;
|
||||
assert(k % group_size == 0);
|
||||
|
||||
const int n_groups = k / group_size;
|
||||
const int blocks_per_group = group_size / QK_TURBO2;
|
||||
|
||||
for (int g = 0; g < n_groups; g++) {
|
||||
const float * grp_src = x + g * group_size;
|
||||
block_turbo2_0 * grp_dst = y + g * blocks_per_group;
|
||||
|
||||
/* 1. L2 norm over the group */
|
||||
float norm_sq = 0.0f;
|
||||
float buf[128];
|
||||
for (int j = 0; j < group_size; j++) {
|
||||
buf[j] = grp_src[j];
|
||||
norm_sq += buf[j] * buf[j];
|
||||
}
|
||||
float grp_norm = sqrtf(norm_sq);
|
||||
float inv_norm = (grp_norm > 1e-10f) ? 1.0f / grp_norm : 0.0f;
|
||||
|
||||
/* 2. Normalize */
|
||||
for (int j = 0; j < group_size; j++) buf[j] *= inv_norm;
|
||||
|
||||
/* 3. Forward WHT rotation */
|
||||
turbo_cpu_fwht(buf, group_size);
|
||||
|
||||
/* 4. Quantize + pack into sub-blocks */
|
||||
float recon_sq = 0.0f;
|
||||
for (int b = 0; b < blocks_per_group; b++) {
|
||||
block_turbo2_0 * blk = &grp_dst[b];
|
||||
const int off = b * QK_TURBO2;
|
||||
|
||||
memset(blk->qs, 0, QK_TURBO2 / 4);
|
||||
|
||||
for (int j = 0; j < QK_TURBO2; j++) {
|
||||
int idx = nearest_centroid_2bit(buf[off + j]);
|
||||
blk->qs[j / 4] |= (idx & 0x3) << ((j % 4) * 2);
|
||||
recon_sq += CENTROIDS_2BIT[idx] * CENTROIDS_2BIT[idx];
|
||||
}
|
||||
}
|
||||
|
||||
/* 5. Corrected norm */
|
||||
float recon_norm = sqrtf(recon_sq);
|
||||
float corrected = (recon_norm > 1e-10f) ? grp_norm / recon_norm : grp_norm;
|
||||
for (int b = 0; b < blocks_per_group; b++) {
|
||||
grp_dst[b].norm = GGML_FP32_TO_FP16(corrected);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void dequantize_row_turbo2_0(const block_turbo2_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
|
||||
assert(k % QK_TURBO2 == 0);
|
||||
const int nb = k / QK_TURBO2;
|
||||
for (int block = 0; block < nb; block++) {
|
||||
float norm = GGML_FP16_TO_FP32(x[block].norm);
|
||||
for (int j = 0; j < QK_TURBO2; j++) {
|
||||
uint8_t idx = (x[block].qs[j/4] >> ((j%4)*2)) & 0x3;
|
||||
y[block * QK_TURBO2 + j] = CENTROIDS_2BIT[idx] * norm;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
size_t quantize_turbo2_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst,
|
||||
int64_t nrows, int64_t n_per_row, const float * imatrix) {
|
||||
GGML_UNUSED(imatrix);
|
||||
assert(n_per_row % QK_TURBO2 == 0);
|
||||
|
||||
size_t row_size = (n_per_row / QK_TURBO2) * sizeof(block_turbo2_0);
|
||||
for (int64_t row = 0; row < nrows; row++) {
|
||||
quantize_row_turbo2_0_ref(
|
||||
src + row * n_per_row,
|
||||
(block_turbo2_0 *)((char *)dst + row * row_size),
|
||||
n_per_row
|
||||
);
|
||||
}
|
||||
return nrows * row_size;
|
||||
}
|
||||
|
||||
/* ---------- TURBO4_0: 3-bit PolarQuant + 1-bit QJL ---------- */
|
||||
|
||||
void quantize_row_turbo4_0_ref(const float * GGML_RESTRICT x, block_turbo4_0 * GGML_RESTRICT y, int64_t k) {
|
||||
turbo_init_rotation();
|
||||
turbo_init_qjl();
|
||||
|
||||
assert(k % QK_TURBO4 == 0);
|
||||
const int nb = k / QK_TURBO4;
|
||||
const int d = QK_TURBO4;
|
||||
|
||||
for (int block = 0; block < nb; block++) {
|
||||
const float * src = x + block * d;
|
||||
|
||||
/* Step 1: Extract norm */
|
||||
float norm_sq = 0.0f;
|
||||
for (int i = 0; i < d; i++) norm_sq += src[i] * src[i];
|
||||
float norm = sqrtf(norm_sq);
|
||||
|
||||
/* Normalize */
|
||||
float normalized[TURBO_D];
|
||||
if (norm > 1e-10f) {
|
||||
const float inv = 1.0f / norm;
|
||||
for (int i = 0; i < d; i++) normalized[i] = src[i] * inv;
|
||||
} else {
|
||||
memset(normalized, 0, d * sizeof(float));
|
||||
}
|
||||
|
||||
/* Step 2: Rotate */
|
||||
float rotated[TURBO_D];
|
||||
matvec(turbo_rotation, normalized, rotated, d);
|
||||
|
||||
#if TURBO4_USE_4BIT
|
||||
/* Step 3: 4-bit quantization (16 centroids) */
|
||||
static const float CENTROIDS_4BIT[16] = {
|
||||
-0.173926f, -0.117195f, -0.089527f, -0.068756f,
|
||||
-0.051262f, -0.035597f, -0.020989f, -0.006938f,
|
||||
0.006938f, 0.020989f, 0.035597f, 0.051262f,
|
||||
0.068756f, 0.089527f, 0.117195f, 0.173926f
|
||||
};
|
||||
uint8_t indices[TURBO_D];
|
||||
for (int i = 0; i < d; i++) {
|
||||
indices[i] = (uint8_t)nearest_centroid_4bit(rotated[i]);
|
||||
}
|
||||
|
||||
/* Norm correction */
|
||||
float recon_norm_sq = 0.0f;
|
||||
for (int i = 0; i < d; i++) {
|
||||
recon_norm_sq += CENTROIDS_4BIT[indices[i]] * CENTROIDS_4BIT[indices[i]];
|
||||
}
|
||||
float recon_norm = sqrtf(recon_norm_sq);
|
||||
float corrected_norm = (recon_norm > 1e-10f) ? norm / recon_norm : norm;
|
||||
y[block].norm = GGML_FP32_TO_FP16(corrected_norm);
|
||||
#else
|
||||
/* Step 3: 3-bit quantization (8 centroids) */
|
||||
uint8_t indices[TURBO_D];
|
||||
for (int i = 0; i < d; i++) {
|
||||
indices[i] = (uint8_t)nearest_centroid_3bit(rotated[i]);
|
||||
}
|
||||
|
||||
/* Step 4: Residual */
|
||||
float reconstructed[TURBO_D];
|
||||
for (int i = 0; i < d; i++) {
|
||||
reconstructed[i] = CENTROIDS_3BIT[indices[i]];
|
||||
}
|
||||
float mse_recon[TURBO_D];
|
||||
matvec(turbo_rotation_t, reconstructed, mse_recon, d);
|
||||
|
||||
float residual[TURBO_D];
|
||||
for (int i = 0; i < d; i++) {
|
||||
residual[i] = normalized[i] - mse_recon[i];
|
||||
}
|
||||
|
||||
/* Step 5: QJL */
|
||||
float projected[TURBO_D];
|
||||
matvec(turbo_qjl_matrix, residual, projected, d);
|
||||
#endif
|
||||
|
||||
/* Pack */
|
||||
#if !TURBO4_USE_4BIT
|
||||
y[block].norm = GGML_FP32_TO_FP16(norm);
|
||||
#endif
|
||||
|
||||
#if TURBO4_USE_4BIT
|
||||
/* 4-bit PolarQuant: nibble pack into qs[64] */
|
||||
memset(y[block].qs, 0, d / 2);
|
||||
for (int i = 0; i < d; i++) {
|
||||
y[block].qs[i / 2] |= (uint8_t)((indices[i] & 0xF) << ((i % 2) * 4));
|
||||
}
|
||||
y[block].rnorm = GGML_FP32_TO_FP16(0.0f);
|
||||
#else
|
||||
/* Legacy 3-bit + QJL: pack 3-bit indices + QJL signs */
|
||||
memset(y[block].qs, 0, d * 3 / 8);
|
||||
for (int i = 0; i < d; i++) {
|
||||
int bit_offset = i * 3;
|
||||
int byte_idx = bit_offset / 8;
|
||||
int bit_pos = bit_offset % 8;
|
||||
uint16_t val = (uint16_t)(indices[i] & 0x7);
|
||||
y[block].qs[byte_idx] |= (uint8_t)(val << bit_pos);
|
||||
if (bit_pos > 5 && byte_idx + 1 < d * 3 / 8) {
|
||||
y[block].qs[byte_idx + 1] |= (uint8_t)(val >> (8 - bit_pos));
|
||||
}
|
||||
}
|
||||
memset(y[block].signs, 0, d / 8);
|
||||
for (int i = 0; i < d; i++) {
|
||||
if (projected[i] >= 0.0f) {
|
||||
y[block].signs[i / 8] |= (1 << (i % 8));
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
void dequantize_row_turbo4_0(const block_turbo4_0 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) {
|
||||
turbo_init_rotation();
|
||||
|
||||
assert(k % QK_TURBO4 == 0);
|
||||
const int nb = k / QK_TURBO4;
|
||||
const int d = QK_TURBO4;
|
||||
|
||||
#if TURBO4_USE_4BIT
|
||||
/* 4-bit PolarQuant: nibble unpack → centroid → inverse rotate → scale */
|
||||
/* TODO: add proper 4-bit centroid table to C code (currently only in Metal) */
|
||||
static const float CENTROIDS_4BIT[16] = {
|
||||
-0.173926f, -0.117195f, -0.089527f, -0.068756f,
|
||||
-0.051262f, -0.035597f, -0.020989f, -0.006938f,
|
||||
0.006938f, 0.020989f, 0.035597f, 0.051262f,
|
||||
0.068756f, 0.089527f, 0.117195f, 0.173926f
|
||||
};
|
||||
for (int block = 0; block < nb; block++) {
|
||||
float norm = GGML_FP16_TO_FP32(x[block].norm);
|
||||
float rotated[QK_TURBO4];
|
||||
for (int i = 0; i < d; i++) {
|
||||
uint8_t idx = (x[block].qs[i / 2] >> ((i % 2) * 4)) & 0xF;
|
||||
rotated[i] = CENTROIDS_4BIT[idx];
|
||||
}
|
||||
float * dst = y + block * d;
|
||||
matvec(turbo_rotation_t, rotated, dst, d);
|
||||
for (int i = 0; i < d; i++) dst[i] *= norm;
|
||||
}
|
||||
#else
|
||||
/* Legacy 3-bit + QJL dequant */
|
||||
turbo_init_qjl();
|
||||
for (int block = 0; block < nb; block++) {
|
||||
float norm = GGML_FP16_TO_FP32(x[block].norm);
|
||||
|
||||
uint8_t indices[TURBO_D];
|
||||
for (int i = 0; i < d; i++) {
|
||||
int bit_offset = i * 3;
|
||||
int byte_idx = bit_offset / 8;
|
||||
int bit_pos = bit_offset % 8;
|
||||
uint16_t raw = (uint16_t)x[block].qs[byte_idx];
|
||||
if (byte_idx + 1 < d * 3 / 8) {
|
||||
raw |= (uint16_t)x[block].qs[byte_idx + 1] << 8;
|
||||
}
|
||||
indices[i] = (uint8_t)((raw >> bit_pos) & 0x7);
|
||||
}
|
||||
|
||||
float signs[TURBO_D];
|
||||
for (int i = 0; i < d; i++) {
|
||||
signs[i] = (x[block].signs[i / 8] & (1 << (i % 8))) ? 1.0f : -1.0f;
|
||||
}
|
||||
|
||||
float rnorm = GGML_FP16_TO_FP32(x[block].rnorm);
|
||||
const float qjl_scale = TURBO_QJL_CONST / (float)d * rnorm;
|
||||
|
||||
float rotated_recon[TURBO_D];
|
||||
for (int i = 0; i < d; i++) {
|
||||
rotated_recon[i] = CENTROIDS_3BIT[indices[i]];
|
||||
}
|
||||
float mse_recon[TURBO_D];
|
||||
matvec(turbo_rotation_t, rotated_recon, mse_recon, d);
|
||||
|
||||
float qjl_recon[TURBO_D];
|
||||
matvec(turbo_qjl_matrix_t, signs, qjl_recon, d);
|
||||
for (int i = 0; i < d; i++) {
|
||||
qjl_recon[i] *= qjl_scale;
|
||||
}
|
||||
|
||||
float * dst = y + block * d;
|
||||
for (int i = 0; i < d; i++) {
|
||||
dst[i] = (mse_recon[i] + qjl_recon[i]) * norm;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
size_t quantize_turbo4_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst,
|
||||
int64_t nrows, int64_t n_per_row, const float * imatrix) {
|
||||
GGML_UNUSED(imatrix);
|
||||
assert(n_per_row % QK_TURBO4 == 0);
|
||||
|
||||
size_t row_size = (n_per_row / QK_TURBO4) * sizeof(block_turbo4_0);
|
||||
for (int64_t row = 0; row < nrows; row++) {
|
||||
quantize_row_turbo4_0_ref(
|
||||
src + row * n_per_row,
|
||||
(block_turbo4_0 *)((char *)dst + row * row_size),
|
||||
n_per_row
|
||||
);
|
||||
}
|
||||
return nrows * row_size;
|
||||
}
|
||||
+63
-2
@@ -749,6 +749,30 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = {
|
||||
.to_float = (ggml_to_float_t) dequantize_row_nvfp4,
|
||||
.from_float_ref = (ggml_from_float_t)quantize_row_nvfp4_ref,
|
||||
},
|
||||
[GGML_TYPE_TURBO3_0] = {
|
||||
.type_name = "turbo3",
|
||||
.blck_size = QK_TURBO3,
|
||||
.type_size = sizeof(block_turbo3_0),
|
||||
.is_quantized = true,
|
||||
.to_float = (ggml_to_float_t) dequantize_row_turbo3_0,
|
||||
.from_float_ref = (ggml_from_float_t) quantize_row_turbo3_0_ref,
|
||||
},
|
||||
[GGML_TYPE_TURBO4_0] = {
|
||||
.type_name = "turbo4",
|
||||
.blck_size = QK_TURBO4,
|
||||
.type_size = sizeof(block_turbo4_0),
|
||||
.is_quantized = true,
|
||||
.to_float = (ggml_to_float_t) dequantize_row_turbo4_0,
|
||||
.from_float_ref = (ggml_from_float_t) quantize_row_turbo4_0_ref,
|
||||
},
|
||||
[GGML_TYPE_TURBO2_0] = {
|
||||
.type_name = "turbo2",
|
||||
.blck_size = QK_TURBO2,
|
||||
.type_size = sizeof(block_turbo2_0),
|
||||
.is_quantized = true,
|
||||
.to_float = (ggml_to_float_t) dequantize_row_turbo2_0,
|
||||
.from_float_ref = (ggml_from_float_t) quantize_row_turbo2_0_ref,
|
||||
},
|
||||
[GGML_TYPE_Q2_K] = {
|
||||
.type_name = "q2_K",
|
||||
.blck_size = QK_K,
|
||||
@@ -1063,6 +1087,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = {
|
||||
"RWKV_WKV7",
|
||||
"SOLVE_TRI",
|
||||
"GATED_DELTA_NET",
|
||||
"TURBO_WHT",
|
||||
|
||||
"UNARY",
|
||||
|
||||
@@ -1080,7 +1105,7 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = {
|
||||
"GLU",
|
||||
};
|
||||
|
||||
static_assert(GGML_OP_COUNT == 96, "GGML_OP_COUNT != 96");
|
||||
static_assert(GGML_OP_COUNT == 97, "GGML_OP_COUNT != 97");
|
||||
|
||||
static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
|
||||
"none",
|
||||
@@ -1173,6 +1198,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
|
||||
"rwkv_wkv7(r, w, k, v, a, b, s)",
|
||||
"A X = B, A triangular, solve X",
|
||||
"gated_delta_net(q, k, v, g, beta, s)",
|
||||
"turbo_wht(a)",
|
||||
|
||||
"unary(x)",
|
||||
|
||||
@@ -1190,7 +1216,7 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = {
|
||||
"glu(x)",
|
||||
};
|
||||
|
||||
static_assert(GGML_OP_COUNT == 96, "GGML_OP_COUNT != 96");
|
||||
static_assert(GGML_OP_COUNT == 97, "GGML_OP_COUNT != 97");
|
||||
|
||||
static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2");
|
||||
|
||||
@@ -6230,6 +6256,38 @@ struct ggml_tensor * ggml_gated_delta_net(
|
||||
return result;
|
||||
}
|
||||
|
||||
// ggml_turbo_wht
|
||||
|
||||
struct ggml_tensor * ggml_turbo_wht(
|
||||
struct ggml_context * ctx,
|
||||
struct ggml_tensor * a,
|
||||
int direction,
|
||||
int group_size,
|
||||
struct ggml_tensor * scale) {
|
||||
GGML_ASSERT(ggml_is_contiguous(a));
|
||||
GGML_ASSERT(a->type == GGML_TYPE_F32);
|
||||
GGML_ASSERT(direction == 0 || direction == 1);
|
||||
|
||||
// Auto-detect group size from tensor dimension if not specified
|
||||
if (group_size == 0) {
|
||||
group_size = (a->ne[0] % 128 == 0) ? 128 : 64;
|
||||
}
|
||||
GGML_ASSERT(group_size == 64 || group_size == 128);
|
||||
GGML_ASSERT(a->ne[0] % group_size == 0);
|
||||
|
||||
struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, a->ne);
|
||||
|
||||
result->op = GGML_OP_TURBO_WHT;
|
||||
result->src[0] = a;
|
||||
result->src[1] = scale; // InnerQ scale_inv (NULL = no scaling)
|
||||
|
||||
// Store direction and group_size in op_params
|
||||
memcpy(result->op_params + 0, &direction, sizeof(int));
|
||||
memcpy(result->op_params + sizeof(int), &group_size, sizeof(int));
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
struct ggml_hash_set ggml_hash_set_new(size_t size) {
|
||||
@@ -7714,6 +7772,9 @@ size_t ggml_quantize_chunk(
|
||||
case GGML_TYPE_IQ1_M: result = quantize_iq1_m (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
|
||||
case GGML_TYPE_IQ4_NL: result = quantize_iq4_nl (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
|
||||
case GGML_TYPE_IQ4_XS: result = quantize_iq4_xs (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
|
||||
case GGML_TYPE_TURBO3_0: result = quantize_turbo3_0(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
|
||||
case GGML_TYPE_TURBO4_0: result = quantize_turbo4_0(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
|
||||
case GGML_TYPE_TURBO2_0: result = quantize_turbo2_0(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break;
|
||||
case GGML_TYPE_F16:
|
||||
{
|
||||
size_t elemsize = sizeof(ggml_fp16_t);
|
||||
|
||||
Executable
+58
@@ -0,0 +1,58 @@
|
||||
#!/bin/bash
|
||||
# SMEM Pre-Dequant Benchmark — M5 Max
|
||||
# Tests SMEM vs baseline at multiple context depths
|
||||
#
|
||||
# BEFORE RUNNING:
|
||||
# 1. cd /Users/tom/local_llms/llama.cpp
|
||||
# 2. git checkout experiment/smem-pre-dequant
|
||||
# 3. Build WITHOUT SMEM first (baseline):
|
||||
# cmake --build build -j12
|
||||
# 4. Run: ./scripts/bench-smem-m5.sh baseline
|
||||
# 5. Build WITH SMEM:
|
||||
# TURBO_SMEM_DEQUANT=1 cmake --build build -j12
|
||||
# 6. Run: ./scripts/bench-smem-m5.sh smem
|
||||
#
|
||||
# Uses Qwen3.5-35B-A3B (MoE, fits in memory, attention-heavy)
|
||||
|
||||
set -e
|
||||
|
||||
LABEL="${1:-baseline}"
|
||||
LLAMA_BENCH="/Users/tom/local_llms/llama.cpp/build/bin/llama-bench"
|
||||
MODEL="/Users/tom/local_llms/models/Qwen3.5-35B-A3B-Q8_0.gguf"
|
||||
OUTFILE="/Users/tom/local_llms/llama.cpp/bench-smem-m5-${LABEL}.txt"
|
||||
|
||||
CONTEXTS=(0 8192 16384 32768)
|
||||
KV_TYPES=("turbo3" "turbo4" "q8_0")
|
||||
|
||||
echo "=== SMEM M5 Benchmark: ${LABEL} ===" | tee "$OUTFILE"
|
||||
echo "Model: $(basename $MODEL)" | tee -a "$OUTFILE"
|
||||
echo "Date: $(date)" | tee -a "$OUTFILE"
|
||||
echo "" | tee -a "$OUTFILE"
|
||||
|
||||
for ctk in "${KV_TYPES[@]}"; do
|
||||
for p in "${CONTEXTS[@]}"; do
|
||||
if [[ "$ctk" == "q8_0" && "$LABEL" == "smem" ]]; then
|
||||
echo "SKIP: q8_0 + smem (q8_0 unaffected by SMEM)" | tee -a "$OUTFILE"
|
||||
continue
|
||||
fi
|
||||
|
||||
depth_label="short"
|
||||
[[ $p -gt 0 ]] && depth_label="${p}"
|
||||
|
||||
echo "--- ${ctk} @ ${depth_label} ---" | tee -a "$OUTFILE"
|
||||
|
||||
ctv="$ctk"
|
||||
$LLAMA_BENCH \
|
||||
-m "$MODEL" \
|
||||
-ngl 99 -fa 1 \
|
||||
-ctk "$ctk" -ctv "$ctv" \
|
||||
-t 1 \
|
||||
-p "$p" -n 128 \
|
||||
2>&1 | tee -a "$OUTFILE"
|
||||
|
||||
echo "" | tee -a "$OUTFILE"
|
||||
done
|
||||
done
|
||||
|
||||
echo "=== Done: ${LABEL} ===" | tee -a "$OUTFILE"
|
||||
echo "Results saved to: $OUTFILE"
|
||||
Executable
+80
@@ -0,0 +1,80 @@
|
||||
#!/bin/bash
|
||||
# TurboQuant quality + speed gate — run BEFORE pushing any changes
|
||||
# Checks: (1) perplexity within 5% of q8_0, (2) context scaling ratio > 0.95
|
||||
#
|
||||
# Usage: bash scripts/turbo-quality-gate.sh
|
||||
# Exit 0 = PASS, Exit 1 = FAIL
|
||||
|
||||
set -e
|
||||
|
||||
LLAMA=${LLAMA:-~/local_llms/llama.cpp/build-turbo/bin}
|
||||
MODEL=${MODEL:-~/local_llms/models/Qwen3.5-35B-A3B-Q8_0.gguf}
|
||||
WIKI=${WIKI:-~/local_llms/llama.cpp/wikitext-2-raw/wiki.test.raw}
|
||||
|
||||
if [ ! -f "$WIKI" ]; then
|
||||
echo "Downloading wikitext-2..."
|
||||
bash ~/local_llms/llama.cpp/scripts/get-wikitext-2.sh
|
||||
fi
|
||||
|
||||
FAIL=0
|
||||
|
||||
echo "========================================"
|
||||
echo " TurboQuant Quality + Speed Gate"
|
||||
echo "========================================"
|
||||
echo ""
|
||||
|
||||
# --- Test 1: Perplexity ---
|
||||
echo "[1/2] Running perplexity check (8 chunks)..."
|
||||
PPL_TURBO=$($LLAMA/llama-perplexity -m $MODEL -f $WIKI -c 512 -ctk turbo3 -ctv turbo3 -fa on --chunks 8 -ngl 99 2>&1 | grep "Final" | grep -oE 'PPL = [0-9.]+' | grep -oE '[0-9.]+')
|
||||
|
||||
if [ -z "$PPL_TURBO" ]; then
|
||||
echo " FAIL: Could not get turbo3 perplexity (crash or timeout)"
|
||||
FAIL=1
|
||||
else
|
||||
BASELINE_PPL=6.111
|
||||
MAX_PPL=$(echo "$BASELINE_PPL * 1.05" | bc)
|
||||
PPL_OK=$(echo "$PPL_TURBO < $MAX_PPL" | bc)
|
||||
if [ "$PPL_OK" -eq 1 ]; then
|
||||
echo " PASS: turbo3 PPL = $PPL_TURBO (< $MAX_PPL, within 5% of q8_0 $BASELINE_PPL)"
|
||||
else
|
||||
echo " FAIL: turbo3 PPL = $PPL_TURBO (> $MAX_PPL, exceeds 5% threshold)"
|
||||
FAIL=1
|
||||
fi
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# --- Test 2: Context Scaling ---
|
||||
echo "[2/2] Running context scaling check (4K prefill)..."
|
||||
TURBO_TPS=$($LLAMA/llama-perplexity -m $MODEL -f $WIKI -c 4096 -ctk turbo3 -ctv turbo3 -fa on --chunks 4 -ngl 99 2>&1 | grep "prompt eval" | grep -oE '[0-9.]+ tokens per second' | grep -oE '[0-9.]+')
|
||||
Q8_TPS=$($LLAMA/llama-perplexity -m $MODEL -f $WIKI -c 4096 -ctk q8_0 -ctv q8_0 -fa on --chunks 4 -ngl 99 2>&1 | grep "prompt eval" | grep -oE '[0-9.]+ tokens per second' | grep -oE '[0-9.]+')
|
||||
|
||||
if [ -z "$TURBO_TPS" ] || [ -z "$Q8_TPS" ]; then
|
||||
echo " FAIL: Could not measure speed (crash or timeout)"
|
||||
echo " turbo3=$TURBO_TPS q8_0=$Q8_TPS"
|
||||
FAIL=1
|
||||
else
|
||||
RATIO=$(echo "scale=4; $TURBO_TPS / $Q8_TPS" | bc)
|
||||
RATIO_OK=$(echo "$RATIO > 0.95" | bc)
|
||||
if [ "$RATIO_OK" -eq 1 ]; then
|
||||
echo " PASS: turbo3/q8_0 = ${RATIO}x at 4K context (> 0.95 threshold)"
|
||||
echo " turbo3 = $TURBO_TPS tok/s, q8_0 = $Q8_TPS tok/s"
|
||||
else
|
||||
echo " FAIL: turbo3/q8_0 = ${RATIO}x at 4K context (< 0.95 threshold)"
|
||||
echo " turbo3 = $TURBO_TPS tok/s, q8_0 = $Q8_TPS tok/s"
|
||||
echo " Context scaling regression detected!"
|
||||
FAIL=1
|
||||
fi
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# --- Summary ---
|
||||
echo "========================================"
|
||||
if [ "$FAIL" -eq 0 ]; then
|
||||
echo " ALL CHECKS PASSED"
|
||||
echo "========================================"
|
||||
exit 0
|
||||
else
|
||||
echo " CHECKS FAILED — DO NOT PUSH"
|
||||
echo "========================================"
|
||||
exit 1
|
||||
fi
|
||||
+27
-2
@@ -3397,8 +3397,16 @@ llama_context * llama_init_from_model(
|
||||
|
||||
if (params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED && ggml_is_quantized(params.type_k)) {
|
||||
const uint32_t blck_size = ggml_blck_size(params.type_k);
|
||||
const bool k_is_turbo = (params.type_k == GGML_TYPE_TURBO2_0 ||
|
||||
params.type_k == GGML_TYPE_TURBO3_0 ||
|
||||
params.type_k == GGML_TYPE_TURBO4_0);
|
||||
for (uint32_t il = 0; il < model->hparams.n_layer; ++il) {
|
||||
if (model->hparams.n_embd_head_k(il) % blck_size != 0) {
|
||||
uint32_t head_k = model->hparams.n_embd_head_k(il);
|
||||
// Turbo types zero-pad heads to next multiple of 128 in llama-kv-cache.cpp
|
||||
if (k_is_turbo && head_k % 128 != 0) {
|
||||
head_k = ((head_k + 127) / 128) * 128;
|
||||
}
|
||||
if (head_k % blck_size != 0) {
|
||||
LLAMA_LOG_ERROR("%s: K cache type %s with block size %u does not divide n_embd_head_k=%u\n",
|
||||
__func__, ggml_type_name(params.type_k), blck_size, model->hparams.n_embd_head_k(il));
|
||||
return nullptr;
|
||||
@@ -3408,8 +3416,17 @@ llama_context * llama_init_from_model(
|
||||
|
||||
if (params.flash_attn_type != LLAMA_FLASH_ATTN_TYPE_DISABLED && ggml_is_quantized(params.type_v)) {
|
||||
const uint32_t blck_size = ggml_blck_size(params.type_v);
|
||||
const bool v_is_turbo = (params.type_v == GGML_TYPE_TURBO2_0 ||
|
||||
params.type_v == GGML_TYPE_TURBO3_0 ||
|
||||
params.type_v == GGML_TYPE_TURBO4_0);
|
||||
const bool is_mla = model->hparams.is_mla();
|
||||
for (uint32_t il = 0; il < model->hparams.n_layer; ++il) {
|
||||
if (model->hparams.n_embd_head_v(il) % blck_size != 0) {
|
||||
uint32_t head_v = model->hparams.n_embd_head_v(il);
|
||||
// Turbo types zero-pad; MLA has no separate V cache (V = view of K)
|
||||
if (v_is_turbo && !is_mla && head_v % 128 != 0) {
|
||||
head_v = ((head_v + 127) / 128) * 128;
|
||||
}
|
||||
if (head_v % blck_size != 0) {
|
||||
LLAMA_LOG_ERROR("%s: V cache type %s with block size %u does not divide n_embd_head_v=%u\n",
|
||||
__func__, ggml_type_name(params.type_v), blck_size, model->hparams.n_embd_head_v(il));
|
||||
return nullptr;
|
||||
@@ -3417,6 +3434,14 @@ llama_context * llama_init_from_model(
|
||||
}
|
||||
}
|
||||
|
||||
// TurboQuant cache types require flash attention — auto-enable if disabled
|
||||
if (params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_DISABLED &&
|
||||
(params.type_k == GGML_TYPE_TURBO3_0 || params.type_k == GGML_TYPE_TURBO4_0 ||
|
||||
params.type_v == GGML_TYPE_TURBO3_0 || params.type_v == GGML_TYPE_TURBO4_0)) {
|
||||
LLAMA_LOG_WARN("%s: turbo cache types require flash_attn — enabling automatically\n", __func__);
|
||||
params.flash_attn_type = LLAMA_FLASH_ATTN_TYPE_ENABLED;
|
||||
}
|
||||
|
||||
if (ggml_is_quantized(params.type_v) && params.flash_attn_type == LLAMA_FLASH_ATTN_TYPE_DISABLED) {
|
||||
LLAMA_LOG_ERROR("%s: V cache quantization requires flash_attn\n", __func__);
|
||||
return nullptr;
|
||||
|
||||
@@ -1958,6 +1958,10 @@ ggml_tensor * llm_graph_context::build_attn_mha(
|
||||
k = ggml_permute(ctx0, k, 0, 2, 1, 3);
|
||||
v = ggml_permute(ctx0, v, 0, 2, 1, 3);
|
||||
|
||||
// TurboQuant note: graph-side Q rotation (pre-rotate-queries) is implemented below
|
||||
// in the flash-attn path. The VEC kernel bug (wrong Q/K stride in
|
||||
// vec_dot_fattn_vec_KQ_turbo3_0) was fixed in fattn-common.cuh to match f16 pattern.
|
||||
|
||||
ggml_tensor * cur;
|
||||
|
||||
const bool use_flash_attn = cparams.flash_attn && kq_b == nullptr;
|
||||
@@ -1984,6 +1988,20 @@ ggml_tensor * llm_graph_context::build_attn_mha(
|
||||
ggml_flash_attn_ext_add_sinks(cur, sinks);
|
||||
ggml_flash_attn_ext_set_prec (cur, GGML_PREC_F32);
|
||||
|
||||
// TurboQuant: inverse WHT on FA output when V values are WHT-rotated.
|
||||
// For MLA, V is a view of K with different ne[0] (e.g. V=512, K=576).
|
||||
// Group size must come from K (which determines the WHT rotation), not V.
|
||||
if (v->type == GGML_TYPE_TURBO3_0 || v->type == GGML_TYPE_TURBO4_0 || v->type == GGML_TYPE_TURBO2_0) {
|
||||
const bool k_is_turbo = (k->type == GGML_TYPE_TURBO3_0 || k->type == GGML_TYPE_TURBO4_0 || k->type == GGML_TYPE_TURBO2_0);
|
||||
const ggml_tensor * group_src = k_is_turbo ? k : v;
|
||||
const int turbo_group = (group_src->ne[0] % 128 == 0) ? 128 : 64;
|
||||
if (cur->ne[0] % turbo_group == 0) {
|
||||
if (!ggml_is_contiguous(cur)) { cur = ggml_cont(ctx0, cur); }
|
||||
ggml_tensor * innerq_scale = mctx ? mctx->get_turbo_innerq_scale_inv() : nullptr;
|
||||
cur = ggml_turbo_wht(ctx0, cur, 1, turbo_group, innerq_scale); // 1 = inverse
|
||||
}
|
||||
}
|
||||
|
||||
if (v_mla) {
|
||||
#if 0
|
||||
// v_mla can be applied as a matrix-vector multiplication with broadcasting across dimension 3 == n_tokens.
|
||||
@@ -2050,6 +2068,18 @@ ggml_tensor * llm_graph_context::build_attn_mha(
|
||||
ggml_tensor * kqv = ggml_mul_mat(ctx0, v, kq);
|
||||
cb(kqv, "kqv", il);
|
||||
|
||||
// TurboQuant: inverse WHT on attention output (non-FA path)
|
||||
if (v->type == GGML_TYPE_TURBO3_0 || v->type == GGML_TYPE_TURBO4_0 || v->type == GGML_TYPE_TURBO2_0) {
|
||||
const bool k_is_turbo = (k->type == GGML_TYPE_TURBO3_0 || k->type == GGML_TYPE_TURBO4_0 || k->type == GGML_TYPE_TURBO2_0);
|
||||
const ggml_tensor * group_src = k_is_turbo ? k : v;
|
||||
const int turbo_group = (group_src->ne[0] % 128 == 0) ? 128 : 64;
|
||||
if (kqv->ne[0] % turbo_group == 0) {
|
||||
if (!ggml_is_contiguous(kqv)) { kqv = ggml_cont(ctx0, kqv); }
|
||||
ggml_tensor * innerq_scale = mctx ? mctx->get_turbo_innerq_scale_inv() : nullptr;
|
||||
kqv = ggml_turbo_wht(ctx0, kqv, 1, turbo_group, innerq_scale);
|
||||
}
|
||||
}
|
||||
|
||||
// for MLA with the absorption optimization, we need to "decompress" from MQA back to MHA
|
||||
if (v_mla) {
|
||||
kqv = ggml_mul_mat(ctx0, v_mla, kqv);
|
||||
@@ -2067,6 +2097,8 @@ ggml_tensor * llm_graph_context::build_attn_mha(
|
||||
}
|
||||
}
|
||||
|
||||
// TurboQuant: graph-side inverse WHT on attention output (undoes V rotation)
|
||||
|
||||
ggml_build_forward_expand(gf, cur);
|
||||
|
||||
return cur;
|
||||
@@ -2227,12 +2259,44 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
ggml_tensor * k = mctx_cur->get_k(ctx0, il);
|
||||
ggml_tensor * v = mctx_cur->get_v(ctx0, il);
|
||||
|
||||
// TurboQuant pre-rotate-queries: O(d log d) WHT rotation via custom op
|
||||
// Q shape: (n_embd_head, n_head, n_tokens)
|
||||
// For zero-padded models (head_dim not 128-aligned), pad Q to match padded K dim first.
|
||||
if (k->type == GGML_TYPE_TURBO3_0 || k->type == GGML_TYPE_TURBO4_0 || k->type == GGML_TYPE_TURBO2_0) {
|
||||
// Pad Q per-head to next multiple of 128 if needed
|
||||
if (q->ne[0] % 128 != 0) {
|
||||
const int64_t pad = ((q->ne[0] + 127) / 128) * 128 - q->ne[0];
|
||||
q = ggml_pad(ctx0, q, pad, 0, 0, 0);
|
||||
}
|
||||
if (!ggml_is_contiguous(q)) { q = ggml_cont(ctx0, q); }
|
||||
ggml_tensor * innerq_scale = mctx_cur->get_turbo_innerq_scale_inv();
|
||||
q = ggml_turbo_wht(ctx0, q, 0, 0, innerq_scale); // 0 = forward, 0 = auto group size from q->ne[0]
|
||||
}
|
||||
|
||||
ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il);
|
||||
cb(cur, "kqv_out", il);
|
||||
|
||||
if (inp->self_v_rot) {
|
||||
cur = ggml_mul_mat_aux(ctx0, cur, inp->self_v_rot);
|
||||
}
|
||||
// TurboQuant: if V was padded, the output has padded dimensions.
|
||||
// Extract original V head_dim after inverse WHT (applied inside build_attn_mha).
|
||||
if (k->type == GGML_TYPE_TURBO3_0 || k->type == GGML_TYPE_TURBO4_0 || k->type == GGML_TYPE_TURBO2_0) {
|
||||
const int64_t orig_v_head = hparams.n_embd_head_v(il);
|
||||
// cur is 2D: (n_embd_head * n_head, n_tokens) after build_attn_mha
|
||||
const int64_t padded_v_head = v->ne[0];
|
||||
if (padded_v_head != orig_v_head) {
|
||||
// Reshape to 4D, extract original head_dim, reshape back to 2D
|
||||
const int64_t n_head_v = hparams.n_head_kv(il);
|
||||
const int64_t n_tokens_cur = cur->ne[1];
|
||||
cur = ggml_reshape_3d(ctx0, cur, padded_v_head, n_head_v, n_tokens_cur);
|
||||
// ggml_view_3d to extract first orig_v_head elements per head
|
||||
cur = ggml_view_3d(ctx0, cur, orig_v_head, n_head_v, n_tokens_cur,
|
||||
cur->nb[1], cur->nb[2], 0);
|
||||
cur = ggml_cont(ctx0, cur);
|
||||
cur = ggml_reshape_2d(ctx0, cur, orig_v_head * n_head_v, n_tokens_cur);
|
||||
}
|
||||
}
|
||||
|
||||
if (wo) {
|
||||
if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) {
|
||||
@@ -2318,9 +2382,39 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
ggml_tensor * k = mctx_cur->get_k(ctx0, il);
|
||||
ggml_tensor * v = ggml_view_4d(ctx0, k, v_cur->ne[0], k->ne[1], k->ne[2], k->ne[3], k->nb[1], k->nb[2], k->nb[3], 0);
|
||||
|
||||
// TurboQuant: pre-rotate Q for K-only (MLA) attention
|
||||
// For zero-padded models, pad Q to match padded K dim first.
|
||||
if (k->type == GGML_TYPE_TURBO3_0 || k->type == GGML_TYPE_TURBO4_0 || k->type == GGML_TYPE_TURBO2_0) {
|
||||
// Pad Q per-head to next multiple of 128 if needed
|
||||
if (q->ne[0] % 128 != 0) {
|
||||
const int64_t pad = ((q->ne[0] + 127) / 128) * 128 - q->ne[0];
|
||||
q = ggml_pad(ctx0, q, pad, 0, 0, 0);
|
||||
}
|
||||
if (!ggml_is_contiguous(q)) { q = ggml_cont(ctx0, q); }
|
||||
ggml_tensor * innerq_scale = mctx_cur->get_turbo_innerq_scale_inv();
|
||||
q = ggml_turbo_wht(ctx0, q, 0, 0, innerq_scale); // 0 = forward, 0 = auto group size
|
||||
}
|
||||
|
||||
ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il);
|
||||
cb(cur, "kqv_out", il);
|
||||
|
||||
// TurboQuant: if V was padded (MLA: V is view of K, may have padded dim),
|
||||
// extract original V head_dim after inverse WHT.
|
||||
if (k->type == GGML_TYPE_TURBO3_0 || k->type == GGML_TYPE_TURBO4_0 || k->type == GGML_TYPE_TURBO2_0) {
|
||||
const int64_t orig_v_head = v_cur->ne[0]; // original V head_dim from model
|
||||
const int64_t padded_v_head = v->ne[0]; // padded V head_dim in cache
|
||||
if (padded_v_head != orig_v_head) {
|
||||
// cur is 2D: (padded_v_head * n_head, n_tokens) after build_attn_mha
|
||||
const int64_t n_head_v = hparams.n_head_kv(il);
|
||||
const int64_t n_tokens_cur = cur->ne[1];
|
||||
cur = ggml_reshape_3d(ctx0, cur, padded_v_head, n_head_v, n_tokens_cur);
|
||||
cur = ggml_view_3d(ctx0, cur, orig_v_head, n_head_v, n_tokens_cur,
|
||||
cur->nb[1], cur->nb[2], 0);
|
||||
cur = ggml_cont(ctx0, cur);
|
||||
cur = ggml_reshape_2d(ctx0, cur, orig_v_head * n_head_v, n_tokens_cur);
|
||||
}
|
||||
}
|
||||
|
||||
if (wo) {
|
||||
if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE) {
|
||||
// GLM4 and GLM4_MOE seem to have numerical issues with half-precision accumulators
|
||||
@@ -2406,12 +2500,37 @@ ggml_tensor * llm_graph_context::build_attn(
|
||||
ggml_tensor * k = mctx_cur->get_k(ctx0, il);
|
||||
ggml_tensor * v = mctx_cur->get_v(ctx0, il);
|
||||
|
||||
// TurboQuant: pre-rotate Q for ISWA attention (pad to 128-aligned if needed)
|
||||
if (k->type == GGML_TYPE_TURBO3_0 || k->type == GGML_TYPE_TURBO4_0 || k->type == GGML_TYPE_TURBO2_0) {
|
||||
if (q->ne[0] % 128 != 0) {
|
||||
const int64_t pad = ((q->ne[0] + 127) / 128) * 128 - q->ne[0];
|
||||
q = ggml_pad(ctx0, q, pad, 0, 0, 0);
|
||||
}
|
||||
if (!ggml_is_contiguous(q)) { q = ggml_cont(ctx0, q); }
|
||||
ggml_tensor * innerq_scale = mctx_cur->get_turbo_innerq_scale_inv();
|
||||
q = ggml_turbo_wht(ctx0, q, 0, 0, innerq_scale);
|
||||
}
|
||||
|
||||
ggml_tensor * cur = build_attn_mha(q, k, v, kq_b, kq_mask, sinks, v_mla, kq_scale, il);
|
||||
cb(cur, "kqv_out", il);
|
||||
|
||||
if (v_rot) {
|
||||
cur = ggml_mul_mat_aux(ctx0, cur, v_rot);
|
||||
}
|
||||
// TurboQuant: if V was padded, extract original V head_dim after inverse WHT
|
||||
if (k->type == GGML_TYPE_TURBO3_0 || k->type == GGML_TYPE_TURBO4_0 || k->type == GGML_TYPE_TURBO2_0) {
|
||||
const int64_t orig_v_head = hparams.n_embd_head_v(il);
|
||||
const int64_t padded_v_head = v->ne[0];
|
||||
if (padded_v_head != orig_v_head) {
|
||||
const int64_t n_head_v = hparams.n_head_kv(il);
|
||||
const int64_t n_tokens_cur = cur->ne[1];
|
||||
cur = ggml_reshape_3d(ctx0, cur, padded_v_head, n_head_v, n_tokens_cur);
|
||||
cur = ggml_view_3d(ctx0, cur, orig_v_head, n_head_v, n_tokens_cur,
|
||||
cur->nb[1], cur->nb[2], 0);
|
||||
cur = ggml_cont(ctx0, cur);
|
||||
cur = ggml_reshape_2d(ctx0, cur, orig_v_head * n_head_v, n_tokens_cur);
|
||||
}
|
||||
}
|
||||
|
||||
if (wo) {
|
||||
cur = build_lora_mm(wo, cur, wo_s);
|
||||
|
||||
+277
-26
@@ -73,6 +73,25 @@ static ggml_tensor * ggml_mul_mat_aux(
|
||||
return res;
|
||||
}
|
||||
|
||||
// InnerQ: cross-TU shared state for CUDA per-channel equalization.
|
||||
// These are defined in ggml-cuda/turbo-innerq.cu (when CUDA is enabled).
|
||||
// When CUDA is not available, we provide stub implementations.
|
||||
#ifndef INNERQ_MAX_CHANNELS
|
||||
#define INNERQ_MAX_CHANNELS 128
|
||||
#endif
|
||||
|
||||
#ifdef GGML_USE_CUDA
|
||||
extern bool g_innerq_finalized;
|
||||
extern float g_innerq_scale_inv_host[INNERQ_MAX_CHANNELS];
|
||||
extern bool turbo_innerq_needs_tensor_update(void);
|
||||
extern void turbo_innerq_mark_tensor_updated(void);
|
||||
#else
|
||||
static bool g_innerq_finalized = false;
|
||||
static float g_innerq_scale_inv_host[INNERQ_MAX_CHANNELS] = {};
|
||||
static bool turbo_innerq_needs_tensor_update(void) { return false; }
|
||||
static void turbo_innerq_mark_tensor_updated(void) {}
|
||||
#endif
|
||||
|
||||
//
|
||||
// llama_kv_cache
|
||||
//
|
||||
@@ -111,7 +130,8 @@ llama_kv_cache::llama_kv_cache(
|
||||
auto it = ctx_map.find(buft);
|
||||
if (it == ctx_map.end()) {
|
||||
ggml_init_params params = {
|
||||
/*.mem_size =*/ size_t(2u*(1 + n_stream)*n_layer_kv*ggml_tensor_overhead()),
|
||||
// +3 for turbo rotation matrices (turbo_rotation + turbo_rotation_inv + turbo_innerq_scale_inv)
|
||||
/*.mem_size =*/ size_t((2u*(1 + n_stream)*n_layer_kv + 3)*ggml_tensor_overhead()),
|
||||
/*.mem_buffer =*/ NULL,
|
||||
/*.no_alloc =*/ true,
|
||||
};
|
||||
@@ -204,11 +224,107 @@ llama_kv_cache::llama_kv_cache(
|
||||
throw std::runtime_error("failed to create ggml context for kv cache");
|
||||
}
|
||||
|
||||
// TurboQuant zero-padding: for models with non-128-aligned head_dim (e.g. DeepSeek
|
||||
// head_dim_k=192), pad each head to the next multiple of 128. The padded zeros don't
|
||||
// affect dot products since WHT preserves inner products:
|
||||
// <WHT(Q_padded), WHT(K_padded)> = <Q_padded, K_padded> = <Q, K> + <0, 0> = <Q, K>
|
||||
const uint32_t n_embd_head_k = hparams.n_embd_head_k(il);
|
||||
|
||||
|
||||
const bool has_k = true;
|
||||
const bool has_v = !is_mla;
|
||||
|
||||
ggml_tensor * k = has_k ? ggml_new_tensor_3d(ctx, type_k, n_embd_k_gqa, kv_size, n_stream) : nullptr;
|
||||
ggml_tensor * v = has_v ? ggml_new_tensor_3d(ctx, type_v, n_embd_v_gqa, kv_size, n_stream) : nullptr;
|
||||
// Layer-adaptive: use higher precision for quality-sensitive layers
|
||||
// Config: TURBO_LAYER_ADAPTIVE env var controls the strategy
|
||||
// 0 = uniform (default)
|
||||
// 1 = q8_0 K+V for first+last 4 layers
|
||||
// 2 = q8_0 K+V for last 8 layers
|
||||
// 5 = Boundary V: first2+last2 V=turbo4, rest V=turbo2 (K unchanged)
|
||||
// 6 = V-only: last 8 V=turbo4, rest V=turbo2 (K unchanged)
|
||||
// 7 = Boundary V (recommended): first2+last2 V=q8_0, rest V=turbo2 (K unchanged)
|
||||
ggml_type layer_type_k = type_k;
|
||||
ggml_type layer_type_v = type_v;
|
||||
{
|
||||
static const int adaptive_mode = [&]() {
|
||||
const char * env = getenv("TURBO_LAYER_ADAPTIVE");
|
||||
if (env) {
|
||||
int mode = atoi(env);
|
||||
if (mode > 0) {
|
||||
LLAMA_LOG_INFO("llama_kv_cache: layer-adaptive mode %d enabled (env)\n", mode);
|
||||
}
|
||||
return mode;
|
||||
}
|
||||
// Auto-enable Boundary V (mode 7) when V is turbo2
|
||||
if (type_v == GGML_TYPE_TURBO2_0 && hparams.n_layer >= 8) {
|
||||
LLAMA_LOG_INFO("llama_kv_cache: Boundary V auto-enabled for turbo2-V (opt-out: TURBO_LAYER_ADAPTIVE=0)\n");
|
||||
return 7;
|
||||
}
|
||||
return 0;
|
||||
}();
|
||||
const bool is_turbo = (type_k == GGML_TYPE_TURBO3_0 || type_k == GGML_TYPE_TURBO4_0 || type_k == GGML_TYPE_TURBO2_0);
|
||||
const bool v_is_turbo = (type_v == GGML_TYPE_TURBO3_0 || type_v == GGML_TYPE_TURBO4_0 || type_v == GGML_TYPE_TURBO2_0);
|
||||
const uint32_t n_layer = hparams.n_layer;
|
||||
if (adaptive_mode == 1 && is_turbo && n_layer >= 8) {
|
||||
if (il < 4 || il >= n_layer - 4) {
|
||||
layer_type_k = GGML_TYPE_Q8_0;
|
||||
layer_type_v = GGML_TYPE_Q8_0;
|
||||
}
|
||||
} else if (adaptive_mode == 2 && is_turbo && n_layer >= 8) {
|
||||
if (il >= n_layer - 8) {
|
||||
layer_type_k = GGML_TYPE_Q8_0;
|
||||
layer_type_v = GGML_TYPE_Q8_0;
|
||||
}
|
||||
} else if (adaptive_mode == 5 && v_is_turbo && n_layer >= 8) {
|
||||
// Boundary V (turbo4 boundaries): first2+last2 V=turbo4, rest V=turbo2
|
||||
const bool is_boundary = (il < 2 || il >= n_layer - 2);
|
||||
layer_type_v = is_boundary ? GGML_TYPE_TURBO4_0 : GGML_TYPE_TURBO2_0;
|
||||
if (il == 0) {
|
||||
LLAMA_LOG_INFO("llama_kv_cache: Boundary V mode 5: first2+last2 V=turbo4, rest V=turbo2\n");
|
||||
}
|
||||
} else if (adaptive_mode == 6 && v_is_turbo && n_layer >= 8) {
|
||||
// V-only: last 8 V=turbo4, rest V=turbo2
|
||||
layer_type_v = (il >= n_layer - 8) ? GGML_TYPE_TURBO4_0 : GGML_TYPE_TURBO2_0;
|
||||
if (il == 0) {
|
||||
LLAMA_LOG_INFO("llama_kv_cache: V-only LA mode 6: last8 V=turbo4, rest V=turbo2\n");
|
||||
}
|
||||
} else if (adaptive_mode == 7 && v_is_turbo && n_layer >= 8) {
|
||||
// Boundary V (recommended): first2+last2 V=q8_0, rest V=turbo2
|
||||
const bool is_boundary = (il < 2 || il >= n_layer - 2);
|
||||
layer_type_v = is_boundary ? GGML_TYPE_Q8_0 : GGML_TYPE_TURBO2_0;
|
||||
if (il == 0) {
|
||||
LLAMA_LOG_INFO("llama_kv_cache: Boundary V mode 7: first2+last2 V=q8_0, rest V=turbo2\n");
|
||||
}
|
||||
}
|
||||
}
|
||||
// For turbo types, pad K head_dim to next multiple of 128 for full WHT groups
|
||||
uint32_t n_embd_k_gqa_eff = n_embd_k_gqa;
|
||||
const bool k_is_turbo = (layer_type_k == GGML_TYPE_TURBO3_0 || layer_type_k == GGML_TYPE_TURBO4_0 || layer_type_k == GGML_TYPE_TURBO2_0);
|
||||
if (k_is_turbo && n_embd_head_k % 128 != 0) {
|
||||
const uint32_t padded_head_k = ((n_embd_head_k + 127) / 128) * 128;
|
||||
const uint32_t n_head_kv = n_embd_k_gqa / n_embd_head_k;
|
||||
n_embd_k_gqa_eff = n_head_kv * padded_head_k;
|
||||
if (il == 0) {
|
||||
LLAMA_LOG_INFO("%s: turbo zero-padding K head_dim %u -> %u (cache %u -> %u)\n",
|
||||
__func__, n_embd_head_k, padded_head_k, n_embd_k_gqa, n_embd_k_gqa_eff);
|
||||
}
|
||||
}
|
||||
|
||||
// For turbo types, pad V head_dim to next multiple of 128 if needed
|
||||
const uint32_t n_embd_head_v = hparams.n_embd_head_v(il);
|
||||
uint32_t n_embd_v_gqa_eff = n_embd_v_gqa;
|
||||
const bool v_is_turbo = (layer_type_v == GGML_TYPE_TURBO3_0 || layer_type_v == GGML_TYPE_TURBO4_0 || layer_type_v == GGML_TYPE_TURBO2_0);
|
||||
if (v_is_turbo && !is_mla && n_embd_head_v % 128 != 0) {
|
||||
const uint32_t padded_head_v = ((n_embd_head_v + 127) / 128) * 128;
|
||||
const uint32_t n_head_kv = n_embd_v_gqa / n_embd_head_v;
|
||||
n_embd_v_gqa_eff = n_head_kv * padded_head_v;
|
||||
if (il == 0) {
|
||||
LLAMA_LOG_INFO("%s: turbo zero-padding V head_dim %u -> %u (cache %u -> %u)\n",
|
||||
__func__, n_embd_head_v, padded_head_v, n_embd_v_gqa, n_embd_v_gqa_eff);
|
||||
}
|
||||
}
|
||||
|
||||
ggml_tensor * k = has_k ? ggml_new_tensor_3d(ctx, layer_type_k, n_embd_k_gqa_eff, kv_size, n_stream) : nullptr;
|
||||
ggml_tensor * v = has_v ? ggml_new_tensor_3d(ctx, layer_type_v, n_embd_v_gqa_eff, kv_size, n_stream) : nullptr;
|
||||
|
||||
has_k && ggml_format_name(k, "cache_k_l%d", il);
|
||||
has_v && ggml_format_name(v, "cache_v_l%d", il);
|
||||
@@ -217,13 +333,26 @@ llama_kv_cache::llama_kv_cache(
|
||||
std::vector<ggml_tensor *> v_stream;
|
||||
|
||||
for (uint32_t s = 0; s < n_stream; ++s) {
|
||||
k_stream.push_back(has_k ? ggml_view_2d(ctx, k, n_embd_k_gqa, kv_size, k->nb[1], s*k->nb[2]) : nullptr);
|
||||
v_stream.push_back(has_v ? ggml_view_2d(ctx, v, n_embd_v_gqa, kv_size, v->nb[1], s*v->nb[2]) : nullptr);
|
||||
k_stream.push_back(has_k ? ggml_view_2d(ctx, k, n_embd_k_gqa_eff, kv_size, k->nb[1], s*k->nb[2]) : nullptr);
|
||||
v_stream.push_back(has_v ? ggml_view_2d(ctx, v, n_embd_v_gqa_eff, kv_size, v->nb[1], s*v->nb[2]) : nullptr);
|
||||
}
|
||||
|
||||
map_layer_ids[il] = layers.size();
|
||||
|
||||
layers.push_back({ il, k, v, k_stream, v_stream, });
|
||||
|
||||
// TurboQuant: create rotation matrix tensors (once, shared across layers)
|
||||
if (turbo_rotation == nullptr &&
|
||||
(type_k == GGML_TYPE_TURBO3_0 || type_k == GGML_TYPE_TURBO4_0 || type_k == GGML_TYPE_TURBO2_0)) {
|
||||
turbo_rotation = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 128, 128);
|
||||
ggml_format_name(turbo_rotation, "turbo_rotation"); // R^T
|
||||
turbo_rotation_inv = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 128, 128);
|
||||
ggml_format_name(turbo_rotation_inv, "turbo_rotation_inv"); // R
|
||||
|
||||
// InnerQ: per-channel scale_inv tensor (128 floats, initialized to all 1.0)
|
||||
turbo_innerq_scale_inv = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, INNERQ_MAX_CHANNELS);
|
||||
ggml_format_name(turbo_innerq_scale_inv, "turbo_innerq_scale_inv");
|
||||
}
|
||||
}
|
||||
|
||||
if (reuse) {
|
||||
@@ -268,6 +397,28 @@ llama_kv_cache::llama_kv_cache(
|
||||
LLAMA_LOG_INFO("%s: %10s KV buffer size = %8.2f MiB\n", __func__, ggml_backend_buffer_name(buf), ggml_backend_buffer_get_size(buf)/1024.0/1024.0);
|
||||
|
||||
ggml_backend_buffer_clear(buf, 0);
|
||||
|
||||
// Fill turbo rotation matrices AFTER buffer clear (clear zeroes everything)
|
||||
if (turbo_rotation != nullptr && turbo_rotation->buffer != nullptr && !model.hparams.no_alloc) {
|
||||
#include "turbo-rotation-data.h"
|
||||
// ggml is column-major; C arrays are row-major. Storing a row-major matrix
|
||||
// into ggml implicitly transposes it. ggml_mul_mat(A, x) computes A^T @ x.
|
||||
// To get R @ q: store R^T → ggml sees (R^T)^T_col = R → mul_mat gives R @ q. Wait no —
|
||||
// store R so ggml col-major reads it as R^T, then mul_mat gives (R^T)^T = R. ✓
|
||||
// Store R for Q forward rotation, R^T for V inverse rotation
|
||||
// ggml_mul_mat(A,x) computes A@x for row-major stored A (verified by test)
|
||||
ggml_backend_tensor_set(turbo_rotation, TURBO_ROTATION_R, 0, 128 * 128 * sizeof(float));
|
||||
ggml_backend_tensor_set(turbo_rotation_inv, TURBO_ROTATION_RT, 0, 128 * 128 * sizeof(float));
|
||||
|
||||
// Initialize InnerQ scale_inv to all 1.0 (identity scaling)
|
||||
if (turbo_innerq_scale_inv != nullptr && turbo_innerq_scale_inv->buffer != nullptr) {
|
||||
float ones[INNERQ_MAX_CHANNELS];
|
||||
for (int i = 0; i < INNERQ_MAX_CHANNELS; i++) ones[i] = 1.0f;
|
||||
ggml_backend_tensor_set(turbo_innerq_scale_inv, ones, 0, INNERQ_MAX_CHANNELS * sizeof(float));
|
||||
}
|
||||
|
||||
LLAMA_LOG_INFO("%s: TurboQuant rotation matrices initialized (128x128)\n", __func__);
|
||||
}
|
||||
ctxs_bufs.emplace_back(std::move(ctx), buf);
|
||||
}
|
||||
|
||||
@@ -337,6 +488,20 @@ void llama_kv_cache::clear(bool data) {
|
||||
for (auto & [_, buf] : ctxs_bufs) {
|
||||
ggml_backend_buffer_clear(buf.get(), 0);
|
||||
}
|
||||
|
||||
// Re-initialize turbo rotation matrices after buffer clear (clear zeroes everything)
|
||||
if (turbo_rotation != nullptr && turbo_rotation->buffer != nullptr) {
|
||||
#include "turbo-rotation-data.h"
|
||||
ggml_backend_tensor_set(turbo_rotation, TURBO_ROTATION_R, 0, 128 * 128 * sizeof(float));
|
||||
ggml_backend_tensor_set(turbo_rotation_inv, TURBO_ROTATION_RT, 0, 128 * 128 * sizeof(float));
|
||||
|
||||
// Re-initialize InnerQ scale_inv to all 1.0
|
||||
if (turbo_innerq_scale_inv != nullptr && turbo_innerq_scale_inv->buffer != nullptr) {
|
||||
float ones[INNERQ_MAX_CHANNELS];
|
||||
for (int i = 0; i < INNERQ_MAX_CHANNELS; i++) ones[i] = 1.0f;
|
||||
ggml_backend_tensor_set(turbo_innerq_scale_inv, ones, 0, INNERQ_MAX_CHANNELS * sizeof(float));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1150,13 +1315,24 @@ ggml_tensor * llama_kv_cache::get_k(ggml_context * ctx, int32_t il, uint32_t n_k
|
||||
const uint64_t kv_size = get_size();
|
||||
const uint64_t n_embd_k_gqa = k->ne[0];
|
||||
|
||||
// For turbo-padded caches, n_embd_k_gqa may be larger than hparams value
|
||||
const bool k_is_turbo = (k->type == GGML_TYPE_TURBO3_0 || k->type == GGML_TYPE_TURBO4_0 || k->type == GGML_TYPE_TURBO2_0);
|
||||
if (k_is_turbo) {
|
||||
assert(n_embd_k_gqa >= hparams.n_embd_k_gqa(il));
|
||||
} else {
|
||||
assert(n_embd_k_gqa == hparams.n_embd_k_gqa(il));
|
||||
}
|
||||
|
||||
// Use padded head_dim for turbo types so the full padded data is returned
|
||||
const uint32_t head_k = hparams.n_embd_head_k(il);
|
||||
const uint32_t head_k_eff = (k_is_turbo && head_k % 128 != 0)
|
||||
? ((head_k + 127) / 128) * 128 : head_k;
|
||||
|
||||
const uint32_t ns = sinfo.s1 - sinfo.s0 + 1;
|
||||
|
||||
return ggml_view_4d(ctx, k,
|
||||
hparams.n_embd_head_k(il), hparams.n_head_kv(il), n_kv, ns,
|
||||
ggml_row_size(k->type, hparams.n_embd_head_k(il)),
|
||||
head_k_eff, hparams.n_head_kv(il), n_kv, ns,
|
||||
ggml_row_size(k->type, head_k_eff),
|
||||
ggml_row_size(k->type, n_embd_k_gqa),
|
||||
ggml_row_size(k->type, n_embd_k_gqa*kv_size),
|
||||
ggml_row_size(k->type, n_embd_k_gqa*kv_size)*sinfo.s0);
|
||||
@@ -1170,16 +1346,22 @@ ggml_tensor * llama_kv_cache::get_v(ggml_context * ctx, int32_t il, uint32_t n_k
|
||||
const uint64_t kv_size = get_size();
|
||||
const uint64_t n_embd_v_gqa = v->ne[0];
|
||||
|
||||
// [TAG_V_CACHE_VARIABLE]
|
||||
// [TAG_V_CACHE_VARIABLE] — for turbo-padded V, cache may be larger
|
||||
assert(n_embd_v_gqa >= hparams.n_embd_v_gqa(il));
|
||||
|
||||
// Use padded head_dim for turbo types
|
||||
const bool v_is_turbo = (v->type == GGML_TYPE_TURBO3_0 || v->type == GGML_TYPE_TURBO4_0 || v->type == GGML_TYPE_TURBO2_0);
|
||||
const uint32_t head_v = hparams.n_embd_head_v(il);
|
||||
const uint32_t head_v_eff = (v_is_turbo && head_v % 128 != 0)
|
||||
? ((head_v + 127) / 128) * 128 : head_v;
|
||||
|
||||
const uint32_t ns = sinfo.s1 - sinfo.s0 + 1;
|
||||
|
||||
if (!v_trans) {
|
||||
// note: v->nb[1] <= v->nb[2]
|
||||
return ggml_view_4d(ctx, v,
|
||||
hparams.n_embd_head_v(il), hparams.n_head_kv(il), n_kv, ns,
|
||||
ggml_row_size(v->type, hparams.n_embd_head_v(il)), // v->nb[1]
|
||||
head_v_eff, hparams.n_head_kv(il), n_kv, ns,
|
||||
ggml_row_size(v->type, head_v_eff), // v->nb[1]
|
||||
ggml_row_size(v->type, n_embd_v_gqa), // v->nb[2]
|
||||
ggml_row_size(v->type, n_embd_v_gqa*kv_size), // v->nb[3]
|
||||
ggml_row_size(v->type, n_embd_v_gqa*kv_size)*sinfo.s0);
|
||||
@@ -1187,8 +1369,8 @@ ggml_tensor * llama_kv_cache::get_v(ggml_context * ctx, int32_t il, uint32_t n_k
|
||||
|
||||
// note: v->nb[1] > v->nb[2]
|
||||
return ggml_view_4d(ctx, v,
|
||||
n_kv, hparams.n_head_kv(il), hparams.n_embd_head_v(il), ns,
|
||||
ggml_row_size(v->type, kv_size*hparams.n_embd_head_v(il)), // v->nb[1]
|
||||
n_kv, hparams.n_head_kv(il), head_v_eff, ns,
|
||||
ggml_row_size(v->type, kv_size*head_v_eff), // v->nb[1]
|
||||
ggml_row_size(v->type, kv_size), // v->nb[2]
|
||||
ggml_row_size(v->type, kv_size*n_embd_v_gqa), // v->nb[3]
|
||||
ggml_row_size(v->type, kv_size*n_embd_v_gqa)*sinfo.s0);
|
||||
@@ -1201,11 +1383,22 @@ ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggm
|
||||
|
||||
ggml_tensor * k = layers[ikv].k;
|
||||
|
||||
const int64_t n_embd_head = k_cur->ne[0];
|
||||
int64_t n_embd_head = k_cur->ne[0];
|
||||
const int64_t n_head = k_cur->ne[1];
|
||||
const int64_t n_tokens = k_cur->ne[2];
|
||||
|
||||
const int64_t n_embd_gqa = n_embd_head*n_head;
|
||||
// Turbo zero-padding: pad each head to next multiple of 128 before merging dims.
|
||||
// k_cur shape here is (n_embd_head, n_head, n_tokens).
|
||||
// ggml_pad pads ne[0] with zeros — exactly what we need per-head.
|
||||
const bool k_is_turbo = (k->type == GGML_TYPE_TURBO3_0 || k->type == GGML_TYPE_TURBO4_0 || k->type == GGML_TYPE_TURBO2_0);
|
||||
const bool k_needs_pad = k_is_turbo && (n_embd_head % 128 != 0);
|
||||
if (k_needs_pad) {
|
||||
const int64_t pad_amount = ((n_embd_head + 127) / 128) * 128 - n_embd_head;
|
||||
k_cur = ggml_pad(ctx, k_cur, pad_amount, 0, 0, 0);
|
||||
n_embd_head = k_cur->ne[0]; // now 128-aligned
|
||||
}
|
||||
|
||||
int64_t n_embd_gqa = n_embd_head * n_head;
|
||||
|
||||
// we can merge dims 0 and 1
|
||||
// TODO: add ggml helper function for this?
|
||||
@@ -1226,7 +1419,16 @@ ggml_tensor * llama_kv_cache::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggm
|
||||
}
|
||||
|
||||
// store the current K values into the cache
|
||||
return ggml_set_rows(ctx, k, k_cur, k_idxs);
|
||||
ggml_tensor * result = ggml_set_rows(ctx, k, k_cur, k_idxs);
|
||||
|
||||
// For turbo: store WHT group size in op_params so the CUDA kernel knows.
|
||||
// With zero-padding, all groups are always full 128-element WHT groups.
|
||||
if (k_is_turbo) {
|
||||
int32_t wht_group = 128; // always 128 with padding
|
||||
memcpy(result->op_params, &wht_group, sizeof(int32_t));
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
ggml_tensor * llama_kv_cache::cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggml_tensor * v_idxs, int32_t il, const slot_info & sinfo) const {
|
||||
@@ -1236,11 +1438,20 @@ ggml_tensor * llama_kv_cache::cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggm
|
||||
|
||||
auto * v = layers[ikv].v;
|
||||
|
||||
const int64_t n_embd_head = v_cur->ne[0];
|
||||
int64_t n_embd_head = v_cur->ne[0];
|
||||
const int64_t n_head = v_cur->ne[1];
|
||||
const int64_t n_tokens = v_cur->ne[2];
|
||||
|
||||
const int64_t n_embd_gqa = n_embd_head*n_head;
|
||||
// Turbo zero-padding: pad V head_dim to next multiple of 128
|
||||
const bool v_is_turbo = (v->type == GGML_TYPE_TURBO3_0 || v->type == GGML_TYPE_TURBO4_0 || v->type == GGML_TYPE_TURBO2_0);
|
||||
const bool v_needs_pad = v_is_turbo && (n_embd_head % 128 != 0);
|
||||
if (v_needs_pad) {
|
||||
const int64_t pad_amount = ((n_embd_head + 127) / 128) * 128 - n_embd_head;
|
||||
v_cur = ggml_pad(ctx, v_cur, pad_amount, 0, 0, 0);
|
||||
n_embd_head = v_cur->ne[0]; // now 128-aligned
|
||||
}
|
||||
|
||||
int64_t n_embd_gqa = n_embd_head * n_head;
|
||||
|
||||
// we can merge dims 0 and 1
|
||||
GGML_ASSERT(ggml_row_size(v_cur->type, n_embd_head) == v_cur->nb[1]);
|
||||
@@ -1261,7 +1472,13 @@ ggml_tensor * llama_kv_cache::cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggm
|
||||
v = ggml_reshape_2d(ctx, v, n_embd_gqa, kv_size*n_stream);
|
||||
}
|
||||
|
||||
return ggml_set_rows(ctx, v, v_cur, v_idxs);
|
||||
ggml_tensor * result = ggml_set_rows(ctx, v, v_cur, v_idxs);
|
||||
// With zero-padding, all groups are always full 128-element WHT groups
|
||||
if (v_is_turbo) {
|
||||
int32_t wht_group = 128; // always 128 with padding
|
||||
memcpy(result->op_params, &wht_group, sizeof(int32_t));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
if (ggml_row_size(v_cur->type, n_embd_gqa) == v_cur->nb[2]) {
|
||||
@@ -1980,10 +2197,11 @@ void llama_kv_cache::state_write_data(llama_io_write_i & io, const cell_ranges_t
|
||||
for (const auto & layer : layers) {
|
||||
const uint32_t il = layer.il;
|
||||
|
||||
const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(il);
|
||||
|
||||
auto * k = layer.k_stream[cr.strm];
|
||||
|
||||
// Use actual tensor width (may be padded for turbo types: e.g. 576→640)
|
||||
const uint32_t n_embd_k_gqa = (uint32_t) k->ne[0];
|
||||
|
||||
// Write key type
|
||||
const int32_t k_type_i = (int32_t) k->type;
|
||||
io.write(&k_type_i, sizeof(k_type_i));
|
||||
@@ -2004,13 +2222,14 @@ void llama_kv_cache::state_write_data(llama_io_write_i & io, const cell_ranges_t
|
||||
for (const auto & layer : layers) {
|
||||
const uint32_t il = layer.il;
|
||||
|
||||
const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(il);
|
||||
|
||||
auto * v = layer.v_stream[cr.strm];
|
||||
if (!v) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Use actual tensor width (may be padded for turbo types)
|
||||
const uint32_t n_embd_v_gqa = (uint32_t) v->ne[0];
|
||||
|
||||
// Write value type
|
||||
const int32_t v_type_i = (int32_t) v->type;
|
||||
io.write(&v_type_i, sizeof(v_type_i));
|
||||
@@ -2212,10 +2431,11 @@ bool llama_kv_cache::state_read_data(llama_io_read_i & io, uint32_t strm, uint32
|
||||
for (const auto & layer : layers) {
|
||||
const uint32_t il = layer.il;
|
||||
|
||||
const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(il);
|
||||
|
||||
auto * k = layer.k_stream[strm];
|
||||
|
||||
// Use actual tensor width (may be padded for turbo types)
|
||||
const uint32_t n_embd_k_gqa = (uint32_t) k->ne[0];
|
||||
|
||||
// Read type of key
|
||||
int32_t k_type_i_ref;
|
||||
io.read(&k_type_i_ref, sizeof(k_type_i_ref));
|
||||
@@ -2252,13 +2472,14 @@ bool llama_kv_cache::state_read_data(llama_io_read_i & io, uint32_t strm, uint32
|
||||
for (const auto & layer : layers) {
|
||||
const uint32_t il = layer.il;
|
||||
|
||||
const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(il);
|
||||
|
||||
auto * v = layer.v_stream[strm];
|
||||
if (!v) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Use actual tensor width (may be padded for turbo types)
|
||||
const uint32_t n_embd_v_gqa = (uint32_t) v->ne[0];
|
||||
|
||||
// Read type of value
|
||||
int32_t v_type_i_ref;
|
||||
io.read(&v_type_i_ref, sizeof(v_type_i_ref));
|
||||
@@ -2416,6 +2637,16 @@ bool llama_kv_cache_context::apply() {
|
||||
kv->apply_ubatch(sinfos[i_cur], ubatches[i_cur]);
|
||||
n_kv = kv->get_n_kv(sinfos[i_cur]);
|
||||
|
||||
// InnerQ: check if CUDA calibration finalized and tensor needs update
|
||||
if (kv->get_turbo_innerq_scale_inv() != nullptr && turbo_innerq_needs_tensor_update()) {
|
||||
ggml_tensor * t = kv->get_turbo_innerq_scale_inv();
|
||||
if (t->buffer != nullptr) {
|
||||
ggml_backend_tensor_set(t, g_innerq_scale_inv_host, 0, INNERQ_MAX_CHANNELS * sizeof(float));
|
||||
turbo_innerq_mark_tensor_updated();
|
||||
LLAMA_LOG_INFO("%s: InnerQ scale_inv tensor updated\n", __func__);
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -2449,6 +2680,26 @@ ggml_tensor * llama_kv_cache_context::get_v(ggml_context * ctx, int32_t il) cons
|
||||
return kv->get_v(ctx, il, n_kv, sinfos[i_cur]);
|
||||
}
|
||||
|
||||
ggml_tensor * llama_kv_cache_context::get_turbo_rotation() const {
|
||||
return kv->get_turbo_rotation();
|
||||
}
|
||||
|
||||
ggml_tensor * llama_kv_cache_context::get_turbo_rotation_inv() const {
|
||||
return kv->get_turbo_rotation_inv();
|
||||
}
|
||||
|
||||
ggml_tensor * llama_kv_cache_context::get_turbo_rot_forward() const {
|
||||
return kv->get_turbo_rotation();
|
||||
}
|
||||
|
||||
ggml_tensor * llama_kv_cache_context::get_turbo_rot_inverse() const {
|
||||
return kv->get_turbo_rotation_inv();
|
||||
}
|
||||
|
||||
ggml_tensor * llama_kv_cache_context::get_turbo_innerq_scale_inv() const {
|
||||
return kv->get_turbo_innerq_scale_inv();
|
||||
}
|
||||
|
||||
ggml_tensor * llama_kv_cache_context::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il) const {
|
||||
return kv->cpy_k(ctx, k_cur, k_idxs, il, sinfos[i_cur]);
|
||||
}
|
||||
|
||||
@@ -165,6 +165,15 @@ public:
|
||||
ggml_tensor * get_k(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const;
|
||||
ggml_tensor * get_v(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const;
|
||||
|
||||
// TurboQuant: get rotation matrices (stored as row-major C arrays)
|
||||
// turbo_rotation = R (forward rotation, for Q pre-rotate-queries)
|
||||
// turbo_rotation_inv = R^T = R^{-1} (inverse rotation, for V output un-rotation)
|
||||
ggml_tensor * get_turbo_rotation() const { return turbo_rotation; }
|
||||
ggml_tensor * get_turbo_rotation_inv() const { return turbo_rotation_inv; }
|
||||
|
||||
// TurboQuant InnerQ: per-channel scale_inv for Q/V equalization
|
||||
ggml_tensor * get_turbo_innerq_scale_inv() const { return turbo_innerq_scale_inv; }
|
||||
|
||||
// store k_cur and v_cur in the cache based on the provided head location
|
||||
ggml_tensor * cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il, const slot_info & sinfo) const;
|
||||
ggml_tensor * cpy_v(ggml_context * ctx, ggml_tensor * v_cur, ggml_tensor * v_idxs, int32_t il, const slot_info & sinfo) const;
|
||||
@@ -270,6 +279,13 @@ private:
|
||||
|
||||
std::vector<kv_layer> layers;
|
||||
|
||||
// TurboQuant rotation matrices (128x128, row-major stored)
|
||||
ggml_tensor * turbo_rotation = nullptr; // R (forward rotation)
|
||||
ggml_tensor * turbo_rotation_inv = nullptr; // R^T = R^{-1} (inverse rotation)
|
||||
|
||||
// TurboQuant InnerQ: per-channel scale_inv for Q/V equalization (128 floats)
|
||||
ggml_tensor * turbo_innerq_scale_inv = nullptr;
|
||||
|
||||
// model layer id -> KV cache layer id
|
||||
std::unordered_map<int32_t, int32_t> map_layer_ids;
|
||||
|
||||
@@ -357,6 +373,17 @@ public:
|
||||
ggml_tensor * get_k(ggml_context * ctx, int32_t il) const;
|
||||
ggml_tensor * get_v(ggml_context * ctx, int32_t il) const;
|
||||
|
||||
// TurboQuant rotation accessors
|
||||
ggml_tensor * get_turbo_rotation() const;
|
||||
ggml_tensor * get_turbo_rotation_inv() const;
|
||||
|
||||
// Override virtual methods from llama_memory_context_i
|
||||
ggml_tensor * get_turbo_rot_forward() const override;
|
||||
ggml_tensor * get_turbo_rot_inverse() const override;
|
||||
|
||||
// TurboQuant InnerQ: per-channel scale_inv for Q/V equalization
|
||||
ggml_tensor * get_turbo_innerq_scale_inv() const override;
|
||||
|
||||
// store k_cur and v_cur in the cache based on the provided head location
|
||||
// note: the heads in k_cur and v_cur should be laid out contiguously in memory
|
||||
// - k_cur [n_embd_head_k, n_head_k, n_tokens]
|
||||
|
||||
@@ -265,6 +265,18 @@ const llama_kv_cache_context * llama_memory_hybrid_context::get_attn() const {
|
||||
return static_cast<const llama_kv_cache_context *>(ctx_attn.get());
|
||||
}
|
||||
|
||||
ggml_tensor * llama_memory_hybrid_context::get_turbo_rot_forward() const {
|
||||
return ctx_attn ? ctx_attn->get_turbo_rot_forward() : nullptr;
|
||||
}
|
||||
|
||||
ggml_tensor * llama_memory_hybrid_context::get_turbo_rot_inverse() const {
|
||||
return ctx_attn ? ctx_attn->get_turbo_rot_inverse() : nullptr;
|
||||
}
|
||||
|
||||
ggml_tensor * llama_memory_hybrid_context::get_turbo_innerq_scale_inv() const {
|
||||
return ctx_attn ? ctx_attn->get_turbo_innerq_scale_inv() : nullptr;
|
||||
}
|
||||
|
||||
const llama_memory_recurrent_context * llama_memory_hybrid_context::get_recr() const {
|
||||
return static_cast<const llama_memory_recurrent_context *>(ctx_recr.get());
|
||||
}
|
||||
|
||||
@@ -120,6 +120,11 @@ public:
|
||||
llama_memory_status get_status() const override;
|
||||
const llama_ubatch & get_ubatch() const override;
|
||||
|
||||
// TurboQuant: delegate to the KV cache context
|
||||
ggml_tensor * get_turbo_rot_forward() const override;
|
||||
ggml_tensor * get_turbo_rot_inverse() const override;
|
||||
ggml_tensor * get_turbo_innerq_scale_inv() const override;
|
||||
|
||||
//
|
||||
// llama_memory_hybrid_context
|
||||
//
|
||||
|
||||
@@ -62,6 +62,15 @@ struct llama_memory_context_i {
|
||||
|
||||
// get the status of the memory context - used for error handling and checking if any updates would be applied
|
||||
virtual llama_memory_status get_status() const = 0;
|
||||
|
||||
// TurboQuant: get rotation tensors for pre-rotate-queries optimization
|
||||
// Returns null for non-turbo memory types. Override in KV cache contexts.
|
||||
virtual ggml_tensor * get_turbo_rot_forward() const { return nullptr; }
|
||||
virtual ggml_tensor * get_turbo_rot_inverse() const { return nullptr; }
|
||||
|
||||
// TurboQuant InnerQ: get per-channel scale_inv tensor for Q/V equalization
|
||||
// Returns nullptr when InnerQ is not active. Override in KV cache contexts.
|
||||
virtual ggml_tensor * get_turbo_innerq_scale_inv() const { return nullptr; }
|
||||
};
|
||||
|
||||
using llama_memory_context_ptr = std::unique_ptr<llama_memory_context_i>;
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
// Pre-computed 32x32 rotation matrices for TurboQuant (group_size=32, seed=42)
|
||||
|
||||
static const float TURBO_ROTATION_R_32[1024] = {
|
||||
-1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f,
|
||||
-1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f,
|
||||
-1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f,
|
||||
1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f,
|
||||
-1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f,
|
||||
1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f,
|
||||
1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f,
|
||||
1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f,
|
||||
1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f,
|
||||
1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f,
|
||||
-1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f,
|
||||
-1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f,
|
||||
-1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f,
|
||||
-1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f,
|
||||
-1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f,
|
||||
1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f,
|
||||
-1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f,
|
||||
-1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f,
|
||||
1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f,
|
||||
-1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f,
|
||||
1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f,
|
||||
-1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f,
|
||||
1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f,
|
||||
-1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f,
|
||||
-1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f,
|
||||
1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f,
|
||||
1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f,
|
||||
1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f,
|
||||
1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f,
|
||||
1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f,
|
||||
1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f,
|
||||
1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f
|
||||
};
|
||||
|
||||
static const float TURBO_ROTATION_RT_32[1024] = {
|
||||
-1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f,
|
||||
1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f,
|
||||
-1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f,
|
||||
-1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f,
|
||||
-1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f,
|
||||
1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f,
|
||||
-1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f,
|
||||
-1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f,
|
||||
-1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f,
|
||||
1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f,
|
||||
-1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f,
|
||||
-1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f,
|
||||
-1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f,
|
||||
-1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f,
|
||||
1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f,
|
||||
-1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f,
|
||||
1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f,
|
||||
1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f,
|
||||
1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f,
|
||||
-1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f,
|
||||
1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f,
|
||||
-1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f,
|
||||
1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f,
|
||||
1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f,
|
||||
1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f,
|
||||
1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f,
|
||||
1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f,
|
||||
1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f,
|
||||
1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f,
|
||||
1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f,
|
||||
-1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f,
|
||||
-1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f, 1.76776695e-01f, 1.76776695e-01f, -1.76776695e-01f
|
||||
};
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,54 @@
|
||||
#include <stdio.h>
|
||||
#include <math.h>
|
||||
#include <string.h>
|
||||
|
||||
extern void quantize_row_turbo3_0_ref(const float * x, void * y, long long k);
|
||||
extern void dequantize_row_turbo3_0(const void * x, float * y, long long k);
|
||||
extern void quantize_row_turbo4_0_ref(const float * x, void * y, long long k);
|
||||
extern void dequantize_row_turbo4_0(const void * x, float * y, long long k);
|
||||
|
||||
int main(void) {
|
||||
const int d = 128;
|
||||
char buf[256];
|
||||
float input[128], output[128];
|
||||
float mse, cosv, ni, no;
|
||||
|
||||
printf("=== TurboQuant C Round-Trip Test ===\n\n");
|
||||
|
||||
/* Test 1: basis vector */
|
||||
memset(input, 0, sizeof(input));
|
||||
input[0] = 1.0f;
|
||||
quantize_row_turbo3_0_ref(input, buf, d);
|
||||
dequantize_row_turbo3_0(buf, output, d);
|
||||
printf("Test 1 (turbo3): e0 = [1, 0, ...]\n");
|
||||
printf(" In: [%.6f, %.6f, %.6f, %.6f]\n", input[0], input[1], input[2], input[3]);
|
||||
printf(" Out: [%.6f, %.6f, %.6f, %.6f]\n", output[0], output[1], output[2], output[3]);
|
||||
mse = cosv = ni = no = 0;
|
||||
for (int i = 0; i < d; i++) { mse += (input[i]-output[i])*(input[i]-output[i]); cosv += input[i]*output[i]; ni += input[i]*input[i]; no += output[i]*output[i]; }
|
||||
printf(" MSE=%.8f Cosine=%.6f OutNorm=%.6f\n\n", mse/d, ni > 0 && no > 0 ? cosv/sqrtf(ni)/sqrtf(no) : 0, sqrtf(no));
|
||||
|
||||
/* Test 2: large-norm vector */
|
||||
for (int i = 0; i < d; i++) input[i] = sinf(i*0.1f+0.5f) * 10.0f;
|
||||
quantize_row_turbo3_0_ref(input, buf, d);
|
||||
dequantize_row_turbo3_0(buf, output, d);
|
||||
printf("Test 2 (turbo3): sin*10\n");
|
||||
printf(" In: [%.4f, %.4f, %.4f, %.4f]\n", input[0], input[1], input[2], input[3]);
|
||||
printf(" Out: [%.4f, %.4f, %.4f, %.4f]\n", output[0], output[1], output[2], output[3]);
|
||||
mse = cosv = ni = no = 0;
|
||||
for (int i = 0; i < d; i++) { mse += (input[i]-output[i])*(input[i]-output[i]); cosv += input[i]*output[i]; ni += input[i]*input[i]; no += output[i]*output[i]; }
|
||||
printf(" MSE=%.8f Cosine=%.6f InNorm=%.2f OutNorm=%.2f\n\n", mse/d, cosv/sqrtf(ni)/sqrtf(no), sqrtf(ni), sqrtf(no));
|
||||
|
||||
/* Test 3: turbo4 */
|
||||
for (int i = 0; i < d; i++) input[i] = cosf(i*0.2f) * 5.0f;
|
||||
quantize_row_turbo4_0_ref(input, buf, d);
|
||||
dequantize_row_turbo4_0(buf, output, d);
|
||||
printf("Test 3 (turbo4): cos*5\n");
|
||||
printf(" In: [%.4f, %.4f, %.4f, %.4f]\n", input[0], input[1], input[2], input[3]);
|
||||
printf(" Out: [%.4f, %.4f, %.4f, %.4f]\n", output[0], output[1], output[2], output[3]);
|
||||
mse = cosv = ni = no = 0;
|
||||
for (int i = 0; i < d; i++) { mse += (input[i]-output[i])*(input[i]-output[i]); cosv += input[i]*output[i]; ni += input[i]*input[i]; no += output[i]*output[i]; }
|
||||
printf(" MSE=%.8f Cosine=%.6f\n\n", mse/d, cosv/sqrtf(ni)/sqrtf(no));
|
||||
|
||||
printf("=== Done ===\n");
|
||||
return 0;
|
||||
}
|
||||
@@ -493,6 +493,15 @@ static ggml_type ggml_type_from_name(const std::string & s) {
|
||||
if (s == "iq4_nl") {
|
||||
return GGML_TYPE_IQ4_NL;
|
||||
}
|
||||
if (s == "turbo2") {
|
||||
return GGML_TYPE_TURBO2_0;
|
||||
}
|
||||
if (s == "turbo3") {
|
||||
return GGML_TYPE_TURBO3_0;
|
||||
}
|
||||
if (s == "turbo4") {
|
||||
return GGML_TYPE_TURBO4_0;
|
||||
}
|
||||
|
||||
return GGML_TYPE_COUNT;
|
||||
}
|
||||
|
||||
@@ -886,8 +886,8 @@ private:
|
||||
|
||||
int n_ctx_slot = llama_n_ctx_seq(ctx_tgt);
|
||||
if (n_ctx_slot > n_ctx_train) {
|
||||
SRV_WRN("the slot context (%d) exceeds the training context of the model (%d) - capping\n", n_ctx_slot, n_ctx_train);
|
||||
n_ctx_slot = n_ctx_train;
|
||||
SRV_WRN("the slot context (%d) exceeds the training context of the model (%d) - using rope scaling to extend\n", n_ctx_slot, n_ctx_train);
|
||||
// Do not cap: caller has configured rope scaling (--rope-scale / --rope-scaling yarn) to handle extended context.
|
||||
}
|
||||
|
||||
slots.clear();
|
||||
|
||||
Reference in New Issue
Block a user