merge: TurboQuant KV cache types onto upstream master (MTP built-in)

2026-05-18 01:52:24 +03:00
parent 726704a160
commit b34ac655f3
72 changed files with 18946 additions and 257 deletions
@@ -0,0 +1,58 @@
+#!/bin/bash
+# SMEM Pre-Dequant Benchmark — M5 Max
+# Tests SMEM vs baseline at multiple context depths
+#
+# BEFORE RUNNING:
+#   1. cd /Users/tom/local_llms/llama.cpp
+#   2. git checkout experiment/smem-pre-dequant
+#   3. Build WITHOUT SMEM first (baseline):
+#      cmake --build build -j12
+#   4. Run: ./scripts/bench-smem-m5.sh baseline
+#   5. Build WITH SMEM:
+#      TURBO_SMEM_DEQUANT=1 cmake --build build -j12
+#   6. Run: ./scripts/bench-smem-m5.sh smem
+#
+# Uses Qwen3.5-35B-A3B (MoE, fits in memory, attention-heavy)
+
+set -e
+
+LABEL="${1:-baseline}"
+LLAMA_BENCH="/Users/tom/local_llms/llama.cpp/build/bin/llama-bench"
+MODEL="/Users/tom/local_llms/models/Qwen3.5-35B-A3B-Q8_0.gguf"
+OUTFILE="/Users/tom/local_llms/llama.cpp/bench-smem-m5-${LABEL}.txt"
+
+CONTEXTS=(0 8192 16384 32768)
+KV_TYPES=("turbo3" "turbo4" "q8_0")
+
+echo "=== SMEM M5 Benchmark: ${LABEL} ===" | tee "$OUTFILE"
+echo "Model: $(basename $MODEL)" | tee -a "$OUTFILE"
+echo "Date: $(date)" | tee -a "$OUTFILE"
+echo "" | tee -a "$OUTFILE"
+
+for ctk in "${KV_TYPES[@]}"; do
+    for p in "${CONTEXTS[@]}"; do
+        if [[ "$ctk" == "q8_0" && "$LABEL" == "smem" ]]; then
+            echo "SKIP: q8_0 + smem (q8_0 unaffected by SMEM)" | tee -a "$OUTFILE"
+            continue
+        fi
+
+        depth_label="short"
+        [[ $p -gt 0 ]] && depth_label="${p}"
+
+        echo "--- ${ctk} @ ${depth_label} ---" | tee -a "$OUTFILE"
+
+        ctv="$ctk"
+        $LLAMA_BENCH \
+            -m "$MODEL" \
+            -ngl 99 -fa 1 \
+            -ctk "$ctk" -ctv "$ctv" \
+            -t 1 \
+            -p "$p" -n 128 \
+            2>&1 | tee -a "$OUTFILE"
+
+        echo "" | tee -a "$OUTFILE"
+    done
+done
+
+echo "=== Done: ${LABEL} ===" | tee -a "$OUTFILE"
+echo "Results saved to: $OUTFILE"
@@ -0,0 +1,80 @@
+#!/bin/bash
+# TurboQuant quality + speed gate — run BEFORE pushing any changes
+# Checks: (1) perplexity within 5% of q8_0, (2) context scaling ratio > 0.95
+#
+# Usage: bash scripts/turbo-quality-gate.sh
+# Exit 0 = PASS, Exit 1 = FAIL
+
+set -e
+
+LLAMA=${LLAMA:-~/local_llms/llama.cpp/build-turbo/bin}
+MODEL=${MODEL:-~/local_llms/models/Qwen3.5-35B-A3B-Q8_0.gguf}
+WIKI=${WIKI:-~/local_llms/llama.cpp/wikitext-2-raw/wiki.test.raw}
+
+if [ ! -f "$WIKI" ]; then
+    echo "Downloading wikitext-2..."
+    bash ~/local_llms/llama.cpp/scripts/get-wikitext-2.sh
+fi
+
+FAIL=0
+
+echo "========================================"
+echo "  TurboQuant Quality + Speed Gate"
+echo "========================================"
+echo ""
+
+# --- Test 1: Perplexity ---
+echo "[1/2] Running perplexity check (8 chunks)..."
+PPL_TURBO=$($LLAMA/llama-perplexity -m $MODEL -f $WIKI -c 512 -ctk turbo3 -ctv turbo3 -fa on --chunks 8 -ngl 99 2>&1 | grep "Final" | grep -oE 'PPL = [0-9.]+' | grep -oE '[0-9.]+')
+
+if [ -z "$PPL_TURBO" ]; then
+    echo "  FAIL: Could not get turbo3 perplexity (crash or timeout)"
+    FAIL=1
+else
+    BASELINE_PPL=6.111
+    MAX_PPL=$(echo "$BASELINE_PPL * 1.05" | bc)
+    PPL_OK=$(echo "$PPL_TURBO < $MAX_PPL" | bc)
+    if [ "$PPL_OK" -eq 1 ]; then
+        echo "  PASS: turbo3 PPL = $PPL_TURBO (< $MAX_PPL, within 5% of q8_0 $BASELINE_PPL)"
+    else
+        echo "  FAIL: turbo3 PPL = $PPL_TURBO (> $MAX_PPL, exceeds 5% threshold)"
+        FAIL=1
+    fi
+fi
+echo ""
+
+# --- Test 2: Context Scaling ---
+echo "[2/2] Running context scaling check (4K prefill)..."
+TURBO_TPS=$($LLAMA/llama-perplexity -m $MODEL -f $WIKI -c 4096 -ctk turbo3 -ctv turbo3 -fa on --chunks 4 -ngl 99 2>&1 | grep "prompt eval" | grep -oE '[0-9.]+ tokens per second' | grep -oE '[0-9.]+')
+Q8_TPS=$($LLAMA/llama-perplexity -m $MODEL -f $WIKI -c 4096 -ctk q8_0 -ctv q8_0 -fa on --chunks 4 -ngl 99 2>&1 | grep "prompt eval" | grep -oE '[0-9.]+ tokens per second' | grep -oE '[0-9.]+')
+
+if [ -z "$TURBO_TPS" ] || [ -z "$Q8_TPS" ]; then
+    echo "  FAIL: Could not measure speed (crash or timeout)"
+    echo "  turbo3=$TURBO_TPS q8_0=$Q8_TPS"
+    FAIL=1
+else
+    RATIO=$(echo "scale=4; $TURBO_TPS / $Q8_TPS" | bc)
+    RATIO_OK=$(echo "$RATIO > 0.95" | bc)
+    if [ "$RATIO_OK" -eq 1 ]; then
+        echo "  PASS: turbo3/q8_0 = ${RATIO}x at 4K context (> 0.95 threshold)"
+        echo "  turbo3 = $TURBO_TPS tok/s, q8_0 = $Q8_TPS tok/s"
+    else
+        echo "  FAIL: turbo3/q8_0 = ${RATIO}x at 4K context (< 0.95 threshold)"
+        echo "  turbo3 = $TURBO_TPS tok/s, q8_0 = $Q8_TPS tok/s"
+        echo "  Context scaling regression detected!"
+        FAIL=1
+    fi
+fi
+echo ""
+
+# --- Summary ---
+echo "========================================"
+if [ "$FAIL" -eq 0 ]; then
+    echo "  ALL CHECKS PASSED"
+    echo "========================================"
+    exit 0
+else
+    echo "  CHECKS FAILED — DO NOT PUSH"
+    echo "========================================"
+    exit 1
+fi