llama : add option to save memory in device buffers (#22679)

* llama : add option to save memory in device buffers * tests : extend llama-save-load-state
2026-05-05 06:35:07 +03:00
parent fa595462ca
commit d6e7b033a4
11 changed files with 402 additions and 58 deletions
@@ -864,6 +864,9 @@ extern "C" {
 // work only with partial states, such as SWA KV cache or recurrent cache (e.g. Mamba)
 #define LLAMA_STATE_SEQ_FLAGS_PARTIAL_ONLY 1

+// keeps the tensor data on device buffers (i.e. not accessible in host memory, but faster save/load)
+#define LLAMA_STATE_SEQ_FLAGS_ON_DEVICE 2
+
    typedef uint32_t llama_state_seq_flags;

    LLAMA_API size_t llama_state_seq_get_size_ext(