server: skip device enumeration in router mode to avoid creating CUDA primary context (#23137)

This commit is contained in:
Pascal
2026-05-16 21:21:06 +02:00
committed by GitHub
parent 6049906133
commit 64b38b561b
3 changed files with 16 additions and 10 deletions
+4 -2
View File
@@ -86,7 +86,10 @@ int main(int argc, char ** argv) {
llama_backend_init();
llama_numa_init(params.numa);
common_params_print_info(params);
// router server never loads a model and must not touch the GPU
// skip device enumeration so the CUDA primary context stays uncreated
const bool is_router_server = params.model.path.empty();
common_params_print_info(params, !is_router_server);
// validate batch size for embeddings
// embeddings require all tokens to be processed in a single ubatch
@@ -126,7 +129,6 @@ int main(int argc, char ** argv) {
server_routes routes(params, ctx_server);
server_tools tools;
bool is_router_server = params.model.path.empty();
std::optional<server_models_routes> models_routes{};
if (is_router_server) {
// setup server instances manager