mikhail/llama.cpp-mtp-turboquant
1
0
Fork 0
Code Issues Pull Requests Actions 32 Packages Projects Releases Wiki Activity

server: skip device enumeration in router mode to avoid creating CUDA primary context (#23137)

Browse Source
This commit is contained in:
Pascal
2026-05-16 21:21:06 +02:00
committed by GitHub
parent 6049906133
commit 64b38b561b
3 changed files with 16 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
common/common.cpp
+11 -7
View File
@@ -373,7 +373,7 @@ void common_init() {
llama_log_set(common_log_default_callback, NULL);
}
void common_params_print_info(const common_params & params) {
void common_params_print_info(const common_params & params, bool print_devices) {
#ifdef NDEBUG
const char * build_type = "";
#else
@@ -382,12 +382,16 @@ void common_params_print_info(const common_params & params) {
LOG_TRC("%s: build %d (%s) with %s for %s%s\n", __func__, llama_build_number(), llama_commit(), llama_compiler(), llama_build_target(), build_type);
LOG_INF("log_info: verbosity = %d (adjust with the `-lv N` CLI arg)\n", common_log_get_verbosity_thold());
LOG_INF("device_info:\n");
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
auto * dev = ggml_backend_dev_get(i);
size_t free, total;
ggml_backend_dev_memory(dev, &free, &total);
LOG_INF(" - %-8s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024);
// device enumeration creates a primary context on CUDA backends, skip it when the caller does not own any device
if (print_devices) {
LOG_INF("device_info:\n");
for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {
auto * dev = ggml_backend_dev_get(i);
size_t free, total;
ggml_backend_dev_memory(dev, &free, &total);
LOG_INF(" - %-8s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev), total / 1024 / 1024, free / 1024 / 1024);
}
}
LOG_INF("%s\n", common_params_get_system_info(params).c_str());
}
common/common.h
+1 -1
View File
@@ -708,7 +708,7 @@ struct common_params {
// initializes the logging system and prints info about the build
void common_init();
void common_params_print_info(const common_params & params);
void common_params_print_info(const common_params & params, bool print_devices = true);
std::string common_params_get_system_info(const common_params & params);
bool parse_cpu_range(const std::string & range, bool(&boolmask)[GGML_MAX_N_THREADS]);