Adrien Gallouët
66199c9f03
Disabling OpenMP generally provides better inference performance (at
least in my testing) but the loading becomes slightly slower.
Benchmark results for `convert_B_packed_format()`:
Before this commit:
N K | No OpenMP OpenMP | Diff | Speedup
------------------------------------------------------------
512 2880 | 640.9us 263.5us | -58.9% | 0.41x
2880 4096 | 2.55ms 261.7us | -89.8% | 0.10x
201088 2880 | 256.44ms 21.61ms | -91.6% | 0.08x
------------------------------------------------------------
Total: 325.43ms vs 31.05ms
After:
N K | No OpenMP OpenMP | Diff | Speedup
------------------------------------------------------------
512 2880 | 1.49ms 263.5us | -82.3% | 0.18x
2880 4096 | 1.55ms 261.7us | -83.1% | 0.17x
201088 2880 | 24.03ms 21.61ms | -10.1% | 0.90x
------------------------------------------------------------
Total: 78.97ms vs 31.05ms
Tested with unsloth/gpt-oss-20b-GGUF:Q4_K_M.
Signed-off-by: Adrien Gallouët <angt@huggingface.co>
116 lines
2.6 KiB
C++
116 lines
2.6 KiB
C++
#pragma once
|
|
|
|
#include "ggml.h"
|
|
#include "ggml-cpu-impl.h"
|
|
|
|
#include <algorithm>
|
|
#include <memory>
|
|
#include <type_traits>
|
|
|
|
#if defined(GGML_USE_OPENMP)
|
|
#include <omp.h>
|
|
#else
|
|
#include <thread>
|
|
#endif
|
|
|
|
#define TILE_M 16
|
|
#define TILE_N 16
|
|
#define TILE_K 32
|
|
#define VNNI_BLK 4
|
|
|
|
#define AMX_BLK_SIZE 32
|
|
|
|
#define TMM0 0
|
|
#define TMM1 1
|
|
#define TMM2 2
|
|
#define TMM3 3
|
|
#define TMM4 4
|
|
#define TMM5 5
|
|
#define TMM6 6
|
|
#define TMM7 7
|
|
|
|
// parallel routines
|
|
template <typename T, typename std::enable_if<std::is_integral<T>::value, int>::type = 0>
|
|
inline T div_up(T x, T y) { return (x + y - 1) / y; }
|
|
|
|
template <typename T>
|
|
inline void balance211(T n, T nth, T ith, T& n_start, T& n_end) {
|
|
#if 0
|
|
// onednn partition pattern
|
|
T& n_my = n_end;
|
|
if (nth <= 1 || n == 0) {
|
|
n_start = 0;
|
|
n_my = n;
|
|
} else {
|
|
T n1 = div_up(n, nth);
|
|
T n2 = n1 - 1;
|
|
T T1 = n - n2 * nth;
|
|
n_my = ith < T1 ? n1 : n2;
|
|
n_start = ith <= T1 ? ith*n1 : T1 * n1 + (ith - T1) * n2;
|
|
}
|
|
n_end += n_start;
|
|
#else
|
|
// pytorch aten partition pattern
|
|
T n_my = div_up(n, nth);
|
|
n_start = ith * n_my;
|
|
n_end = std::min(n_start + n_my, n);
|
|
#endif
|
|
}
|
|
|
|
template <typename func_t>
|
|
inline void parallel_for(int n, const func_t & f) {
|
|
if (n <= 0) {
|
|
return;
|
|
}
|
|
#if defined(GGML_USE_OPENMP)
|
|
#pragma omp parallel
|
|
{
|
|
int nth = omp_get_num_threads();
|
|
int ith = omp_get_thread_num();
|
|
int tbegin, tend;
|
|
balance211(n, nth, ith, tbegin, tend);
|
|
f(tbegin, tend);
|
|
}
|
|
#else
|
|
int nth = std::thread::hardware_concurrency();
|
|
if (nth <= 1) {
|
|
f(0, n);
|
|
return;
|
|
}
|
|
if (nth > n) {
|
|
nth = n;
|
|
}
|
|
std::vector<std::thread> threads;
|
|
threads.reserve(nth);
|
|
for (int ith = 0; ith < nth; ++ith) {
|
|
threads.emplace_back([&f, n, ith, nth] {
|
|
int tbegin, tend;
|
|
balance211(n, nth, ith, tbegin, tend);
|
|
f(tbegin, tend);
|
|
});
|
|
}
|
|
for (auto & t : threads) {
|
|
t.join();
|
|
}
|
|
#endif
|
|
}
|
|
|
|
template <typename func_t>
|
|
inline void parallel_for_ggml(const ggml_compute_params * params, int n, const func_t & f) {
|
|
int tbegin, tend;
|
|
balance211(n, params->nth, params->ith, tbegin, tend);
|
|
f(tbegin, tend);
|
|
}
|
|
|
|
// quantized types that have AMX support
|
|
inline bool qtype_has_amx_kernels(const enum ggml_type type) {
|
|
// TODO: fix padding for vnni format
|
|
return (type == GGML_TYPE_Q4_0) ||
|
|
(type == GGML_TYPE_Q4_1) ||
|
|
(type == GGML_TYPE_Q8_0) ||
|
|
(type == GGML_TYPE_Q4_K) ||
|
|
(type == GGML_TYPE_Q5_K) ||
|
|
(type == GGML_TYPE_Q6_K) ||
|
|
(type == GGML_TYPE_IQ4_XS);
|
|
}
|