{"slug": "llama-cpp-gfx1151-optimizations-for-qwen3-8-27b", "title": "llama.cpp gfx1151 optimizations for Qwen3.8 27B", "summary": "A developer contributed optimizations to llama.cpp for AMD's gfx1151 GPU, specifically targeting Qwen3.8 27B's SSM convolution input pattern. The patch introduces a fast LDS-transpose path for dim-0 concatenation when the second source tensor is transposed-contiguous, staging a 64x64 tile through shared memory to achieve fully coalesced memory access. Benchmarks on the Qwen3.5 shape show throughput improved from 35 GB/s to 70 GB/s.", "body_md": "| diff --git a/ggml/src/ggml-cuda/concat.cu b/ggml/src/ggml-cuda/concat.cu | |\n| index 6df89013c..e5337ec30 100644 | |\n| --- a/ggml/src/ggml-cuda/concat.cu | |\n| +++ b/ggml/src/ggml-cuda/concat.cu | |\n| @@ -79,6 +79,76 @@ static void concat_cont_cuda(const T * x, | |\n| concat_cont<<>>(x, y, dst, ne00, ne01, ne02, ne0, ne1, ne2); | |\n| } | |\n| +// Fast LDS-transpose path for a dim-0 concat whose src1 is transposed-contiguous | |\n| +// (nb[1] == element size < nb[0]). That is the qwen35 ssm_conv input pattern: | |\n| +// concat(conv_state[3, d_inner], transpose(x)[n_tokens, d_inner]) along dim 0. | |\n| +// | |\n| +// The generic non-cont kernel walks i0 with consecutive threads, so for a | |\n| +// transposed src1 every lane of a wave reads a different row -> one memory | |\n| +// transaction per lane on both the load and the store side. Staging a 32x32 | |\n| +// tile through LDS makes both sides fully coalesced. | |\n| +#define CONCAT_TRANSPOSE_TILE 64 // 64x64 measured best on gfx1151 (35 -> 70 GB/s on the qwen35 shape) | |\n| +#define CONCAT_TRANSPOSE_NTX 64 | |\n| +#define CONCAT_TRANSPOSE_NTY 4 | |\n| + | |\n| +template | |\n| +static __global__ void __launch_bounds__(CONCAT_TRANSPOSE_NTX * CONCAT_TRANSPOSE_NTY) | |\n| + concat_dim0_transposed_src1( | |\n| + const char * __restrict__ src0, | |\n| + const char * __restrict__ src1, | |\n| + char * __restrict__ dst, | |\n| + int64_t ne00, | |\n| + int64_t ne0, | |\n| + int64_t ne1, | |\n| + int64_t ne2, | |\n| + uint64_t nb00, uint64_t nb01, uint64_t nb02, uint64_t nb03, | |\n| + uint64_t nb10, uint64_t nb11, uint64_t nb12, uint64_t nb13, | |\n| + uint64_t nb0, uint64_t nb1, uint64_t nb2, uint64_t nb3) { | |\n| + constexpr int TILE = CONCAT_TRANSPOSE_TILE; | |\n| + constexpr int NTX = CONCAT_TRANSPOSE_NTX; | |\n| + constexpr int NTY = CONCAT_TRANSPOSE_NTY; | |\n| + __shared__ T tile[TILE][TILE + 1]; // +1 to avoid LDS bank conflicts on the transposed read | |\n| + | |\n| + const int tx = threadIdx.x; | |\n| + const int ty = threadIdx.y; | |\n| + const int i23 = blockIdx.z; | |\n| + const int i2 = i23 % (int) ne2; | |\n| + const int i3 = i23 / (int) ne2; | |\n| + const int64_t i0_base = (int64_t) blockIdx.x * TILE; | |\n| + const int64_t i1_base = (int64_t) blockIdx.y * TILE; | |\n| + | |\n| + // load: consecutive tx -> consecutive i1, i.e. consecutive addresses in src1 | |\n| +#pragma unroll | |\n| + for (int c = 0; c < TILE; c += NTX) { | |\n| + const int64_t load_i1 = i1_base + c + tx; | |\n| + for (int r = ty; r < TILE; r += NTY) { | |\n| + const int64_t load_i0 = i0_base + r; | |\n| + T v = T(0); | |\n| + if (load_i0 < ne0 && load_i1 < ne1) { | |\n| + if (load_i0 < ne00) { | |\n| + v = *(const T *)(src0 + i3*nb03 + i2*nb02 + load_i1*nb01 + load_i0*nb00); | |\n| + } else { | |\n| + v = *(const T *)(src1 + i3*nb13 + i2*nb12 + (load_i0 - ne00)*nb10 + load_i1*nb11); | |\n| + } | |\n| + } | |\n| + tile[r][c + tx] = v; | |\n| + } | |\n| + } | |\n| + __syncthreads(); | |\n| + | |\n| + // store: consecutive tx -> consecutive i0, i.e. consecutive addresses in dst | |\n| +#pragma unroll | |\n| + for (int c = 0; c < TILE; c += NTX) { | |\n| + const int64_t store_i0 = i0_base + c + tx; | |\n| + for (int r = ty; r < TILE; r += NTY) { | |\n| + const int64_t store_i1 = i1_base + r; | |\n| + if (store_i0 < ne0 && store_i1 < ne1) { | |\n| + *(T *)(dst + i3*nb3 + i2*nb2 + store_i1*nb1 + store_i0*nb0) = tile[c + tx][r]; | |\n| + } | |\n| + } | |\n| + } | |\n| +} | |\n| + | |\n| // non-contiguous kernel (slow) | |\n| template | |\n| static __global__ void __launch_bounds__(CUDA_CONCAT_BLOCK_SIZE) | |\n| @@ -164,6 +234,25 @@ static void concat_cuda(const ggml_tensor * src0, const ggml_tensor * src1, ggml | |\n| GGML_ASSERT(!ggml_is_quantized(src0->type)); | |\n| dim3 grid_dim(dst->ne[1], dst->ne[2], dst->ne[3]); | |\n| + // transposed-src1 dim-0 concat (qwen35 ssm_conv input): stage through LDS | |\n| + if (dim == 0 | |\n| + && src1->nb[1] == sizeof(T) && src1->nb[0] > src1->nb[1] | |\n| + && src0->nb[0] == sizeof(T) | |\n| + && src0->ne[1] == dst->ne[1] && src1->ne[1] == dst->ne[1]) { | |\n| + constexpr int TILE = CONCAT_TRANSPOSE_TILE; | |\n| + const dim3 grid((dst->ne[0] + TILE - 1) / TILE, | |\n| + (dst->ne[1] + TILE - 1) / TILE, | |\n| + (unsigned) (dst->ne[2] * dst->ne[3])); | |\n| + const dim3 block(CONCAT_TRANSPOSE_NTX, CONCAT_TRANSPOSE_NTY, 1); | |\n| + concat_dim0_transposed_src1<<>>( | |\n| + (const char *) src0->data, (const char *) src1->data, (char *) dst->data, | |\n| + src0->ne[0], dst->ne[0], dst->ne[1], dst->ne[2], | |\n| + src0->nb[0], src0->nb[1], src0->nb[2], src0->nb[3], | |\n| + src1->nb[0], src1->nb[1], src1->nb[2], src1->nb[3], | |\n| + dst->nb[0], dst->nb[1], dst->nb[2], dst->nb[3]); | |\n| + return; | |\n| + } | |\n| + | |\n| auto launch_kernel = [&](auto dim) { | |\n| concat_non_cont<<>>( | |\n| (const char *) src0->data, (const char *) src1->data, (char *) dst->data, | |\n| diff --git a/ggml/src/ggml-cuda/fattn-tile.cuh b/ggml/src/ggml-cuda/fattn-tile.cuh | |\n| index d1164b852..6611f8096 100644 | |\n| --- a/ggml/src/ggml-cuda/fattn-tile.cuh | |\n| +++ b/ggml/src/ggml-cuda/fattn-tile.cuh | |\n| @@ -312,8 +312,25 @@ static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_am | |\n| return 0; | |\n| } | |\n| +// gfx1151, DKQ = DV = 256 (the qwen35 attention head): nbatch_fa 64 -> 128. | |\n| +// | |\n| +// The generic RDNA table's 64 is the right answer only while the kernel is spilling: with | |\n| +// the staging loops fully unrolled this shape sits at 256 VGPRs with 5 spilled, and every | |\n| +// config that wants more registers is punished for that rather than on its merits. Once the | |\n| +// unroll is capped (see GGML_FA_TILE_UNROLL_STAGE) the optimum moves -- 128 was the *worst* | |\n| +// point of the original sweep and is the best one now. LDS goes 37888 -> ~33800 B, still | |\n| +// three workgroups per WGP. Measured 8.65 -> 9.40 TFLOPS at 512x4096 and 8.93 -> 9.18 at | |\n| +// 512x33280. | |\n| +static constexpr __host__ __device__ uint32_t ggml_cuda_fattn_tile_get_config_amd_rdna3_5(const int DKQ, const int DV, const int ncols) { | |\n| + GGML_CUDA_FATTN_TILE_CONFIG_CASE(256, 256, 32, 256, 2, 128, 64) | |\n| + return ggml_cuda_fattn_tile_get_config_amd_rdna(DKQ, DV, ncols); | |\n| +} | |\n| + | |\n| static __host__ uint32_t ggml_cuda_fattn_tile_get_config(const int DKQ, const int DV, const int ncols, const int cc) { | |\n| if (GGML_CUDA_CC_IS_AMD(cc)) { | |\n| + if (GGML_CUDA_CC_IS_RDNA3_5(cc)) { | |\n| + return ggml_cuda_fattn_tile_get_config_amd_rdna3_5(DKQ, DV, ncols); | |\n| + } | |\n| if (GGML_CUDA_CC_IS_RDNA(cc)) { | |\n| return ggml_cuda_fattn_tile_get_config_amd_rdna(DKQ, DV, ncols); | |\n| } | |\n| @@ -327,7 +344,9 @@ static __host__ uint32_t ggml_cuda_fattn_tile_get_config(const int DKQ, const in | |\n| static constexpr __device__ uint32_t ggml_cuda_fattn_tile_get_config(const int DKQ, const int DV, const int ncols) { | |\n| #ifdef GGML_USE_HIP | |\n| -#ifdef RDNA | |\n| +#ifdef RDNA3_5 | |\n| + return ggml_cuda_fattn_tile_get_config_amd_rdna3_5(DKQ, DV, ncols); | |\n| +#elif defined(RDNA) | |\n| return ggml_cuda_fattn_tile_get_config_amd_rdna(DKQ, DV, ncols); | |\n| #else | |\n| return ggml_cuda_fattn_tile_get_config_amd(DKQ, DV, ncols); | |\n| @@ -482,6 +501,19 @@ static __device__ __forceinline__ void flash_attn_tile_load_tile( | |\n| // Function that performs a single iteration in for the KQ matrix multiplication: | |\n| template | |\n| +// gfx1151: these two loops stage 24 half2 of K/Q (resp. V/KQ) per iteration, and fully | |\n| +// unrolling them lets the scheduler keep several iterations of loads in flight. Combined | |\n| +// with this target's raised -amdgpu-unroll-threshold-local that pushes the DKQ=256 kernel | |\n| +// to 256 VGPRs with 5 spilled, and the spills then stop it keeping *any* further loads in | |\n| +// flight -- the kernel runs at 45% instruction-issue utilisation. Capping the unroll takes | |\n| +// it to 127 VGPRs with no spills and +23% at 32k context. Measured on RDNA3.5 only, so it | |\n| +// is scoped there; other targets keep the full unroll. | |\n| +#if defined(GGML_USE_HIP) && defined(RDNA3_5) | |\n| +#define GGML_FA_TILE_UNROLL_STAGE _Pragma(\"unroll 4\") | |\n| +#else | |\n| +#define GGML_FA_TILE_UNROLL_STAGE _Pragma(\"unroll\") | |\n| +#endif | |\n| + | |\n| static __device__ __forceinline__ void flash_attn_tile_iter_KQ( | |\n| T_vec_dot * const Q_tmp, | |\n| const half2 * const __restrict__ K_h2, | |\n| @@ -504,7 +536,7 @@ static __device__ __forceinline__ void flash_attn_tile_iter_KQ( | |\n| #ifdef FAST_FP16_AVAILABLE | |\n| static_assert((nbatch_K/2) % cpy_ne == 0, \"bad nbatch_K\"); | |\n| -#pragma unroll | |\n| +GGML_FA_TILE_UNROLL_STAGE | |\n| for (int k_KQ_1 = 0; k_KQ_1 < nbatch_K/2; k_KQ_1 += cpy_ne) { | |\n| __align__(16) half2 K_k[nbatch_fa/(np*warp_size)][cpy_ne]; | |\n| __align__(16) half2 Q_k[cpw][cpy_ne]; | |\n| @@ -723,7 +755,7 @@ static __device__ __forceinline__ void flash_attn_tile_iter( | |\n| __syncthreads(); | |\n| #ifdef FAST_FP16_AVAILABLE | |\n| -#pragma unroll | |\n| +GGML_FA_TILE_UNROLL_STAGE | |\n| for (int k1 = 0; k1 < nbatch_V; k1 += np) { | |\n| __align__(16) half2 V_k[(DVp/2)/warp_size]; | |\n| __align__(16) half2 KQ_k[cpw]; | |\n| @@ -755,7 +787,7 @@ static __device__ __forceinline__ void flash_attn_tile_iter( | |\n| } | |\n| } | |\n| #else | |\n| -#pragma unroll | |\n| +GGML_FA_TILE_UNROLL_STAGE | |\n| for (int k1 = 0; k1 < nbatch_V; k1 += np) { | |\n| __align__(16) float2 V_k[(DVp/2)/warp_size]; | |\n| __align__(16) float KQ_k[cpw]; | |\n| diff --git a/ggml/src/ggml-cuda/fattn-wmma-d256.cu b/ggml/src/ggml-cuda/fattn-wmma-d256.cu | |\n| new file mode 100644 | |\n| index 000000000..908eb00bb | |\n| --- /dev/null | |\n| +++ b/ggml/src/ggml-cuda/fattn-wmma-d256.cu | |\n| @@ -0,0 +1,55 @@ | |\n| +#include \"fattn-wmma-d256.cuh\" | |\n| +#include | |\n| + | |\n| +template | |\n| +static void launch_wmma_d256(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { | |\n| + constexpr int DKQ = 256; | |\n| + constexpr int DV = 256; | |\n| + launch_fattn( | |\n| + ctx, dst, flash_attn_wmma_d256, | |\n| + /*nwarps =*/ FA_WMMA_D256_NWARPS, | |\n| + /*nbytes_shared=*/ getenv(\"GGML_FA_LDS_ABLATE\") | |\n| + ? (size_t) atoi(getenv(\"GGML_FA_LDS_ABLATE\")) : ggml_cuda_fattn_wmma_d256_nbytes_shared(), | |\n| + /*nbatch_fa =*/ FA_WMMA_D256_FA, | |\n| + // false: the kernel dequantises K/V itself, so launch_fattn must not pre-convert | |\n| + // the cache to f16 -- that conversion is exactly the DRAM traffic we are avoiding. | |\n| + /*need_f16_K =*/ false, | |\n| + /*need_f16_V =*/ false, | |\n| + /*stream_k =*/ false, | |\n| + /*warp_size =*/ 32); | |\n| +} | |\n| + | |\n| +void ggml_cuda_flash_attn_ext_wmma_d256(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { | |\n| + const ggml_tensor * Q = dst->src[0]; | |\n| + const ggml_tensor * K = dst->src[1]; | |\n| + | |\n| + GGML_ASSERT(Q->ne[0] == 256); | |\n| + | |\n| + const int gqa_ratio = Q->ne[2] / K->ne[2]; | |\n| + | |\n| + // Widest power-of-two head group that divides the GQA ratio, capped so that | |\n| + // ncols1 = 64/ncols2 still leaves at least one Q column per block. | |\n| + int ncols2 = 1; | |\n| + while (gqa_ratio % (2*ncols2) == 0 && ncols2 < 8) { | |\n| + ncols2 *= 2; | |\n| + } | |\n| + | |\n| + constexpr int NC = FA_WMMA_D256_NC; | |\n| + const ggml_type tK = K->type; | |\n| + const ggml_type tV = dst->src[2]->type; | |\n| + | |\n| +#define FA_WMMA_DISPATCH_KV(N1, N2) \\ | |\n| + if (tK == GGML_TYPE_F16 && tV == GGML_TYPE_F16 ) { launch_wmma_d256(ctx, dst); return; } \\ | |\n| + if (tK == GGML_TYPE_Q8_0 && tV == GGML_TYPE_Q8_0) { launch_wmma_d256(ctx, dst); return; } \\ | |\n| + if (tK == GGML_TYPE_Q4_0 && tV == GGML_TYPE_Q4_0) { launch_wmma_d256(ctx, dst); return; } \\ | |\n| + GGML_ABORT(\"unsupported KV type combination\"); | |\n| + | |\n| + switch (ncols2) { | |\n| + case 1: FA_WMMA_DISPATCH_KV(NC/1, 1); break; | |\n| + case 2: FA_WMMA_DISPATCH_KV(NC/2, 2); break; | |\n| + case 4: FA_WMMA_DISPATCH_KV(NC/4, 4); break; | |\n| + case 8: FA_WMMA_DISPATCH_KV(NC/8, 8); break; | |\n| + default: GGML_ABORT(\"fatal error\"); | |\n| + } | |\n| +#undef FA_WMMA_DISPATCH_KV | |\n| +} | |\n| diff --git a/ggml/src/ggml-cuda/fattn-wmma-d256.cuh b/ggml/src/ggml-cuda/fattn-wmma-d256.cuh | |\n| new file mode 100644 | |\n| index 000000000..88187a552 | |\n| --- /dev/null | |\n| +++ b/ggml/src/ggml-cuda/fattn-wmma-d256.cuh | |\n| @@ -0,0 +1,426 @@ | |\n| +#pragma once | |\n| + | |\n| +#include \"common.cuh\" | |\n| +#include \"fattn-common.cuh\" | |\n| + | |\n| +// WMMA flash attention for RDNA3/3.5 at head size 256. | |\n| +// | |\n| +// The generic tile kernel emits no WMMA at all, and fattn-mma-f16 gives every warp the | |\n| +// full DV in registers, which at DV=256 spills >1 kB/lane to scratch on a 256-VGPR wave. | |\n| +// This kernel splits DV across warps instead, so the O accumulator is 64 VGPRs/lane. | |\n| +// | |\n| +// Fixed shape: 8 warps, 64 columns per block, 32 KV rows per iteration. | |\n| +// NQB = 4 q blocks, NKB = 2 kv blocks, NQB*NKB == 8 -> one 16x16 S tile per warp. | |\n| +// O phase: warp w owns q block w/NKB and DV slice [(w%NKB)*128, +128) = 8 tiles. | |\n| +// | |\n| +// WMMA fragment layout (wave32, gfx11): | |\n| +// A: lane t supplies row (t % 16) of A, all 16 k | |\n| +// B: lane t supplies column (t % 16) of B, all 16 k | |\n| +// D: lane t holds COLUMN (t % 16); its 8 slots are rows 2*l + t/16 | |\n| + | |\n| +#define FA_WMMA_D256_D 256 | |\n| +#ifndef FA_WMMA_D256_NC | |\n| +#define FA_WMMA_D256_NC 64 | |\n| +#endif | |\n| +#ifndef FA_WMMA_D256_FA | |\n| +#define FA_WMMA_D256_FA 32 | |\n| +#endif | |\n| +#define FA_WMMA_D256_NQB (FA_WMMA_D256_NC / 16) | |\n| +#define FA_WMMA_D256_NKB (FA_WMMA_D256_FA / 16) | |\n| +// KSPLIT warps cooperate on each S tile by splitting the head dimension. It buys waves per | |\n| +// workgroup without any extra LDS, which is what this kernel is short of: the Q tile alone | |\n| +// is 33 kB, so only 2 workgroups fit per WGP however the rest is arranged. | |\n| +#define FA_WMMA_D256_KSPLIT 1 | |\n| +#define FA_WMMA_D256_NWARPS (FA_WMMA_D256_NQB * FA_WMMA_D256_NKB * FA_WMMA_D256_KSPLIT) | |\n| +#define FA_WMMA_D256_NSW (FA_WMMA_D256_NKB * FA_WMMA_D256_KSPLIT) // warps sharing a q block | |\n| +#define FA_WMMA_D256_DVW (FA_WMMA_D256_D / FA_WMMA_D256_NSW) | |\n| +#define FA_WMMA_D256_OT (FA_WMMA_D256_DVW / 16) | |\n| +#define FA_WMMA_D256_KPAD 8 | |\n| +#define FA_WMMA_D256_FPAD 8 | |\n| + | |\n| +static constexpr size_t ggml_cuda_fattn_wmma_d256_nbytes_shared() { | |\n| + // Q is staged through LDS once and then held in registers, so its footprint overlaps | |\n| + // the K/V/P working set instead of adding to it. Keeping Q resident would put the | |\n| + // block at 60 kB and 2 workgroups per WGP; this way it is 3, which measured -20% on | |\n| + // the long-context shape. | |\n| + const size_t nb_Q = (size_t) FA_WMMA_D256_NC * (FA_WMMA_D256_D + FA_WMMA_D256_KPAD) * 2; | |\n| + const size_t nb_K = (size_t) FA_WMMA_D256_FA * (FA_WMMA_D256_D + FA_WMMA_D256_KPAD) * 2; | |\n| + const size_t nb_V = (size_t) FA_WMMA_D256_D * (FA_WMMA_D256_FA + FA_WMMA_D256_FPAD) * 2; | |\n| + const size_t nb_S = (size_t) FA_WMMA_D256_NQB * FA_WMMA_D256_NKB * 32 * 8 * sizeof(float); | |\n| + size_t nb_KV = nb_K > nb_V ? nb_K : nb_V; | |\n| + if (nb_S > nb_KV) { nb_KV = nb_S; } | |\n| + const size_t nb_P = (size_t) FA_WMMA_D256_NC * (FA_WMMA_D256_FA + FA_WMMA_D256_FPAD) * 2; | |\n| + const size_t nb_R = (size_t) 2 * FA_WMMA_D256_NQB * FA_WMMA_D256_NKB * 16 * sizeof(float); | |\n| + return nb_Q + nb_KV + nb_P + nb_R; | |\n| +} | |\n| + | |\n| +#if defined(AMD_WMMA_AVAILABLE) && defined(RDNA3) | |\n| + | |\n| +typedef _Float16 fa_half16 __attribute__((ext_vector_type(16))); | |\n| +typedef float fa_float8 __attribute__((ext_vector_type(8))); | |\n| + | |\n| +union fa_frag16 { fa_half16 h; int4 v[2]; }; | |\n| + | |\n| +static __device__ __forceinline__ fa_half16 fa_load_frag(const _Float16 * p) { | |\n| + fa_frag16 f; | |\n| + f.v[0] = *(const int4 *) (p + 0); | |\n| + f.v[1] = *(const int4 *) (p + 8); | |\n| + return f.h; | |\n| +} | |\n| + | |\n| +// Read 8 consecutive head-dimension elements of one K/V row, dequantising on the way in. | |\n| +// The point of doing this here rather than letting launch_fattn pre-convert the cache to | |\n| +// f16 is that the DRAM traffic is what bounds this kernel: at head size 256 the arithmetic | |\n| +// intensity is (columns per block)/2 MAC per byte, so halving the bytes doubles the roofline. | |\n| +template | |\n| +static __device__ __forceinline__ int4 fa_load8(const char * __restrict__ row, int i0) { | |\n| + int4 out; | |\n| + _Float16 * o = (_Float16 *) &out; | |\n| + if constexpr (type_KV == GGML_TYPE_F16) { | |\n| + out = *(const int4 *) (row + i0*2); | |\n| + } else if constexpr (type_KV == GGML_TYPE_Q8_0) { | |\n| + const block_q8_0 * b = (const block_q8_0 *) row + i0/QK8_0; | |\n| + const float d = __half2float(b->d); | |\n| + const int j0 = i0 % QK8_0; | |\n| +#pragma unroll | |\n| + for (int j = 0; j < 8; ++j) { o[j] = (_Float16) (d * (float) b->qs[j0 + j]); } | |\n| + } else { | |\n| + static_assert(type_KV == GGML_TYPE_Q4_0, \"unsupported KV type\"); | |\n| + const block_q4_0 * b = (const block_q4_0 *) row + i0/QK4_0; | |\n| + const float d = __half2float(b->d); | |\n| + const int j0 = i0 % QK4_0; | |\n| + const int sh = (j0 / 16) * 4; // low nibbles hold the first half of the block | |\n| + const uint8_t * q = b->qs + (j0 % 16); | |\n| +#pragma unroll | |\n| + for (int j = 0; j < 8; ++j) { o[j] = (_Float16) (d * (float) (int) (((q[j] >> sh) & 0x0F) - 8)); } | |\n| + } | |\n| + return out; | |\n| +} | |\n| + | |\n| +// Single element, for the transposed V staging where each lane walks kv, not the head dim. | |\n| +template | |\n| +static __device__ __forceinline__ _Float16 fa_load1(const char * __restrict__ row, int i) { | |\n| + if constexpr (type_KV == GGML_TYPE_F16) { | |\n| + return *(const _Float16 *) (row + i*2); | |\n| + } else if constexpr (type_KV == GGML_TYPE_Q8_0) { | |\n| + const block_q8_0 * b = (const block_q8_0 *) row + i/QK8_0; | |\n| + return (_Float16) (__half2float(b->d) * (float) b->qs[i % QK8_0]); | |\n| + } else { | |\n| + static_assert(type_KV == GGML_TYPE_Q4_0, \"unsupported KV type\"); | |\n| + const block_q4_0 * b = (const block_q4_0 *) row + i/QK4_0; | |\n| + const int j = i % QK4_0; | |\n| + return (_Float16) (__half2float(b->d) * (float) (int) (((b->qs[j % 16] >> ((j/16)*4)) & 0x0F) - 8)); | |\n| + } | |\n| +} | |\n| + | |\n| +template | |\n| +__launch_bounds__(FA_WMMA_D256_NWARPS*32, 1) | |\n| +static __global__ void flash_attn_wmma_d256( | |\n| + const char * Q_ptr, | |\n| + const char * K_ptr, | |\n| + const char * V_ptr, | |\n| + const char * mask_ptr, | |\n| + const char * sinks_ptr, | |\n| + const int * KV_max_ptr, | |\n| + float * dst_ptr, | |\n| + float2 * dst_meta_ptr, | |\n| + const float scale, | |\n| + const float max_bias, | |\n| + const float m0, | |\n| + const float m1, | |\n| + const uint32_t n_head_log2, | |\n| + const float logit_softcap, | |\n| + const int32_t ne00, const uint3 ne01, const int32_t ne02, const int32_t ne03, | |\n| + const int32_t nb01, const int32_t nb02, const int32_t nb03, | |\n| + const int32_t ne10, const int32_t ne11, const int32_t ne12, const int32_t ne13, | |\n| + const int32_t nb11, const int32_t nb12, const int64_t nb13, | |\n| + const int32_t nb21, const int32_t nb22, const int64_t nb23, | |\n| + const int32_t ne31, const int32_t ne32, const int32_t ne33, | |\n| + const int32_t nb31, const int32_t nb32, const int64_t nb33) { | |\n| + constexpr int D = FA_WMMA_D256_D; | |\n| + constexpr int NC = FA_WMMA_D256_NC; | |\n| + constexpr int FAB = FA_WMMA_D256_FA; | |\n| + constexpr int NWARPS = FA_WMMA_D256_NWARPS; | |\n| + constexpr int NQB = FA_WMMA_D256_NQB; | |\n| + constexpr int NKB = FA_WMMA_D256_NKB; | |\n| + constexpr int KSPLIT = FA_WMMA_D256_KSPLIT; | |\n| + constexpr int NSW = FA_WMMA_D256_NSW; | |\n| + constexpr int DVW = FA_WMMA_D256_DVW; | |\n| + constexpr int OT = FA_WMMA_D256_OT; | |\n| + constexpr int KPAD = FA_WMMA_D256_KPAD; | |\n| + constexpr int FPAD = FA_WMMA_D256_FPAD; | |\n| + constexpr int WS = 32; | |\n| + constexpr int NTHR = NWARPS*WS; | |\n| + | |\n| + static_assert(DKQ == D && DV == D, \"this kernel is head size 256 only\"); | |\n| + static_assert(ncols1*ncols2 == NC, \"bad column count\"); | |\n| + static_assert(NQB*NKB*KSPLIT == NWARPS, \"bad warp count\"); | |\n| + static_assert(D % (16*KSPLIT) == 0, \"bad head split\"); | |\n| + | |\n| + GGML_UNUSED_VARS(max_bias, m0, m1, n_head_log2, logit_softcap, ne00, ne10, ne12, ne13, | |\n| + ne31, ne32, nb32, nb33, sinks_ptr); | |\n| + | |\n| + const int tid = threadIdx.x + threadIdx.y*WS; | |\n| + const int warp = threadIdx.y; | |\n| + const int lane = threadIdx.x; | |\n| + const int l16 = lane % 16; | |\n| + const int lhi = lane / 16; | |\n| + | |\n| + const int col_Q_0 = blockIdx.x * ncols1; | |\n| + | |\n| + const int sequence = blockIdx.z / (ne02/ncols2); | |\n| + const int head0 = blockIdx.z*ncols2 - sequence*ne02; | |\n| + const int gqa_ratio = ne02 / ne12; | |\n| + | |\n| + const float * Q_f = (const float *) (Q_ptr + nb03*sequence + nb02* head0); | |\n| + const char * K_base = K_ptr + nb13*sequence + nb12*(head0 / gqa_ratio); | |\n| + const char * V_base = V_ptr + nb23*sequence + nb22*(head0 / gqa_ratio); | |\n| + const half * maskh = mask_ptr ? (const half *) (mask_ptr + nb33*(sequence % ne33)) : nullptr; | |\n| + | |\n| + const int stride_mask = nb31 / sizeof(half); | |\n| + | |\n| + extern __shared__ char fa_smem[]; | |\n| + _Float16 * Qs = (_Float16 *) fa_smem; // [NC][D+KPAD] | |\n| + _Float16 * Ks = Qs + NC*(D + KPAD); // [FAB][D+KPAD] | |\n| + _Float16 * Vt = Ks; // [D][FAB+FPAD], aliases Ks | |\n| + constexpr size_t kv_halves = (size_t) FAB*(D + KPAD) > (size_t) D*(FAB + FPAD) | |\n| + ? (size_t) FAB*(D + KPAD) : (size_t) D*(FAB + FPAD); | |\n| + _Float16 * Ps = Ks + kv_halves; // [NC][FAB+FPAD] | |\n| + float * Rm = (float *) (Ps + NC*(FAB + FPAD)); // [NQB*NKB][16] | |\n| + float * Rl = Rm + NQB*NKB*16; // [NQB*NKB][16] | |\n| + | |\n| + ggml_cuda_pdl_sync(); | |\n| + | |\n| + // ---- Q into LDS, pre-scaled and converted to f16 ---- | |\n| + // column jc -> q position col_Q_0 + jc/ncols2, head head0 + jc%ncols2 | |\n| + for (int idx = tid*8; idx < NC*D; idx += NTHR*8) { | |\n| + const int jc = idx / D; | |\n| + const int i0 = idx % D; | |\n| + const int j = jc / ncols2; | |\n| + const int c = jc % ncols2; | |\n| + const float * src = Q_f + c*(nb02/sizeof(float)) + fastmodulo(col_Q_0 + j, ne01)*(nb01/sizeof(float)) + i0; | |\n| + fa_frag16 f; | |\n| +#pragma unroll | |\n| + for (int i = 0; i < 8; ++i) { | |\n| + ((_Float16 *) &f.v[0])[i] = (_Float16) (src[i] * scale); | |\n| + } | |\n| + *(int4 *) &Qs[jc*(D + KPAD) + i0] = f.v[0]; | |\n| + } | |\n| + | |\n| + const int s_tile = warp % (NQB*NKB); | |\n| + const int k_half = warp / (NQB*NKB); | |\n| + const int qb = s_tile / NKB; | |\n| + const int s_kb = s_tile % NKB; | |\n| + // warps sharing this q block: NSW of them, each taking one DV slice in the O phase | |\n| + const int o_slot = (s_tile % NKB) + NKB*k_half; | |\n| + const int o_d0 = o_slot * DVW; | |\n| + | |\n| + fa_float8 Oacc[OT]; | |\n| +#pragma unroll | |\n| + for (int i = 0; i < OT; ++i) { Oacc[i] = (fa_float8)(0.0f); } | |\n| + | |\n| + float m_run[8], l_run[8]; | |\n| +#pragma unroll | |\n| + for (int l = 0; l < 8; ++l) { m_run[l] = -FLT_MAX/2.0f; l_run[l] = 0.0f; } | |\n| + | |\n| + __syncthreads(); | |\n| + | |\n| + const int k_VKQ_max = KV_max_ptr ? KV_max_ptr[sequence*gridDim.x + blockIdx.x] : ne11; | |\n| + | |\n| + for (int k_VKQ_0 = blockIdx.y*FAB; k_VKQ_0 < k_VKQ_max; k_VKQ_0 += gridDim.y*FAB) { | |\n| + const int kv_sup = k_VKQ_max - k_VKQ_0 < FAB ? k_VKQ_max - k_VKQ_0 : FAB; | |\n| + | |\n| + // ---- K into LDS ---- | |\n| + for (int idx = tid*8; idx < FAB*D; idx += NTHR*8) { | |\n| + const int kv = idx / D; | |\n| + const int i0 = idx % D; | |\n| + int4 v = make_int4(0, 0, 0, 0); | |\n| + if (kv < kv_sup) { | |\n| + v = fa_load8(K_base + (size_t)(k_VKQ_0 + kv)*nb11, i0); | |\n| + } | |\n| + *(int4 *) &Ks[kv*(D + KPAD) + i0] = v; | |\n| + } | |\n| + __syncthreads(); | |\n| + | |\n| + // ---- S = Q * K^T, one tile per warp ---- | |\n| + fa_float8 Sacc = (fa_float8)(0.0f); | |\n| +#pragma unroll | |\n| + for (int k0 = k_half*(D/KSPLIT); k0 < (k_half + 1)*(D/KSPLIT); k0 += 16) { | |\n| + const fa_half16 a = fa_load_frag(&Qs[(qb *16 + l16)*(D + KPAD) + k0]); | |\n| + const fa_half16 b = fa_load_frag(&Ks[(s_kb*16 + l16)*(D + KPAD) + k0]); | |\n| + Sacc = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a, b, Sacc); | |\n| + } | |\n| + // combine the KSPLIT partial head ranges; K is dead now so its buffer is scratch | |\n| + if constexpr (KSPLIT > 1) { | |\n| + float * Sred = (float *) Ks; | |\n| + __syncthreads(); | |\n| + if (k_half != 0) { | |\n| +#pragma unroll | |\n| + for (int l = 0; l < 8; ++l) { Sred[(s_tile*32 + lane)*8 + l] = Sacc[l]; } | |\n| + } | |\n| + __syncthreads(); | |\n| + if (k_half == 0) { | |\n| +#pragma unroll | |\n| + for (int l = 0; l < 8; ++l) { Sacc[l] += Sred[(s_tile*32 + lane)*8 + l]; } | |\n| +#pragma unroll | |\n| + for (int l = 0; l < 8; ++l) { Sred[(s_tile*32 + lane)*8 + l] = Sacc[l]; } | |\n| + } | |\n| + __syncthreads(); | |\n| + if (k_half != 0) { | |\n| +#pragma unroll | |\n| + for (int l = 0; l < 8; ++l) { Sacc[l] = Sred[(s_tile*32 + lane)*8 + l]; } | |\n| + } | |\n| + } | |\n| + // slot l -> block column jc = qb*16 + 2*l + lhi, kv = s_kb*16 + l16 | |\n| + | |\n| + // ---- mask / out-of-bounds ---- | |\n| + const int kv_loc = s_kb*16 + l16; | |\n| +#pragma unroll | |\n| + for (int l = 0; l < 8; ++l) { | |\n| + const int jc = qb*16 + 2*l + lhi; | |\n| + float s = Sacc[l]; | |\n| + if (kv_loc >= kv_sup) { | |\n| + s = -FLT_MAX/2.0f; | |\n| + } else if (maskh) { | |\n| + s += __half2float(maskh[(size_t)(col_Q_0 + jc/ncols2)*stride_mask + k_VKQ_0 + kv_loc]); | |\n| + } | |\n| + Sacc[l] = s; | |\n| + } | |\n| + | |\n| + // ---- row max: reduce over the 16 lanes of the subgroup, then over NKB warps ---- | |\n| + float m_t[8]; | |\n| +#pragma unroll | |\n| + for (int l = 0; l < 8; ++l) { m_t[l] = Sacc[l]; } | |\n| +#pragma unroll | |\n| + for (int off = 1; off < 16; off <<= 1) { | |\n| +#pragma unroll | |\n| + for (int l = 0; l < 8; ++l) { m_t[l] = fmaxf(m_t[l], __shfl_xor_sync(0xFFFFFFFF, m_t[l], off, WS)); } | |\n| + } | |\n| + if (l16 == 0 && k_half == 0) { | |\n| +#pragma unroll | |\n| + for (int l = 0; l < 8; ++l) { Rm[s_tile*16 + 2*l + lhi] = m_t[l]; } | |\n| + } | |\n| + __syncthreads(); | |\n| + | |\n| + float m_new[8], rescale[8]; | |\n| +#pragma unroll | |\n| + for (int l = 0; l < 8; ++l) { | |\n| + float mx = Rm[(qb*NKB + 0)*16 + 2*l + lhi]; | |\n| +#pragma unroll | |\n| + for (int w = 1; w < NKB; ++w) { mx = fmaxf(mx, Rm[(qb*NKB + w)*16 + 2*l + lhi]); } | |\n| + m_new[l] = fmaxf(m_run[l], mx); | |\n| + rescale[l] = expf(m_run[l] - m_new[l]); | |\n| + m_run[l] = m_new[l]; | |\n| + } | |\n| + | |\n| + // ---- P = exp(S - m), row sums, P into LDS ---- | |\n| + float l_t[8]; | |\n| +#pragma unroll | |\n| + for (int l = 0; l < 8; ++l) { | |\n| + const float p = expf(Sacc[l] - m_new[l]); | |\n| + l_t[l] = p; | |\n| + if (k_half == 0) { | |\n| + Ps[(qb*16 + 2*l + lhi)*(FAB + FPAD) + s_kb*16 + l16] = (_Float16) p; | |\n| + } | |\n| + } | |\n| +#pragma unroll | |\n| + for (int off = 1; off < 16; off <<= 1) { | |\n| +#pragma unroll | |\n| + for (int l = 0; l < 8; ++l) { l_t[l] += __shfl_xor_sync(0xFFFFFFFF, l_t[l], off, WS); } | |\n| + } | |\n| + if (l16 == 0 && k_half == 0) { | |\n| +#pragma unroll | |\n| + for (int l = 0; l < 8; ++l) { Rl[s_tile*16 + 2*l + lhi] = l_t[l]; } | |\n| + } | |\n| + | |\n| +#pragma unroll | |\n| + for (int t = 0; t < OT; ++t) { | |\n| +#pragma unroll | |\n| + for (int l = 0; l < 8; ++l) { Oacc[t][l] *= rescale[l]; } | |\n| + } | |\n| + __syncthreads(); // Rl ready and K no longer needed | |\n| + | |\n| +#pragma unroll | |\n| + for (int l = 0; l < 8; ++l) { | |\n| + float sum = 0.0f; | |\n| +#pragma unroll | |\n| + for (int w = 0; w < NKB; ++w) { sum += Rl[(qb*NKB + w)*16 + 2*l + lhi]; } | |\n| + l_run[l] = l_run[l]*rescale[l] + sum; | |\n| + } | |\n| + | |\n| + // ---- V transposed into the K buffer: Vt[dv][kv] ---- | |\n| + // one lane owns 8 consecutive kv of a single dv, so the LDS write is one int4 | |\n| + for (int w = tid; w < (FAB/8)*D; w += NTHR) { | |\n| + const int dv = w % D; | |\n| + const int kc = w / D; | |\n| + fa_frag16 g; | |\n| +#pragma unroll | |\n| + for (int j = 0; j < 8; ++j) { | |\n| + const int kv = kc*8 + j; | |\n| + ((_Float16 *) &g.v[0])[j] = kv < kv_sup | |\n| + ? fa_load1(V_base + (size_t)(k_VKQ_0 + kv)*nb21, dv) : (_Float16) 0.0f; | |\n| + } | |\n| + *(int4 *) &Vt[(size_t) dv*(FAB + FPAD) + kc*8] = g.v[0]; | |\n| + } | |\n| + __syncthreads(); | |\n| + | |\n| + // ---- O += P * V ---- | |\n| +#pragma unroll | |\n| + for (int t = 0; t < OT; ++t) { | |\n| + const int d0 = o_d0 + t*16; | |\n| +#pragma unroll | |\n| + for (int k0 = 0; k0 < FAB; k0 += 16) { | |\n| + const fa_half16 a = fa_load_frag(&Ps[(qb*16 + l16)*(FAB + FPAD) + k0]); | |\n| + const fa_half16 b = fa_load_frag(&Vt[(size_t)(d0 + l16)*(FAB + FPAD) + k0]); | |\n| + Oacc[t] = __builtin_amdgcn_wmma_f32_16x16x16_f16_w32(a, b, Oacc[t]); | |\n| + } | |\n| + } | |\n| + __syncthreads(); | |\n| + } | |\n| + | |\n| + // ---- write back ---- | |\n| +#pragma unroll | |\n| + for (int l = 0; l < 8; ++l) { | |\n| + const int jc = qb*16 + 2*l + lhi; | |\n| + const int j = jc / ncols2; | |\n| + const int c = jc % ncols2; | |\n| + | |\n| + if (ncols1 > 1 && col_Q_0 + j >= int(ne01.z)) { | |\n| + continue; | |\n| + } | |\n| + | |\n| + const float out_scale = gridDim.y == 1 ? 1.0f/l_run[l] : 1.0f; | |\n| + const int j_dst_unrolled = ((sequence*int(ne01.z) + col_Q_0 + j)*ne02 + head0 + c)*gridDim.y + blockIdx.y; | |\n| + | |\n| +#pragma unroll | |\n| + for (int t = 0; t < OT; ++t) { | |\n| + dst_ptr[(size_t) j_dst_unrolled*DV + o_d0 + t*16 + l16] = Oacc[t][l] * out_scale; | |\n| + } | |\n| + | |\n| + if (gridDim.y != 1 && l16 == 0 && o_d0 == 0) { | |\n| + dst_meta_ptr[j_dst_unrolled] = make_float2(m_run[l], l_run[l]); | |\n| + } | |\n| + } | |\n| +} | |\n| + | |\n| +#else // AMD_WMMA_AVAILABLE && RDNA3 | |\n| + | |\n| +template | |\n| +__launch_bounds__(FA_WMMA_D256_NWARPS*32, 1) | |\n| +static __global__ void flash_attn_wmma_d256( | |\n| + const char *, const char *, const char *, const char *, const char *, | |\n| + const int *, float *, float2 *, | |\n| + const float, const float, const float, const float, const uint32_t, const float, | |\n| + const int32_t, const uint3, const int32_t, const int32_t, | |\n| + const int32_t, const int32_t, const int32_t, | |\n| + const int32_t, const int32_t, const int32_t, const int32_t, | |\n| + const int32_t, const int32_t, const int64_t, | |\n| + const int32_t, const int32_t, const int64_t, | |\n| + const int32_t, const int32_t, const int32_t, | |\n| + const int32_t, const int32_t, const int64_t) { | |\n| + NO_DEVICE_CODE; | |\n| +} | |\n| + | |\n| +#endif // AMD_WMMA_AVAILABLE && RDNA3 | |\n| + | |\n| +void ggml_cuda_flash_attn_ext_wmma_d256(ggml_backend_cuda_context & ctx, ggml_tensor * dst); | |\n| diff --git a/ggml/src/ggml-cuda/fattn.cu b/ggml/src/ggml-cuda/fattn.cu | |\n| index ab7a3b297..f95c57b01 100644 | |\n| --- a/ggml/src/ggml-cuda/fattn.cu | |\n| +++ b/ggml/src/ggml-cuda/fattn.cu | |\n| @@ -3,7 +3,9 @@ | |\n| #include \"fattn-mma-f16.cuh\" | |\n| #include \"fattn-tile.cuh\" | |\n| #include \"fattn-vec.cuh\" | |\n| +#include \"fattn-wmma-d256.cuh\" | |\n| #include \"fattn.cuh\" | |\n| +#include | |\n| template | |\n| static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { | |\n| @@ -333,6 +335,7 @@ enum best_fattn_kernel { | |\n| BEST_FATTN_KERNEL_TILE = 200, | |\n| BEST_FATTN_KERNEL_VEC = 100, | |\n| BEST_FATTN_KERNEL_MMA_F16 = 400, | |\n| + BEST_FATTN_KERNEL_WMMA_D256 = 500, | |\n| }; | |\n| static bool ggml_cuda_fattn_kv_type_supported(ggml_type type) { | |\n| @@ -511,6 +514,28 @@ static best_fattn_kernel ggml_cuda_get_best_fattn_kernel(const int device, const | |\n| } | |\n| } | |\n| + // gfx1151, head size 256: the tile kernel emits no WMMA at all there, so use the | |\n| + // dedicated DV-split WMMA kernel. It needs 64 columns per block to pay for its LDS | |\n| + // footprint, so only for batches that can fill them, and only for the plain f16 case. | |\n| + float fa_logit_softcap = 0.0f; | |\n| + memcpy(&fa_logit_softcap, (const float *) KQV->op_params + 2, sizeof(float)); | |\n| + // Opt-in: 1.3-1.4x the tile kernel on short/medium KV in isolation, but its 60 kB LDS | |\n| + // footprint fits only 2 workgroups per WGP against the tile kernel's 3, and at long | |\n| + // context that costs more than the matrix units gain. See docs/exp205. | |\n| + // | |\n| + // Quantised K/V are read natively (no pre-conversion to f16) so that the DRAM traffic | |\n| + // actually drops. Measured: it does not pay. q8_0 and q4_0 are 1.7-1.8x *slower* than | |\n| + // f16 here, because this kernel is bound by instruction issue and memory latency rather | |\n| + // than by bytes, and per-element dequantisation costs more than the bytes it saves. | |\n| + if (GGML_CUDA_CC_IS_RDNA3_5(cc) && getenv(\"GGML_CUDA_FA_WMMA256\") | |\n| + && Q->ne[0] == 256 && V->ne[0] == 256 | |\n| + && K->type == V->type | |\n| + && (K->type == GGML_TYPE_F16 || K->type == GGML_TYPE_Q8_0 || K->type == GGML_TYPE_Q4_0) | |\n| + && mask && !dst->src[4] && max_bias == 0.0f && fa_logit_softcap == 0.0f | |\n| + && gqa_opt_applies && Q->ne[1] * gqa_ratio_eff >= FA_WMMA_D256_NC) { | |\n| + return BEST_FATTN_KERNEL_WMMA_D256; | |\n| + } | |\n| + | |\n| // AMD WMMA is always faster than the tile kernel if the full tile width of 16 can be utilized. | |\n| if ((amd_wmma_available(cc) && gqa_opt_applies && Q->ne[0] <= 128) && Q->ne[0] != 40 && Q->ne[0] != 72 && Q->ne[1] * gqa_ratio_eff > 8) { | |\n| return BEST_FATTN_KERNEL_MMA_F16; | |\n| @@ -553,6 +578,8 @@ size_t ggml_cuda_flash_attn_ext_get_alloc_size(int device, const ggml_tensor * d | |\n| need_f16_K = true; | |\n| need_f16_V = true; | |\n| break; | |\n| + case BEST_FATTN_KERNEL_WMMA_D256: | |\n| + break; // reads K/V natively, quantised or not | |\n| case BEST_FATTN_KERNEL_VEC: | |\n| need_f16_K = K->type == GGML_TYPE_F32; | |\n| need_f16_V = V->type == GGML_TYPE_F32; | |\n| @@ -567,8 +594,35 @@ size_t ggml_cuda_flash_attn_ext_get_alloc_size(int device, const ggml_tensor * d | |\n| return f16_extra.end - (uintptr_t) dst->data; | |\n| } | |\n| -void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { | |\n| - ggml_cuda_set_device(ctx.device); | |\n| +// Max Q columns per flash-attention launch, 0 to disable splitting. | |\n| +// One launch with many Q columns puts many column tiles on the GPU at the same time. The tiles | |\n| +// then read different parts of K/V and stop sharing them in cache, which only hurts when the KV | |\n| +// cache is much larger than the last level cache. Splitting Q keeps the number of concurrent | |\n| +// tiles low. On gfx1151 with ubatch 2048 this is 1.4x faster at 32k context and 1.8x at 64k. | |\n| +static int ggml_cuda_fattn_q_chunk(const ggml_tensor * dst) { | |\n| + static const int chunk = getenv(\"GGML_CUDA_FA_QCHUNK\") ? atoi(getenv(\"GGML_CUDA_FA_QCHUNK\")) : 256; | |\n| + | |\n| + if (chunk <= 0 || !GGML_CUDA_CC_IS_RDNA3_5(ggml_cuda_info().devices[ggml_cuda_get_device()].cc)) { | |\n| + return 0; | |\n| + } | |\n| + | |\n| + // Only worth it if K/V does not fit in cache anyway, else the extra launches just add overhead. | |\n| + static const int min_kv = getenv(\"GGML_CUDA_FA_QCHUNK_MINKV\") ? atoi(getenv(\"GGML_CUDA_FA_QCHUNK_MINKV\")) : 8192; | |\n| + | |\n| + if (dst->src[1]->ne[1] < min_kv) { | |\n| + return 0; | |\n| + } | |\n| + | |\n| + // Q rows are dst dimension 2. If dst has more than one dimension 3 then a row range of dst is | |\n| + // not one continuous block, but the kernels write dst as continuous memory. Do not split. | |\n| + if (dst->ne[3] != 1) { | |\n| + return 0; | |\n| + } | |\n| + | |\n| + return chunk; | |\n| +} | |\n| + | |\n| +static void ggml_cuda_flash_attn_ext_dispatch(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { | |\n| switch (ggml_cuda_get_best_fattn_kernel(ggml_cuda_get_device(), dst)) { | |\n| case BEST_FATTN_KERNEL_NONE: | |\n| GGML_ABORT(\"fatal error\"); | |\n| @@ -581,6 +635,50 @@ void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst | |\n| case BEST_FATTN_KERNEL_MMA_F16: | |\n| ggml_cuda_flash_attn_ext_mma_f16(ctx, dst); | |\n| break; | |\n| + case BEST_FATTN_KERNEL_WMMA_D256: | |\n| + ggml_cuda_flash_attn_ext_wmma_d256(ctx, dst); | |\n| + break; | |\n| + } | |\n| +} | |\n| + | |\n| +void ggml_cuda_flash_attn_ext(ggml_backend_cuda_context & ctx, ggml_tensor * dst) { | |\n| + ggml_cuda_set_device(ctx.device); | |\n| + | |\n| + const ggml_tensor * Q = dst->src[0]; | |\n| + const ggml_tensor * mask = dst->src[3]; | |\n| + | |\n| + const int chunk = ggml_cuda_fattn_q_chunk(dst); | |\n| + if (chunk == 0 || Q->ne[1] <= chunk) { | |\n| + ggml_cuda_flash_attn_ext_dispatch(ctx, dst); | |\n| + return; | |\n| + } | |\n| + | |\n| + // Q rows are dst rows in dimension 2, mask rows are in dimension 1. | |\n| + ggml_tensor Q_chunk = *Q; | |\n| + ggml_tensor mask_chunk = mask ? *mask : ggml_tensor(); | |\n| + ggml_tensor dst_chunk = *dst; | |\n| + | |\n| + dst_chunk.src[0] = &Q_chunk; | |\n| + if (mask) { | |\n| + dst_chunk.src[3] = &mask_chunk; | |\n| + } | |\n| + | |\n| + for (int64_t i0 = 0; i0 < Q->ne[1]; i0 += chunk) { | |\n| + const int64_t ncols = std::min(chunk, Q->ne[1] - i0); | |\n| + | |\n| + Q_chunk.ne[1] = ncols; | |\n| + Q_chunk.data = (char *) Q->data + i0*Q->nb[1]; | |\n| + | |\n| + // Keep all remaining mask rows so that reads past ncols stay in bounds. | |\n| + if (mask) { | |\n| + mask_chunk.ne[1] = mask->ne[1] - i0; | |\n| + mask_chunk.data = (char *) mask->data + i0*mask->nb[1]; | |\n| + } | |\n| + | |\n| + dst_chunk.ne[2] = ncols; | |\n| + dst_chunk.data = (char *) dst->data + i0*dst->nb[2]; | |\n| + | |\n| + ggml_cuda_flash_attn_ext_dispatch(ctx, &dst_chunk); | |\n| } | |\n| } | |\n| diff --git a/ggml/src/ggml-cuda/gated_delta_net.cu b/ggml/src/ggml-cuda/gated_delta_net.cu | |\n| index 1b431a724..97ab34724 100644 | |\n| --- a/ggml/src/ggml-cuda/gated_delta_net.cu | |\n| +++ b/ggml/src/ggml-cuda/gated_delta_net.cu | |\n| @@ -1,6 +1,17 @@ | |\n| #include \"gated_delta_net.cuh\" | |\n| +#include | |\n| #include \"ggml-cuda/common.cuh\" | |\n| +// Lanes cooperating on one state column in the column-parallel kernel. | |\n| +// 0 disables it and falls back to the warp-per-column kernel. Overridable for tuning. | |\n| +static int ggml_cuda_gdn_split() { | |\n| + static const int split = [] { | |\n| + const char * s = getenv(\"GGML_CUDA_GDN_SPLIT\"); | |\n| + return s ? atoi(s) : 2; | |\n| + }(); | |\n| + return split; | |\n| +} | |\n| + | |\n| template | |\n| __global__ void __launch_bounds__((ggml_cuda_get_physical_warp_size() < S_v ? ggml_cuda_get_physical_warp_size() : S_v) * 4, 2) | |\n| gated_delta_net_cuda(const float * q, | |\n| @@ -166,6 +177,152 @@ gated_delta_net_cuda(const float * q, | |\n| } | |\n| } | |\n| +// Column-parallel variant of the kernel above. | |\n| +// | |\n| +// The warp-per-column kernel gives one *warp* one state column and splits the column's | |\n| +// rows across the warp's lanes. Every warp therefore reads the head's whole q and k | |\n| +// vector, and with the columns spread over S_v/nwarps blocks that means q and k get | |\n| +// re-read S_v times per head per token. That redundancy, not the arithmetic, is what | |\n| +// bounds the kernel on prefill. | |\n| +// | |\n| +// Here a block owns *all* S_v columns of one head, so q/k/g are read exactly once per | |\n| +// head per token into LDS. Each column is handled by SPLIT adjacent lanes, so a thread | |\n| +// holds S_v/SPLIT state values in registers and the row reduction costs log2(SPLIT) | |\n| +// shuffles instead of log2(warp_size). SPLIT trades registers (and hence occupancy) | |\n| +// against those shuffles. | |\n| +template | |\n| +__global__ void __launch_bounds__(S_v * SPLIT, 1) | |\n| +gated_delta_net_col_cuda(const float * __restrict__ q, | |\n| + const float * __restrict__ k, | |\n| + const float * __restrict__ v, | |\n| + const float * __restrict__ g, | |\n| + const float * __restrict__ beta, | |\n| + const float * __restrict__ curr_state, | |\n| + float * __restrict__ dst, | |\n| + float * __restrict__ state, | |\n| + int64_t H, | |\n| + int64_t n_tokens, | |\n| + int64_t n_seqs, | |\n| + int64_t sq1, int64_t sq2, int64_t sq3, | |\n| + int64_t sv1, int64_t sv2, int64_t sv3, | |\n| + int64_t sb1, int64_t sb2, int64_t sb3, | |\n| + const uint3 neqk1_magic, | |\n| + const uint3 rq3_magic, | |\n| + float scale, | |\n| + int64_t state_slot_stride, | |\n| + int K) { | |\n| + GGML_UNUSED(n_seqs); | |\n| + constexpr int RPT = S_v / SPLIT; // state rows held per thread | |\n| + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); | |\n| + static_assert(S_v % SPLIT == 0, \"SPLIT must divide S_v\"); | |\n| + static_assert(SPLIT <= warp_size, \"the row reduction must stay inside one wave\"); | |\n| + | |\n| + const uint32_t h_idx = blockIdx.x; | |\n| + const uint32_t sequence = blockIdx.y; | |\n| + const int tid = threadIdx.x; | |\n| + const int col = tid / SPLIT; // SPLIT adjacent lanes share a column | |\n| + const int part = tid % SPLIT; | |\n| + const int row0 = part * RPT; | |\n| + | |\n| + const uint32_t iq1 = fastmodulo(h_idx, neqk1_magic); | |\n| + const uint32_t iq3 = fastdiv(sequence, rq3_magic); | |\n| + | |\n| + float * attn_data = dst + (sequence * n_tokens * H + h_idx) * S_v; | |\n| + | |\n| + const int64_t state_in_offset = sequence * H * S_v * S_v + h_idx * S_v * S_v; | |\n| + const int64_t state_out_offset = (sequence * H + h_idx) * S_v * S_v; | |\n| + state += state_out_offset; | |\n| + curr_state += state_in_offset + col * S_v + row0; | |\n| + | |\n| + // state is stored transposed: M[col][i] = S[i][col], so row `col` is contiguous | |\n| + float s_col[RPT]; | |\n| + | |\n| + // q/k (and the per-row gate for KDA) are identical for every column, so stage them in | |\n| + // LDS once per token. Double-buffered so one barrier per token is enough. | |\n| + __shared__ float sk[2][S_v]; | |\n| + __shared__ float sq[2][S_v]; | |\n| + __shared__ float sg[2][KDA ? S_v : 1]; | |\n| + | |\n| + ggml_cuda_pdl_sync(); | |\n| +#pragma unroll | |\n| + for (int j = 0; j < RPT; j++) { | |\n| + s_col[j] = curr_state[j]; | |\n| + } | |\n| + | |\n| + for (int t = 0; t < n_tokens; t++) { | |\n| + const float * __restrict__ q_t = q + iq3 * sq3 + t * sq2 + iq1 * sq1; | |\n| + const float * __restrict__ k_t = k + iq3 * sq3 + t * sq2 + iq1 * sq1; | |\n| + const float * __restrict__ v_t = v + sequence * sv3 + t * sv2 + h_idx * sv1; | |\n| + | |\n| + const int64_t gb_offset = sequence * sb3 + t * sb2 + h_idx * sb1; | |\n| + const float beta_val = beta[gb_offset]; | |\n| + const float * __restrict__ g_t = g + gb_offset * (KDA ? S_v : 1); | |\n| + | |\n| + const int buf = t & 1; | |\n| + if (tid < S_v) { | |\n| + sk[buf][tid] = k_t[tid]; | |\n| + sq[buf][tid] = q_t[tid]; | |\n| + if (KDA) { | |\n| + sg[buf][tid] = expf(g_t[tid]); | |\n| + } | |\n| + } | |\n| + __syncthreads(); | |\n| + | |\n| + const float * __restrict__ lk = sk[buf] + row0; | |\n| + const float * __restrict__ lq = sq[buf] + row0; | |\n| + const float * __restrict__ lg = sg[buf] + (KDA ? row0 : 0); | |\n| + | |\n| + const float g_val = KDA ? 0.0f : expf(*g_t); | |\n| + | |\n| + float kv = 0.0f; | |\n| +#pragma unroll | |\n| + for (int j = 0; j < RPT; j++) { | |\n| + kv = KDA ? fmaf(lg[j] * s_col[j], lk[j], kv) : fmaf(s_col[j], lk[j], kv); | |\n| + } | |\n| +#pragma unroll | |\n| + for (int o = 1; o < SPLIT; o <<= 1) { | |\n| + kv += __shfl_xor_sync(0xFFFFFFFF, kv, o, warp_size); | |\n| + } | |\n| + | |\n| + const float delta = (v_t[col] - (KDA ? kv : g_val * kv)) * beta_val; | |\n| + | |\n| + float attn = 0.0f; | |\n| +#pragma unroll | |\n| + for (int j = 0; j < RPT; j++) { | |\n| + s_col[j] = fmaf(KDA ? lg[j] : g_val, s_col[j], lk[j] * delta); | |\n| + attn = fmaf(s_col[j], lq[j], attn); | |\n| + } | |\n| +#pragma unroll | |\n| + for (int o = 1; o < SPLIT; o <<= 1) { | |\n| + attn += __shfl_xor_sync(0xFFFFFFFF, attn, o, warp_size); | |\n| + } | |\n| + | |\n| + if (part == 0) { | |\n| + attn_data[col] = attn * scale; | |\n| + } | |\n| + attn_data += S_v * H; | |\n| + | |\n| + if constexpr (keep_rs_t) { | |\n| + const int target_slot = (int) n_tokens - 1 - t; | |\n| + if (target_slot >= 0 && target_slot < K) { | |\n| + float * slot = state + target_slot * state_slot_stride + col * S_v + row0; | |\n| +#pragma unroll | |\n| + for (int j = 0; j < RPT; j++) { | |\n| + slot[j] = s_col[j]; | |\n| + } | |\n| + } | |\n| + } | |\n| + } | |\n| + | |\n| + if constexpr (!keep_rs_t) { | |\n| + float * out = state + col * S_v + row0; | |\n| +#pragma unroll | |\n| + for (int j = 0; j < RPT; j++) { | |\n| + out[j] = s_col[j]; | |\n| + } | |\n| + } | |\n| +} | |\n| + | |\n| template | |\n| static void launch_gated_delta_net( | |\n| const float * q_d, const float * k_d, const float * v_d, | |\n| @@ -186,6 +343,42 @@ static void launch_gated_delta_net( | |\n| const uint3 neqk1_magic = init_fastdiv_values(neqk1); | |\n| const uint3 rq3_magic = init_fastdiv_values(rq3); | |\n| + // Column-parallel kernel: one block per (head, sequence), all S_v columns inside it. | |\n| + // Needs at least a full wave per block, so only used from S_v = 64 up. | |\n| + { | |\n| + const int gdn_split = ggml_cuda_gdn_split(); | |\n| + // For a single token the warp-per-column kernel wins: there is no q/k reuse to | |\n| + // recover and its narrower blocks start faster. | |\n| + if (gdn_split > 0 && n_tokens >= 16 && (S_v == 64 || S_v == 128)) { | |\n| + const dim3 grid_col(H, n_seqs, 1); | |\n| + const dim3 block_col(S_v * gdn_split, 1, 1); | |\n| + const ggml_cuda_kernel_launch_params lp = ggml_cuda_kernel_launch_params(grid_col, block_col, 0, stream); | |\n| +#define GDN_COL_LAUNCH(SV, SP) \\ | |\n| + ggml_cuda_kernel_launch(gated_delta_net_col_cuda, lp, \\ | |\n| + q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, \\ | |\n| + n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, \\ | |\n| + sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K) | |\n| + if (S_v == 64) { | |\n| + switch (gdn_split) { | |\n| + case 1: GDN_COL_LAUNCH(64, 1); return; | |\n| + case 2: GDN_COL_LAUNCH(64, 2); return; | |\n| + case 4: GDN_COL_LAUNCH(64, 4); return; | |\n| + case 8: GDN_COL_LAUNCH(64, 8); return; | |\n| + default: break; | |\n| + } | |\n| + } else { | |\n| + switch (gdn_split) { | |\n| + case 1: GDN_COL_LAUNCH(128, 1); return; | |\n| + case 2: GDN_COL_LAUNCH(128, 2); return; | |\n| + case 4: GDN_COL_LAUNCH(128, 4); return; | |\n| + case 8: GDN_COL_LAUNCH(128, 8); return; | |\n| + default: break; | |\n| + } | |\n| + } | |\n| +#undef GDN_COL_LAUNCH | |\n| + } | |\n| + } | |\n| + | |\n| const ggml_cuda_kernel_launch_params launch_params = ggml_cuda_kernel_launch_params(grid_dims, block_dims, 0, stream); | |\n| switch (S_v) { | |\n| case 16: | |\n| diff --git a/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh b/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh | |\n| index 180b2d937..9838fd175 100644 | |\n| --- a/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh | |\n| +++ b/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh | |\n| @@ -1,290 +1,290 @@ | |\n| static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_config_rdna3_5(ggml_type type, int J, bool fallback) { | |\n| CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_Q1_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_Q1_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q1_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q1_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q1_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q1_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q1_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q1_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q1_0, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_Q2_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_Q2_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q2_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q2_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q2_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q2_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q2_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q2_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q2_0, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_Q4_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_Q4_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q4_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q4_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q4_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q4_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q4_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q4_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q4_0, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_Q4_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_Q4_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q4_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q4_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q4_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q4_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q4_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q4_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q4_1, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_Q5_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_Q5_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q5_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q5_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q5_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q5_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q5_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q5_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q5_0, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_Q5_1, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_Q5_1, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q5_1, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q5_1, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q5_1, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q5_1, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q5_1, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q5_1, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q5_1, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_Q8_0, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_Q8_0, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q8_0, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q8_0, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q8_0, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q8_0, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q8_0, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q8_0, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q8_0, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| // --------------------------------------------------------------------------------------------- | |\n| CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_Q2_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_Q2_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q2_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q2_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q2_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q2_K, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q2_K, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_Q3_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_Q3_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q3_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q3_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q3_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q3_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q3_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q3_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q3_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_Q4_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_Q4_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q4_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q4_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_Q5_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_Q5_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q5_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q5_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_Q6_K, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_Q6_K, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_Q6_K, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_Q6_K, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q6_K, MMQ_ITER_K, false, false); | |\n| // --------------------------------------------------------------------------------------------- | |\n| CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ1_S, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ2_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ2_XS, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ2_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ2_S, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q3_K, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ3_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ3_XXS, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ3_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ3_S, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ4_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ4_XS, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_IQ4_NL, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_IQ4_NL, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); | |\n| // --------------------------------------------------------------------------------------------- | |\n| CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_MXFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_MXFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_MXFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_MXFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_MXFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_MXFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_MXFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_MXFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_MXFP4, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_1, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); | |\n| - CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); | |\n| + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, true); | |\n| CASE(GGML_TYPE_NVFP4, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); | |\n| CASE(GGML_TYPE_NVFP4, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_NVFP4, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_NVFP4, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_NVFP4, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_NVFP4, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_NVFP4, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); | |\n| - CASE(GGML_TYPE_NVFP4, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); | |\n| + CASE(GGML_TYPE_NVFP4, 128, 2, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_NVFP4, MMQ_ITER_K, false, false); | |\n| - return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true); | |\n| + return ggml_cuda_mmq_config(GGML_TYPE_COUNT, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, 256, false, true); | |\n| } | |\n| diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu | |\n| index 0589e65bd..3a9d732ca 100644 | |\n| --- a/ggml/src/ggml-cuda/mmvq.cu | |\n| +++ b/ggml/src/ggml-cuda/mmvq.cu | |\n| @@ -63,12 +63,17 @@ static constexpr __host__ __device__ int get_vdr_mmvq(ggml_type type) { | |\n| } | |\n| } | |\n| +#ifndef GGML_MMVQ_RDNA35_NWARPS | |\n| +#define GGML_MMVQ_RDNA35_NWARPS 2 | |\n| +#endif | |\n| + | |\n| enum mmvq_parameter_table_id { | |\n| MMVQ_PARAMETERS_GENERIC = 0, | |\n| MMVQ_PARAMETERS_TURING, | |\n| MMVQ_PARAMETERS_GCN, | |\n| MMVQ_PARAMETERS_RDNA2, | |\n| MMVQ_PARAMETERS_RDNA3_0, | |\n| + MMVQ_PARAMETERS_RDNA3_5, | |\n| MMVQ_PARAMETERS_RDNA4 | |\n| }; | |\n| @@ -77,7 +82,9 @@ static constexpr __device__ mmvq_parameter_table_id get_device_table_id() { | |\n| return MMVQ_PARAMETERS_RDNA4; | |\n| #elif defined(RDNA3_0) | |\n| return MMVQ_PARAMETERS_RDNA3_0; | |\n| -#elif defined(RDNA2) || defined(RDNA3_5) | |\n| +#elif defined(RDNA3_5) | |\n| + return MMVQ_PARAMETERS_RDNA3_5; | |\n| +#elif defined(RDNA2) | |\n| return MMVQ_PARAMETERS_RDNA2; | |\n| #elif defined(GCN) || defined(CDNA) | |\n| return MMVQ_PARAMETERS_GCN; | |\n| @@ -95,7 +102,10 @@ static __host__ mmvq_parameter_table_id get_device_table_id(int cc) { | |\n| if (GGML_CUDA_CC_IS_RDNA3_0(cc)) { | |\n| return MMVQ_PARAMETERS_RDNA3_0; | |\n| } | |\n| - if (GGML_CUDA_CC_IS_RDNA2(cc) || GGML_CUDA_CC_IS_RDNA3_5(cc)) { | |\n| + if (GGML_CUDA_CC_IS_RDNA3_5(cc)) { | |\n| + return MMVQ_PARAMETERS_RDNA3_5; | |\n| + } | |\n| + if (GGML_CUDA_CC_IS_RDNA2(cc)) { | |\n| return MMVQ_PARAMETERS_RDNA2; | |\n| } | |\n| if (GGML_CUDA_CC_IS_GCN(cc) || GGML_CUDA_CC_IS_CDNA(cc)) { | |\n| @@ -406,6 +416,31 @@ static constexpr __host__ __device__ int calc_nwarps(ggml_type type, int ncols_d | |\n| } | |\n| return 1; | |\n| } | |\n| + if (table_id == MMVQ_PARAMETERS_RDNA3_5) { | |\n| + // Strix Halo: mat-vec decode is bandwidth bound, and one warp per block does not | |\n| + // issue enough loads to keep the memory system busy. GGML_MMVQ_RDNA35_NWARPS | |\n| + // warps cooperate on a row instead. | |\n| + if (ncols_dst == 1) { | |\n| + switch (type) { | |\n| + case GGML_TYPE_Q4_0: | |\n| + case GGML_TYPE_Q4_1: | |\n| + case GGML_TYPE_Q5_0: | |\n| + case GGML_TYPE_Q5_1: | |\n| + case GGML_TYPE_Q8_0: | |\n| + case GGML_TYPE_Q2_K: | |\n| + case GGML_TYPE_Q3_K: | |\n| + case GGML_TYPE_Q4_K: | |\n| + case GGML_TYPE_Q5_K: | |\n| + case GGML_TYPE_Q6_K: | |\n| + case GGML_TYPE_IQ4_NL: | |\n| + case GGML_TYPE_IQ4_XS: | |\n| + return GGML_MMVQ_RDNA35_NWARPS; | |\n| + default: | |\n| + return 1; | |\n| + } | |\n| + } | |\n| + return 1; | |\n| + } | |\n| if (table_id == MMVQ_PARAMETERS_RDNA3_0) { | |\n| // RDNA3 (W7900): stricter whitelist than RDNA4. | |\n| // Q2_K / Q5_K / IQ4_XS regress in full quant sweeps. | |\n| diff --git a/ggml/src/ggml-cuda/quantize.cu b/ggml/src/ggml-cuda/quantize.cu | |\n| index bcc772395..dd42db0a0 100644 | |\n| --- a/ggml/src/ggml-cuda/quantize.cu | |\n| +++ b/ggml/src/ggml-cuda/quantize.cu | |\n| @@ -1,4 +1,5 @@ | |\n| #include \"quantize.cuh\" | |\n| +#include | |\n| #include | |\n| #if defined(BLACKWELL_MMA_AVAILABLE) | |\n| @@ -458,19 +459,25 @@ template | |\n| static __global__ void quantize_mmq_q8_1( | |\n| const float * __restrict__ x, const int32_t * __restrict__ ids, void * __restrict__ vy, | |\n| const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, | |\n| - const int64_t ne0, const int ne1, const int ne2, const int n_expert_used) { | |\n| + const int64_t ne0, const int ne1, const int ne2, const int n_expert_used, const int chunks) { | |\n| constexpr int vals_per_scale = ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6 ? 64 : 32; | |\n| constexpr int vals_per_sum = ds_layout == MMQ_Q8_1_DS_LAYOUT_D2S6 ? 16 : 32; | |\n| - const int64_t i0 = ((int64_t)blockDim.x*blockIdx.y + threadIdx.x)*4; | |\n| + ggml_cuda_pdl_sync(); | |\n| + | |\n| + // Each block walks `chunks` consecutive column chunks of one row. One chunk is only | |\n| + // 4*blockDim.x floats, so a chunk per block leaves the reads scattered over ne1 | |\n| + // separate rows; walking several keeps a longer contiguous run per row. | |\n| + for (int chunk = 0; chunk < chunks; ++chunk) { | |\n| + | |\n| + const int64_t i0 = ((int64_t)blockDim.x*(blockIdx.y*(int64_t)chunks + chunk) + threadIdx.x)*4; | |\n| if (i0 >= ne0) { | |\n| - return; | |\n| + break; | |\n| } | |\n| const int64_t i00 = i0; | |\n| - ggml_cuda_pdl_sync(); | |\n| int64_t base_idx; | |\n| if constexpr (scatter) { | |\n| @@ -529,7 +536,7 @@ static __global__ void quantize_mmq_q8_1( | |\n| const int64_t i = ids[(int64_t) blockIdx.x * n_expert_used + slot]; | |\n| ib = k_block*ne1 + i; | |\n| } else { | |\n| - const int64_t ib0 = blockIdx.z*((int64_t)gridDim.x*gridDim.y*blockDim.x/QK8_1); // first block of channel | |\n| + const int64_t ib0 = blockIdx.z*(ne0/QK8_1_MMQ)*(int64_t)ne1; // first block of channel | |\n| ib = ib0 + k_block*ne1 + blockIdx.x; | |\n| } | |\n| @@ -552,6 +559,8 @@ static __global__ void quantize_mmq_q8_1( | |\n| } | |\n| } | |\n| } | |\n| + | |\n| + } // chunk | |\n| GGML_UNUSED(n_expert_used); | |\n| } | |\n| @@ -572,6 +581,14 @@ void quantize_row_q8_1_cuda( | |\n| GGML_UNUSED(type_src0); | |\n| } | |\n| +static int ggml_cuda_quantize_mmq_chunks() { | |\n| + static const int chunks = [] { | |\n| + const char * s = getenv(\"GGML_CUDA_QUANTIZE_CHUNKS\"); | |\n| + return s ? atoi(s) : 2; | |\n| + }(); | |\n| + return chunks; | |\n| +} | |\n| + | |\n| void quantize_mmq_q8_1_cuda( | |\n| const float * x, const int32_t * ids, void * vy, const ggml_type type_src0, | |\n| const int64_t ne00, const int64_t s01, const int64_t s02, const int64_t s03, | |\n| @@ -580,21 +597,23 @@ void quantize_mmq_q8_1_cuda( | |\n| GGML_ASSERT(ne0 % QK8_1_MMQ == 0); | |\n| // ne1 tends to assume the highest values, therefore use it as the \"x\" dimension of the CUDA grid: | |\n| - const int64_t block_num_y = (ne0 + 4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1) / (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ); | |\n| + const int64_t block_num_y_full = (ne0 + 4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1) / (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ); | |\n| + const int chunks = ggml_cuda_quantize_mmq_chunks(); | |\n| + const int64_t block_num_y = (block_num_y_full + chunks - 1) / chunks; | |\n| const dim3 num_blocks(ne1, block_num_y, ne2*ne3); | |\n| const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1); | |\n| switch (mmq_get_q8_1_ds_layout(type_src0)) { | |\n| case MMQ_Q8_1_DS_LAYOUT_D4: | |\n| quantize_mmq_q8_1 | |\n| - <<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0); | |\n| + <<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0, chunks); | |\n| break; | |\n| case MMQ_Q8_1_DS_LAYOUT_DS4: | |\n| quantize_mmq_q8_1 | |\n| - <<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0); | |\n| + <<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0, chunks); | |\n| break; | |\n| case MMQ_Q8_1_DS_LAYOUT_D2S6: | |\n| quantize_mmq_q8_1 | |\n| - <<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0); | |\n| + <<>>(x, ids, vy, ne00, s01, s02, s03, ne0, ne1, ne2, /*n_expert_used=*/0, chunks); | |\n| break; | |\n| default: | |\n| GGML_ABORT(\"fatal error\"); | |\n| @@ -611,20 +630,21 @@ void quantize_scatter_mmq_q8_1_cuda( | |\n| GGML_ASSERT(ne0 % QK8_1_MMQ == 0); | |\n| const int64_t block_num_y = (ne0 + 4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ - 1) / (4*CUDA_QUANTIZE_BLOCK_SIZE_MMQ); | |\n| + const int chunks = 1; | |\n| const dim3 num_blocks(n_tokens, block_num_y, 1); | |\n| const dim3 block_size(CUDA_QUANTIZE_BLOCK_SIZE_MMQ, 1, 1); | |\n| switch (mmq_get_q8_1_ds_layout(type_src0)) { | |\n| case MMQ_Q8_1_DS_LAYOUT_D4: | |\n| quantize_mmq_q8_1<<>>( | |\n| - x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used); | |\n| + x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used, chunks); | |\n| break; | |\n| case MMQ_Q8_1_DS_LAYOUT_DS4: | |\n| quantize_mmq_q8_1<<>>( | |\n| - x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used); | |\n| + x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used, chunks); | |\n| break; | |\n| case MMQ_Q8_1_DS_LAYOUT_D2S6: | |\n| quantize_mmq_q8_1<<>>( | |\n| - x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used); | |\n| + x, ids_src1_inv, vy, ne00, /*s01=*/0, /*s02=*/stride_token, /*s03=*/0, ne0, /*ne1=*/(int) nrows_dst, /*ne2=*/1, n_expert_used, chunks); | |\n| break; | |\n| default: | |\n| GGML_ABORT(\"fatal error\"); |", "url": "https://wpnews.pro/news/llama-cpp-gfx1151-optimizations-for-qwen3-8-27b", "canonical_source": "https://gist.github.com/pedapudi/f414fbcd838610cc6bd8319c11a6cc87", "published_at": "2026-08-16 04:59:27+00:00", "updated_at": "2026-08-16 18:41:42.745134+00:00", "lang": "en", "topics": ["machine-learning", "large-language-models", "developer-tools"], "entities": ["llama.cpp", "Qwen3.8", "AMD", "gfx1151"], "alternates": {"html": "https://wpnews.pro/news/llama-cpp-gfx1151-optimizations-for-qwen3-8-27b", "markdown": "https://wpnews.pro/news/llama-cpp-gfx1151-optimizations-for-qwen3-8-27b.md", "text": "https://wpnews.pro/news/llama-cpp-gfx1151-optimizations-for-qwen3-8-27b.txt", "jsonld": "https://wpnews.pro/news/llama-cpp-gfx1151-optimizations-for-qwen3-8-27b.jsonld"}}