From 3b35c96d3d4ede67128df4371d9419ca0555eccf Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 26 Sep 2026 20:24:16 +1000 Subject: [PATCH 01/65] Bonsai 2 27B (PTQ1_0) on Intel Arc B580: SYCL + Vulkan performance work - SYCL: PTQ1_0 weights repacked to 2-bit at load and multiplied on XMX via the TernSYCL int8 x int2 DPAS kernels (vendored, BSD 3-Clause), activations quantized to int8 with round-to-nearest (GGML_SYCL_PTQ1_T2=ffn|all). - SYCL: decode attention for q4_0 KV serving 1-4 query tokens per launch; blocked gated delta-net; PTQ1_0 mat-vec kernels (single column, decode-once multi column, same-input merge, fused gate/up); narrow concat. - Vulkan: matching PTQ1_0 multi-column, decode attention, gated delta-net and coopmat work. - Speculative decoding: draft KV cache type (-ctkd/-ctvd) honoured by the MTP draft context, LLAMA_ARG_SPEC_DRAFT_UBATCH to shrink its compute buffer. - docs/bonsai-arc-b580.md: results at 128K on a B580, build and run instructions, switches, credits. Based on the bonsai-combo branch of professorpalmer/llama.cpp-ada-ternary (PrismML PR #221). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 753323e6025a6fe866d0fb11a3fd5e9ab99d785d on Torchit1/llama.cpp arc-b580) Excluded from this port: .github docs ggml/src/ggml-vulkan Conflicts resolved against prism: fwht.cpp keeps prism's FWHT (#302) and adds this commit's signed variant beside it as ggml_sycl_op_fwht_signed; ggml_sycl_op_mul_mat_sycl takes fp16 for PQ2_0/PTQ1_0 as before (#278) plus this commit's large-batch rule; the T2 path runs before the PQ2_0 XMX path (#294), which still serves PQ2_0/PTQ1_0 when GGML_SYCL_PTQ1_T2 is unset. --- common/speculative.cpp | 9 + ggml/src/ggml-sycl/common.hpp | 3 + ggml/src/ggml-sycl/concat.cpp | 17 + ggml/src/ggml-sycl/convert.cpp | 118 +++ ggml/src/ggml-sycl/convert.hpp | 5 + ggml/src/ggml-sycl/dequantize.hpp | 52 + ggml/src/ggml-sycl/dmmv.cpp | 38 +- ggml/src/ggml-sycl/fattn-dec.cpp | 794 ++++++++++++++ ggml/src/ggml-sycl/fattn-dec.hpp | 7 + ggml/src/ggml-sycl/fattn.cpp | 15 +- ggml/src/ggml-sycl/fusion.cpp | 14 +- ggml/src/ggml-sycl/fwht.cpp | 198 ++++ ggml/src/ggml-sycl/fwht.hpp | 5 + ggml/src/ggml-sycl/gated_delta_net.cpp | 463 ++++++++- ggml/src/ggml-sycl/gated_delta_net.hpp | 5 + ggml/src/ggml-sycl/gemm.hpp | 48 + ggml/src/ggml-sycl/ggml-sycl.cpp | 531 +++++++++- ggml/src/ggml-sycl/mmvq.cpp | 982 +++++++++++++++++- ggml/src/ggml-sycl/mmvq.hpp | 11 + ggml/src/ggml-sycl/ptq1-t2.cpp | 294 ++++++ ggml/src/ggml-sycl/ptq1-t2.hpp | 18 + ggml/src/ggml-sycl/quantize.hpp | 35 + ggml/src/ggml-sycl/quants.hpp | 39 + ggml/src/ggml-sycl/ternsycl/LICENSE.md | 29 + ggml/src/ggml-sycl/ternsycl/NOTICE | 8 + ggml/src/ggml-sycl/ternsycl/epilogue_dev.hpp | 71 ++ .../src/ggml-sycl/ternsycl/int2_int8_dpas.hpp | 435 ++++++++ ggml/src/ggml-sycl/ternsycl/xe2.hpp | 163 +++ ggml/src/ggml-sycl/vecdotq.hpp | 206 ++++ ggml/src/ggml-sycl/w8a8.cpp | 146 +++ ggml/src/ggml-sycl/w8a8.hpp | 10 + src/llama-context.cpp | 4 + tests/test-backend-ops.cpp | 68 +- 33 files changed, 4799 insertions(+), 42 deletions(-) create mode 100644 ggml/src/ggml-sycl/fattn-dec.cpp create mode 100644 ggml/src/ggml-sycl/fattn-dec.hpp create mode 100644 ggml/src/ggml-sycl/ptq1-t2.cpp create mode 100644 ggml/src/ggml-sycl/ptq1-t2.hpp create mode 100644 ggml/src/ggml-sycl/ternsycl/LICENSE.md create mode 100644 ggml/src/ggml-sycl/ternsycl/NOTICE create mode 100644 ggml/src/ggml-sycl/ternsycl/epilogue_dev.hpp create mode 100644 ggml/src/ggml-sycl/ternsycl/int2_int8_dpas.hpp create mode 100644 ggml/src/ggml-sycl/ternsycl/xe2.hpp create mode 100644 ggml/src/ggml-sycl/w8a8.cpp create mode 100644 ggml/src/ggml-sycl/w8a8.hpp diff --git a/common/speculative.cpp b/common/speculative.cpp index 2d73e48bc8cc..23843abe0139 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -3490,6 +3490,15 @@ common_params common_base_params_to_speculative(const common_params & params) { result.cache_type_k = params_spec.cache_type_k; result.cache_type_v = params_spec.cache_type_v; + // ARC-LAB: the draft context inherits the target's physical batch (2048 on the Arc server = a 784 MiB compute + // buffer for an MTP layer that drafts 1-3 tokens); its prompt catch-up already runs in n_ubatch chunks. + if (const char * ub = getenv("LLAMA_ARG_SPEC_DRAFT_UBATCH")) { + const int32_t n = atoi(ub); + if (n > 0 && n < result.n_ubatch) { + result.n_ubatch = n; + } + } + // the K-cache mean-center bias is calibrated for the target model // (per-head/channel K layout); the draft model has a different K // geometry, so it must not inherit the bias diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hpp index 8f2106d83088..0ee0689da04d 100644 --- a/ggml/src/ggml-sycl/common.hpp +++ b/ggml/src/ggml-sycl/common.hpp @@ -60,6 +60,8 @@ void ggml_sycl_host_free(void* ptr); extern int g_ggml_sycl_debug; extern int g_ggml_sycl_enable_optimize; +extern int g_ggml_sycl_quant_f16_gemm; +extern int g_ggml_sycl_quant_s4_gemm; extern int g_ggml_sycl_enable_fusion; extern int g_ggml_sycl_enable_esimd; extern int g_ggml_sycl_prioritize_dmmv; @@ -225,6 +227,7 @@ inline dpct::err0 ggml_sycl_set_device(const int device) try { struct optimize_feature { bool reorder=false; bool xmx_pq2=false; // PQ2_0 rewritten into the XMX layout (pq2_xmx.hpp); only that path can read it + bool t2=false; // ARC-LAB: PTQ1_0 repacked to the TernSYCL 2-bit layout (ptq1-t2.cpp) }; struct sycl_device_info { diff --git a/ggml/src/ggml-sycl/concat.cpp b/ggml/src/ggml-sycl/concat.cpp index bd5f3b2ceb30..2392e81cb78e 100644 --- a/ggml/src/ggml-sycl/concat.cpp +++ b/ggml/src/ggml-sycl/concat.cpp @@ -94,6 +94,23 @@ template static void concat_T_sycl(const T *x, const T *y, T *dst, int ne00, int ne01, int ne02, int ne0, int ne1, int ne2, int dim, queue_ptr stream) { + if (dim == 0 && ne0 < SYCL_CONCAT_BLOCK_SIZE) { + // ARC-LAB: narrow rows (delta-net conv state: 3 + 1 columns x 10240 channels) launched one work-group of 256 per + // row with 4 active lanes (12.5 us per call on the B580); one lane per element instead. + const int total = ne0 * ne1 * ne2; + const int nblk = (total + SYCL_CONCAT_BLOCK_SIZE - 1) / SYCL_CONCAT_BLOCK_SIZE; + stream->parallel_for(sycl::nd_range<1>(nblk * SYCL_CONCAT_BLOCK_SIZE, SYCL_CONCAT_BLOCK_SIZE), + [=](sycl::nd_item<1> it) { + const int i = it.get_global_id(0); + if (i >= total) { + return; + } + const int i0 = i % ne0; + const int row = i / ne0; // i1 + i2 * ne1 + dst[i] = i0 < ne00 ? x[row * ne00 + i0] : y[row * (ne0 - ne00) + i0 - ne00]; + }); + return; + } int num_blocks = (ne0 + SYCL_CONCAT_BLOCK_SIZE - 1) / SYCL_CONCAT_BLOCK_SIZE; sycl::range<3> gridDim(ne2, ne1, num_blocks); switch (dim) { diff --git a/ggml/src/ggml-sycl/convert.cpp b/ggml/src/ggml-sycl/convert.cpp index b046091512ea..b87c50f9eca7 100644 --- a/ggml/src/ggml-sycl/convert.cpp +++ b/ggml/src/ggml-sycl/convert.cpp @@ -170,6 +170,35 @@ static void dequantize_row_q4_0_sycl_reorder(const void *vx, dst_t *y, const int } +template +static void dequantize_row_pq2_0_sycl_reorder(const void * vx, dst_t * y, const int64_t k, dpct::queue_ptr stream) { + GGML_ASSERT(k % QK_PQ2_0 == 0); + const uint8_t * qs = (const uint8_t *) vx; + const sycl::half * d = (const sycl::half *) (qs + k / 4); + stream->parallel_for(sycl::range<1>(k / 4), [=](sycl::id<1> i) { + const float dv = d[i / (QK_PQ2_0 / 4)]; + const uint8_t q = qs[i]; +#pragma unroll + for (int j = 0; j < 4; ++j) { + y[i * 4 + j] = (float) (((q >> (2 * j)) & 3) - 1) * dv; + } + }); +} + +template +static void dequantize_row_ptq1_0_sycl_reorder(const void * vx, dst_t * y, const int64_t k, dpct::queue_ptr stream) { + GGML_ASSERT(k % QK_PTQ1_0 == 0); + const int64_t nb = k / QK_PTQ1_0; + const uint8_t * qs = (const uint8_t *) vx; + const uint8_t * qh = qs + nb * 24; + const sycl::half * d = (const sycl::half *) (qs + nb * 26); + stream->parallel_for(sycl::range<1>(k), [=](sycl::id<1> id) { + const int64_t i = id[0]; + const int64_t ib = i / QK_PTQ1_0; + y[i] = (float) (ptq1_0_trit(qs + ib * 24, qh + ib * 2, (int) (i % QK_PTQ1_0)) - 1) * (float) d[ib]; + }); +} + template static void dequantize_row_q8_0_sycl_reorder(const void *vx, dst_t *y, const int64_t k, dpct::queue_ptr stream) { @@ -663,6 +692,18 @@ to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst) { return dequantize_block_sycl; case GGML_TYPE_Q2_0: return dequantize_block_sycl; + case GGML_TYPE_PTQ1_0: + if (dst->src[0]->extra && + ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + return dequantize_row_ptq1_0_sycl_reorder; + } + return dequantize_block_sycl; + case GGML_TYPE_PQ2_0: + if (dst->src[0]->extra && + ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + return dequantize_row_pq2_0_sycl_reorder; + } + return dequantize_block_sycl; case GGML_TYPE_Q4_0: if (dst->src[0]->extra && ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) { @@ -757,6 +798,18 @@ to_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type, ggml_tensor *dst) { return dequantize_block_sycl; case GGML_TYPE_Q2_0: return dequantize_block_sycl; + case GGML_TYPE_PTQ1_0: + if (dst->src[0]->extra && + ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + return dequantize_row_ptq1_0_sycl_reorder; + } + return dequantize_block_sycl; + case GGML_TYPE_PQ2_0: + if (dst->src[0]->extra && + ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + return dequantize_row_pq2_0_sycl_reorder; + } + return dequantize_block_sycl; case GGML_TYPE_Q4_0: if (dst->src[0]->extra && ((ggml_tensor_extra_gpu*)dst->src[0]->extra)->optimized_feature.reorder) { @@ -887,3 +940,68 @@ to_fp16_nc_sycl_t ggml_get_to_fp16_nc_sycl(ggml_type type) { return nullptr; } } + +void ggml_sycl_quant_to_s4(ggml_type type, bool reordered, const void * vx, uint8_t * w4, float * scales, + int64_t nrows, int64_t ncols, dpct::queue_ptr stream) { + GGML_ASSERT(type == GGML_TYPE_PQ2_0 || type == GGML_TYPE_PTQ1_0); + GGML_ASSERT(ncols % 128 == 0); + const int64_t nb = nrows * ncols / 128; + const uint8_t * base = (const uint8_t *) vx; + const bool ptq = type == GGML_TYPE_PTQ1_0; + // one work item = 8 elements = one uint32 of s4 + stream->parallel_for(sycl::range<1>(nb * 16), [=](sycl::id<1> id) { + const int64_t ib = id[0] / 16; + const int c = (int) (id[0] % 16); + int v[8]; + sycl::half d; + if (ptq) { + const uint8_t * qs; + const uint8_t * qh; + if (reordered) { + qs = base + ib * 24; + qh = base + nb * 24 + ib * 2; + d = ((const sycl::half *) (base + nb * 26))[ib]; + } else { + const block_ptq1_0 * x = (const block_ptq1_0 *) vx + ib; + qs = x->qs; + qh = x->qh; + d = x->d; + } + // trit t of byte b = ((b * 3^t) mod 256) * 3 >> 8 + constexpr uint32_t pow3[5] = { 1, 3, 9, 27, 81 }; + if (c < 10) { // elements t*16 + m, 8 consecutive m + const uint32_t p = pow3[c / 2]; + const int m = (c % 2) * 8; +#pragma unroll + for (int i = 0; i < 8; ++i) v[i] = (int) ((((qs[m + i] * p) & 0xFF) * 3) >> 8) - 1; + } else if (c < 15) { // elements 80 + t*8 + m, m = 0..7 + const uint32_t p = pow3[c - 10]; +#pragma unroll + for (int i = 0; i < 8; ++i) v[i] = (int) ((((qs[16 + i] * p) & 0xFF) * 3) >> 8) - 1; + } else { // elements 120 + t*2 + h +#pragma unroll + for (int i = 0; i < 8; ++i) v[i] = (int) ((((qh[i % 2] * pow3[i / 2]) & 0xFF) * 3) >> 8) - 1; + } + } else { + const uint8_t * qs; + if (reordered) { + qs = base + ib * 32; + d = ((const sycl::half *) (base + nb * 32))[ib]; + } else { + const block_pq2_0 * x = (const block_pq2_0 *) vx + ib; + qs = x->qs; + d = x->d; + } +#pragma unroll + for (int i = 0; i < 8; ++i) v[i] = ((qs[2 * c + i / 4] >> (2 * (i % 4))) & 3) - 1; + } + uint32_t out = 0; +#pragma unroll + for (int i = 0; i < 8; ++i) out |= (uint32_t) (v[i] & 0xF) << (4 * i); + ((uint32_t *) w4)[id[0]] = out; + if (c == 0) { + const int64_t bpr = ncols / 128; + scales[(ib % bpr) * nrows + ib / bpr] = d; // [group][row] + } + }); +} diff --git a/ggml/src/ggml-sycl/convert.hpp b/ggml/src/ggml-sycl/convert.hpp index 8de79d10ff69..bbd9fe124184 100644 --- a/ggml/src/ggml-sycl/convert.hpp +++ b/ggml/src/ggml-sycl/convert.hpp @@ -21,6 +21,11 @@ typedef to_t_sycl_t to_fp32_sycl_t; typedef to_t_sycl_t to_fp16_sycl_t; to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst); + +// PQ2_0 / PTQ1_0 rows -> packed s4 (low nibble = even element) + f32 scales laid out [group][row]. +// All their values (-1..2) are exact in s4. +void ggml_sycl_quant_to_s4(ggml_type type, bool reordered, const void * vx, uint8_t * w4, float * scales, + int64_t nrows, int64_t ncols, dpct::queue_ptr stream); to_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type, ggml_tensor * dst); #ifdef GGML_SYCL_HAS_BF16 diff --git a/ggml/src/ggml-sycl/dequantize.hpp b/ggml/src/ggml-sycl/dequantize.hpp index 6bb4dd4515a9..90d2ac1f89d8 100644 --- a/ggml/src/ggml-sycl/dequantize.hpp +++ b/ggml/src/ggml-sycl/dequantize.hpp @@ -47,6 +47,58 @@ static __dpct_inline__ void dequantize_q2_0(const void *vx, const int64_t ib, #endif // GGML_SYCL_F16 } +static __dpct_inline__ void dequantize_pq2_0(const void *vx, const int64_t ib, + const int iqs, dfloat2 &v) { + const block_pq2_0 * x = (const block_pq2_0 *) vx; + + const dfloat d = x[ib].d; + + const int byte_idx = iqs / 4; + const int shift = (iqs % 4) * 2; + const uint8_t vui = x[ib].qs[byte_idx]; + + v.x() = (vui >> shift) & 3; + v.y() = (vui >> (shift + 2)) & 3; + +#ifdef GGML_SYCL_F16 + v.s0() = ((dfloat)v.s0() - 1.0f) * d; + v.s1() = ((dfloat)v.s1() - 1.0f) * d; +#else + v.x() = ((dfloat)v.x() - 1.0f) * d; + v.y() = ((dfloat)v.y() - 1.0f) * d; +#endif // GGML_SYCL_F16 +} + +// Trit of element e (0..127) in a PTQ1_0 block: 16 bytes x 5, 8 bytes x 5, then 2 qh bytes x 4. +static __dpct_inline__ int ptq1_0_trit(const uint8_t * qs, const uint8_t * qh, const int e) { + uint32_t v; + int t; + if (e < 80) { + v = qs[e % 16]; + t = e / 16; + } else if (e < 120) { + v = qs[16 + (e - 80) % 8]; + t = (e - 80) / 8; + } else { + v = qh[(e - 120) % 2]; + t = (e - 120) / 2; + } + for (int i = 0; i < t; ++i) { + v = (v * 3) & 0xFF; + } + return (int) ((v * 3) >> 8); +} + +static __dpct_inline__ void dequantize_ptq1_0(const void *vx, const int64_t ib, + const int iqs, dfloat2 &v) { + const block_ptq1_0 * x = (const block_ptq1_0 *) vx; + + const dfloat d = x[ib].d; + + v.x() = (dfloat) (ptq1_0_trit(x[ib].qs, x[ib].qh, iqs + 0) - 1) * d; + v.y() = (dfloat) (ptq1_0_trit(x[ib].qs, x[ib].qh, iqs + 1) - 1) * d; +} + static __dpct_inline__ void dequantize_q4_0(const void *vx, const int64_t ib, const int iqs, dfloat2 &v) { const block_q4_0 * x = (const block_q4_0 *) vx; diff --git a/ggml/src/ggml-sycl/dmmv.cpp b/ggml/src/ggml-sycl/dmmv.cpp index d47d6831a359..4995ffece14f 100644 --- a/ggml/src/ggml-sycl/dmmv.cpp +++ b/ggml/src/ggml-sycl/dmmv.cpp @@ -238,6 +238,34 @@ static void convert_mul_mat_vec_f16_sycl(const void *vx, const dfloat *y, } #ifdef GGML_SYCL_DMMV_HAS_BF16 +// ARC-LAB: small-output bf16 mat-vec (Bonsai's delta-net gate projections: 48 x 5120, 96 calls per token). +// The generic dequantize_mul_mat_vec gives each row one sub-group, so 48 rows used 48 sub-groups and reached +// ~25 GB/s on the B580 (~20 us per call). Here each row gets a 256-wide work-group that splits K, loads 8 bf16 +// at a time and reduces over the group. +static void bf16_gemv_rowsplit_sycl(const void * vx, const dfloat * y, float * dst, const int ncols, + const int nrows, dpct::queue_ptr stream) { + constexpr int WG = 256; + stream->parallel_for( + sycl::nd_range<1>(sycl::range<1>((size_t) nrows * WG), sycl::range<1>(WG)), + [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + const int row = it.get_group(0); + const int tid = it.get_local_id(0); + const uint16_t * x = (const uint16_t *) vx + (size_t) row * ncols; + float sum = 0.0f; + for (int i = tid * 8; i < ncols; i += WG * 8) { + const sycl::vec w = *(const sycl::vec *) (x + i); +#pragma unroll + for (int j = 0; j < 8; ++j) { + sum += sycl::bit_cast((uint32_t) w[j] << 16) * (float) y[i + j]; + } + } + sum = sycl::reduce_over_group(it.get_group(), sum, sycl::plus()); + if (tid == 0) { + dst[row] = sum; + } + }); +} + static void convert_mul_mat_vec_bf16_sycl(const void *vx, const dfloat *y, float *dst, const int ncols, const int nrows, @@ -2209,9 +2237,15 @@ void ggml_sycl_op_dequantize_mul_mat_vec( convert_mul_mat_vec_f16_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); break; #ifdef GGML_SYCL_DMMV_HAS_BF16 - case GGML_TYPE_BF16: - convert_mul_mat_vec_bf16_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); + case GGML_TYPE_BF16: { + static const bool rowsplit = getenv("GGML_SYCL_BF16_ROWSPLIT_OFF") == nullptr; + if (rowsplit && row_diff <= 512 && ne00 % 8 == 0 && ((uintptr_t) src0_dd_i % 16) == 0) { + bf16_gemv_rowsplit_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); + } else { + convert_mul_mat_vec_bf16_sycl(src0_dd_i, src1_dfloat, dst_dd_i, ne00, row_diff, stream); + } break; + } #endif default: printf("ggml_sycl_op_dequantize_mul_mat_vec unsupported GGML_TYPE %d\n", src0->type); diff --git a/ggml/src/ggml-sycl/fattn-dec.cpp b/ggml/src/ggml-sycl/fattn-dec.cpp new file mode 100644 index 000000000000..72c90d05c3d9 --- /dev/null +++ b/ggml/src/ggml-sycl/fattn-dec.cpp @@ -0,0 +1,794 @@ +// ARC-LAB decode attention for a q4_0 KV cache (generation and MTP verify batches of 1-4 query tokens). +// +// The TILE path first converts the whole K and V cache to f16 on every call and then reads that copy +// (Bonsai 27B at 16K context: 5.9 + 11.4 ms per generated token over 16 attention layers). Here each +// work-group owns one KV head and a slice of the context, reads the q4_0 blocks directly, and serves all +// G query heads of that KV head (GQA) and all NQ query tokens from the one read (flash-decoding): +// - the scaled query rows (G * NQ of them) sit in SLM; +// - per tile of TK keys: thread (c, key) computes the G scores of query token c against its key, +// straight from the q4_0 nibbles; per-row online softmax with a finite running max (a fully masked +// tile or slice gets zero weight, never NaN); thread d accumulates P.V for head dim d of every row; +// - slices are merged by flash_attn_combine_results (same partial layout as the vec kernel). + +#include +#include "dpct/helper.hpp" +#include "common.hpp" +#include "fattn-common.hpp" +#include "fattn-dec.hpp" +#include + +namespace { +constexpr int DEC_D = 256; // head size (K and V) +constexpr int DEC_WG = 256; // work-group size = DEC_D: thread d owns output dim d +constexpr float DEC_MINF = -1.0e30f; // finite "minus infinity" for the running max +} + +// v3 kernel, kept for 4-token batches: the pipelined kernel below is slower there (436 vs 508 us per layer at +// 16K; see README) and faster at 1-2 tokens (16K 273 -> 256, 48K 944 -> 742 us). +template +static void fattn_dec_q4_0_v3(const char * Q, const char * K, const char * V, const char * mask, float * dst, + float * parts, sycl::float2 * meta, float scale, int ne01, int ne02, int ne11, + int nkvh, int ne03, int64_t nb01, int64_t nb02, int64_t nb03, int64_t nb11, + int64_t nb12, int64_t nb13, int64_t nb21, int64_t nb22, int64_t nb23, int64_t nb31, + int64_t nb33, int ne33, int nsplit, int chunk, dpct::queue_ptr stream) { + constexpr int R = G * NQ; // query rows per KV head + constexpr int RP = (R + 3) / 4 * 4; // padded row stride of the transposed scores + constexpr int TK = DEC_WG / NQ; // keys per tile: thread (c, jj) = (tid / TK, tid % TK) + constexpr int NB = DEC_D / QK4_0; + constexpr int KW = DEC_D / QK4_0 * sizeof(block_q4_0) / 4; // 36 dwords per q4_0 K row (D 256) + constexpr int KWP = KW + 1; // padded SLM row stride (bank conflicts) + static_assert(TK % 16 == 0 && (DEC_D / QK4_0 * sizeof(block_q4_0)) % 4 == 0, "shape"); + + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor sQ(sycl::range<1>(R * DEC_D / 4), cgh); + sycl::local_accessor sK(sycl::range<1>(TK * KWP), cgh); + sycl::local_accessor sS(sycl::range<1>(TK * RP / 4), cgh); // [key][row], transposed + sycl::local_accessor sM(sycl::range<1>(R), cgh); + sycl::local_accessor sL(sycl::range<1>(R), cgh); + sycl::local_accessor sA(sycl::range<1>(RP), cgh); + cgh.parallel_for( + sycl::nd_range<3>(sycl::range<3>(ne03, nkvh, (size_t) nsplit * DEC_WG), sycl::range<3>(1, 1, DEC_WG)), + [=](sycl::nd_item<3> it) [[sycl::reqd_sub_group_size(16)]] { + const int tid = it.get_local_id(2); + const int split = it.get_group(2); + const int kvh = it.get_group(1); + const int seq = it.get_group(0); + auto sg = it.get_sub_group(); + const int w = tid / 16; + const int lane = tid % 16; + float * sSf = (float *) &sS[0]; + + // query rows r = c * G + g: token c, head kvh * G + g; pre-scaled like the other FA kernels + for (int e = tid; e < R * DEC_D / 4; e += DEC_WG) { + const int r = e / (DEC_D / 4), d4 = e % (DEC_D / 4), c = r / G, g = r % G; + sycl::float4 qv(0.0f); + if (c < ne01) { + qv = *(const sycl::float4 *) (Q + seq * nb03 + c * nb01 + (int64_t) (kvh * G + g) * nb02 + d4 * sizeof(sycl::float4)); + } + sQ[e] = qv * scale; + } + if (tid < R) { + sM[tid] = DEC_MINF; + sL[tid] = 0.0f; + } + if (tid < RP) { + sA[tid] = 1.0f; + } + + const int k_begin = split * chunk; + const int k_end = sycl::min(k_begin + chunk, ne11); + const char * Kh = K + seq * nb13 + kvh * nb12; + const char * Vh = V + seq * nb23 + kvh * nb22; + const sycl::half * mrow = mask ? (const sycl::half *) (mask + (seq % ne33) * nb33) : nullptr; + + float o[RP]; +#pragma unroll + for (int r = 0; r < RP; ++r) { + o[r] = 0.0f; + } + + const int c = tid / TK; + const int jj = tid % TK; + for (int t0 = k_begin; t0 < k_end; t0 += TK) { + // K tile -> SLM: raw q4_0 rows, coalesced aligned dwords + for (int e = tid; e < TK * KW; e += DEC_WG) { + const int kk = e / KW, wd = e % KW, j = t0 + kk; + sK[kk * KWP + wd] = j < k_end ? ((const uint32_t *) (Kh + j * nb11))[wd] : 0u; + } + it.barrier(sycl::access::fence_space::local_space); + + // scores: this thread's key against the G heads of query token c + { + const int j = t0 + jj; + float s[G]; +#pragma unroll + for (int g = 0; g < G; ++g) { + s[g] = 0.0f; + } + if (j < k_end && c < ne01) { + const int kr = jj * KWP; + const int q0 = c * G * (DEC_D / 4); +#pragma unroll 2 + for (int b = 0; b < NB; ++b) { + // block b = bytes [18b, 18b + 18): scale (2 bytes) then 16 bytes of nibbles + auto byte_at = [&](int o) -> uint32_t { + return (sK[kr + o / 4] >> (8 * (o % 4))) & 0xFFu; + }; + const int o0 = b * (int) sizeof(block_q4_0); + const uint16_t hb = (uint16_t) (byte_at(o0) | (byte_at(o0 + 1) << 8)); + const float dk = static_cast(sycl::bit_cast(hb)); +#pragma unroll + for (int i = 0; i < QK4_0 / 2; i += 4) { + sycl::float4 k0, k1; +#pragma unroll + for (int u = 0; u < 4; ++u) { + const uint32_t by = byte_at(o0 + 2 + i + u); + k0[u] = (float) ((int) (by & 0xF) - 8) * dk; + k1[u] = (float) ((int) (by >> 4) - 8) * dk; + } + const int d4 = (b * QK4_0 + i) / 4; +#pragma unroll + for (int g = 0; g < G; ++g) { + const sycl::float4 qa = sQ[q0 + g * (DEC_D / 4) + d4]; + const sycl::float4 qb = sQ[q0 + g * (DEC_D / 4) + d4 + QK4_0 / 8]; + s[g] += sycl::dot(qa, k0) + sycl::dot(qb, k1); + } + } + } + const float mv = mrow ? static_cast(mrow[(c * nb31) / (int64_t) sizeof(sycl::half) + j]) : 0.0f; +#pragma unroll + for (int g = 0; g < G; ++g) { + s[g] += mv; + } + } else { +#pragma unroll + for (int g = 0; g < G; ++g) { + s[g] = -INFINITY; + } + } +#pragma unroll + for (int g = 0; g < G; ++g) { + sSf[jj * RP + c * G + g] = s[g]; + } + } + it.barrier(sycl::access::fence_space::local_space); + + // V tile -> SLM (reusing the K tile buffer; scores are done with it) + static_assert(sizeof(block_q4_0) * NB % 4 == 0, "V rows are dword multiples"); + for (int e = tid; e < TK * KW; e += DEC_WG) { + const int kk = e / KW, wd = e % KW, j = t0 + kk; + sK[kk * KWP + wd] = j < k_end ? ((const uint32_t *) (Vh + j * nb21))[wd] : 0u; + } + + // online softmax, one row per subgroup at a time + for (int r = w; r < R; r += DEC_WG / 16) { + float mt = -INFINITY; + for (int k = lane; k < TK; k += 16) { + mt = sycl::fmax(mt, sSf[k * RP + r]); + } + mt = sycl::reduce_over_group(sg, mt, sycl::maximum()); + const float m_old = sM[r]; + const float m_new = sycl::fmax(m_old, mt); // finite: m_old starts at DEC_MINF + float lt = 0.0f; + for (int k = lane; k < TK; k += 16) { + const float p = sycl::native::exp(sSf[k * RP + r] - m_new); // exp(-inf) = 0 + sSf[k * RP + r] = p; + lt += p; + } + lt = sycl::reduce_over_group(sg, lt, sycl::plus()); + if (lane == 0) { + const float a = sycl::native::exp(m_old - m_new); + sA[r] = a; + sL[r] = sL[r] * a + lt; + sM[r] = m_new; + } + } + it.barrier(sycl::access::fence_space::local_space); + + // P.V for head dim d = tid (padded rows R..RP-1 have P = 0 from the -inf scores) + { + const int d = tid; + const int b = d / QK4_0; + const int wi = d % QK4_0; + const int nk = sycl::min(TK, k_end - t0); +#pragma unroll + for (int r = 0; r < RP; ++r) { + o[r] *= sA[r]; + } + const int ob = b * (int) sizeof(block_q4_0); // scale bytes ob, ob + 1 + const int oq = ob + 2 + wi % (QK4_0 / 2); // this dim's nibble byte + const int sh = wi < QK4_0 / 2 ? 0 : 4; +#pragma unroll 4 + for (int k = 0; k < nk; ++k) { + const int vr = k * KWP; + const uint32_t sw = sK[vr + ob / 4] >> (8 * (ob % 4)); // ob even: both scale bytes in one dword + const float dv = static_cast(sycl::bit_cast((uint16_t) (sw & 0xFFFFu))); + const int byte = (sK[vr + oq / 4] >> (8 * (oq % 4))) & 0xFF; + const float vv = (float) (((byte >> sh) & 0xF) - 8) * dv; +#pragma unroll + for (int r4 = 0; r4 < RP / 4; ++r4) { + const sycl::float4 p = sS[k * (RP / 4) + r4]; + o[4 * r4 + 0] += p[0] * vv; + o[4 * r4 + 1] += p[1] * vv; + o[4 * r4 + 2] += p[2] * vv; + o[4 * r4 + 3] += p[3] * vv; + } + } + } + it.barrier(sycl::access::fence_space::local_space); + } + +#pragma unroll + for (int r = 0; r < R; ++r) { + const int cr = r / G; + if (cr >= ne01) { + continue; + } + const int64_t jdu = ((int64_t) seq * ne01 + cr) * ne02 + kvh * G + r % G; + if (nsplit == 1) { + const float l = sL[r]; + dst[jdu * DEC_D + tid] = l > 0.0f ? o[r] / l : 0.0f; + } else { + parts[(jdu * nsplit + split) * DEC_D + tid] = o[r]; + if (tid == 0) { + meta[jdu * nsplit + split] = sycl::float2(sM[r], sL[r]); + } + } + } + }); + }); +} + +template +static void fattn_dec_q4_0(const char * Q, const char * K, const char * V, const char * mask, float * dst, + float * parts, sycl::float2 * meta, float scale, int ne01, int ne02, int ne11, + int nkvh, int ne03, int64_t nb01, int64_t nb02, int64_t nb03, int64_t nb11, + int64_t nb12, int64_t nb13, int64_t nb21, int64_t nb22, int64_t nb23, int64_t nb31, + int64_t nb33, int ne33, int nsplit, int chunk, dpct::queue_ptr stream) { + constexpr int R = G * NQ; // query rows per KV head + constexpr int RP = (R + 3) / 4 * 4; // padded row stride of the transposed scores + constexpr int NRG = DEC_WG / TK; // row groups in the score phase: thread (rg, jj) = (tid / TK, tid % TK) + constexpr int RPT = R / NRG; // rows per thread in the score phase + constexpr int NB = DEC_D / QK4_0; + constexpr int KW = DEC_D / QK4_0 * sizeof(block_q4_0) / 4; // 36 dwords per q4_0 row (D 256) + constexpr int KWP = KW + 1; // padded SLM row stride (bank conflicts) + constexpr int PKW = TK * KW / DEC_WG; // dwords of a K or V tile per thread + static_assert(TK % 16 == 0 && R % NRG == 0 && (TK * KW) % DEC_WG == 0 && G % RPT == 0, "shape"); // a thread's rows share one token + + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor sQ(sycl::range<1>(R * DEC_D / 4), cgh); + sycl::local_accessor sK(sycl::range<1>(TK * KWP), cgh); // K tile, then V tile + sycl::local_accessor sS(sycl::range<1>(TK * RP / 4), cgh); // [key][row], transposed + sycl::local_accessor sM(sycl::range<1>(R), cgh); + sycl::local_accessor sL(sycl::range<1>(R), cgh); + sycl::local_accessor sA(sycl::range<1>(RP), cgh); + cgh.parallel_for( + sycl::nd_range<3>(sycl::range<3>(ne03, nkvh, (size_t) nsplit * DEC_WG), sycl::range<3>(1, 1, DEC_WG)), + [=](sycl::nd_item<3> it) [[sycl::reqd_sub_group_size(16)]] { + const int tid = it.get_local_id(2); + const int split = it.get_group(2); + const int kvh = it.get_group(1); + const int seq = it.get_group(0); + auto sg = it.get_sub_group(); + const int w = tid / 16; + const int lane = tid % 16; + float * sSf = (float *) &sS[0]; + + const int k_begin = split * chunk; + const int k_end = sycl::min(k_begin + chunk, ne11); + const char * Kh = K + seq * nb13 + kvh * nb12; + const char * Vh = V + seq * nb23 + kvh * nb22; + const sycl::half * mrow = mask ? (const sycl::half *) (mask + (seq % ne33) * nb33) : nullptr; + + // software pipeline: a tile's raw rows are loaded into registers one phase before they are stored + // to SLM (V during Q.K, the next K during softmax + P.V), so the loads overlap compute + uint32_t pk[PKW], pv[PKW]; + auto load_rows = [&](const char * base, int64_t nb, int t0, uint32_t * reg) { +#pragma unroll + for (int i = 0; i < PKW; ++i) { + const int e = tid + i * DEC_WG, kk = e / KW, wd = e % KW, j = t0 + kk; + reg[i] = j < k_end ? ((const uint32_t *) (base + j * nb))[wd] : 0u; + } + }; + auto store_rows = [&](const uint32_t * reg) { +#pragma unroll + for (int i = 0; i < PKW; ++i) { + const int e = tid + i * DEC_WG, kk = e / KW, wd = e % KW; + sK[kk * KWP + wd] = reg[i]; + } + }; + if constexpr (PIPE) { + load_rows(Kh, nb11, k_begin, pk); + } + + // query rows r = c * G + g: token c, head kvh * G + g; pre-scaled like the other FA kernels + for (int e = tid; e < R * DEC_D / 4; e += DEC_WG) { + const int r = e / (DEC_D / 4), d4 = e % (DEC_D / 4), c = r / G, g = r % G; + sycl::float4 qv(0.0f); + if (c < ne01) { + qv = *(const sycl::float4 *) (Q + seq * nb03 + c * nb01 + (int64_t) (kvh * G + g) * nb02 + d4 * sizeof(sycl::float4)); + } + sQ[e] = qv * scale; + } + if (tid < R) { + sM[tid] = DEC_MINF; + sL[tid] = 0.0f; + } + if (tid < RP) { + sA[tid] = 1.0f; + } + + float o[RP]; +#pragma unroll + for (int r = 0; r < RP; ++r) { + o[r] = 0.0f; + } + + const int rg = tid / TK; + const int jj = tid % TK; + const int r0 = rg * RPT; + for (int t0 = k_begin; t0 < k_end; t0 += TK) { + if constexpr (!PIPE) { + load_rows(Kh, nb11, t0, pk); + } + store_rows(pk); // K tile t + it.barrier(sycl::access::fence_space::local_space); + if constexpr (PIPE) { + load_rows(Vh, nb21, t0, pv); // V tile t, in flight during Q.K + } + + // scores of rows r0 .. r0 + RPT - 1 against this thread's key + { + const int j = t0 + jj; + float s[RPT]; +#pragma unroll + for (int i = 0; i < RPT; ++i) { + s[i] = 0.0f; + } + if (j < k_end) { + const int kr = jj * KWP; + const int qb = r0 * (DEC_D / 4); // hoisted row base (v3 form): row rr at qb + rr * 64 +#pragma unroll 2 + for (int b = 0; b < NB; ++b) { + auto byte_at = [&](int o) -> uint32_t { + return (sK[kr + o / 4] >> (8 * (o % 4))) & 0xFFu; + }; + const int o0 = b * (int) sizeof(block_q4_0); + const uint16_t hb = (uint16_t) (byte_at(o0) | (byte_at(o0 + 1) << 8)); + const float dk = static_cast(sycl::bit_cast(hb)); +#pragma unroll + for (int i = 0; i < QK4_0 / 2; i += 4) { + sycl::float4 k0, k1; +#pragma unroll + for (int u = 0; u < 4; ++u) { + const uint32_t by = byte_at(o0 + 2 + i + u); + k0[u] = (float) ((int) (by & 0xF) - 8) * dk; + k1[u] = (float) ((int) (by >> 4) - 8) * dk; + } + const int d4 = (b * QK4_0 + i) / 4; +#pragma unroll + for (int rr = 0; rr < RPT; ++rr) { + const sycl::float4 qa = sQ[qb + rr * (DEC_D / 4) + d4]; + const sycl::float4 qc = sQ[qb + rr * (DEC_D / 4) + d4 + QK4_0 / 8]; + s[rr] += sycl::dot(qa, k0) + sycl::dot(qc, k1); + } + } + } + } + const int c = r0 / G; // all RPT rows of this thread belong to token c + const float mv = (j < k_end && c < ne01 && mrow) ? static_cast(mrow[(c * nb31) / (int64_t) sizeof(sycl::half) + j]) : 0.0f; +#pragma unroll + for (int rr = 0; rr < RPT; ++rr) { + sSf[jj * RP + r0 + rr] = (j < k_end && c < ne01) ? s[rr] + mv : -INFINITY; + } + } + it.barrier(sycl::access::fence_space::local_space); + + if constexpr (!PIPE) { + load_rows(Vh, nb21, t0, pv); + } + store_rows(pv); // V tile t -> SLM (the K tile is done) + if constexpr (PIPE) { + if (t0 + TK < k_end) { + load_rows(Kh, nb11, t0 + TK, pk); // K tile t + 1, in flight during softmax + P.V + } + } + + // online softmax, one row per subgroup at a time + for (int r = w; r < R; r += DEC_WG / 16) { + float mt = -INFINITY; + for (int k = lane; k < TK; k += 16) { + mt = sycl::fmax(mt, sSf[k * RP + r]); + } + mt = sycl::reduce_over_group(sg, mt, sycl::maximum()); + const float m_old = sM[r]; + const float m_new = sycl::fmax(m_old, mt); // finite: m_old starts at DEC_MINF + float lt = 0.0f; + for (int k = lane; k < TK; k += 16) { + const float p = sycl::native::exp(sSf[k * RP + r] - m_new); // exp(-inf) = 0 + sSf[k * RP + r] = p; + lt += p; + } + lt = sycl::reduce_over_group(sg, lt, sycl::plus()); + if (lane == 0) { + const float a = sycl::native::exp(m_old - m_new); + sA[r] = a; + sL[r] = sL[r] * a + lt; + sM[r] = m_new; + } + } + it.barrier(sycl::access::fence_space::local_space); + + // P.V for head dim d = tid (padded rows R..RP-1 are never written out) + { + const int d = tid; + const int b = d / QK4_0; + const int wi = d % QK4_0; + const int nk = sycl::min(TK, k_end - t0); +#pragma unroll + for (int r = 0; r < RP; ++r) { + o[r] *= sA[r]; + } + const int ob = b * (int) sizeof(block_q4_0); // scale bytes ob, ob + 1 + const int oq = ob + 2 + wi % (QK4_0 / 2); // this dim's nibble byte + const int sh = wi < QK4_0 / 2 ? 0 : 4; +#pragma unroll 4 + for (int k = 0; k < nk; ++k) { + const int vr = k * KWP; + const uint32_t sw = sK[vr + ob / 4] >> (8 * (ob % 4)); // ob even: both scale bytes in one dword + const float dv = static_cast(sycl::bit_cast((uint16_t) (sw & 0xFFFFu))); + const int byte = (sK[vr + oq / 4] >> (8 * (oq % 4))) & 0xFF; + const float vv = (float) (((byte >> sh) & 0xF) - 8) * dv; +#pragma unroll + for (int r4 = 0; r4 < RP / 4; ++r4) { + const sycl::float4 p = sS[k * (RP / 4) + r4]; + o[4 * r4 + 0] += p[0] * vv; + o[4 * r4 + 1] += p[1] * vv; + o[4 * r4 + 2] += p[2] * vv; + o[4 * r4 + 3] += p[3] * vv; + } + } + } + it.barrier(sycl::access::fence_space::local_space); + } + +#pragma unroll + for (int r = 0; r < R; ++r) { + const int cr = r / G; + if (cr >= ne01) { + continue; + } + const int64_t jdu = ((int64_t) seq * ne01 + cr) * ne02 + kvh * G + r % G; + if (nsplit == 1) { + const float l = sL[r]; + dst[jdu * DEC_D + tid] = l > 0.0f ? o[r] / l : 0.0f; + } else { + parts[(jdu * nsplit + split) * DEC_D + tid] = o[r]; + if (tid == 0) { + meta[jdu * nsplit + split] = sycl::float2(sM[r], sL[r]); + } + } + } + }); + }); +} + +// ARC-LAB XMX variant for 3-4 query tokens (MTP verify): the scalar kernels are f32-ALU bound there (Q.K 163 + P.V +// 121 of 436 us per layer at 16K). Per tile of TK keys: K dequantized to f16 in SLM ([key][dim]); S^T = K Q^T on +// joint_matrix (A = 8 keys x 16 dims, B = Q^T 16 dims x 16 rows) stored straight into the transposed score layout; +// per-row online softmax writes P (f16, [row][key]); V dequantized into the same buffer; O += P V on joint_matrix +// (subgroup s owns head dims 16 s .. +15 for all row tiles), rows rescaled through the coordinate-aware apply. +template +static void fattn_dec_q4_0_xmx(const char * Q, const char * K, const char * V, const char * mask, float * dst, + float * parts, sycl::float2 * meta, float scale, int ne01, int ne02, int ne11, + int nkvh, int ne03, int64_t nb01, int64_t nb02, int64_t nb03, int64_t nb11, + int64_t nb12, int64_t nb13, int64_t nb21, int64_t nb22, int64_t nb23, int64_t nb31, + int64_t nb33, int ne33, int nsplit, int chunk, dpct::queue_ptr stream) { + namespace jm = sycl::ext::oneapi::experimental::matrix; + namespace ijm = sycl::ext::intel::experimental::matrix; + constexpr int R = G * NQ; // query rows per KV head (24) + constexpr int RP = (R + 15) / 16 * 16; // rows padded to whole 16-wide B tiles (32) + constexpr int MT = R / 8; // 8-row tiles of real rows (3) + constexpr int NB = DEC_D / QK4_0; + constexpr int DS = DEC_D + 16; // f16 row stride of the K / V tile ([key][dim]) + constexpr int PS = TK + 16; // f16 row stride of P ([row][key]) + constexpr int NSG = DEC_WG / 16; // 16 subgroups + constexpr int KW = DEC_D / QK4_0 * sizeof(block_q4_0) / 4; // 36 dwords per raw q4_0 row + constexpr int KWP = KW + 1; + static_assert(R % 8 == 0 && TK % 16 == 0 && (TK / 8) * (RP / 16) <= NSG && DEC_D / 16 == NSG, "shape"); + + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor sQT(sycl::range<1>(DEC_D * RP), cgh); // Q^T: [dim][row], scaled + sycl::local_accessor sKV(sycl::range<1>(TK * DS), cgh); // K tile, then V tile + sycl::local_accessor sRaw(sycl::range<1>(TK * KWP), cgh); // raw q4_0 rows of the tile + sycl::local_accessor sS(sycl::range<1>(TK * RP), cgh); // S^T: [key][row] + sycl::local_accessor sP(sycl::range<1>(RP * PS), cgh); // P: [row][key] + sycl::local_accessor sM(sycl::range<1>(R), cgh); + sycl::local_accessor sL(sycl::range<1>(R), cgh); + sycl::local_accessor sA(sycl::range<1>(R), cgh); + cgh.parallel_for( + sycl::nd_range<3>(sycl::range<3>(ne03, nkvh, (size_t) nsplit * DEC_WG), sycl::range<3>(1, 1, DEC_WG)), + [=](sycl::nd_item<3> it) [[sycl::reqd_sub_group_size(16)]] { + const int tid = it.get_local_id(2); + const int split = it.get_group(2); + const int kvh = it.get_group(1); + const int seq = it.get_group(0); + auto sg = it.get_sub_group(); + const int w = tid / 16; + const int lane = tid % 16; + auto pQT = sQT.template get_multi_ptr(); + auto pKV = sKV.template get_multi_ptr(); + auto pS = sS.template get_multi_ptr(); + auto pP = sP.template get_multi_ptr(); + + // Q^T (padded rows are zero) + for (int e = tid; e < DEC_D * RP; e += DEC_WG) { + const int d = e / RP, r = e % RP, c = r / G, g = r % G; + float qv = 0.0f; + if (r < R && c < ne01) { + qv = *(const float *) (Q + seq * nb03 + c * nb01 + (int64_t) (kvh * G + g) * nb02 + d * sizeof(float)) * scale; + } + sQT[e] = sycl::half(qv); + } + if (tid < R) { + sM[tid] = DEC_MINF; + sL[tid] = 0.0f; + sA[tid] = 1.0f; + } + + const int k_begin = split * chunk; + const int k_end = sycl::min(k_begin + chunk, ne11); + const char * Kh = K + seq * nb13 + kvh * nb12; + const char * Vh = V + seq * nb23 + kvh * nb22; + const sycl::half * mrow = mask ? (const sycl::half *) (mask + (seq % ne33) * nb33) : nullptr; + + // O accumulators: subgroup w owns head dims 16 w .. 16 w + 15, all MT row tiles + jm::joint_matrix O[MT]; +#pragma unroll + for (int mt = 0; mt < MT; ++mt) { + jm::joint_matrix_fill(sg, O[mt], 0.0f); + } + + // q4_0 rows -> f16 tile in two steps: coalesced dword copy of the raw rows, then an SLM -> SLM + // conversion where each work item writes 8 dims of one key as one 16-byte vector + auto dequant_tile = [&](const char * base, int64_t nb, int t0) { + for (int e = tid; e < TK * KW; e += DEC_WG) { + const int kk = e / KW, wd = e % KW, j = t0 + kk; + sRaw[kk * KWP + wd] = j < k_end ? ((const uint32_t *) (base + j * nb))[wd] : 0u; + } + it.barrier(sycl::access::fence_space::local_space); + for (int e = tid; e < TK * (DEC_D / 8); e += DEC_WG) { + const int kk = e / (DEC_D / 8), g = e % (DEC_D / 8), b = g / 4, part = g % 4; + const int row = kk * KWP; + auto byte_at = [&](int o) -> uint32_t { return (sRaw[row + o / 4] >> (8 * (o % 4))) & 0xFFu; }; + const int ob = b * (int) sizeof(block_q4_0); + const uint32_t sw = sRaw[row + ob / 4] >> (8 * (ob % 4)); // ob even: both scale bytes in one dword + const float dd = static_cast(sycl::bit_cast((uint16_t) (sw & 0xFFFFu))); + const int o0 = ob + 2 + (part % 2) * 8; + const int sh = part < 2 ? 0 : 4; + sycl::vec v; +#pragma unroll + for (int u = 0; u < 8; ++u) { + v[u] = sycl::half((float) ((int) ((byte_at(o0 + u) >> sh) & 0xFu) - 8) * dd); + } + v.store(0, pKV + (kk * DS + g * 8)); + } + }; + + it.barrier(sycl::access::fence_space::local_space); + + for (int t0 = k_begin; t0 < k_end; t0 += TK) { + dequant_tile(Kh, nb11, t0); + it.barrier(sycl::access::fence_space::local_space); + + // S^T tile (keys 8 kt .. +7, rows 16 nt .. +15) per subgroup + if (w < (TK / 8) * (RP / 16)) { + const int kt = w % (TK / 8), nt = w / (TK / 8); + jm::joint_matrix St; + jm::joint_matrix_fill(sg, St, 0.0f); +#pragma unroll 4 + for (int dk = 0; dk < DEC_D; dk += 16) { + jm::joint_matrix Ak; + jm::joint_matrix Bq; + jm::joint_matrix_load(sg, Ak, pKV + (kt * 8) * DS + dk, DS); + jm::joint_matrix_load(sg, Bq, pQT + dk * RP + nt * 16, RP); + jm::joint_matrix_mad(sg, St, Ak, Bq, St); + } + jm::joint_matrix_store(sg, St, pS + (kt * 8) * RP + nt * 16, RP, jm::layout::row_major); + } + it.barrier(sycl::access::fence_space::local_space); + + // online softmax per row (mask, -inf for keys past the slice) -> P [row][key] f16 + for (int r = w; r < R; r += NSG) { + const int c = r / G; + float sv[TK / 16]; + float mt = -INFINITY; +#pragma unroll + for (int i = 0; i < TK / 16; ++i) { + const int k = lane + 16 * i, j = t0 + k; + float v = -INFINITY; + if (j < k_end && c < ne01) { + v = sS[k * RP + r] + (mrow ? static_cast(mrow[(c * nb31) / (int64_t) sizeof(sycl::half) + j]) : 0.0f); + } + sv[i] = v; + mt = sycl::fmax(mt, v); + } + mt = sycl::reduce_over_group(sg, mt, sycl::maximum()); + const float m_old = sM[r]; + const float m_new = sycl::fmax(m_old, mt); + float lt = 0.0f; +#pragma unroll + for (int i = 0; i < TK / 16; ++i) { + const float pv = sycl::native::exp(sv[i] - m_new); + sP[r * PS + lane + 16 * i] = sycl::half(pv); + lt += pv; + } + lt = sycl::reduce_over_group(sg, lt, sycl::plus()); + if (lane == 0) { + const float a = sycl::native::exp(m_old - m_new); + sA[r] = a; + sL[r] = sL[r] * a + lt; + sM[r] = m_new; + } + } + it.barrier(sycl::access::fence_space::local_space); // S reads done, K tile free + + dequant_tile(Vh, nb21, t0); + it.barrier(sycl::access::fence_space::local_space); + + // O = diag(alpha) O + P V for head dims 16 w .. +15 +#pragma unroll + for (int mt = 0; mt < MT; ++mt) { + ijm::joint_matrix_apply(sg, O[mt], [=](float & x, size_t row, size_t) { x *= sA[mt * 8 + row]; }); + } +#pragma unroll + for (int kk = 0; kk < TK; kk += 16) { + jm::joint_matrix Bv; + jm::joint_matrix_load(sg, Bv, pKV + kk * DS + w * 16, DS); +#pragma unroll + for (int mt = 0; mt < MT; ++mt) { + jm::joint_matrix Ap; + jm::joint_matrix_load(sg, Ap, pP + (mt * 8) * PS + kk, PS); + jm::joint_matrix_mad(sg, O[mt], Ap, Bv, O[mt]); + } + } + it.barrier(sycl::access::fence_space::local_space); + } + + // write: rows r = mt * 8 + row -> (token c, head kvh * G + g); dims 16 w + col +#pragma unroll + for (int mt = 0; mt < MT; ++mt) { + ijm::joint_matrix_apply(sg, O[mt], [=](float & x, size_t row, size_t col) { + const int r = mt * 8 + (int) row, c = r / G; + if (c < ne01) { + const int64_t jdu = ((int64_t) seq * ne01 + c) * ne02 + kvh * G + r % G; + const int d = w * 16 + (int) col; + if (nsplit == 1) { + const float l = sL[r]; + dst[jdu * DEC_D + d] = l > 0.0f ? x / l : 0.0f; + } else { + parts[(jdu * nsplit + split) * DEC_D + d] = x; + } + } + }); + } + if (nsplit > 1 && tid < R && tid / G < ne01) { + const int r = tid; + const int64_t jdu = ((int64_t) seq * ne01 + r / G) * ne02 + kvh * G + r % G; + meta[jdu * nsplit + split] = sycl::float2(sM[r], sL[r]); + } + }); + }); +} + +bool ggml_sycl_flash_attn_ext_dec_supported(const ggml_tensor * dst) { + static const bool off = getenv("GGML_SYCL_FA_DEC_OFF") != nullptr; + if (off) { + return false; + } + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + const ggml_tensor * sinks = dst->src[4]; + float max_bias = 0.0f, softcap = 0.0f; + memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float)); + memcpy(&softcap, (const float *) dst->op_params + 2, sizeof(float)); + return K->type == GGML_TYPE_Q4_0 && V->type == GGML_TYPE_Q4_0 && Q->type == GGML_TYPE_F32 && + K->ne[0] == DEC_D && V->ne[0] == DEC_D && Q->ne[0] == DEC_D && + Q->ne[2] == 6 * K->ne[2] && V->ne[2] == K->ne[2] && + Q->ne[1] >= 1 && Q->ne[1] <= 4 && Q->ne[3] == K->ne[3] && V->ne[3] == K->ne[3] && // 5-8: TILE is faster (v3<6,8> 1172-1240 vs 1049 us @16K) + !sinks && max_bias == 0.0f && softcap == 0.0f && + (!mask || (mask->type == GGML_TYPE_F16 && mask->ne[2] == 1)) && + Q->nb[0] == sizeof(float) && K->nb[0] == ggml_type_size(K->type) && V->nb[0] == ggml_type_size(V->type) && + K->nb[1] % 4 == 0 && K->nb[2] % 4 == 0 && K->nb[3] % 4 == 0 && ((uintptr_t) K->data) % 4 == 0 && + V->nb[1] % 4 == 0 && V->nb[2] % 4 == 0 && V->nb[3] % 4 == 0 && ((uintptr_t) V->data) % 4 == 0 && + Q->nb[1] % 16 == 0 && Q->nb[2] % 16 == 0 && Q->nb[3] % 16 == 0 && ((uintptr_t) Q->data) % 16 == 0; +} + +void ggml_sycl_flash_attn_ext_dec(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { + const ggml_tensor * Q = dst->src[0]; + const ggml_tensor * K = dst->src[1]; + const ggml_tensor * V = dst->src[2]; + const ggml_tensor * mask = dst->src[3]; + + float scale = 1.0f; + memcpy(&scale, (const float *) dst->op_params + 0, sizeof(float)); + + const int ne01 = Q->ne[1]; + const int nq = ne01 <= 1 ? 1 : ne01 <= 2 ? 2 : ne01 <= 4 ? 4 : 8; + const int tk = nq == 8 ? 32 : nq == 4 ? 64 : 128; // keys per tile (must match the instantiations below) + const int ne11 = K->ne[1]; + + // slices: enough work-groups to fill the GPU (~64 slices per KV head), at least one tile each + static const int target = getenv("GGML_SYCL_FA_DEC_SPLITS") ? atoi(getenv("GGML_SYCL_FA_DEC_SPLITS")) : 64; + const int ntiles = (ne11 + tk - 1) / tk; + int nsplit = std::max(1, std::min(target, ntiles)); + const int chunk = ((ntiles + nsplit - 1) / nsplit) * tk; + nsplit = (ne11 + chunk - 1) / chunk; + + ggml_sycl_pool_alloc parts(ctx.pool()); + ggml_sycl_pool_alloc meta(ctx.pool()); + if (nsplit > 1) { + parts.alloc((size_t) nsplit * ggml_nelements(dst)); + meta.alloc((size_t) nsplit * ggml_nrows(dst)); + } + + dpct::queue_ptr stream = ctx.stream(); + const char * mdata = mask ? (const char *) mask->data : nullptr; + const int64_t nb31 = mask ? mask->nb[1] : 0; + const int64_t nb33 = mask ? mask->nb[3] : 0; + const int ne33 = mask ? (int) mask->ne[3] : 1; + +#define FATTN_DEC_CALL(NQ_) fattn_dec_q4_0<6, NQ_, (NQ_ == 4 ? 64 : 128), (NQ_ != 4)>( /* PIPE: see v4 notes */(const char *) Q->data, (const char *) K->data, (const char *) V->data, \ + mdata, (float *) dst->data, parts.get(), meta.get(), scale, ne01, (int) Q->ne[2], ne11, (int) K->ne[2], \ + (int) Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], V->nb[1], V->nb[2], V->nb[3], nb31, \ + nb33, ne33, nsplit, chunk, stream) + switch (nq) { + case 1: FATTN_DEC_CALL(1); break; + case 2: FATTN_DEC_CALL(2); break; + case 8: // 5-8 tokens (MTP depth 4+, short n-gram drafts): v3 kernel, 48 rows, 32-key tiles, ~59 KB SLM + fattn_dec_q4_0_v3<6, 8>((const char *) Q->data, (const char *) K->data, (const char *) V->data, mdata, + (float *) dst->data, parts.get(), meta.get(), scale, ne01, (int) Q->ne[2], ne11, (int) K->ne[2], + (int) Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], V->nb[1], V->nb[2], V->nb[3], + nb31, nb33, ne33, nsplit, chunk, stream); + break; + default: + if (static const int xmx = getenv("GGML_SYCL_FA_DEC_XMX") ? atoi(getenv("GGML_SYCL_FA_DEC_XMX")) : 0; xmx) { // ARC-LAB opt-in +#define FATTN_DEC_XMX(TK_) fattn_dec_q4_0_xmx<6, 4, TK_>((const char *) Q->data, (const char *) K->data, (const char *) V->data, mdata, \ + (float *) dst->data, parts.get(), meta.get(), scale, ne01, (int) Q->ne[2], ne11, (int) K->ne[2], \ + (int) Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], V->nb[1], V->nb[2], V->nb[3], \ + nb31, nb33, ne33, nsplit, chunk, stream) + if (xmx == 32) { + FATTN_DEC_XMX(32); + } else { + FATTN_DEC_XMX(64); + } +#undef FATTN_DEC_XMX + break; + } + fattn_dec_q4_0_v3<6, 4>((const char *) Q->data, (const char *) K->data, (const char *) V->data, mdata, + (float *) dst->data, parts.get(), meta.get(), scale, ne01, (int) Q->ne[2], ne11, (int) K->ne[2], + (int) Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], V->nb[1], V->nb[2], V->nb[3], + nb31, nb33, ne33, nsplit, chunk, stream); + break; + } +#undef FATTN_DEC_CALL + + if (nsplit > 1) { + const int D = DEC_D; + const sycl::range<3> grid(Q->ne[3], Q->ne[2], (size_t) ne01 * D); + const size_t nbytes_shared = nsplit * sizeof(sycl::float2); + float * parts_p = parts.get(); + sycl::float2 * meta_p = meta.get(); + float * dst_p = (float *) dst->data; + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor lm(sycl::range<1>(nbytes_shared), cgh); + cgh.parallel_for(sycl::nd_range<3>(grid, sycl::range<3>(1, 1, D)), + [=](sycl::nd_item<3>) [[sycl::reqd_sub_group_size(16)]] { + flash_attn_combine_results(parts_p, meta_p, dst_p, nsplit, + lm.get_multi_ptr().get()); + }); + }); + } +} diff --git a/ggml/src/ggml-sycl/fattn-dec.hpp b/ggml/src/ggml-sycl/fattn-dec.hpp new file mode 100644 index 000000000000..32760be8b001 --- /dev/null +++ b/ggml/src/ggml-sycl/fattn-dec.hpp @@ -0,0 +1,7 @@ +#pragma once + +#include "common.hpp" + +// ARC-LAB decode attention straight from a q4_0 KV cache (see fattn-dec.cpp). +bool ggml_sycl_flash_attn_ext_dec_supported(const ggml_tensor * dst); +void ggml_sycl_flash_attn_ext_dec(ggml_backend_sycl_context & ctx, ggml_tensor * dst); diff --git a/ggml/src/ggml-sycl/fattn.cpp b/ggml/src/ggml-sycl/fattn.cpp index a85eb721f6af..9ccac7fa548e 100644 --- a/ggml/src/ggml-sycl/fattn.cpp +++ b/ggml/src/ggml-sycl/fattn.cpp @@ -19,6 +19,7 @@ #include "fattn-vec.hpp" #include "fattn.hpp" #include "fattn-onednn.hpp" +#include "fattn-dec.hpp" #define FATTN_VEC_CASE(D, type_K, type_V) \ @@ -98,6 +99,7 @@ enum best_fattn_kernel { BEST_FATTN_KERNEL_NONE = 0, BEST_FATTN_KERNEL_VEC = 100, BEST_FATTN_KERNEL_ONEDNN = 150, // oneDNN SDPA: native F16 (PR #25222) + BEST_FATTN_KERNEL_DEC = 160, // ARC-LAB: q4_0 KV decode, 1-4 query tokens (fattn-dec.cpp) BEST_FATTN_KERNEL_TILE = 200, BEST_FATTN_KERNEL_MKL = 300, }; @@ -112,6 +114,11 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const if(!g_ggml_sycl_enable_flash_attention) return BEST_FATTN_KERNEL_NONE; + // ARC-LAB: generation / MTP-verify batches on a q4_0 cache read it directly (TILE dequantizes the whole cache per call) + if (ggml_sycl_flash_attn_ext_dec_supported(dst)) { + return BEST_FATTN_KERNEL_DEC; + } + const ggml_tensor * KQV = dst; const ggml_tensor * Q = dst->src[0]; const ggml_tensor * K = dst->src[1]; @@ -262,7 +269,10 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const } } } else { - if (Q->ne[1] <= 2) { + // like the CUDA backend: with GQA packing available, the tile kernel reads each quantized K/V head once + // for all its query heads; the vec kernel re-reads it per query head (gqa_ratio x the KV traffic) + static const bool gqa_tile = ggml_sycl_get_env("GGML_SYCL_FA_QUANT_GQA_TILE", 1) != 0; + if (Q->ne[1] <= 2 && !(gqa_tile && gqa_opt_applies)) { return BEST_FATTN_KERNEL_VEC; } } @@ -324,6 +334,9 @@ void ggml_sycl_flash_attn_ext(ggml_backend_sycl_context & ctx, ggml_tensor * dst case BEST_FATTN_KERNEL_MKL: ggml_sycl_flash_attn_ext_mkl(ctx, dst); break; + case BEST_FATTN_KERNEL_DEC: + ggml_sycl_flash_attn_ext_dec(ctx, dst); + break; } // --- Output fingerprint (GGML_SYCL_MKL_FA_DIAG=1) --- diff --git a/ggml/src/ggml-sycl/fusion.cpp b/ggml/src/ggml-sycl/fusion.cpp index 709bc8ca2a22..7f1c577e79ba 100644 --- a/ggml/src/ggml-sycl/fusion.cpp +++ b/ggml/src/ggml-sycl/fusion.cpp @@ -1,4 +1,5 @@ #include "fusion.hpp" +#include "mmvq.hpp" #include @@ -29,8 +30,17 @@ static bool ggml_sycl_should_fuse_mul_mat_glu(const ggml_tensor * gate, const gg return false; } - // only q4_K has a fused reorder GEMV so far, and it walks whole super-blocks - if (wu->type != GGML_TYPE_Q4_K || wu->ne[0] % QK_K != 0) { + // types with a fused reorder GEMV; each walks whole blocks + const bool type_ok = (wu->type == GGML_TYPE_Q4_K && wu->ne[0] % QK_K == 0) || + (wu->type == GGML_TYPE_PQ2_0 && wu->ne[0] % QK_PQ2_0 == 0) || + (wu->type == GGML_TYPE_PTQ1_0 && wu->ne[0] % QK_PTQ1_0 == 0); + if (!type_ok) { + return false; + } + // single-token PQ2_0 decode is faster unfused (its own 2-row GEMV); fuse for 2..8 columns. ARC-LAB: PTQ1_0 has a + // single-column fused kernel (pairs core, GGML_SYCL_PTQ1_GLU1=0 turns it off) + if (wu->type != GGML_TYPE_Q4_K && act->ne[1] < 2 && + !(wu->type == GGML_TYPE_PTQ1_0 && ggml_sycl_ptq1_0_glu_n1_enabled())) { return false; } diff --git a/ggml/src/ggml-sycl/fwht.cpp b/ggml/src/ggml-sycl/fwht.cpp index fb48d7fec70d..19b5d2e9f978 100644 --- a/ggml/src/ggml-sycl/fwht.cpp +++ b/ggml/src/ggml-sycl/fwht.cpp @@ -402,3 +402,201 @@ bool ggml_sycl_op_fwht(ggml_backend_sycl_context & ctx, const ggml_tensor * src, return false; } } + +// ARC-LAB (Torchit1/llama.cpp arc-b580): FWHT with the Hadamard sign flip done during the load +// (dst = H * (src * signs)), F32 or F16 input. Kept beside the general FWHT above, which carries the +// wide and Kronecker sizes from upstream; only the fused sign-flip entry point below uses it. +namespace fwht_signed_impl { + +template +static void fwht_kernel(const T * __restrict__ src, float * __restrict__ dst, const int64_t n_rows, + const float scale, const float * __restrict__ signs, const int n_blk, + const sycl::nd_item<2> & item) { + const sycl::sub_group sg = item.get_sub_group(); + + const int64_t r = item.get_global_id(0); + if (r >= n_rows) { + return; + } + + src += r * N; + dst += r * N; + + constexpr int el_w = N / WARP_SIZE; + static_assert(el_w >= 1 && N % WARP_SIZE == 0, "row must be a whole number of sub-group widths"); + + float reg[el_w]; + const int lane = sg.get_local_linear_id(); + + const float * signs_row = has_signs ? signs + (r % n_blk) * N : nullptr; +#pragma unroll + for (int i = 0; i < el_w; ++i) { + reg[i] = (float) src[i * WARP_SIZE + lane] * scale; + if constexpr (has_signs) { + reg[i] *= signs_row[i * WARP_SIZE + lane]; + } + } + + // Butterflies inside the sub-group. The partner of a lane with bit h clear is the + // lower index of the pair, so it takes the sum and the upper takes lower - upper. +#pragma unroll + for (int h = 1; h < WARP_SIZE; h *= 2) { +#pragma unroll + for (int j = 0; j < el_w; ++j) { + const float val = reg[j]; + const float val2 = dpct::permute_sub_group_by_xor(sg, val, h, WARP_SIZE); + + reg[j] = (lane & h) == 0 ? val + val2 : val2 - val; + } + } + + // Butterflies across registers: h is a multiple of WARP_SIZE, so the partner of + // element i*WARP_SIZE + lane lives in reg[i + h/WARP_SIZE] on the same lane. +#pragma unroll + for (int h = WARP_SIZE; h < N; h *= 2) { + const int step = h / WARP_SIZE; +#pragma unroll + for (int j = 0; j < el_w; j += 2 * step) { +#pragma unroll + for (int k = 0; k < step; ++k) { + const float x = reg[j + k]; + const float y = reg[j + k + step]; + + reg[j + k] = x + y; + reg[j + k + step] = x - y; + } + } + } + +#pragma unroll + for (int i = 0; i < el_w; ++i) { + dst[i * WARP_SIZE + lane] = reg[i]; + } +} + +// Wide rows: one work-group per row, butterflies through local memory. +template +static void fwht_kernel_slm(const T * __restrict__ src, float * __restrict__ dst, const float scale, + const float * __restrict__ signs, const int n_blk, float * s, const sycl::nd_item<1> & item) { + const int64_t r = item.get_group(0); + const int tid = item.get_local_id(0); + const int nt = item.get_local_range(0); + + src += r * N; + dst += r * N; + const float * signs_row = has_signs ? signs + (r % n_blk) * N : nullptr; + + for (int i = tid; i < N; i += nt) { + float v = (float) src[i] * scale; + if constexpr (has_signs) { + v *= signs_row[i]; + } + s[i] = v; + } + item.barrier(sycl::access::fence_space::local_space); + + for (int h = 1; h < N; h *= 2) { + for (int idx = tid; idx < N / 2; idx += nt) { + const int j = (idx / h) * 2 * h + idx % h; + const float x = s[j]; + const float y = s[j + h]; + s[j] = x + y; + s[j + h] = x - y; + } + item.barrier(sycl::access::fence_space::local_space); + } + + for (int i = tid; i < N; i += nt) { + dst[i] = s[i]; + } +} + +template +static void launch_fwht(const T * src, float * dst, const int64_t n_rows, const float scale, const float * signs, + const int n_blk, dpct::queue_ptr stream) { + if constexpr (N <= 1024) { + constexpr int rows_per_block = 4; + + const int64_t num_blocks = (n_rows + rows_per_block - 1) / rows_per_block; + + // dim 1 is the fastest-varying, so a sub-group is exactly one row's WARP_SIZE lanes. + const sycl::range<2> global(num_blocks * rows_per_block, WARP_SIZE); + const sycl::range<2> local(rows_per_block, WARP_SIZE); + + stream->parallel_for(sycl::nd_range<2>(global, local), + [=](sycl::nd_item<2> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + fwht_kernel(src, dst, n_rows, scale, signs, n_blk, item); + }); + } else { + constexpr int nt = 256; + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor s(sycl::range<1>(N), cgh); + cgh.parallel_for(sycl::nd_range<1>(n_rows * nt, nt), [=](sycl::nd_item<1> item) { + fwht_kernel_slm(src, dst, scale, signs, n_blk, + s.get_multi_ptr().get(), item); + }); + }); + } +} + +template +static bool fwht_dispatch(const T * src_d, float * dst_d, const int n, const int64_t rows, const float scale, + const float * signs, const int n_blk, dpct::queue_ptr stream) { + switch (n) { +#define FWHT_CASE(NN) \ + case NN: \ + if (signs) { \ + launch_fwht(src_d, dst_d, rows, scale, signs, n_blk, stream); \ + } else { \ + launch_fwht(src_d, dst_d, rows, scale, nullptr, 1, stream); \ + } \ + return true; + FWHT_CASE(64) + FWHT_CASE(128) + FWHT_CASE(256) + FWHT_CASE(512) + FWHT_CASE(1024) + FWHT_CASE(2048) + FWHT_CASE(4096) + FWHT_CASE(8192) +#undef FWHT_CASE + default: + return false; + } +} + +static bool fwht_run(ggml_backend_sycl_context & ctx, const ggml_tensor * src, ggml_tensor * dst, + const ggml_tensor * signs_t) { + if ((src->type != GGML_TYPE_F32 && src->type != GGML_TYPE_F16) || dst->type != GGML_TYPE_F32) { + return false; + } + if (ggml_nelements(src) != ggml_nelements(dst) || !ggml_is_contiguous(src) || !ggml_is_contiguous(dst)) { + return false; + } + + const int n = (int) dst->ne[0]; + const int64_t rows = ggml_nelements(dst) / n; + + const float * signs = nullptr; + int n_blk = 1; + if (signs_t) { + if (signs_t->type != GGML_TYPE_F32 || !ggml_is_contiguous(signs_t) || signs_t->ne[0] % n != 0) { + return false; + } + signs = (const float *) signs_t->data; + n_blk = (int) (signs_t->ne[0] / n); + } + + const float scale = 1.0f / std::sqrt((float) n); + if (src->type == GGML_TYPE_F32) { + return fwht_dispatch((const float *) src->data, (float *) dst->data, n, rows, scale, signs, n_blk, ctx.stream()); + } + return fwht_dispatch((const sycl::half *) src->data, (float *) dst->data, n, rows, scale, signs, n_blk, ctx.stream()); +} + +} // namespace fwht_signed_impl + +bool ggml_sycl_op_fwht_signed(ggml_backend_sycl_context & ctx, const ggml_tensor * src, const ggml_tensor * signs, + ggml_tensor * dst) { + return fwht_signed_impl::fwht_run(ctx, src, dst, signs); +} diff --git a/ggml/src/ggml-sycl/fwht.hpp b/ggml/src/ggml-sycl/fwht.hpp index cd238cfaf37b..7dee0ffe38ef 100644 --- a/ggml/src/ggml-sycl/fwht.hpp +++ b/ggml/src/ggml-sycl/fwht.hpp @@ -9,4 +9,9 @@ // ordinary mat-mul dispatch. bool ggml_sycl_op_fwht(ggml_backend_sycl_context & ctx, const ggml_tensor * src, ggml_tensor * dst); +// Same, with the Hadamard sign flip (dst = H * (src * signs)) done during the load. +// signs: F32, ne[0] a multiple of the block width, repeating per block row. +bool ggml_sycl_op_fwht_signed(ggml_backend_sycl_context & ctx, const ggml_tensor * src, const ggml_tensor * signs, + ggml_tensor * dst); + #endif // GGML_SYCL_FWHT_HPP diff --git a/ggml/src/ggml-sycl/gated_delta_net.cpp b/ggml/src/ggml-sycl/gated_delta_net.cpp index 8468bbf5bd07..740a3a1efded 100644 --- a/ggml/src/ggml-sycl/gated_delta_net.cpp +++ b/ggml/src/ggml-sycl/gated_delta_net.cpp @@ -1,3 +1,4 @@ +#include #include #include "dpct/helper.hpp" #include "common.hpp" @@ -30,7 +31,9 @@ void gated_delta_net_sycl(const float * q, const sycl::uint3 rq3_magic, float scale, int64_t state_slot_stride, - int K) { + int K, + const int32_t * s_ids, + int64_t s_row_stride) { auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>(); const uint32_t h_idx = item_ct1.get_group(2); const uint32_t sequence = item_ct1.get_group(1); @@ -45,7 +48,9 @@ void gated_delta_net_sycl(const float * q, // input state holds s0 only [S_v, S_v, H, n_seqs] — seq stride is D = H * S_v * S_v. // output state layout (per-slot D * n_seqs) — same per-(seq,head) offset as before. - const int64_t state_in_offset = sequence * H * S_v * S_v + h_idx * S_v * S_v; + // fused gather: read this sequence's live state straight from its cache row s_ids[sequence] + const int64_t state_in_offset = (s_ids ? (int64_t) s_ids[sequence] * s_row_stride : (int64_t) sequence * H * S_v * S_v) + + h_idx * S_v * S_v; const int64_t state_out_offset = (sequence * H + h_idx) * S_v * S_v; state += state_out_offset; curr_state += state_in_offset + col * S_v; @@ -162,6 +167,195 @@ void gated_delta_net_sycl(const float * q, } } +// ARC-LAB token-blocked sequential kernel (non-KDA, S_v = 128). Same per-token recurrence and per-column +// arithmetic/reduction order as gated_delta_net_sycl, but the sequential kernel stalls on global-memory latency +// every token (~0.4 us/step on the B580 even with the GPU mostly idle) and needs ~5 rounds of threads at 48 heads. +// Here each subgroup owns NC columns and shares its k/q registers across them (CUDA's cols_per_warp), and the +// work-group stages T tokens of k, q, v, exp(g), beta in SLM; the next block's global loads are issued into +// registers before the current block is computed, so their latency overlaps T steps of compute. +template +static void gdn_blocked_sycl(const float * q, const float * k, const float * v, const float * g, const float * beta, + const float * curr_state, float * dst, float * state, int64_t H, int64_t n_tokens, + int64_t n_seqs, int64_t sq1, int64_t sq2, int64_t sq3, int64_t sv1, int64_t sv2, + int64_t sv3, int64_t sb1, int64_t sb2, int64_t sb3, int64_t neqk1, int64_t rq3, + float scale, int64_t state_slot_stride, int K, const int32_t * s_ids, + int64_t s_row_stride, dpct::queue_ptr stream) { + constexpr int S = 128, L = 16, RPL = S / L; + constexpr int NCW = NC * NW, WGS = NW * L, ncb = S / NCW; + constexpr int PK = T * S / WGS, PV = T * NCW / WGS; // per-thread prefetch registers + static_assert((T * S) % WGS == 0 && (T * NCW) % WGS == 0 && T <= WGS && S % NCW == 0, "gdn_blocked shape"); + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor sk(sycl::range<1>(T * S), cgh); + sycl::local_accessor sq(sycl::range<1>(T * S), cgh); + sycl::local_accessor sv(sycl::range<1>(T * NCW), cgh); + sycl::local_accessor sgb(sycl::range<1>(2 * T), cgh); + cgh.parallel_for(sycl::nd_range<1>(sycl::range<1>((size_t) (n_seqs * H * ncb * WGS)), sycl::range<1>(WGS)), + [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(16)]] { + const int tid = it.get_local_id(0); + const int lane = tid % L; + const int w = tid / L; + const int64_t grp = it.get_group(0); + const int64_t seq = grp / (H * ncb); + const int64_t h = (grp / ncb) % H; + const int colb = (int) (grp % ncb) * NCW; // first column of the work-group + const int col0 = colb + w * NC; // first column of this subgroup + const int64_t iq1 = h % neqk1; + const int64_t iq3 = seq / rq3; + + const float * qb = q + iq3 * sq3 + iq1 * sq1; + const float * kb = k + iq3 * sq3 + iq1 * sq1; + const float * vb = v + seq * sv3 + h * sv1 + colb; + const float * gp = g + seq * sb3 + h * sb1; + const float * bp = beta + seq * sb3 + h * sb1; + float * attn = dst + (seq * n_tokens * H + h) * S; + float * st = state + (seq * H + h) * S * S; + const float * cs = curr_state + (s_ids ? (int64_t) s_ids[seq] * s_row_stride : seq * H * S * S) + h * S * S; + + float s[NC][RPL]; +#pragma unroll + for (int c = 0; c < NC; ++c) { +#pragma unroll + for (int r = 0; r < RPL; ++r) { + s[c][r] = cs[(col0 + c) * S + r * L + lane]; + } + } + + float pk[PK], pq[PK], pv[PV], pg = 0.0f, pb = 0.0f; + auto fetch = [&](int64_t t0) { +#pragma unroll + for (int j = 0; j < PK; ++j) { + const int e = tid + j * WGS, t = e / S, i = e % S; + const bool ok = t0 + t < n_tokens; + pk[j] = ok ? kb[(t0 + t) * sq2 + i] : 0.0f; + pq[j] = ok ? qb[(t0 + t) * sq2 + i] : 0.0f; + } +#pragma unroll + for (int j = 0; j < PV; ++j) { + const int e = tid + j * WGS, t = e / NCW, c = e % NCW; + pv[j] = t0 + t < n_tokens ? vb[(t0 + t) * sv2 + c] : 0.0f; + } + if (tid < T && t0 + tid < n_tokens) { + pg = gp[(t0 + tid) * sb2]; + pb = bp[(t0 + tid) * sb2]; + } + }; + fetch(0); + + for (int64_t t0 = 0; t0 < n_tokens; t0 += T) { + it.barrier(sycl::access::fence_space::local_space); // previous block's SLM reads are done +#pragma unroll + for (int j = 0; j < PK; ++j) { + sk[tid + j * WGS] = pk[j]; + sq[tid + j * WGS] = pq[j]; + } +#pragma unroll + for (int j = 0; j < PV; ++j) { + sv[tid + j * WGS] = pv[j]; + } + if (tid < T) { + sgb[tid] = sycl::native::exp(pg); + sgb[T + tid] = pb; + } + it.barrier(sycl::access::fence_space::local_space); + if (t0 + T < n_tokens) { + fetch(t0 + T); // in flight while this block is computed + } + + const int nt = (int) sycl::min((int64_t) T, n_tokens - t0); + for (int t = 0; t < nt; ++t) { + float kr[RPL], qr[RPL]; +#pragma unroll + for (int r = 0; r < RPL; ++r) { + kr[r] = sk[t * S + r * L + lane]; + qr[r] = sq[t * S + r * L + lane]; + } + const float g_val = sgb[t]; + const float beta_val = sgb[T + t]; + float out = 0.0f; +#pragma unroll + for (int c = 0; c < NC; ++c) { + float kv_shard = 0.0f; +#pragma unroll + for (int r = 0; r < RPL; ++r) { + kv_shard += s[c][r] * kr[r]; + } + const float kv_col = warp_reduce_sum(kv_shard); + const float delta_col = (sv[t * NCW + w * NC + c] - g_val * kv_col) * beta_val; + float attn_partial = 0.0f; +#pragma unroll + for (int r = 0; r < RPL; ++r) { + s[c][r] = g_val * s[c][r] + kr[r] * delta_col; + attn_partial += s[c][r] * qr[r]; + } + const float attn_col = warp_reduce_sum(attn_partial); // same sum on every lane + if (lane == c) { + out = attn_col; + } + } + if (lane < NC) { + attn[(t0 + t) * S * H + col0 + lane] = out * scale; + } + if constexpr (keep_rs_t) { + const int target_slot = (int) (n_tokens - 1 - (t0 + t)); + if (target_slot >= 0 && target_slot < K) { + float * cst = st + target_slot * state_slot_stride; +#pragma unroll + for (int c = 0; c < NC; ++c) { +#pragma unroll + for (int r = 0; r < RPL; ++r) { + cst[(col0 + c) * S + r * L + lane] = s[c][r]; + } + } + } + } + } + } + + if constexpr (!keep_rs_t) { +#pragma unroll + for (int c = 0; c < NC; ++c) { +#pragma unroll + for (int r = 0; r < RPL; ++r) { + st[(col0 + c) * S + r * L + lane] = s[c][r]; + } + } + } + }); + }); +} + +template +static void launch_gdn_blocked(int variant, const float * q, const float * k, const float * v, const float * g, + const float * beta, const float * s0, float * dst, float * state, int64_t H, + int64_t n_tokens, int64_t n_seqs, int64_t sq1, int64_t sq2, int64_t sq3, int64_t sv1, + int64_t sv2, int64_t sv3, int64_t sb1, int64_t sb2, int64_t sb3, int64_t neqk1, + int64_t rq3, float scale, int64_t state_slot_stride, int K, const int32_t * s_ids, + int64_t s_row_stride, dpct::queue_ptr stream) { +#define GDN_BLK(NC, NW, T) gdn_blocked_sycl(q, k, v, g, beta, s0, dst, state, H, n_tokens, n_seqs, \ + sq1, sq2, sq3, sv1, sv2, sv3, sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, s_ids, s_row_stride, stream) + if (variant < 0) { + // auto: widest NC that still leaves ~768 subgroups (~60% of the B580's 1280 thread slots) to hide latency. + // B580, 1024 tokens (old kernel -> this): 48 heads 3383 -> 1688 us (NC 8, T 8); 32 heads 2762 -> 1024 (NC 4); + // 4 heads 403 -> 312 (NC 1). NC 8 at 32 heads = 1767, NC 4 at 48 heads = 1934 (more than one round of threads). + const int64_t cols = H * n_seqs * 128; + variant = cols >= 768 * 8 ? 9 : cols >= 768 * 4 ? 3 : cols >= 768 * 2 ? 4 : 5; + } + switch (variant) { // lab sweep (GGML_SYCL_GDN_BLK_VARIANT); NC columns/subgroup, NW subgroups, T tokens/block + case 1: GDN_BLK(8, 16, 16); break; + case 2: GDN_BLK(8, 8, 16); break; + case 3: GDN_BLK(4, 16, 16); break; + case 4: GDN_BLK(2, 16, 16); break; + case 5: GDN_BLK(1, 16, 16); break; + case 6: GDN_BLK(4, 32, 16); break; + case 7: GDN_BLK(4, 8, 32); break; + case 8: GDN_BLK(8, 8, 8); break; + case 9: GDN_BLK(8, 16, 8); break; + case 10: GDN_BLK(4, 16, 8); break; + default: GDN_BLK(4, 8, 16); break; + } +#undef GDN_BLK +} + template static void launch_gated_delta_net(const float * q_d, const float * k_d, @@ -189,8 +383,21 @@ static void launch_gated_delta_net(const float * q_d, float scale, int64_t state_slot_stride, int K, + const int32_t * s_ids, + int64_t s_row_stride, dpct::queue_ptr stream) { //TODO: Add chunked kernel for even faster pre-fill + if constexpr (!KDA) { + static const bool blk_off = getenv("GGML_SYCL_GDN_BLOCKED_OFF") != nullptr; + static const int blk_variant = getenv("GGML_SYCL_GDN_BLK_VARIANT") ? atoi(getenv("GGML_SYCL_GDN_BLK_VARIANT")) : -1; + static const int blk_min = getenv("GGML_SYCL_GDN_BLK_MIN") ? atoi(getenv("GGML_SYCL_GDN_BLK_MIN")) : 16; + if (!blk_off && S_v == 128 && n_tokens >= blk_min) { + launch_gdn_blocked(blk_variant, q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, + n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, sb1, sb2, sb3, neqk1, rq3, scale, + state_slot_stride, K, s_ids, s_row_stride, stream); + return; + } + } const int warp_size = ggml_sycl_info().devices[ggml_sycl_get_device()].warp_size; const int num_warps = 4; @@ -208,7 +415,7 @@ static void launch_gated_delta_net(const float * q_d, [=](sycl::nd_item<3> /*item_ct1*/) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { gated_delta_net_sycl(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, sq1, sq2, sq3, sv1, sv2, sv3, sb1, sb2, - sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K); + sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K, s_ids, s_row_stride); }); } break; @@ -219,7 +426,7 @@ static void launch_gated_delta_net(const float * q_d, [=](sycl::nd_item<3> /*item_ct1*/) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { gated_delta_net_sycl(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, sq1, sq2, sq3, sv1, sv2, sv3, sb1, sb2, - sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K); + sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K, s_ids, s_row_stride); }); } break; @@ -230,7 +437,7 @@ static void launch_gated_delta_net(const float * q_d, [=](sycl::nd_item<3> /*item_ct1*/) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { gated_delta_net_sycl( q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, sq1, sq2, - sq3, sv1, sv2, sv3, sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K); + sq3, sv1, sv2, sv3, sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K, s_ids, s_row_stride); }); } break; @@ -242,7 +449,7 @@ static void launch_gated_delta_net(const float * q_d, [=](sycl::nd_item<3> /*item_ct1*/) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { gated_delta_net_sycl( q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, H, n_tokens, sq1, sq2, - sq3, sv1, sv2, sv3, sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K); + sq3, sv1, sv2, sv3, sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, state_slot_stride, K, s_ids, s_row_stride); }); } break; @@ -253,6 +460,203 @@ static void launch_gated_delta_net(const float * q_d, } } +// ARC-LAB: chunked prefill for the scalar-gate gated delta rule (S_k = S_v = 128, one sequence). +// Per token: delta = beta * (v - gamma * S^T k), S <- gamma * S + k delta^T, o = scale * S^T q, gamma = exp(g). +// Per chunk of C tokens (UT / WY form, as build_delta_net_chunking), with G = cumsum(g) inside the chunk: +// A[t][s] = beta_t exp(G_t - G_s) k_t.k_s (s < t) +// R = beta * (V - exp(G) * (K S0)) -> Delta = (I + A)^-1 R (forward substitution) +// O = scale * (exp(G) * (Q S0) + (exp(G_t - G_s) q_t.k_s)_{s<=t} Delta) +// S_C = exp(G_last) S0 + K^T (exp(G_last - G) * Delta) +// One work-group per (head, block of W value columns): the value columns of S evolve independently, the C x C +// terms are recomputed per block. q/k of a chunk are shared in SLM by all W columns; every product is fused, so +// the serial chain is one step per chunk instead of one per token. fp32 throughout. +namespace { +constexpr int GDN_CH_C = 16; // tokens per chunk +constexpr int GDN_CH_W = 32; // value columns per work-group +constexpr int GDN_CH_S = 128; // head size +constexpr int GDN_CH_KP = GDN_CH_S + 1; // padded SLM row stride of the K / Q tiles +constexpr int GDN_CH_WG = 256; +} + +template +static void gdn_chunked_sycl_t(const float * q, const float * k, const float * v, const float * g, const float * beta, + const float * s_in, float * dst, float * s_out, int64_t H, int64_t n_proc, + int64_t sq1, int64_t sq2, int64_t sv1, int64_t sv2, int64_t sb1, int64_t sb2, + int64_t neqk1, float scale, const int32_t * s_ids, int64_t s_row_stride, + dpct::queue_ptr stream) { + constexpr int S = GDN_CH_S, KP = GDN_CH_KP, WG = GDN_CH_WG; + constexpr int ncb = S / W; + static_assert((C * W) % WG == 0 || C * W < WG, "C*W vs WG"); + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor sS(sycl::range<1>(S * W), cgh); + sycl::local_accessor sK(sycl::range<1>(C * KP), cgh); + sycl::local_accessor sQ(sycl::range<1>(C * KP), cgh); + sycl::local_accessor sV(sycl::range<1>(C * W), cgh); + sycl::local_accessor sD(sycl::range<1>(C * W), cgh); + sycl::local_accessor sA(sycl::range<1>(C * C), cgh); + sycl::local_accessor sQK(sycl::range<1>(C * C), cgh); + sycl::local_accessor sG(sycl::range<1>(C), cgh); + sycl::local_accessor sB(sycl::range<1>(C), cgh); + cgh.parallel_for(sycl::nd_range<1>(sycl::range<1>((size_t) H * ncb * WG), sycl::range<1>(WG)), + [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(16)]] { + const int tid = it.get_local_id(0); + const int grp = it.get_group(0); + const int h = grp / ncb; + const int col0 = (grp % ncb) * W; + const int64_t hk = h % neqk1; + + const float * sin = s_in + (s_ids ? (int64_t) s_ids[0] * s_row_stride : (int64_t) 0) + (int64_t) h * S * S; + for (int e = tid; e < S * W; e += WG) { + const int c = e / S, i = e % S; + sS[i * W + c] = sin[(int64_t) (col0 + c) * S + i]; + } + it.barrier(sycl::access::fence_space::local_space); + + for (int64_t t0 = 0; t0 < n_proc; t0 += C) { + const int nc = (int) sycl::min((int64_t) C, n_proc - t0); + for (int e = tid; e < C * S; e += WG) { + const int t = e / S, i = e % S; + float kv = 0.0f, qv = 0.0f; + if (t < nc) { + const int64_t off = (t0 + t) * sq2 + hk * sq1 + i; + kv = k[off]; + qv = q[off]; + } + sK[t * KP + i] = kv; + sQ[t * KP + i] = qv; + } + for (int e = tid; e < C * W; e += WG) { + const int t = e / W, c = e % W; + sV[e] = t < nc ? v[(t0 + t) * sv2 + h * sv1 + col0 + c] : 0.0f; + } + if (tid < C) { + // padded tokens: g = 0 keeps G at the last real value, beta = 0 zeroes their delta + sG[tid] = tid < nc ? g[(t0 + tid) * sb2 + h * sb1] : 0.0f; + sB[tid] = tid < nc ? beta[(t0 + tid) * sb2 + h * sb1] : 0.0f; + } + it.barrier(sycl::access::fence_space::local_space); + if (tid == 0) { + float acc = 0.0f; + for (int t = 0; t < C; ++t) { + acc += sG[t]; + sG[t] = acc; + } + } + it.barrier(sycl::access::fence_space::local_space); + + // C x C terms: A (strictly lower, beta and decay folded in) and the decayed q.k (lower incl. diag) + for (int e = tid; e < 2 * C * C; e += WG) { + const int which = e / (C * C), ts = e % (C * C), t = ts / C, s2 = ts % C; + const int ra = which ? t * KP : t * KP; + float d = 0.0f; + if (which == 0) { + for (int i = 0; i < S; ++i) d += sK[ra + i] * sK[s2 * KP + i]; + sA[ts] = s2 < t ? sB[t] * sycl::exp(sG[t] - sG[s2]) * d : 0.0f; + } else { + for (int i = 0; i < S; ++i) d += sQ[ra + i] * sK[s2 * KP + i]; + sQK[ts] = s2 <= t ? sycl::exp(sG[t] - sG[s2]) * d : 0.0f; + } + } + // K S0 and Q S0 for this column block (C x W each), sharing the S0 reads + float qs_reg[(C * W + WG - 1) / WG]; +#pragma unroll + for (int r = 0; r < (C * W + WG - 1) / WG; ++r) { + const int e = tid + r * WG, t = e / W, c = e % W; + if (e >= C * W) break; + float ks = 0.0f, qs = 0.0f; + for (int i = 0; i < S; ++i) { + const float sv = sS[i * W + c]; + ks += sK[t * KP + i] * sv; + qs += sQ[t * KP + i] * sv; + } + const float eg = sycl::exp(sG[t]); + sD[e] = sB[t] * (sV[e] - eg * ks); // R + qs_reg[r] = eg * qs; + } + it.barrier(sycl::access::fence_space::local_space); + + // Delta = (I + A)^-1 R: forward substitution, one column per work-item + if (tid < W) { + const int c = tid; + for (int t = 1; t < C; ++t) { + float r = sD[t * W + c]; + for (int s2 = 0; s2 < t; ++s2) r -= sA[t * C + s2] * sD[s2 * W + c]; + sD[t * W + c] = r; + } + } + it.barrier(sycl::access::fence_space::local_space); + +#pragma unroll + for (int r = 0; r < (C * W + WG - 1) / WG; ++r) { + const int e = tid + r * WG, t = e / W, c = e % W; + if (e < C * W && t < nc) { + float o = qs_reg[r]; + for (int s2 = 0; s2 <= t; ++s2) o += sQK[t * C + s2] * sD[s2 * W + c]; + dst[((t0 + t) * H + h) * S + col0 + c] = o * scale; + } + } + + const float glast = sG[C - 1]; + const float egl = sycl::exp(glast); + float dec[C]; +#pragma unroll + for (int t = 0; t < C; ++t) dec[t] = sycl::exp(glast - sG[t]); + for (int e = tid; e < S * W; e += WG) { + const int i = e / W, c = e % W; + float acc = egl * sS[e]; +#pragma unroll + for (int t = 0; t < C; ++t) acc += sK[t * KP + i] * dec[t] * sD[t * W + c]; + sS[e] = acc; + } + it.barrier(sycl::access::fence_space::local_space); + } + + float * so = s_out + (int64_t) h * S * S; + for (int e = tid; e < S * W; e += WG) { + const int c = e / S, i = e % S; + so[(int64_t) (col0 + c) * S + i] = sS[i * W + c]; + } + }); + }); +} + +static void gdn_chunked_sycl(const float * q, const float * k, const float * v, const float * g, const float * beta, + const float * s_in, float * dst, float * s_out, int64_t H, int64_t n_proc, + int64_t sq1, int64_t sq2, int64_t sv1, int64_t sv2, int64_t sb1, int64_t sb2, + int64_t neqk1, float scale, const int32_t * s_ids, int64_t s_row_stride, + dpct::queue_ptr stream) { + static const int variant = getenv("GGML_SYCL_GDN_CH_VARIANT") ? atoi(getenv("GGML_SYCL_GDN_CH_VARIANT")) : 0; // lab sweep +#define GDN_CH_CALL(CC, WW) gdn_chunked_sycl_t(q, k, v, g, beta, s_in, dst, s_out, H, n_proc, sq1, sq2, sv1, sv2, sb1, sb2, neqk1, scale, s_ids, s_row_stride, stream) + switch (variant) { + case 1: GDN_CH_CALL(16, 64); break; + case 2: GDN_CH_CALL(32, 32); break; + case 3: GDN_CH_CALL(32, 16); break; + case 4: GDN_CH_CALL(8, 32); break; + case 5: GDN_CH_CALL(16, 16); break; + case 6: GDN_CH_CALL(8, 64); break; + default: GDN_CH_CALL(16, 32); break; + } +#undef GDN_CH_CALL +} + +// GET_ROWS (recurrent-state gather) -> GATED_DELTA_NET src[5] registrations, per context, reset per graph: +// the gather is skipped and the kernel indexes the cache rows directly (port of the CUDA #220 fusion). +struct ggml_sycl_gdn_gather { + const float * base; + const int32_t * ids; + int64_t row_stride; +}; +static std::unordered_map> g_gdn_gathers; + +void ggml_sycl_gdn_gather_reset(const ggml_backend_sycl_context & ctx) { + g_gdn_gathers[&ctx].clear(); +} + +void ggml_sycl_gdn_gather_set(const ggml_backend_sycl_context & ctx, const ggml_tensor * gdn, const float * base, + const int32_t * ids, int64_t row_stride) { + g_gdn_gathers[&ctx][gdn] = { base, ids, row_stride }; +} + static void ggml_sycl_op_gated_delta_net_impl(ggml_backend_sycl_context & ctx, ggml_tensor * dst, const ggml_sycl_gated_delta_net_fused_cache * cache) { ggml_tensor * src_q = dst->src[0]; @@ -288,8 +692,19 @@ static void ggml_sycl_op_gated_delta_net_impl(ggml_backend_sycl_context & ctx, g const float * g_d = (const float *) src_g->data; const float * b_d = (const float *) src_beta->data; - const float * s_d = (const float *) src_state->data; - float * dst_d = (float *) dst->data; + const float * s_d = (const float *) src_state->data; + float * dst_d = (float *) dst->data; + const int32_t * s_ids = nullptr; + int64_t s_row_stride = 0; + { + auto & m = g_gdn_gathers[&ctx]; + auto it = m.find(dst); + if (it != m.end()) { + s_d = it->second.base; + s_ids = it->second.ids; + s_row_stride = it->second.row_stride; + } + } GGML_ASSERT(ggml_is_contiguous_rows(src_q)); GGML_ASSERT(ggml_is_contiguous_rows(src_k)); @@ -331,21 +746,45 @@ static void ggml_sycl_op_gated_delta_net_impl(ggml_backend_sycl_context & ctx, g if (keep_rs) { launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream); + sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, s_ids, s_row_stride, stream); } else { launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream); + sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, s_ids, s_row_stride, stream); } } else { + // ARC-LAB chunked prefill (OPT-IN, GGML_SYCL_GDN_CHUNKED=1): all but the last K tokens (the snapshot window) + // go through gdn_chunked_sycl; the sequential kernel then finishes the tail from that state. Correct, but on + // the B580 every shape measured slower than the sequential kernel (scalar SLM math, no XMX) - see README. + static const bool chunk_off = getenv("GGML_SYCL_GDN_CHUNKED") == nullptr; + const int64_t n_tail = keep_rs ? (int64_t) K : 0; + const int64_t n_proc = n_tokens - n_tail; + if (!chunk_off && S_v == GDN_CH_S && n_seqs == 1 && neq3 == 1 && n_proc >= 64) { + if (n_tail == 0) { + float * so = state_d; // final state straight to its destination (dst tail or cache slot 0) + gdn_chunked_sycl(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, so, H, n_proc, sq1, sq2, sv1, sv2, sb1, sb2, + neqk1, scale, s_ids, s_row_stride, stream); + return; + } + ggml_sycl_pool_alloc mid(ctx.pool(), (size_t) H * S_v * S_v); + gdn_chunked_sycl(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, mid.get(), H, n_proc, sq1, sq2, sv1, sv2, sb1, sb2, + neqk1, scale, s_ids, s_row_stride, stream); + // tail: the sequential kernel on tokens [n_proc, n_tokens) from the mid state; n_seqs == 1 so shifting + // the base pointers by n_proc tokens keeps every layout (dst row = t * H + h) + launch_gated_delta_net(q_d + n_proc * sq2, k_d + n_proc * sq2, v_d + n_proc * sv2, + g_d + n_proc * sb2, b_d + n_proc * sb2, mid.get(), dst_d + n_proc * S_v * H, state_d, + S_v, H, n_tail, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, + sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, nullptr, 0, stream); + return; + } if (keep_rs) { launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream); + sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, s_ids, s_row_stride, stream); } else { launch_gated_delta_net(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, state_d, S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, - sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, stream); + sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, s_ids, s_row_stride, stream); } } } diff --git a/ggml/src/ggml-sycl/gated_delta_net.hpp b/ggml/src/ggml-sycl/gated_delta_net.hpp index 7903b8e06df2..da5dda3f4969 100644 --- a/ggml/src/ggml-sycl/gated_delta_net.hpp +++ b/ggml/src/ggml-sycl/gated_delta_net.hpp @@ -17,3 +17,8 @@ void ggml_sycl_gated_delta_net(ggml_backend_sycl_context & ctx, ggml_tensor * ds // same op, but writes the snapshot(s) into the cache instead of dst (see ggml_sycl_try_gdn_cache_fusion) void ggml_sycl_op_gated_delta_net_fused_cache(ggml_backend_sycl_context & ctx, ggml_tensor * dst, ggml_sycl_gated_delta_net_fused_cache cache); + +// fused recurrent-state gather (see ggml_sycl_try_gdn_gather_skip): reset per graph, set per GDN node +void ggml_sycl_gdn_gather_reset(const ggml_backend_sycl_context & ctx); +void ggml_sycl_gdn_gather_set(const ggml_backend_sycl_context & ctx, const ggml_tensor * gdn, const float * base, + const int32_t * ids, int64_t row_stride); diff --git a/ggml/src/ggml-sycl/gemm.hpp b/ggml/src/ggml-sycl/gemm.hpp index c202da110beb..94bc58efe026 100644 --- a/ggml/src/ggml-sycl/gemm.hpp +++ b/ggml/src/ggml-sycl/gemm.hpp @@ -81,6 +81,54 @@ class DnnlGemmWrapper { matmul_prim.execute(stream, matmul_args); } + // dst[t * m + r] = sum_k act[t * k + c] * w[r][c] * scale[c / 128][r]; w is s4, row r contiguous along k. + static void gemm_s4_group(ggml_backend_sycl_context & ctx, int m, int n, int k, const sycl::half * act, + const uint8_t * w4, const float * scales, float * dst, const queue_ptr & q) { + auto stream = ctx.stream_dnnl(q); + auto eng = ctx.engine_dnnl(q); + + const auto act_md = dnnl::memory::desc({ n, k }, dt::f16, tag::ab); + const auto w_md = dnnl::memory::desc({ k, m }, dt::s4, tag::ba); + const auto dst_md = dnnl::memory::desc({ n, m }, dt::f32, tag::ab); + const auto sc_md = dnnl::memory::desc({ k / 128, m }, dt::f32, tag::ab); // oneDNN reads scales as dense ab, whatever the tag + + dnnl::primitive_attr attr; + attr.set_scratchpad_mode(dnnl::scratchpad_mode::user); + attr.set_scales(DNNL_ARG_WEIGHTS, (1 << 0) | (1 << 1), { 128, 1 }, dt::f32); + attr.set_fpmath_mode(dnnl::fpmath_mode::f16, true); + + auto pd = dnnl::matmul::primitive_desc(eng, act_md, w_md, dst_md, attr); + auto prim = dnnl::matmul(pd); + + std::unordered_map args; + args.insert({ DNNL_ARG_SRC, dnnl::memory(act_md, eng, const_cast(act)) }); + args.insert({ DNNL_ARG_WEIGHTS, dnnl::memory(w_md, eng, const_cast(w4)) }); + args.insert({ DNNL_ARG_DST, dnnl::memory(pd.dst_desc(), eng, dst) }); + args.insert({ DNNL_ARG_ATTR_SCALES | DNNL_ARG_WEIGHTS, dnnl::memory(sc_md, eng, const_cast(scales)) }); + args.insert({ DNNL_ARG_SCRATCHPAD, ctx.get_scratchpad_mem(pd.scratchpad_desc(), eng, q) }); + prim.execute(stream, args); + } + + // dst[t * m + r] = sum_k act[t * k + c] * w[r][c], all int8, int32 accumulate, f32 out (no scales) + static void gemm_s8(ggml_backend_sycl_context & ctx, int m, int n, int k, const int8_t * act, const int8_t * w, + float * dst, const queue_ptr & q) { + auto stream = ctx.stream_dnnl(q); + auto eng = ctx.engine_dnnl(q); + const auto act_md = dnnl::memory::desc({ n, k }, dt::s8, tag::ab); + const auto w_md = dnnl::memory::desc({ k, m }, dt::s8, tag::ba); + const auto dst_md = dnnl::memory::desc({ n, m }, dt::f32, tag::ab); + dnnl::primitive_attr attr; + attr.set_scratchpad_mode(dnnl::scratchpad_mode::user); + auto pd = dnnl::matmul::primitive_desc(eng, act_md, w_md, dst_md, attr); + auto prim = dnnl::matmul(pd); + std::unordered_map args; + args.insert({ DNNL_ARG_SRC, dnnl::memory(act_md, eng, const_cast(act)) }); + args.insert({ DNNL_ARG_WEIGHTS, dnnl::memory(w_md, eng, const_cast(w)) }); + args.insert({ DNNL_ARG_DST, dnnl::memory(pd.dst_desc(), eng, dst) }); + args.insert({ DNNL_ARG_SCRATCHPAD, ctx.get_scratchpad_mem(pd.scratchpad_desc(), eng, q) }); + prim.execute(stream, args); + } + static void row_gemm(ggml_backend_sycl_context & ctx, int m, int n, int k, const void * a, dt at, const void * b, dt bt, void * c, dt ct, const queue_ptr & q) { diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 97b5f35659f7..e6e141bed967 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -11,6 +11,7 @@ // #include +#include #include #include #include @@ -51,6 +52,7 @@ #include "ggml.h" #include "ggml-sycl.h" +#include "ptq1-t2.hpp" #include "ggml-impl.h" #include "ggml-backend-impl.h" @@ -60,6 +62,7 @@ #include "ggml-sycl/common.hpp" #include "ggml-sycl/element_wise.hpp" #include "ggml-sycl/fwht.hpp" +#include "ggml-sycl/w8a8.hpp" #include "ggml-sycl/gemm.hpp" #include "ggml-sycl/getrows.hpp" #include "ggml-sycl/norm.hpp" @@ -91,6 +94,9 @@ static bool g_sycl_loaded = false; int g_ggml_sycl_debug = 0; int g_ggml_sycl_enable_optimize = 1; +int g_ggml_sycl_quant_f16_gemm = 1; +int g_ggml_sycl_quant_s4_gemm = 1; +int g_ggml_sycl_quant_w8a8 = 1; int g_ggml_sycl_enable_graph = 0; int g_ggml_sycl_enable_dnn = 1; int g_ggml_sycl_fa_onednn = 1; @@ -326,6 +332,9 @@ static void ggml_check_sycl() try { if (!initialized) { g_ggml_sycl_debug = ggml_sycl_get_env("GGML_SYCL_DEBUG", 0); g_ggml_sycl_enable_optimize = ggml_sycl_get_env("GGML_SYCL_ENABLE_OPT", 1); + g_ggml_sycl_quant_f16_gemm = ggml_sycl_get_env("GGML_SYCL_QUANT_F16_GEMM", 1); + g_ggml_sycl_quant_s4_gemm = ggml_sycl_get_env("GGML_SYCL_QUANT_S4_GEMM", 1); + g_ggml_sycl_quant_w8a8 = ggml_sycl_get_env("GGML_SYCL_QUANT_W8A8", 1); g_ggml_sycl_enable_graph = ggml_sycl_get_env("GGML_SYCL_ENABLE_GRAPH", 0); g_ggml_sycl_enable_dnn = ggml_sycl_get_env("GGML_SYCL_ENABLE_DNN", 1); g_ggml_sycl_fa_onednn = ggml_sycl_get_env("GGML_SYCL_FA_ONEDNN", 1); @@ -420,6 +429,8 @@ static void ggml_check_sycl() try { #endif GGML_LOG_INFO(" GGML_SYCL_ENABLE_OPT: %d\n", g_ggml_sycl_enable_optimize); + GGML_LOG_INFO(" GGML_SYCL_QUANT_F16_GEMM: %d\n", g_ggml_sycl_quant_f16_gemm); + GGML_LOG_INFO(" GGML_SYCL_QUANT_S4_GEMM: %d\n", g_ggml_sycl_quant_s4_gemm); #if defined(GGML_SYCL_SUPPORT_VMM) GGML_LOG_INFO(" GGML_SYCL_ENABLE_VMM: %d\n", g_ggml_sycl_enable_vmm); @@ -608,6 +619,8 @@ ggml_backend_sycl_buffer_init_tensor(ggml_backend_buffer_t buffer, switch (tensor->type) { case GGML_TYPE_Q4_0: case GGML_TYPE_Q8_0: + case GGML_TYPE_PQ2_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: @@ -1004,6 +1017,11 @@ static size_t ggml_backend_sycl_buffer_type_get_alloc_size(ggml_backend_buffer_t size += ggml_row_size(tensor->type, MATRIX_ROW_PADDING - ne0 % MATRIX_ROW_PADDING); } } + // ARC-LAB: room for the in-place TernSYCL 2-bit repack (GGML_SYCL_PTQ1_T2) + if (tensor->type == GGML_TYPE_PTQ1_0 && tensor->ne[2] == 1 && tensor->ne[3] == 1 && + ggml_sycl_t2_wants(tensor->name, tensor->ne[0], tensor->ne[1])) { + size = std::max(size, ggml_sycl_t2_bytes(tensor->ne[0], tensor->ne[1])); + } const auto * buft_ctx = (const ggml_backend_sycl_buffer_type_context *) buft->context; if (ggml_sycl_ptq1_xmx_expands(buft_ctx->device, tensor)) { @@ -2726,8 +2744,36 @@ inline void ggml_sycl_op_mul_mat_sycl( #ifdef GGML_SYCL_F16 bool use_fp16 = true; // TODO(Yu) SYCL capability check #else - // ternary weights are exact in FP16 and the FP16 GEMM is much faster than FP32 - bool use_fp16 = src0->type == GGML_TYPE_PQ2_0 || src0->type == GGML_TYPE_PTQ1_0; + // ternary weights are exact in FP16 and the FP16 GEMM is much faster than FP32; other quantized weights + // take it for large batches too (ARC-LAB: fp16 GEMM runs on XMX, fp32 GEMM is ALU bound) + bool use_fp16 = src0->type == GGML_TYPE_PQ2_0 || src0->type == GGML_TYPE_PTQ1_0 || + (g_ggml_sycl_quant_f16_gemm && ggml_is_quantized(src0->type) && src1_ncols > 32); +#endif + +#if GGML_SYCL_DNNL + // PQ2_0 / PTQ1_0 batches: repack to s4 + group scales and let oneDNN decompress inside the GEMM (XMX), + // instead of writing and reading back a full fp16 copy of the weights. + if (g_ggml_sycl_quant_w8a8 && g_ggml_sycl_enable_dnn && (src0->type == GGML_TYPE_PQ2_0 || src0->type == GGML_TYPE_PTQ1_0) && + src1_ncols > 32 && ggml_is_contiguous(src0) && row_diff == src0->ne[1] && ldc == row_diff && + dst->op_params[0] == GGML_PREC_DEFAULT) { + const bool reordered = src0->extra && ((ggml_tensor_extra_gpu *) src0->extra)->optimized_feature.reorder; + if (ggml_sycl_w8a8_mul_mat(ctx, src0->type, reordered, src0_dd_i, src1_ddf_i, dst_dd_i, row_diff, src1_ncols, + ne10, stream)) { + return; + } + } + if (g_ggml_sycl_quant_s4_gemm && g_ggml_sycl_enable_dnn && (src0->type == GGML_TYPE_PQ2_0 || src0->type == GGML_TYPE_PTQ1_0) && + src1_ncols > 32 && ggml_is_contiguous(src0) && row_diff == src0->ne[1] && ldc == row_diff && + dst->op_params[0] == GGML_PREC_DEFAULT) { + const bool reordered = src0->extra && ((ggml_tensor_extra_gpu *) src0->extra)->optimized_feature.reorder; + ggml_sycl_pool_alloc w4(ctx.pool(), row_diff * ne00 / 2); + ggml_sycl_pool_alloc scales(ctx.pool(), row_diff * ne00 / 128); + ggml_sycl_quant_to_s4(src0->type, reordered, src0_dd_i, w4.get(), scales.get(), row_diff, ne00, stream); + ggml_sycl_pool_alloc src1_as_f16(ctx.pool(), src1_ncols * ne10); + ggml_get_to_fp16_sycl(src1->type, dst)(src1_ddf_i, src1_as_f16.get(), src1_ncols * ne10, stream); + DnnlGemmWrapper::gemm_s4_group(ctx, row_diff, src1_ncols, ne10, src1_as_f16.get(), w4.get(), scales.get(), dst_dd_i, stream); + return; + } #endif #if GGML_SYCL_DNNL && defined(GGML_SYCL_HAS_BF16) @@ -3852,6 +3898,8 @@ inline bool ggml_sycl_supports_reorder_mmvq(enum ggml_type type) { switch (type) { case GGML_TYPE_Q1_0: case GGML_TYPE_Q4_0: + case GGML_TYPE_PQ2_0: + case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q8_0: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: @@ -4044,6 +4092,80 @@ static bool reorder_qw_q4_0(uint8_t * data_device, const int ncols, const int nr return true; } +static bool reorder_qw_pq2_0(uint8_t * data_device, const int ncols, const int nrows, size_t size, size_t offset, + dpct::queue_ptr stream) { + sycl_reorder_temp_buffer tmp(stream, size); + if (!tmp) { + GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size); + return false; + } + uint8_t * tmp_buf = static_cast(tmp.ptr); + + sycl::event copy_event; + SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size))); + if (!g_ggml_sycl_use_async_mem_op) { + copy_event.wait(); + } + + GGML_ASSERT((size % sizeof(block_pq2_0) == 0)); + GGML_ASSERT((offset % sizeof(block_pq2_0) == 0)); + int offset_blks = offset / sizeof(block_pq2_0); + auto qs_ptr = data_device + offset_blks * QK_PQ2_0 / 4; + auto d_ptr = (sycl::half *) (qs_ptr + ncols * nrows / 4) + offset_blks; + + auto reorder_event = stream->parallel_for( + size / sizeof(block_pq2_0), + [=](auto i) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + const block_pq2_0 * x = (const block_pq2_0 *) tmp_buf; + const int ib = i; + + for (int j = 0; j < QK_PQ2_0 / 4; j++) { + *(qs_ptr + ib * QK_PQ2_0 / 4 + j) = x[ib].qs[j]; + } + *(d_ptr + ib) = x[ib].d; + }); + if (!g_ggml_sycl_use_async_mem_op) { + reorder_event.wait_and_throw(); + } + return true; +} + +static bool reorder_qw_ptq1_0(uint8_t * data_device, size_t size, dpct::queue_ptr stream) { + sycl_reorder_temp_buffer tmp(stream, size); + if (!tmp) { + GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__, size); + return false; + } + uint8_t * tmp_buf = static_cast(tmp.ptr); + + sycl::event copy_event; + SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, data_device, size))); + if (!g_ggml_sycl_use_async_mem_op) { + copy_event.wait(); + } + + GGML_ASSERT((size % sizeof(block_ptq1_0) == 0)); + const int64_t nb = size / sizeof(block_ptq1_0); + auto qs_ptr = data_device; + auto qh_ptr = data_device + nb * 24; + auto d_ptr = (sycl::half *) (data_device + nb * 26); + + auto reorder_event = stream->parallel_for(nb, [=](auto i) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + const block_ptq1_0 * x = (const block_ptq1_0 *) tmp_buf; + const int64_t ib = i; + for (int j = 0; j < 24; j++) { + qs_ptr[ib * 24 + j] = x[ib].qs[j]; + } + qh_ptr[ib * 2 + 0] = x[ib].qh[0]; + qh_ptr[ib * 2 + 1] = x[ib].qh[1]; + d_ptr[ib] = x[ib].d; + }); + if (!g_ggml_sycl_use_async_mem_op) { + reorder_event.wait_and_throw(); + } + return true; +} + static bool reorder_qw_q8_0(uint8_t * data_device, const int ncols, const int nrows, size_t size, size_t offset, dpct::queue_ptr stream) { sycl_reorder_temp_buffer tmp(stream, size); @@ -4488,6 +4610,10 @@ static bool reorder_qw(const ggml_tensor * src0, dpct::queue_ptr stream) { return reorder_qw_q4_0(data_device, ncols, nrows, size, 0, stream); case GGML_TYPE_Q8_0: return reorder_qw_q8_0(data_device, ncols, nrows, size, 0, stream); + case GGML_TYPE_PQ2_0: + return reorder_qw_pq2_0(data_device, ncols, nrows, size, 0, stream); + case GGML_TYPE_PTQ1_0: + return reorder_qw_ptq1_0(data_device, size, stream); case GGML_TYPE_Q2_K: return reorder_qw_q2_k(data_device, size, 0, stream); case GGML_TYPE_Q3_K: @@ -4624,9 +4750,105 @@ static bool ggml_sycl_pq2_xmx_use(ggml_backend_sycl_context & ctx, const ggml_te return true; } +// ARC-LAB: small-output bf16 mat-mul for 2..8 activation columns (speculative-decoding verify batches through Bonsai's +// delta-net gate projections, 48 x 5120). The generic path converted the whole bf16 matrix and ran a GEMM per call. +// One 256-wide work-group per output row reads the row once (8 bf16 per load) and dots it with every column. +static bool ggml_sycl_bf16_small_ok(const ggml_tensor * src0, const ggml_tensor * src1, const ggml_tensor * dst) { + static const bool off = getenv("GGML_SYCL_BF16_ROWSPLIT_OFF") != nullptr; + return !off && src0->type == GGML_TYPE_BF16 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 && + src0->ne[1] <= 512 && src1->ne[1] >= 2 && src1->ne[1] <= 8 && src0->ne[0] % 8 == 0 && + src1->ne[2] * src1->ne[3] == 1 && src0->ne[2] * src0->ne[3] == 1 && + ggml_is_contiguous(src0) && ggml_is_contiguous_rows(src1) && ggml_is_contiguous(dst) && + ((uintptr_t) src0->data % 16) == 0 && !ggml_backend_buffer_is_sycl_split(src0->buffer); +} + +static void ggml_sycl_mul_mat_bf16_small(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { + constexpr int WG = 256; + const int ncols_x = src0->ne[0]; + const int nrows = src0->ne[1]; + const int ncols_y = src1->ne[1]; + const int64_t sy = src1->nb[1] / sizeof(float); + const int64_t sd = dst->nb[1] / sizeof(float); + const uint16_t * x = (const uint16_t *) src0->data; + const float * y = (const float *) src1->data; + float * d = (float *) dst->data; + ctx.stream()->parallel_for( + sycl::nd_range<1>(sycl::range<1>((size_t) nrows * WG), sycl::range<1>(WG)), + [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + const int row = it.get_group(0); + const int tid = it.get_local_id(0); + const uint16_t * xr = x + (size_t) row * ncols_x; + float sum[8] = { 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f }; + for (int i = tid * 8; i < ncols_x; i += WG * 8) { + const sycl::vec w = *(const sycl::vec *) (xr + i); + float wf[8]; +#pragma unroll + for (int j = 0; j < 8; ++j) { + wf[j] = sycl::bit_cast((uint32_t) w[j] << 16); + } +#pragma unroll + for (int c = 0; c < 8; ++c) { + if (c < ncols_y) { + const float * yc = y + c * sy + i; +#pragma unroll + for (int j = 0; j < 8; ++j) { + sum[c] += wf[j] * yc[j]; + } + } + } + } +#pragma unroll + for (int c = 0; c < 8; ++c) { + if (c < ncols_y) { + const float v = sycl::reduce_over_group(it.get_group(), sum[c], sycl::plus()); + if (tid == 0) { + d[c * sd + row] = v; + } + } + } + }); +} + +// ARC-LAB: PTQ1_0 weights chosen by GGML_SYCL_PTQ1_T2 are repacked in place (their buffer was sized for it) on first use +// and from then on only this path may read them. +static bool ggml_sycl_t2_tensor(const ggml_tensor * w) { + if (w->type != GGML_TYPE_PTQ1_0 || w->ne[2] != 1 || w->ne[3] != 1 || !w->extra) { + return false; + } + const auto * extra = static_cast(w->extra); + return extra->optimized_feature.t2 || ggml_sycl_t2_wants(w->name, w->ne[0], w->ne[1]); +} + +static bool ggml_sycl_t2_mul_mat_try(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, + ggml_tensor * dst) { + if (!ggml_sycl_t2_tensor(src0)) { + return false; + } + auto * extra = static_cast(src0->extra); + GGML_ASSERT(!extra->optimized_feature.reorder && "PTQ1_0 T2 tensor was reordered"); + const int64_t K = src0->ne[0], N = src0->ne[1]; + GGML_ASSERT(src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 && src1->ne[0] == K && src1->nb[0] == sizeof(float)); + GGML_ASSERT(ggml_is_contiguous(dst) && dst->ne[0] == N); + GGML_ASSERT(src1->ne[3] == 1 && (src1->ne[2] == 1 || src1->nb[2] == src1->ne[1] * src1->nb[1])); + dpct::queue_ptr stream = ctx.stream(); + if (!extra->optimized_feature.t2) { + ggml_sycl_t2_repack(*stream, src0->data, K, N); + extra->optimized_feature.t2 = true; + } + // tokens: ne[1] x ne[2] rows of K floats (row stride nb[1]; planes back to back) + const int64_t M = src1->ne[1] * src1->ne[2]; + ggml_sycl_t2_mul_mat(*stream, src0->data, (const float *) src1->data, src1->nb[1] / sizeof(float), (float *) dst->data, + M, N, K); + return true; +} + static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); + if (dst->op == GGML_OP_MUL_MAT && ggml_sycl_t2_mul_mat_try(ctx, src0, src1, dst)) { + return; + } + // Handle HADAMARAD hint given from further up the pipeline and pass it to the correct // kernel. // @@ -4664,6 +4886,28 @@ static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor min_compute_capability = ggml_sycl_info().devices[ctx.device].cc; } + // Small batches (e.g. speculative verify) on quantized weights: run MMVQ on chunks of columns + // instead of dequantizing all of src0 to fp32 (the output matrix is ~5 GB as fp32). + if (!split && ggml_is_quantized(src0->type) && ggml_is_contiguous(src0) && src0->ne[2] == 1 && src0->ne[3] == 1 && + src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 && src1->ne[2] == 1 && src1->ne[3] == 1 && + src1->ne[1] > MMVQ_MAX_BATCH_SIZE && src1->nb[0] == sizeof(float) && + (src1->ne[1] <= 4 * MMVQ_MAX_BATCH_SIZE || (size_t) src0->ne[0] * src0->ne[1] * sizeof(float) > (1ull << 30))) { + for (int64_t c0 = 0; c0 < src1->ne[1]; c0 += MMVQ_MAX_BATCH_SIZE) { + const int64_t nc = std::min(MMVQ_MAX_BATCH_SIZE, src1->ne[1] - c0); + ggml_tensor src1_c = *src1; + ggml_tensor dst_c = *dst; + src1_c.ne[1] = nc; + src1_c.data = (char *) src1->data + c0 * src1->nb[1]; + src1_c.nb[2] = src1_c.nb[3] = nc * src1->nb[1]; + dst_c.ne[1] = nc; + dst_c.data = (char *) dst->data + c0 * dst->nb[1]; + dst_c.nb[2] = dst_c.nb[3] = nc * dst->nb[1]; + dst_c.src[1] = &src1_c; + ggml_sycl_mul_mat(ctx, src0, &src1_c, &dst_c); + } + return; + } + // check data types and tensor shapes for custom matrix multiplication kernels: bool use_dequantize_mul_mat_vec = can_use_dequantize_mul_mat_vec(src0, src1, dst); @@ -4714,6 +4958,8 @@ static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor } else if (!split && src0->type == GGML_TYPE_F16 && !ggml_is_transposed(src0) && !ggml_is_transposed(src1) && src1->ne[2] * src1->ne[3] > 1) { // KQ + KQV multi-batch ggml_sycl_mul_mat_batched_sycl(ctx, src0, src1, dst); + } else if (!split && ggml_sycl_bf16_small_ok(src0, src1, dst)) { + ggml_sycl_mul_mat_bf16_small(ctx, src0, src1, dst); } else if (use_dequantize_mul_mat_vec) { opt_for_reorder(&ctx, src0, src1, dst, mul_mat_algo::DMMV); ggml_sycl_op_mul_mat(ctx, src0, src1, dst, ggml_sycl_op_dequantize_mul_mat_vec); @@ -4721,7 +4967,12 @@ static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor opt_for_reorder(&ctx, src0, src1, dst, mul_mat_algo::MMVQ); ggml_tensor_extra_gpu * extra = static_cast(src0->extra); if (extra && extra->optimized_feature.reorder) { - ggml_sycl_op_mul_mat(ctx, src0, src1, dst, ggml_sycl_op_mul_mat_vec_q); + if (src0->type == GGML_TYPE_PTQ1_0 && ggml_sycl_ptq1_0_ncols_dec_enabled() && src1->ne[1] >= 2 && src1->ne[1] <= 8) { + // ARC-LAB: decode-once PTQ1_0 multi-column kernel reads the chunk-interleaved activation layout + ggml_sycl_op_mul_mat(ctx, src0, src1, dst, ggml_sycl_op_mul_mat_vec_q); + } else { + ggml_sycl_op_mul_mat(ctx, src0, src1, dst, ggml_sycl_op_mul_mat_vec_q); + } } else { ggml_sycl_op_mul_mat(ctx, src0, src1, dst, ggml_sycl_op_mul_mat_vec_q); } @@ -4735,9 +4986,13 @@ static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor // Fused dense-FFN mat-vec for the {mul_mat(gate), mul_mat(up), GLU} subgraph at node_idx. // Returns false if it declined, in which case the caller runs the three nodes normally. static bool ggml_sycl_mul_mat_glu_mmvq_fused(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int node_idx) { - if (!ggml_sycl_can_fuse(cgraph, node_idx, { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT, GGML_OP_GLU }, {})) { + static const bool fuse_off = getenv("GGML_SYCL_GLU_FUSE_OFF") != nullptr; // ARC-LAB A/B switch + if (fuse_off || !ggml_sycl_can_fuse(cgraph, node_idx, { GGML_OP_MUL_MAT, GGML_OP_MUL_MAT, GGML_OP_GLU }, {})) { return false; } + if (ggml_sycl_t2_tensor(cgraph->nodes[node_idx]->src[0]) || ggml_sycl_t2_tensor(cgraph->nodes[node_idx + 1]->src[0])) { + return false; // ARC-LAB: 2-bit repacked weights (ptq1-t2.cpp) run unfused + } ggml_tensor * glu = cgraph->nodes[node_idx + 2]; ggml_tensor * gate = glu->src[0]; @@ -4783,8 +5038,15 @@ static bool ggml_sycl_mul_mat_glu_mmvq_fused(ggml_backend_sycl_context & ctx, gg (size_t) ne11 * src1_padded_cols * sizeof(block_q8_1) / QK8_1); char * src1_ddq = src1_q8_alloc.get(); - quantize_row_q8_1_sycl((const float *) act->data, src1_ddq, (int) ne00, (int) ne11, - src1_padded_cols, stream); + if (wu->type == GGML_TYPE_PTQ1_0 && ggml_sycl_ptq1_0_ncols_dec_enabled() && ne11 <= 8 && + !(ne11 == 1 && ggml_sycl_ptq1_0_glu_n1_enabled())) { + // ARC-LAB: the decode-once PTQ1_0 fused kernel reads the chunk-interleaved activation layout + quantize_row_q8_1_sycl((const float *) act->data, src1_ddq, (int) ne00, + (int) ne11, src1_padded_cols, stream); + } else { + quantize_row_q8_1_sycl((const float *) act->data, src1_ddq, (int) ne00, (int) ne11, + src1_padded_cols, stream); + } return ggml_sycl_mul_mat_vec_q_glu_reorder(wu->type, ggml_get_glu_op(glu), wu->data, wg->data, src1_ddq, (float *) glu->data, (int) ne00, (int) wu->ne[1], (int) ne11, @@ -5780,11 +6042,234 @@ static int ggml_sycl_try_gdn_cache_fusion(const ggml_cgraph * cgraph, int node_i return skip; } +// GET_ROWS that gathers a recurrent state row feeding only GATED_DELTA_NET src[5] (possibly through a RESHAPE): +// skip it and let the kernel read the cache row directly. Single sequence only. GGML_SYCL_GDN_GATHER=0 disables. +static bool ggml_sycl_try_gdn_gather_skip(ggml_backend_sycl_context & ctx, const ggml_cgraph * cgraph, int node_idx) { + static const bool enabled = ggml_sycl_get_env("GGML_SYCL_GDN_GATHER", 1) != 0; + if (!enabled) { + return false; + } + const ggml_tensor * gr = cgraph->nodes[node_idx]; + if (gr->op != GGML_OP_GET_ROWS || gr->type != GGML_TYPE_F32 || (gr->flags & GGML_TENSOR_FLAG_OUTPUT) || + !ggml_is_contiguous(gr)) { + return false; + } + const ggml_tensor * cache = gr->src[0]; + const ggml_tensor * ids = gr->src[1]; + if (cache->type != GGML_TYPE_F32 || ids->type != GGML_TYPE_I32 || !cache->data || !ids->data || + (ids->buffer && ggml_backend_buffer_is_host(ids->buffer)) || cache->nb[0] != sizeof(float) || + cache->nb[1] % sizeof(float) != 0 || !ggml_is_contiguous(ids) || ggml_nelements(ids) != 1 || + gr->ne[1] != 1 || gr->ne[2] != 1 || gr->ne[3] != 1 || gr->ne[0] != cache->ne[0]) { + return false; + } + if (ggml_node_get_use_count(cgraph, node_idx) != 1) { + return false; + } + const ggml_tensor * cur = gr; + for (int j = node_idx + 1; j < cgraph->n_nodes; ++j) { + const ggml_tensor * n = cgraph->nodes[j]; + if (n->op == GGML_OP_GATED_DELTA_NET && n->src[5] == cur) { + const ggml_tensor * v = n->src[2]; + const int64_t D = v->ne[0] * v->ne[0] * v->ne[1]; + if (gr->ne[0] != D || v->ne[3] != 1 || ggml_nelements(cur) != D) { + return false; + } + ggml_sycl_gdn_gather_set(ctx, n, (const float *) cache->data, (const int32_t *) ids->data, + (int64_t) (cache->nb[1] / sizeof(float))); + return true; + } + if (n->op == GGML_OP_RESHAPE && n->src[0] == cur) { + if (ggml_node_get_use_count(cgraph, j) != 1) { + return false; + } + cur = n; + continue; + } + for (int s = 0; s < GGML_MAX_SRC; ++s) { + if (n->src[s] == cur || (n->view_src != nullptr && n->view_src == gr)) { + return false; + } + } + } + return false; +} + +// ARC-LAB: single-column PTQ1_0 mat-vecs that read the same activation (GDN qkv + z, attention q + k + v) in ONE launch +// with ONE quantization (in-model each launch costs ~10 us beyond its weight bytes). The graph interleaves them with +// other nodes, so later members are pulled forward to node i: only when their output overlaps no tensor that a node in +// between reads or writes (the allocator reuses dead memory). GGML_SYCL_PTQ1_MULTI=0 turns it off. +static bool ggml_sycl_ptq1_multi_eligible(ggml_backend_sycl_context & ctx, const ggml_tensor * mm) { + if (mm->op != GGML_OP_MUL_MAT || (mm->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { + return false; + } + const ggml_tensor * w = mm->src[0]; + const ggml_tensor * act = mm->src[1]; + if (w->type != GGML_TYPE_PTQ1_0 || act->type != GGML_TYPE_F32 || mm->type != GGML_TYPE_F32) { + return false; + } + // 1 column: pairs kernel (rows % 4); 2..8 (MTP verify): decode-once kernel (rows % 2) + if (act->ne[1] < 1 || act->ne[1] > 8 || act->ne[2] != 1 || act->ne[3] != 1 || w->ne[2] != 1 || w->ne[3] != 1 || + w->ne[0] % QK_PTQ1_0 != 0 || w->ne[1] % 4 != 0 || mm->ne[0] != w->ne[1]) { + return false; + } + if (act->ne[1] > 1 && !ggml_sycl_ptq1_0_ncols_dec_enabled()) { + return false; + } + static const bool ncols_on = !getenv("GGML_SYCL_PTQ1_MULTI_NCOLS") || atoi(getenv("GGML_SYCL_PTQ1_MULTI_NCOLS")) != 0; + if (act->ne[1] > 1 && !ncols_on) { + return false; + } + if (!ggml_is_contiguous(w) || !ggml_is_contiguous(act) || !ggml_is_contiguous(mm)) { + return false; + } + if (ggml_backend_buffer_is_sycl_split(w->buffer) || g_ggml_sycl_prioritize_dmmv || !should_reorder_tensor(ctx, mm)) { + return false; + } + const auto * extra = static_cast(w->extra); + return extra && extra->optimized_feature.reorder; // reordered on its first unfused run +} + +static bool ggml_sycl_tensors_overlap(const ggml_tensor * a, const ggml_tensor * b) { + if (!a || !b || !a->data || !b->data) { + return false; + } + const char * a0 = (const char *) a->data; + const char * b0 = (const char *) b->data; + return a0 < b0 + ggml_nbytes(b) && b0 < a0 + ggml_nbytes(a); +} + +// A member whose output memory is still in use by nodes in between (GDN z: the allocator hands its buffer to the conv / +// delta-net intermediates) is computed into a private scratch slot instead and copied to its tensor at its own node. +static constexpr int PTQ1_MULTI_SLOTS = 4; +static constexpr size_t PTQ1_MULTI_SLOT_BYTES = 256 * 1024; + +static float * ggml_sycl_ptq1_multi_slot(queue_ptr stream) { + static std::map scratch; // per device + static int next = 0; + const int dev = ggml_sycl_get_device(); + float *& buf = scratch[dev]; + if (!buf) { + buf = (float *) sycl::malloc_device(PTQ1_MULTI_SLOTS * PTQ1_MULTI_SLOT_BYTES, *stream); + if (!buf) { + return nullptr; + } + } + next = (next + 1) % PTQ1_MULTI_SLOTS; + return buf + next * (PTQ1_MULTI_SLOT_BYTES / sizeof(float)); +} + +static bool ggml_sycl_try_ptq1_multi(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int i, + std::vector & done, std::vector & pending) { + static const bool on = !getenv("GGML_SYCL_PTQ1_MULTI") || atoi(getenv("GGML_SYCL_PTQ1_MULTI")) != 0; + static int log = getenv("GGML_SYCL_PTQ1_MULTI_LOG") ? atoi(getenv("GGML_SYCL_PTQ1_MULTI_LOG")) : 0; + ggml_tensor * mm0 = cgraph->nodes[i]; + if (!on || !ggml_sycl_ptq1_multi_eligible(ctx, mm0)) { + return false; + } + const ggml_tensor * act = mm0->src[1]; + int members[3] = { i, -1, -1 }; + bool staged[3] = { false, false, false }; + int nm = 1; + const int jend = std::min(cgraph->n_nodes, i + 512); + for (int j = i + 1; j < jend && nm < 3; ++j) { + ggml_tensor * mj = cgraph->nodes[j]; + if (done[j] || mj->op != GGML_OP_MUL_MAT || mj->src[1] != act || mj->src[0]->ne[0] != mm0->src[0]->ne[0] || + !ggml_sycl_ptq1_multi_eligible(ctx, mj)) { + continue; + } + bool safe = true; + for (int k = i + 1; k < j && safe; ++k) { + const ggml_tensor * nk = cgraph->nodes[k]; + bool member = false; + for (int m = 1; m < nm; ++m) { + member = member || members[m] == k; + } + if (member) { + continue; + } + safe = !ggml_sycl_tensors_overlap(mj, nk); + for (int s = 0; s < GGML_MAX_SRC && safe; ++s) { + safe = !ggml_sycl_tensors_overlap(mj, nk->src[s]); + } + } + for (int m = 0; m < nm && safe; ++m) { + safe = !ggml_sycl_tensors_overlap(mj, cgraph->nodes[members[m]]) && !ggml_sycl_tensors_overlap(mj, act); + } + if (safe || ggml_nbytes(mj) <= PTQ1_MULTI_SLOT_BYTES) { + staged[nm] = !safe; + members[nm++] = j; + } + } + if (nm < 2) { + return false; + } + + const int64_t ne00 = mm0->src[0]->ne[0]; + const int ne11 = (int) act->ne[1]; + const int src1_padded_cols = GGML_PAD((int) ne00, MATRIX_ROW_PADDING); + const int stride_y_bytes = src1_padded_cols * (int) sizeof(block_q8_1) / QK8_1; + const queue_ptr stream = ctx.stream(); + ggml_sycl_pool_alloc q8(ctx.pool(), (size_t) ne11 * stride_y_bytes); + if (ne11 == 1) { + quantize_row_q8_1_sycl((const float *) act->data, q8.get(), (int) ne00, 1, + src1_padded_cols, stream); + } else { + quantize_row_q8_1_sycl((const float *) act->data, q8.get(), (int) ne00, ne11, + src1_padded_cols, stream); + } + const void * vx[3]; + float * dst[3]; + int nrows[3]; + for (int m = 0; m < nm; ++m) { + ggml_tensor * t = cgraph->nodes[members[m]]; + vx[m] = t->src[0]->data; + dst[m] = (float *) t->data; + nrows[m] = (int) t->src[0]->ne[1]; + if (staged[m]) { + dst[m] = ggml_sycl_ptq1_multi_slot(stream); + if (!dst[m]) { + return false; + } + } + } + const bool launched = ne11 == 1 ? + ggml_sycl_ptq1_0_mul_mat_vec_multi(vx, dst, nrows, nm, q8.get(), (int) ne00, stream) : + ggml_sycl_ptq1_0_mul_mat_vec_multi_ncols(vx, dst, nrows, nm, q8.get(), (int) ne00, ne11, stride_y_bytes, stream); + if (!launched) { + return false; + } + if (log > 0) { + GGML_LOG_INFO("[ptq1-multi] %s", mm0->name); + for (int m = 1; m < nm; ++m) { + GGML_LOG_INFO(" + %s (+%d%s)", cgraph->nodes[members[m]]->name, members[m] - i, staged[m] ? ", staged" : ""); + } + GGML_LOG_INFO("\n"); + --log; + } + for (int m = 1; m < nm; ++m) { + done[members[m]] = 1; + if (staged[m]) { + pending[members[m]] = dst[m]; // copied into the tensor when the loop reaches it + } + } + return true; +} + static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * sycl_ctx, ggml_cgraph * cgraph) { ggml_sycl_set_main_device(sycl_ctx->device); + ggml_sycl_gdn_gather_reset(*sycl_ctx); + // ARC-LAB: nodes already computed by ggml_sycl_try_ptq1_multi, and staged outputs still to copy in + std::vector done(cgraph->n_nodes, 0); + std::vector pending(cgraph->n_nodes, nullptr); for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; + if (done[i]) { + if (pending[i]) { + SYCL_CHECK(CHECK_TRY_ERROR(sycl_ctx->stream()->memcpy(node->data, pending[i], ggml_nbytes(node)))); + } + continue; + } if (ggml_sycl_is_view_or_noop(node)) { continue; } @@ -5805,6 +6290,9 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc } } #endif + if (node->op == GGML_OP_GET_ROWS && ggml_sycl_try_gdn_gather_skip(*sycl_ctx, cgraph, i)) { + continue; + } // gated_delta_net -> cpy: scatter recurrent-state snapshots into the cache if (node->op == GGML_OP_GATED_DELTA_NET) { ggml_sycl_gated_delta_net_fused_cache fused_state_cpy; @@ -5815,6 +6303,25 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc continue; } } + // Hadamard sign flip + reshape + FWHT-hint matmul: apply the signs in the transform's load + if (node->op == GGML_OP_MUL && + ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_MUL, GGML_OP_RESHAPE, GGML_OP_MUL_MAT }, { i + 2 })) { + const ggml_tensor * reshape = cgraph->nodes[i + 1]; + ggml_tensor * mm = cgraph->nodes[i + 2]; + const ggml_tensor * x = node->src[0]; + const ggml_tensor * signs = node->src[1]; + + const bool pattern_ok = ggml_get_op_params_i32(mm, 1) == GGML_HINT_SRC0_IS_HADAMARD && + mm->src[1] == reshape && reshape->src[0] == node && signs->ne[1] == 1 && signs->ne[2] == 1 && + signs->ne[3] == 1 && signs->type == GGML_TYPE_F32 && + (x->type == GGML_TYPE_F32 || x->type == GGML_TYPE_F16) && node->type == x->type && + ggml_is_contiguous(x) && ggml_is_contiguous(signs) && signs->ne[0] == x->ne[0] && + signs->ne[0] % mm->src[0]->ne[0] == 0; + if (pattern_ok && ggml_sycl_op_fwht_signed(*sycl_ctx, x, signs, mm)) { + i += 2; + continue; + } + } if (node->op == GGML_OP_RMS_NORM && ggml_sycl_can_fuse(cgraph, i, { GGML_OP_RMS_NORM, GGML_OP_MUL }, {})) { ggml_sycl_op_rms_norm_fused(*sycl_ctx, node, cgraph->nodes[i + 1]); @@ -5832,6 +6339,9 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc i += 2; continue; } + if (node->op == GGML_OP_MUL_MAT && ggml_sycl_try_ptq1_multi(*sycl_ctx, cgraph, i, done, pending)) { + continue; + } bool ok = ggml_sycl_compute_forward(*sycl_ctx, node); if (!ok) { @@ -5854,11 +6364,6 @@ static bool check_graph_compatibility(ggml_cgraph * cgraph) { switch (node_op) { default: break; - case GGML_OP_CONCAT: - // ggml_sycl_op_concat() does a blocking host wait after memcpy operations, - // but wait() can't be called on the events returned by a queue recording - // to a graph. - [[fallthrough]]; case GGML_OP_MUL_MAT_ID: // ggml_sycl_mul_mat_id() does a blocking host wait on the sycl queue after // submitting a memcpy operation, but wait() can't be called on a queue that @@ -5867,6 +6372,10 @@ static bool check_graph_compatibility(ggml_cgraph * cgraph) { ggml_op_name(node_op)); return false; case GGML_OP_MUL_MAT: + // oneMKL gemm (multi-token batches) cannot be recorded to a graph. + if (cgraph->nodes[i]->src[1]->ne[1] > 1) { + return false; + } // We cannot use graphs with ggml_sycl_mul_mat() when SYCL async memory allocation extensions are not available, // as SYCL malloc / free and host wait calls are not supported when recording to a graph which are all present // in reordering. diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cpp index 36c4c1778769..4e96f068d24c 100644 --- a/ggml/src/ggml-sycl/mmvq.cpp +++ b/ggml/src/ggml-sycl/mmvq.cpp @@ -737,6 +737,752 @@ static void reorder_mul_mat_vec_q4_0_q8_1_sycl_switch_ncols( } } +static void reorder_mul_mat_vec_pq2_0_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, + const int nrows, dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_PQ2_0 == 0); + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder>(vx, vy, dst, ncols, nrows, + nd_item); + }); + }); +} + +template +static void reorder_mul_mat_vec_pq2_0_q8_1_sycl_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_PQ2_0 == 0); + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder_ncols, ncols_dst>( + vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, + /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item); + }); + }); +} + +static void reorder_mul_mat_vec_pq2_0_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + switch (ncols_dst) { + case 1: reorder_mul_mat_vec_pq2_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; + case 2: reorder_mul_mat_vec_pq2_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 3: reorder_mul_mat_vec_pq2_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 4: reorder_mul_mat_vec_pq2_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 5: reorder_mul_mat_vec_pq2_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 6: reorder_mul_mat_vec_pq2_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 7: reorder_mul_mat_vec_pq2_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 8: reorder_mul_mat_vec_pq2_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + default: GGML_ABORT("unsupported ncols_dst=%d for PQ2_0 reorder multi-col MMVQ", ncols_dst); + } +} + +// PTQ1_0 single-column: one lane takes a whole block, so a sub-group takes R rows and spreads all +// R * blocks_per_row blocks over its lanes. Rows of 40 blocks leave no idle lane at R = 2. +template +static void mul_mat_vec_ptq1_0_reorder_rows(const void * __restrict__ vx, const void * __restrict__ vy, + float * __restrict__ dst, const int ncols, const int nrows, + const sycl::nd_item<3> & nd_item) { + using block_type = ggml_sycl_reordered::block_q_t; + + const auto sg = nd_item.get_sub_group(); + const int row0 = (nd_item.get_group_linear_id() * sg.get_group_linear_range() + sg.get_group_linear_id()) * R; + if (row0 >= nrows) { + return; + } + + const int blocks_per_row = ncols / QK_PTQ1_0; + const int nblocks = nrows * blocks_per_row; + const int lane = sg.get_local_linear_id(); + + float acc[R] = {}; + for (int i = lane; i < R * blocks_per_row; i += WARP_SIZE) { + const int r = i / blocks_per_row; + const int b = i - r * blocks_per_row; + if (row0 + r >= nrows) { + continue; + } + const int ibx = (row0 + r) * blocks_per_row + b; + const int8_t * q8 = (const int8_t *) vy + b * QK_PTQ1_0; + const sycl::half2 * ds = (const sycl::half2 *) ((const char *) vy + ncols + b * block_type::block_to_q8_1_ratio() * sizeof(sycl::half2)); + const float v = reorder_vec_dot_q_sycl()( + vx, block_type::get_block_offset(ibx, nblocks), block_type::get_d_offset(nrows, ncols, ibx), q8, ds, 0); +#pragma unroll + for (int k = 0; k < R; ++k) { + acc[k] += r == k ? v : 0.0f; + } + } + +#pragma unroll + for (int k = 0; k < R; ++k) { + const float sum = sycl::reduce_over_group(sg, acc[k], std::plus<>()); + if (lane == k && row0 + k < nrows) { + dst[row0 + k] = sum; + } + } +} + +template +static void launch_mul_mat_vec_ptq1_0_reorder_rows(const void * vx, const void * vy, float * dst, const int ncols, + const int nrows, dpct::queue_ptr stream) { + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, R * GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_ptq1_0_reorder_rows(vx, vy, dst, ncols, nrows, nd_item); + }); + }); +} + +// ARC-LAB: single-column PTQ1_0 mat-vec in the layout of the Vulkan shader (mul_mat_vecq_ptq1_0.comp, 47 us vs 60 on the +// 17408x5120 FFN matrix): two lanes per 128-weight block (lane 0: qs dwords 0..2, lane 1: dwords 3..5 + qh); a block's +// activations are loaded ONCE and reused for the NR rows of the subgroup (the per-lane-block kernel reloads 128 B of +// activations per row, over 4x the weight bytes); trits decoded per level on two bytes at a time in 16-bit lanes and fed +// straight to dp4a against pre-masked activation words, so the residue byte multiplies a zero (no repacking). +// Core: rows row0 .. row0 + NR - 1 of NM same-shape matrices (NM = 2: gate and up) against one activation column; +// temp[m][n] holds this lane's partial sums (not yet reduced over the subgroup). +template +static inline void ptq1_0_pairs_core(const uint8_t * const (&bases)[NM], const void * __restrict__ vy, const int ncols, + const int nrows, const int row0, const int lane, float (&temp)[NM][NR]) { + const int bpr = ncols / QK_PTQ1_0; + const int nblocks = nrows * bpr; + const int sub = lane & 1; + + const uint32_t * y32 = static_cast(vy); + const sycl::half2 * yds = (const sycl::half2 *) ((const char *) vy + ncols); + +#pragma unroll + for (int m = 0; m < NM; ++m) { +#pragma unroll + for (int n = 0; n < NR; ++n) { + temp[m][n] = 0.0f; + } + } + + for (int b = lane >> 1; b < bpr; b += WARP_SIZE / 2) { + const uint32_t * yb = y32 + b * (QK_PTQ1_0 / 4); + const sycl::float2 ds0 = yds[b * 4 + 0].convert(); + const sycl::float2 ds1 = yds[b * 4 + 1].convert(); + const sycl::float2 ds2 = yds[b * 4 + 2].convert(); + const sycl::float2 ds3 = yds[b * 4 + 3].convert(); + // accA: this lane's first qs dword (sub-block t/2); accB: the other two (lane 0: t/2, lane 1: 2,2,3,3,3) + const float sA0 = ds0.x(), sA1 = ds1.x(), sA2 = ds2.x(); + const float sB0 = sub ? ds2.x() : ds0.x(); + const float sB1 = sub ? ds3.x() : ds1.x(); + const float sB2 = sub ? ds3.x() : ds2.x(); + const float bias = sub ? (ds2.y() + ds3.y()) : (ds0.y() + ds1.y()); + + // activation words per trit level, pre-masked: yh = bytes 1,3; yl = bytes 0,2 moved to 1,3 + int yl0[5], yh0[5], yl1[5], yh1[5], yl2[5], yh2[5]; +#pragma unroll + for (int t = 0; t < 5; ++t) { + const uint32_t a0 = sub ? yb[4 * t + 3] : yb[4 * t]; + const uint32_t a1 = sub ? yb[20 + 2 * t] : yb[4 * t + 1]; + const uint32_t a2 = sub ? yb[21 + 2 * t] : yb[4 * t + 2]; + yh0[t] = (int) (a0 & 0xFF00FF00u); + yh1[t] = (int) (a1 & 0xFF00FF00u); + yh2[t] = (int) (a2 & 0xFF00FF00u); + yl0[t] = (int) ((a0 << 8) & 0xFF00FF00u); + yl1[t] = (int) ((a1 << 8) & 0xFF00FF00u); + yl2[t] = (int) ((a2 << 8) & 0xFF00FF00u); + } + const int yq0 = (int) yb[30]; + const int yq1 = (int) yb[31]; + +#pragma unroll + for (int m = 0; m < NM; ++m) { + const uint8_t * base = bases[m]; +#pragma unroll + for (int n = 0; n < NR; ++n) { + const int row = row0 + n; + if (row >= nrows) { + break; + } + const int ibx = row * bpr + b; + const uint32_t * q32 = (const uint32_t *) (base + ibx * 24) + 3 * sub; + const uint32_t w0 = q32[0], w1 = q32[1], w2 = q32[2]; + const uint8_t * qh = base + nblocks * 24 + ibx * 2; + const float da = (float) *(reinterpret_cast(base + nblocks * 26 + ibx * 2)); + + uint32_t v0l = w0 & 0x00FF00FFu, v0h = (w0 >> 8) & 0x00FF00FFu; + uint32_t v1l = w1 & 0x00FF00FFu, v1h = (w1 >> 8) & 0x00FF00FFu; + uint32_t v2l = w2 & 0x00FF00FFu, v2h = (w2 >> 8) & 0x00FF00FFu; + int accA0 = 0, accA1 = 0, accA2 = 0, accB0 = 0, accB1 = 0, accB2 = 0; + auto level = [](uint32_t & vl, uint32_t & vh, int yl, int yh, int & acc) { + const uint32_t wl = vl * 3u, wh = vh * 3u; + vl = wl & 0x00FF00FFu; + vh = wh & 0x00FF00FFu; + acc = dpct::dp4a((int) wl, yl, acc); + acc = dpct::dp4a((int) wh, yh, acc); + }; + level(v0l, v0h, yl0[0], yh0[0], accA0); level(v1l, v1h, yl1[0], yh1[0], accB0); level(v2l, v2h, yl2[0], yh2[0], accB0); + level(v0l, v0h, yl0[1], yh0[1], accA0); level(v1l, v1h, yl1[1], yh1[1], accB0); level(v2l, v2h, yl2[1], yh2[1], accB0); + level(v0l, v0h, yl0[2], yh0[2], accA1); level(v1l, v1h, yl1[2], yh1[2], accB1); level(v2l, v2h, yl2[2], yh2[2], accB1); + level(v0l, v0h, yl0[3], yh0[3], accA1); level(v1l, v1h, yl1[3], yh1[3], accB1); level(v2l, v2h, yl2[3], yh2[3], accB1); + level(v0l, v0h, yl0[4], yh0[4], accA2); level(v1l, v1h, yl1[4], yh1[4], accB2); level(v2l, v2h, yl2[4], yh2[4], accB2); + { + // qh trits -> elements 120 + 2m + h, sub-block 3 (lane 1 only; lane 0 decodes zeros) + uint32_t vq = sub ? ((uint32_t) qh[0] | ((uint32_t) qh[1] << 16)) : 0u; + uint32_t wq = vq * 3u; + vq = wq & 0x00FF00FFu; + int q01 = (int) (((wq >> 8) & 0x3u) | ((wq >> 16) & 0x300u)); + wq = vq * 3u; + vq = wq & 0x00FF00FFu; + q01 |= (int) (((wq << 8) & 0x30000u) | (wq & 0x03000000u)); + wq = vq * 3u; + vq = wq & 0x00FF00FFu; + int q23 = (int) (((wq >> 8) & 0x3u) | ((wq >> 16) & 0x300u)); + wq = vq * 3u; + q23 |= (int) (((wq << 8) & 0x30000u) | (wq & 0x03000000u)); + accB2 = dpct::dp4a(q01, yq0, accB2); + accB2 = dpct::dp4a(q23, yq1, accB2); + } + const float sum = sA0 * accA0 + sA1 * accA1 + sA2 * accA2 + sB0 * accB0 + sB1 * accB1 + sB2 * accB2 - bias; + temp[m][n] += da * sum; + } + } + } +} + +template +static void mul_mat_vec_ptq1_0_pairs(const void * __restrict__ vx, const void * __restrict__ vy, float * __restrict__ dst, + const int ncols, const int nrows, const sycl::nd_item<3> & nd_item) { + const auto sg = nd_item.get_sub_group(); + const int lane = sg.get_local_linear_id(); + const int row0 = (nd_item.get_group_linear_id() * sg.get_group_linear_range() + sg.get_group_linear_id()) * NR; + if (row0 >= nrows) { + return; + } + const uint8_t * const bases[1] = { static_cast(vx) }; + float temp[1][NR]; + ptq1_0_pairs_core(bases, vy, ncols, nrows, row0, lane, temp); +#pragma unroll + for (int n = 0; n < NR; ++n) { + const float s = sycl::reduce_over_group(sg, temp[0][n], std::plus<>()); + if (lane == 0 && row0 + n < nrows) { + dst[row0 + n] = s; + } + } +} + +// ARC-LAB: up to 3 matrices that share one activation (GDN qkv + z, attention q + k + v) in ONE launch: the row-group +// index space is the matrices stacked, each subgroup serves NR rows of one of them. A launch costs ~10 us beyond its +// weight bytes on the B580 (in-model trace: 1 MB attn_k 12 us), so merging saves the tail of every extra launch and +// one activation quantization. Row counts must be multiples of NR. +struct ptq1_0_multi_args { + const void * vx[3]; + float * dst[3]; + int nrows[3]; +}; + +template +static void mul_mat_vec_ptq1_0_pairs_multi(const ptq1_0_multi_args a, const void * __restrict__ vy, const int ncols, + const int nrows_total, const sycl::nd_item<3> & nd_item) { + const auto sg = nd_item.get_sub_group(); + const int lane = sg.get_local_linear_id(); + int row0 = (nd_item.get_group_linear_id() * sg.get_group_linear_range() + sg.get_group_linear_id()) * NR; + if (row0 >= nrows_total) { + return; + } + int m = 0; + if (row0 >= a.nrows[0]) { + row0 -= a.nrows[0]; + m = 1; + if (row0 >= a.nrows[1]) { + row0 -= a.nrows[1]; + m = 2; + } + } + const int nrows = a.nrows[m]; + const uint8_t * const bases[1] = { static_cast(a.vx[m]) }; + float * const dst = a.dst[m]; + float temp[1][NR]; + ptq1_0_pairs_core(bases, vy, ncols, nrows, row0, lane, temp); +#pragma unroll + for (int n = 0; n < NR; ++n) { + const float s = sycl::reduce_over_group(sg, temp[0][n], std::plus<>()); + if (lane == 0 && row0 + n < nrows) { + dst[row0 + n] = s; + } + } +} + +// ARC-LAB: single-column gate/up + GLU with the pairs core: each subgroup dots NR rows of BOTH matrices against one load +// of the activations and writes act(gate) * up. +template +static void mul_mat_vec_ptq1_0_pairs_glu(const void * __restrict__ vup, const void * __restrict__ vgate, + const void * __restrict__ vy, float * __restrict__ dst, const int ncols, + const int nrows, const ggml_glu_op glu_op, const sycl::nd_item<3> & nd_item) { + const auto sg = nd_item.get_sub_group(); + const int lane = sg.get_local_linear_id(); + const int row0 = (nd_item.get_group_linear_id() * sg.get_group_linear_range() + sg.get_group_linear_id()) * NR; + if (row0 >= nrows) { + return; + } + const uint8_t * const bases[2] = { static_cast(vup), static_cast(vgate) }; + float temp[2][NR]; + ptq1_0_pairs_core(bases, vy, ncols, nrows, row0, lane, temp); +#pragma unroll + for (int n = 0; n < NR; ++n) { + const float u = sycl::reduce_over_group(sg, temp[0][n], std::plus<>()); + const float g = sycl::reduce_over_group(sg, temp[1][n], std::plus<>()); + if (lane == 0 && row0 + n < nrows) { + dst[row0 + n] = u * (glu_op == GGML_GLU_OP_SWIGLU ? op_silu(g) : op_gelu(g)); + } + } +} + +template +static void launch_ptq1_0_pairs(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, + dpct::queue_ptr stream) { + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, NR * GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_ptq1_0_pairs(vx, vy, dst, ncols, nrows, nd_item); + }); + }); +} + +template +static void launch_ptq1_0_pairs_multi(const ptq1_0_multi_args & a, const void * vy, const int ncols, const int nrows_total, + dpct::queue_ptr stream) { + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows_total, NR * GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + const ptq1_0_multi_args args = a; + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_ptq1_0_pairs_multi(args, vy, ncols, nrows_total, nd_item); + }); + }); +} + +template +static void launch_ptq1_0_pairs_glu(const void * vup, const void * vgate, const void * vy, float * dst, const int ncols, + const int nrows, const ggml_glu_op glu_op, dpct::queue_ptr stream) { + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, NR * GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_ptq1_0_pairs_glu(vup, vgate, vy, dst, ncols, nrows, glu_op, nd_item); + }); + }); +} + +// ARC-LAB: n matrices (<= 3) sharing one SoA-quantized activation column, one launch. False = caller runs them apart. +bool ggml_sycl_ptq1_0_mul_mat_vec_multi(const void * const * vx, float * const * dst, const int * nrows, const int nm, + const void * vy, const int ncols, dpct::queue_ptr stream) { + constexpr int NR = 4; + if (nm < 2 || nm > 3 || ncols % QK_PTQ1_0 != 0) { + return false; + } + ptq1_0_multi_args a = {}; + int total = 0; + for (int m = 0; m < nm; ++m) { + if (nrows[m] % NR != 0) { + return false; + } + a.vx[m] = vx[m]; + a.dst[m] = dst[m]; + a.nrows[m] = nrows[m]; + total += nrows[m]; + } + launch_ptq1_0_pairs_multi(a, vy, ncols, total, stream); + return true; +} + +// ARC-LAB: single-column fused gate/up + GLU (pairs core). GGML_SYCL_PTQ1_GLU1=0 declines (unfused path). +bool ggml_sycl_ptq1_0_glu_n1_enabled() { + static const bool on = !getenv("GGML_SYCL_PTQ1_GLU1") || atoi(getenv("GGML_SYCL_PTQ1_GLU1")) != 0; + return on; +} + +static void reorder_mul_mat_vec_ptq1_0_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, + const int nrows, dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_PTQ1_0 == 0); + // ARC-LAB: Vulkan-style kernel (two lanes per block, activations reused over NR rows), 4 rows per subgroup by default. + // Whole model (llama-bench tg128, B580): old 36.96, NR 2 36.62, NR 4 37.54, NR 8 34.99 t/s. test-backend-ops favours + // NR 8 (10240x5120 36.0 -> 27.0 us) but reruns one matrix that stays in the 18 MB L2; in the model the weights stream + // from memory and NR 8 lost 7% of mat-vec time (unitrace). GGML_SYCL_PTQ1_PAIRS=0 restores the per-lane-block kernel. + static const int pairs_env = getenv("GGML_SYCL_PTQ1_PAIRS") ? atoi(getenv("GGML_SYCL_PTQ1_PAIRS")) : -1; + const int pairs = pairs_env >= 0 ? pairs_env : 4; + if (pairs == 2) { launch_ptq1_0_pairs<2>(vx, vy, dst, ncols, nrows, stream); return; } + if (pairs == 4) { launch_ptq1_0_pairs<4>(vx, vy, dst, ncols, nrows, stream); return; } + if (pairs == 8) { launch_ptq1_0_pairs<8>(vx, vy, dst, ncols, nrows, stream); return; } + // two rows per sub-group only when one row leaves lanes idle in the last round + if ((ncols / QK_PTQ1_0) % WARP_SIZE == 0) { + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder>(vx, vy, dst, ncols, + nrows, nd_item); + }); + }); + } else { + launch_mul_mat_vec_ptq1_0_reorder_rows<2>(vx, vy, dst, ncols, nrows, stream); + } +} + +bool ggml_sycl_ptq1_0_ncols_dec_enabled() { + static const bool off = getenv("GGML_SYCL_PTQ1_NCOLS_DEC_OFF") != nullptr; + return !off; +} + +// ARC-LAB: PTQ1_0 multi-column mat-vec (MTP verify, n-gram drafts, fused gate/up) that decodes each block's trits +// ONCE and reuses them for every column; mul_mat_vec_q_reorder_ncols re-decodes per column (at 4 columns a matrix +// cost 2.5x the single-column time). R rows per sub-group keeps every lane busy when a row's block count is not a +// multiple of the sub-group (Bonsai K 5120 = 40 blocks). +template // PROBE: lab-only compile-time timing variants +static inline void ptq1_0_ncols_dec_body(const void * __restrict__ vx, const void * __restrict__ vgate, + const void * __restrict__ vy, float * __restrict__ dst, const int ncols, + const int nrows, const int stride_col_y_bytes, const int stride_col_dst, + const ggml_glu_op glu_op, const sycl::sub_group & sg, const int row0) { + using block_type = ggml_sycl_reordered::block_q_t; + + const int blocks_per_row = ncols / QK_PTQ1_0; + const int nblocks = nrows * blocks_per_row; + const int lane = sg.get_local_linear_id(); + + float acc[R][ncols_dst] = {}; + [[maybe_unused]] float accg[has_fusion ? R : 1][has_fusion ? ncols_dst : 1] = {}; + + auto load_decode = [&](const void * w, const std::pair bx, const std::pair dof, int * qw) -> float { + const uint8_t * base = static_cast(w); + uint8_t qs[24]; +#pragma unroll + for (int k = 0; k < 3; ++k) { + *(sycl::uint2 *) (qs + 8 * k) = ((const sycl::uint2 *) (base + bx.first))[k]; + } + const uint8_t * qh = base + bx.second; + if constexpr (PROBE == 1) { // no trit decode +#pragma unroll + for (int k = 0; k < 32; ++k) { + qw[k] = (int) qs[k % 24] | ((int) qh[k & 1] << 8); + } + } else { + ptq1_0_decode_block(qs, (uint32_t) qh[0] | ((uint32_t) qh[1] << 16), qw); + } + return (float) *(reinterpret_cast(base + dof.first)); + }; + + for (int i = lane; i < R * blocks_per_row; i += WARP_SIZE) { + const int r = i / blocks_per_row; + const int b = i - r * blocks_per_row; + if (row0 + r >= nrows) { + continue; + } + const int ibx = (row0 + r) * blocks_per_row + b; + const auto bx = block_type::get_block_offset(ibx, nblocks); + const auto dof = block_type::get_d_offset(nrows, ncols, ibx); + + int qw[32]; + const float d = load_decode(vx, bx, dof, qw); + [[maybe_unused]] int qg[has_fusion ? 32 : 1]; + [[maybe_unused]] float dg = 0.0f; + if constexpr (has_fusion) { + dg = load_decode(vgate, bx, dof, qg); + } + + // activations in the quantize_and_reorder_q8_1_ptq1_il layout: chunk c of block b at [c * blocks_per_row + b] + auto dot_col = [&](const int * q, const sycl::int4 * xb, const sycl::half2 * dsb, const float dd) -> float { + int si[4] = { 0, 0, 0, 0 }; +#pragma unroll + for (int c = 0; c < 8; ++c) { + const sycl::int4 xv = xb[c * blocks_per_row]; +#pragma unroll + for (int u = 0; u < 4; ++u) { + si[c >> 1] = dpct::dp4a(q[4 * c + u], xv[u], si[c >> 1]); + } + } + float a = 0.0f; +#pragma unroll + for (int k = 0; k < 4; ++k) { + const sycl::float2 dsf = dsb[k * blocks_per_row].convert(); + a += si[k] * dsf.x() - dsf.y(); + } + return dd * a; + }; + +#pragma unroll + for (int j = 0; j < ncols_dst; ++j) { + const char * vy_j = (const char *) vy + j * stride_col_y_bytes; + const sycl::int4 * xb = (const sycl::int4 *) vy_j + (PROBE == 2 ? 0 : b); // PROBE 2: hot activations + const sycl::half2 * dsb = (const sycl::half2 *) (vy_j + ncols) + b; + float v; + if constexpr (PROBE == 3) { // dot only for column 0; later columns still load their activations + if (j == 0) { + v = dot_col(qw, xb, dsb, d); + } else { + int sink = 0; +#pragma unroll + for (int c = 0; c < 8; ++c) { + const sycl::int4 xv = xb[c * blocks_per_row]; + sink ^= xv[0] ^ xv[1] ^ xv[2] ^ xv[3]; + } + v = (float) sink * 1e-30f + (float) dsb[0].x() * 1e-30f; + } + } else { + v = dot_col(qw, xb, dsb, d); + } +#pragma unroll + for (int k = 0; k < R; ++k) { + acc[k][j] += r == k ? v : 0.0f; + } + if constexpr (has_fusion) { + const float vg = dot_col(qg, xb, dsb, dg); +#pragma unroll + for (int k = 0; k < R; ++k) { + accg[k][j] += r == k ? vg : 0.0f; + } + } + } + } + +#pragma unroll + for (int k = 0; k < R; ++k) { +#pragma unroll + for (int j = 0; j < ncols_dst; ++j) { + float sum = sycl::reduce_over_group(sg, acc[k][j], std::plus<>()); + if constexpr (has_fusion) { + const float gate = sycl::reduce_over_group(sg, accg[k][j], std::plus<>()); + sum *= glu_op == GGML_GLU_OP_SWIGLU ? op_silu(gate) : op_gelu(gate); + } + if (lane == 0 && row0 + k < nrows) { + dst[j * stride_col_dst + row0 + k] = sum; + } + } + } +} + +template +static void mul_mat_vec_ptq1_0_ncols_dec(const void * __restrict__ vx, const void * __restrict__ vgate, + const void * __restrict__ vy, float * __restrict__ dst, const int ncols, + const int nrows, const int stride_col_y_bytes, const int stride_col_dst, + const ggml_glu_op glu_op, const sycl::nd_item<3> & nd_item) { + const auto sg = nd_item.get_sub_group(); + const int row0 = (nd_item.get_group_linear_id() * sg.get_group_linear_range() + sg.get_group_linear_id()) * R; + if (row0 >= nrows) { + return; + } + ptq1_0_ncols_dec_body(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, + stride_col_dst, glu_op, sg, row0); +} + +static bool ptq1_0_ncols_dec_enabled() { + return ggml_sycl_ptq1_0_ncols_dec_enabled(); +} + +template +static void launch_ptq1_0_ncols_dec(const void * vx, const void * vgate, const void * vy, float * dst, const int ncols, + const int nrows, const int stride_col_y_bytes, const int stride_col_dst, + const ggml_glu_op glu_op, dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_PTQ1_0 == 0); + constexpr size_t num_subgroups = WARP_SIZE; + auto go = [&](auto rc) { + constexpr int R = decltype(rc)::value; + const int block_num_y = ceil_div(nrows, R * GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_ptq1_0_ncols_dec( + vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, glu_op, nd_item); + }); + }); + }; + static const int probe = getenv("GGML_SYCL_PTQ1_PROBE") ? atoi(getenv("GGML_SYCL_PTQ1_PROBE")) : 0; // lab timing only + if (!has_fusion && probe >= 1 && probe <= 3 && (ncols / QK_PTQ1_0) % WARP_SIZE != 0) { + auto launch_probe = [&](auto pc) { + using PC = decltype(pc); + const int block_num_y = ceil_div(nrows, 2 * GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_ptq1_0_ncols_dec<2, ncols_dst, false, PC::value>(vx, vgate, vy, dst, ncols, nrows, + stride_col_y_bytes, stride_col_dst, glu_op, nd_item); + }); + }); + }; + if (probe == 1) { + launch_probe(std::integral_constant{}); + } else if (probe == 2) { + launch_probe(std::integral_constant{}); + } else { + launch_probe(std::integral_constant{}); + } + return; + } + static const bool fused_r1 = getenv("GGML_SYCL_PTQ1_FUSED_R1") != nullptr; // ARC-LAB lab knob + if ((ncols / QK_PTQ1_0) % WARP_SIZE == 0 || (has_fusion && fused_r1)) { + go(std::integral_constant{}); + } else { + go(std::integral_constant{}); + } +} + +// ARC-LAB: the same-input merge (ggml_sycl_ptq1_0_mul_mat_vec_multi) for 2..8 columns (MTP verify / n-gram drafts): +// subgroups over the stacked rows of up to 3 matrices, each served by the decode-once body; activations in the ptq1_il +// layout, every dst contiguous (column stride = its row count). Row counts must be multiples of R. +template +static void mul_mat_vec_ptq1_0_ncols_dec_multi(const ptq1_0_multi_args a, const void * __restrict__ vy, const int ncols, + const int nrows_total, const int stride_col_y_bytes, + const sycl::nd_item<3> & nd_item) { + const auto sg = nd_item.get_sub_group(); + int row0 = (nd_item.get_group_linear_id() * sg.get_group_linear_range() + sg.get_group_linear_id()) * R; + if (row0 >= nrows_total) { + return; + } + int m = 0; + if (row0 >= a.nrows[0]) { + row0 -= a.nrows[0]; + m = 1; + if (row0 >= a.nrows[1]) { + row0 -= a.nrows[1]; + m = 2; + } + } + ptq1_0_ncols_dec_body(a.vx[m], nullptr, vy, a.dst[m], ncols, a.nrows[m], stride_col_y_bytes, + a.nrows[m], GGML_GLU_OP_SWIGLU, sg, row0); +} + +template +static void launch_ptq1_0_ncols_dec_multi(const ptq1_0_multi_args & a, const void * vy, const int ncols, const int total, + const int stride_col_y_bytes, const bool r1, dpct::queue_ptr stream) { + constexpr size_t num_subgroups = WARP_SIZE; + const ptq1_0_multi_args args = a; + auto go = [&](auto rc) { + constexpr int R = decltype(rc)::value; + const int block_num_y = ceil_div(total, R * GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_ptq1_0_ncols_dec_multi(args, vy, ncols, total, + stride_col_y_bytes, nd_item); + }); + }); + }; + if (r1) { + go(std::integral_constant{}); + } else { + go(std::integral_constant{}); + } +} + +bool ggml_sycl_ptq1_0_mul_mat_vec_multi_ncols(const void * const * vx, float * const * dst, const int * nrows, + const int nm, const void * vy, const int ncols, const int ncols_dst, + const int stride_col_y_bytes, dpct::queue_ptr stream) { + if (nm < 2 || nm > 3 || ncols % QK_PTQ1_0 != 0 || ncols_dst < 2 || ncols_dst > 8 || !ptq1_0_ncols_dec_enabled()) { + return false; + } + const bool r1 = (ncols / QK_PTQ1_0) % WARP_SIZE == 0; // as launch_ptq1_0_ncols_dec: 2 rows when lanes would idle + const int R = r1 ? 1 : 2; + ptq1_0_multi_args a = {}; + int total = 0; + for (int m = 0; m < nm; ++m) { + if (nrows[m] % R != 0) { + return false; + } + a.vx[m] = vx[m]; + a.dst[m] = dst[m]; + a.nrows[m] = nrows[m]; + total += nrows[m]; + } + switch (ncols_dst) { + case 2: launch_ptq1_0_ncols_dec_multi<2>(a, vy, ncols, total, stride_col_y_bytes, r1, stream); break; + case 3: launch_ptq1_0_ncols_dec_multi<3>(a, vy, ncols, total, stride_col_y_bytes, r1, stream); break; + case 4: launch_ptq1_0_ncols_dec_multi<4>(a, vy, ncols, total, stride_col_y_bytes, r1, stream); break; + case 5: launch_ptq1_0_ncols_dec_multi<5>(a, vy, ncols, total, stride_col_y_bytes, r1, stream); break; + case 6: launch_ptq1_0_ncols_dec_multi<6>(a, vy, ncols, total, stride_col_y_bytes, r1, stream); break; + case 7: launch_ptq1_0_ncols_dec_multi<7>(a, vy, ncols, total, stride_col_y_bytes, r1, stream); break; + case 8: launch_ptq1_0_ncols_dec_multi<8>(a, vy, ncols, total, stride_col_y_bytes, r1, stream); break; + } + return true; +} + +template +static void reorder_mul_mat_vec_ptq1_0_q8_1_sycl_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_PTQ1_0 == 0); + if (ptq1_0_ncols_dec_enabled()) { + launch_ptq1_0_ncols_dec(vx, nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, + GGML_GLU_OP_SWIGLU, stream); + return; + } + constexpr size_t num_subgroups = WARP_SIZE; + const int block_num_y = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups); + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_reorder_ncols, ncols_dst>( + vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, + /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item); + }); + }); +} + +static void reorder_mul_mat_vec_ptq1_0_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y_bytes, const int stride_col_dst, + dpct::queue_ptr stream) { + switch (ncols_dst) { + case 1: reorder_mul_mat_vec_ptq1_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; + case 2: reorder_mul_mat_vec_ptq1_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 3: reorder_mul_mat_vec_ptq1_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 4: reorder_mul_mat_vec_ptq1_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 5: reorder_mul_mat_vec_ptq1_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 6: reorder_mul_mat_vec_ptq1_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 7: reorder_mul_mat_vec_ptq1_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + case 8: reorder_mul_mat_vec_ptq1_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break; + default: GGML_ABORT("unsupported ncols_dst=%d for PTQ1_0 reorder multi-col MMVQ", ncols_dst); + } +} + static void mul_mat_vec_q4_0_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK4_0 == 0); @@ -1458,6 +2204,126 @@ static void mul_mat_vec_q2_0_q8_1_sycl_switch_ncols( } } +static void mul_mat_vec_pq2_0_q8_1_sycl(const void * vx, const void * vy, + float * dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_PQ2_0 == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q( + vx, vy, dst, ncols, nrows, item_ct1); + }); + }); +} + +template +static void mul_mat_vec_pq2_0_q8_1_sycl_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_PQ2_0 == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_ncols( + vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); + }); + }); +} + +static void mul_mat_vec_pq2_0_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + switch (ncols_dst) { + case 1: mul_mat_vec_pq2_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; + case 2: mul_mat_vec_pq2_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 3: mul_mat_vec_pq2_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 4: mul_mat_vec_pq2_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 5: mul_mat_vec_pq2_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 6: mul_mat_vec_pq2_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 7: mul_mat_vec_pq2_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 8: mul_mat_vec_pq2_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + default: GGML_ABORT("unsupported ncols_dst=%d for PQ2_0 multi-col MMVQ", ncols_dst); + } +} + +static void mul_mat_vec_ptq1_0_q8_1_sycl(const void * vx, const void * vy, + float * dst, const int ncols, + const int nrows, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_PTQ1_0 == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q( + vx, vy, dst, ncols, nrows, item_ct1); + }); + }); +} + +template +static void mul_mat_vec_ptq1_0_q8_1_sycl_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + GGML_ASSERT(ncols % QK_PTQ1_0 == 0); + const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; + const sycl::range<3> block_nums(1, 1, block_num_y); + const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); + + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for( + sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q_ncols( + vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); + }); + }); +} + +static void mul_mat_vec_ptq1_0_q8_1_sycl_switch_ncols( + const void * vx, const void * vy, float * dst, + const int ncols, const int nrows, const int ncols_dst, + const int stride_col_y, const int stride_col_dst, + dpct::queue_ptr stream) { + switch (ncols_dst) { + case 1: mul_mat_vec_ptq1_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; + case 2: mul_mat_vec_ptq1_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 3: mul_mat_vec_ptq1_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 4: mul_mat_vec_ptq1_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 5: mul_mat_vec_ptq1_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 6: mul_mat_vec_ptq1_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 7: mul_mat_vec_ptq1_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + case 8: mul_mat_vec_ptq1_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; + default: GGML_ABORT("unsupported ncols_dst=%d for PTQ1_0 multi-col MMVQ", ncols_dst); + } +} + static void mul_mat_vec_q2_K_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, @@ -2503,6 +3369,62 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens mul_mat_vec_q2_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); } break; + case GGML_TYPE_PQ2_0: + if ((ggml_tensor_extra_gpu *) dst->src[0]->extra && + ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { + const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs; + const int stride_col_dst = dst->ne[0]; + GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_pq2_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols); + reorder_mul_mat_vec_pq2_0_q8_1_sycl_switch_ncols( + src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, + src1_ncols, stride_col_y_bytes, stride_col_dst, stream); + return; + } else { + GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_pq2_0_q8_1_sycl\n"); + reorder_mul_mat_vec_pq2_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); + } + } else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { + const int stride_col_y = src1_padded_col_size / QK8_1; + const int stride_col_dst = dst->ne[0]; + GGML_SYCL_DEBUG("Calling mul_mat_vec_pq2_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols); + mul_mat_vec_pq2_0_q8_1_sycl_switch_ncols( + src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, + src1_ncols, stride_col_y, stride_col_dst, stream); + return; + } else if (i == 0 || src1_ncols == 1) { + GGML_SYCL_DEBUG("Calling mul_mat_vec_pq2_0_q8_1_sycl\n"); + mul_mat_vec_pq2_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); + } + break; + case GGML_TYPE_PTQ1_0: + if ((ggml_tensor_extra_gpu *) dst->src[0]->extra && + ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { + if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { + const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs; + const int stride_col_dst = dst->ne[0]; + GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_ptq1_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols); + reorder_mul_mat_vec_ptq1_0_q8_1_sycl_switch_ncols( + src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, + src1_ncols, stride_col_y_bytes, stride_col_dst, stream); + return; + } else { + GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_ptq1_0_q8_1_sycl\n"); + reorder_mul_mat_vec_ptq1_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); + } + } else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { + const int stride_col_y = src1_padded_col_size / QK8_1; + const int stride_col_dst = dst->ne[0]; + GGML_SYCL_DEBUG("Calling mul_mat_vec_ptq1_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols); + mul_mat_vec_ptq1_0_q8_1_sycl_switch_ncols( + src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, + src1_ncols, stride_col_y, stride_col_dst, stream); + return; + } else if (i == 0 || src1_ncols == 1) { + GGML_SYCL_DEBUG("Calling mul_mat_vec_ptq1_0_q8_1_sycl\n"); + mul_mat_vec_ptq1_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); + } + break; case GGML_TYPE_Q2_K: if ((ggml_tensor_extra_gpu *) dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) { @@ -2992,7 +3914,7 @@ static void launch_mul_mat_vec_q_reorder_glu(const void * vx, const void * vgate const int ncols, const int nrows, const int stride_col_y_bytes, const int stride_col_dst, const ggml_glu_op glu_op, dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_K == 0); + GGML_ASSERT(ncols % ggml_sycl_reordered::block_q_t::traits::qk == 0); constexpr size_t num_subgroups = WARP_SIZE; @@ -3010,18 +3932,11 @@ static void launch_mul_mat_vec_q_reorder_glu(const void * vx, const void * vgate }); } -bool ggml_sycl_mul_mat_vec_q_glu_reorder(enum ggml_type src0_type, enum ggml_glu_op glu_op, const void * vx, - const void * vgate, const void * vy, float * dst, int ncols, int nrows, - int ncols_dst, int stride_col_y_bytes, int stride_col_dst, - dpct::queue_ptr stream) { - if (src0_type != GGML_TYPE_Q4_K) { - return false; - } - if (glu_op != GGML_GLU_OP_SWIGLU && glu_op != GGML_GLU_OP_GEGLU) { - return false; - } - - using vec_dot = reorder_vec_dot_q_sycl; +template +static bool mul_mat_vec_q_glu_reorder_type(enum ggml_glu_op glu_op, const void * vx, const void * vgate, const void * vy, + float * dst, int ncols, int nrows, int ncols_dst, int stride_col_y_bytes, + int stride_col_dst, dpct::queue_ptr stream) { + using vec_dot = reorder_vec_dot_q_sycl; switch (ncols_dst) { case 1: @@ -3059,4 +3974,45 @@ bool ggml_sycl_mul_mat_vec_q_glu_reorder(enum ggml_type src0_type, enum ggml_glu default: return false; } + return false; +} + +bool ggml_sycl_mul_mat_vec_q_glu_reorder(enum ggml_type src0_type, enum ggml_glu_op glu_op, const void * vx, + const void * vgate, const void * vy, float * dst, int ncols, int nrows, + int ncols_dst, int stride_col_y_bytes, int stride_col_dst, + dpct::queue_ptr stream) { + if (glu_op != GGML_GLU_OP_SWIGLU && glu_op != GGML_GLU_OP_GEGLU) { + return false; + } + switch (src0_type) { + case GGML_TYPE_Q4_K: + return mul_mat_vec_q_glu_reorder_type(glu_op, vx, vgate, vy, dst, ncols, nrows, ncols_dst, + stride_col_y_bytes, stride_col_dst, stream); + case GGML_TYPE_PQ2_0: + return mul_mat_vec_q_glu_reorder_type(glu_op, vx, vgate, vy, dst, ncols, nrows, ncols_dst, + stride_col_y_bytes, stride_col_dst, stream); + case GGML_TYPE_PTQ1_0: + if (ncols_dst == 1 && ggml_sycl_ptq1_0_glu_n1_enabled()) { + static const int glu_nr = getenv("GGML_SYCL_PTQ1_GLU1_NR") ? atoi(getenv("GGML_SYCL_PTQ1_GLU1_NR")) : 2; + if (glu_nr == 4) { + launch_ptq1_0_pairs_glu<4>(vx, vgate, vy, dst, ncols, nrows, glu_op, stream); + } else { + launch_ptq1_0_pairs_glu<2>(vx, vgate, vy, dst, ncols, nrows, glu_op, stream); + } + return true; + } + if (ptq1_0_ncols_dec_enabled() && ncols_dst >= 1 && ncols_dst <= 8) { +#define PTQ1_DEC_GLU(N) case N: launch_ptq1_0_ncols_dec(vx, vgate, vy, dst, ncols, nrows, stride_col_y_bytes, \ + stride_col_dst, glu_op, stream); return true; + switch (ncols_dst) { + PTQ1_DEC_GLU(1) PTQ1_DEC_GLU(2) PTQ1_DEC_GLU(3) PTQ1_DEC_GLU(4) + PTQ1_DEC_GLU(5) PTQ1_DEC_GLU(6) PTQ1_DEC_GLU(7) PTQ1_DEC_GLU(8) + } +#undef PTQ1_DEC_GLU + } + return mul_mat_vec_q_glu_reorder_type(glu_op, vx, vgate, vy, dst, ncols, nrows, ncols_dst, + stride_col_y_bytes, stride_col_dst, stream); + default: + return false; + } } diff --git a/ggml/src/ggml-sycl/mmvq.hpp b/ggml/src/ggml-sycl/mmvq.hpp index 9d2f5645ecf3..c3f04bbbc1d0 100644 --- a/ggml/src/ggml-sycl/mmvq.hpp +++ b/ggml/src/ggml-sycl/mmvq.hpp @@ -73,4 +73,15 @@ bool ggml_sycl_mul_mat_vec_q_glu_reorder( int stride_col_dst, // floats between output columns in dst dpct::queue_ptr stream); +// ARC-LAB: PTQ1_0 decode-once multi-column mat-vec on (GGML_SYCL_PTQ1_NCOLS_DEC_OFF unset); its activations must be +// quantized with quantize_and_reorder_q8_1_ptq1_il (plain path: 2..8 columns; fused gate/up path: 1..8 columns) +bool ggml_sycl_ptq1_0_ncols_dec_enabled(); +// ARC-LAB: single-column PTQ1_0 helpers (pairs kernel) +bool ggml_sycl_ptq1_0_glu_n1_enabled(); +bool ggml_sycl_ptq1_0_mul_mat_vec_multi(const void * const * vx, float * const * dst, const int * nrows, const int nm, + const void * vy, const int ncols, dpct::queue_ptr stream); +bool ggml_sycl_ptq1_0_mul_mat_vec_multi_ncols(const void * const * vx, float * const * dst, const int * nrows, + const int nm, const void * vy, const int ncols, const int ncols_dst, + const int stride_col_y_bytes, dpct::queue_ptr stream); + #endif // GGML_SYCL_MMVQ_HPP diff --git a/ggml/src/ggml-sycl/ptq1-t2.cpp b/ggml/src/ggml-sycl/ptq1-t2.cpp new file mode 100644 index 000000000000..0df2e5f50fb4 --- /dev/null +++ b/ggml/src/ggml-sycl/ptq1-t2.cpp @@ -0,0 +1,294 @@ +// ARC-LAB: PTQ1_0 on XMX through the TernSYCL int2 x int8 DPAS kernels (ternsycl/, BSD 3-Clause, libxsmm/TernSYCL +// @5e162d8). The base-3 PTQ1_0 packing (1.625 + 0.125 bits/weight) has to be decoded on the ALUs before any DPAS, which +// makes the 4-column verify mat-vec compute-bound (~190 GB/s on the B580); 2-bit two's-complement codes (2.125 bits) +// feed the s8 x s2 DPAS directly. B580 standalone, 2 GiB rotating weights: 17408x5120 m=1 56 us, m=4 61 us (vs 70 / +// ~100 in-model for PTQ1_0). The weight grows 31%; activations become int8 with one fp16 scale per 128 (q8_1 is per 32). +#include "ptq1-t2.hpp" + +#include "ternsycl/int2_int8_dpas.hpp" + +#include +#include +#include +#include + +namespace { + +constexpr int QK = 128; +constexpr int64_t BLK_BYTES = 28; // block_ptq1_0: qs[24], qh[2], fp16 d + +int t2_mode() { + static const int mode = [] { + const char * e = getenv("GGML_SYCL_PTQ1_T2"); + if (!e) { + return 0; + } + if (!strcmp(e, "all")) { + return 2; + } + if (!strcmp(e, "ffn")) { + return 1; + } + return 0; + }(); + return mode; +} + +// growing per-process scratch for the int8 activations and their scales (in-order queue: reuse is safe) +struct scratch { + void * p = nullptr; + size_t bytes = 0; +}; + +void * scratch_get(sycl::queue & q, size_t bytes) { + static std::mutex mtx; + static scratch s; + std::lock_guard lock(mtx); + if (bytes > s.bytes) { + if (s.p) { + q.wait(); + sycl::free(s.p, q); + } + s.bytes = bytes + bytes / 4; + s.p = sycl::malloc_device(s.bytes, q); + if (!s.p) { + fprintf(stderr, "ptq1-t2: no device memory for %zu bytes of activation scratch\n", s.bytes); + abort(); + } + } + return s.p; +} + +} // namespace + +bool ggml_sycl_t2_wants(const char * name, int64_t K, int64_t N) { + const int mode = t2_mode(); + if (mode == 0 || K % QK != 0 || N % 16 != 0) { + return false; + } + if (mode == 2) { + return true; + } + return name && (strstr(name, "ffn_gate.") || strstr(name, "ffn_up.") || strstr(name, "ffn_down.")); +} + +size_t ggml_sycl_t2_bytes(int64_t K, int64_t N) { + return (size_t) (K / 16) * N * 4 + (size_t) (K / QK) * N * 2; +} + +namespace { +void t2_precompile(sycl::queue & q); +} + +void ggml_sycl_t2_repack(sycl::queue & q, void * data, int64_t K, int64_t N) { + static std::once_flag compiled; + std::call_once(compiled, [&] { t2_precompile(q); }); + const int64_t nb = K / QK; + const size_t src_bytes = (size_t) N * nb * BLK_BYTES; + uint8_t * tmp = (uint8_t *) sycl::malloc_device(src_bytes, q); + if (!tmp) { + fprintf(stderr, "%s: no device memory for a %zu-byte repack buffer\n", __func__, src_bytes); + abort(); + } + q.memcpy(tmp, data, src_bytes).wait(); + uint32_t * B = (uint32_t *) data; + uint16_t * SB = (uint16_t *) ((char *) data + (size_t) (K / 16) * N * 4); + q.parallel_for(sycl::range<2>((size_t) (K / 16), (size_t) N), [=](sycl::item<2> it) { + const int64_t kp = it[0], n = it[1]; + const int64_t g = kp / 8; + const uint8_t * b = tmp + (n * nb + g) * BLK_BYTES; + const int e0 = (int) (kp % 8) * 16; + uint32_t w = 0; + for (int j = 0; j < 16; ++j) { + const int e = e0 + j; + uint32_t v; + int lvl; + if (e < 80) { + v = b[e % 16]; + lvl = e / 16; + } else if (e < 120) { + v = b[16 + (e - 80) % 8]; + lvl = (e - 80) / 8; + } else { + v = b[24 + (e - 120) % 2]; + lvl = (e - 120) / 2; + } + uint32_t digit = 0; + for (int t = 0; t <= lvl; ++t) { // base-3 fixed-point digits, as ptq1_0_decode_block + const uint32_t x = v * 3; + digit = x >> 8; + v = x & 0xFF; + } + const uint32_t code = digit == 2 ? 1u : (digit == 1 ? 0u : 3u); // weight = digit - 1, 2-bit two's complement + w |= code << (2 * j); + } + B[kp * N + n] = w; + if (kp % 8 == 0) { + SB[g * N + n] = (uint16_t) (b[26] | (b[27] << 8)); + } + }).wait(); + sycl::free(tmp, q); +} + +namespace { + +// GGML_SYCL_PTQ1_T2_S32: activation scales per 32 (q8_1 granularity) instead of per 128: 1 = GEMV (<= 8 tokens) only, +// 2 = GEMV + GEMM; 0 = TernSYCL's per 128 +int t2_s32() { + static const int v = getenv("GGML_SYCL_PTQ1_T2_S32") ? atoi(getenv("GGML_SYCL_PTQ1_T2_S32")) : 0; + return v; +} + +// fp32 activations -> int8 Aq [M, K] + fp16 SA [K/128, ldsa(M)] (TernSYCL QMODE 0: SA = 127 / absmax, saturate) +// G = 128 (one scale per 16 lanes x 8 values) or 32 (a scale per 4 lanes); SA [K/G, ldsa(M)] +void quant_a(sycl::queue & q, const float * x, int64_t x_stride, int64_t M, int64_t K, int8_t * Aq, uint16_t * SA, + const int G = QK) { + const int lda = int8dpas::ldsa((int) M); + q.parallel_for(sycl::nd_range<2>({ (size_t) M, (size_t) (K / QK) * 16 }, { 1, 16 }), + [=](sycl::nd_item<2> it) [[sycl::reqd_sub_group_size(16)]] { + const auto sg = it.get_sub_group(); + const int lane = sg.get_local_linear_id(); + const int64_t m = it.get_global_id(0); + const int64_t g = it.get_global_id(1) / 16; + const float * a = x + m * x_stride + g * QK + 8 * lane; + float v[8], mx = 0.0f; +#pragma unroll + for (int i = 0; i < 8; ++i) { + v[i] = a[i]; + mx = sycl::fmax(mx, sycl::fabs(v[i])); + } + if (G == QK) { + mx = sycl::reduce_over_group(sg, mx, sycl::maximum()); + } else { // 32-value groups = 4 lanes + mx = sycl::fmax(mx, sycl::permute_group_by_xor(sg, mx, 1)); + mx = sycl::fmax(mx, sycl::permute_group_by_xor(sg, mx, 2)); + } + const sycl::half sh = sycl::half(127.0f / sycl::fmax(mx, int8dpas::EPS)); + if (G == QK ? lane == 0 : (lane & 3) == 0) { + const int64_t gi = G == QK ? g : g * 4 + lane / 4; + SA[gi * lda + m] = sycl::bit_cast(sh); + } + const float s = (float) sh; + uint64_t p = 0; +#pragma unroll + for (int i = 0; i < 8; ++i) { + // round to nearest (TernSYCL truncates: RTZ biases every product toward zero) + p |= (uint64_t) (uint8_t) (int8_t) sycl::rint(sycl::clamp(v[i] * s, -128.0f, 127.0f)) << (8 * i); + } + *(uint64_t *) (Aq + m * K + g * QK + 8 * lane) = p; + }); +} + +template +void gemv(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const uint32_t * B, const uint16_t * SB, float * C, + int M, int N, int K) { + using Kern = int8dpas::Gemv; + const size_t wgn = 16 * 2; + const Epi epi{ nullptr, nullptr, 0, 1 }; + const sycl::range<2> local(1, Kern::WG); + const sycl::range<2> global((M + SGM - 1) / SGM, (N + wgn - 1) / wgn * Kern::WG); + q.parallel_for(sycl::nd_range<2>(global, local), + Kern{ nullptr, (const signed char *) Aq, SA, B, SB, C, epi, M, N, K }); +} + +template +void gemm(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const uint32_t * B, const uint16_t * SB, float * C, + int M, int N, int K) { + constexpr int MT_M = 8, MT_N = 128, WG_M = 8, WG_N = 2; + const size_t tm = MT_M * WG_M, tn = MT_N * WG_N; + const Epi epi{ nullptr, nullptr, 0, 1 }; + const sycl::range<2> local(1, 16 * WG_M * WG_N); + const sycl::range<2> global((M + tm - 1) / tm, (N + tn - 1) / tn * local[1]); + q.parallel_for(sycl::nd_range<2>(global, local), + int8dpas::GemmMT{ + nullptr, (const signed char *) Aq, SA, B, SB, C, epi, M, N, K }); +} + +} // namespace + +namespace { +template +void t2_dispatch(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const uint32_t * B, const uint16_t * SB, + float * dst, int m, int n, int k, bool use_gemm) { + if (use_gemm) { + gemm(q, Aq, SA, B, SB, dst, m, n, k); + } else if (m == 1) { + n <= 8192 ? gemv<1, 4, NS>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<1, 2, NS>(q, Aq, SA, B, SB, dst, m, n, k); + } else if (m == 2) { + n <= 8192 ? gemv<2, 4, NS>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<2, 2, NS>(q, Aq, SA, B, SB, dst, m, n, k); + } else if (m <= 4) { + n <= 8192 ? gemv<4, 4, NS>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<4, 2, NS>(q, Aq, SA, B, SB, dst, m, n, k); + } else { + n <= 8192 ? gemv<8, 4, NS>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<8, 2, NS>(q, Aq, SA, B, SB, dst, m, n, k); + } +} +} // namespace + +void ggml_sycl_t2_mul_mat(sycl::queue & q, const void * w, const float * x, int64_t x_stride, float * dst, int64_t M, + int64_t N, int64_t K) { + // batches > 8 (n-gram verify, prompts): GemmMT reads each weight once per 64 rows. Its SA read, A read and C write + // were switched from 2D block I/O to plain / sub-group block access (the 2D forms misbehaved in this JIT build). + static const bool gemm_off = getenv("GGML_SYCL_PTQ1_T2_GEMM_OFF") != nullptr; + const bool use_gemm = M > 8 && !gemm_off; + const bool s32 = use_gemm ? t2_s32() >= 2 : t2_s32() >= 1; + + const int lda = int8dpas::ldsa((int) M); + const size_t aq_bytes = (size_t) M * K; + const size_t sa_off = (aq_bytes + 255) & ~(size_t) 255; + const size_t sa_bytes = (size_t) (K / 32) * lda * 2 + 256; + char * s = (char *) scratch_get(q, sa_off + sa_bytes); + int8_t * Aq = (int8_t *) s; + uint16_t * SA = (uint16_t *) (s + sa_off); + const uint32_t * B = (const uint32_t *) w; + const uint16_t * SB = (const uint16_t *) ((const char *) w + (size_t) (K / 16) * N * 4); + + quant_a(q, x, x_stride, M, K, Aq, SA, s32 ? 32 : QK); + if (s32) { + t2_dispatch<4>(q, Aq, SA, B, SB, dst, (int) M, (int) N, (int) K, use_gemm); + } else { + t2_dispatch<1>(q, Aq, SA, B, SB, dst, (int) M, (int) N, (int) K, use_gemm); + } +} + +namespace { +// run every kernel once on a tiny problem while the model loads (the first weight repack happens in the warm-up +// decode): the JIT of GemmMT alone took ~35 s at the first prompt otherwise +void t2_precompile(sycl::queue & q) { + constexpr int K = 128, N = 32, M = 16; + char * w = (char *) sycl::malloc_device(ggml_sycl_t2_bytes(K, N), q); + float * x = sycl::malloc_device(M * K, q); + float * d = sycl::malloc_device(M * N, q); + int8_t * aq = sycl::malloc_device(M * K, q); + uint16_t * sa = sycl::malloc_device(int8dpas::ldsa(M) * (K / 32) + 64, q); + q.memset(w, 0, ggml_sycl_t2_bytes(K, N)); + q.memset(x, 0, sizeof(float) * M * K); + const uint32_t * B = (const uint32_t *) w; + const uint16_t * SB = (const uint16_t *) (w + (K / 16) * N * 4); + quant_a(q, x, K, M, K, aq, sa, QK); + quant_a(q, x, K, M, K, aq, sa, 32); + for (int m : { 1, 2, 4, 8, M }) { + for (int ns = 0; ns < 2; ++ns) { + // both GEMV tiles (the N > 8192 one is picked by n, so call the templates directly) + if (ns == 0) { + t2_dispatch<1>(q, aq, sa, B, SB, d, m, N, K, m > 8); + } else { + t2_dispatch<4>(q, aq, sa, B, SB, d, m, N, K, m > 8); + } + } + } + gemv<1, 2>(q, aq, sa, B, SB, d, 1, N, K); + gemv<2, 2>(q, aq, sa, B, SB, d, 2, N, K); + gemv<4, 2>(q, aq, sa, B, SB, d, 4, N, K); + gemv<8, 2>(q, aq, sa, B, SB, d, 8, N, K); + gemv<1, 2, 4>(q, aq, sa, B, SB, d, 1, N, K); + gemv<2, 2, 4>(q, aq, sa, B, SB, d, 2, N, K); + gemv<4, 2, 4>(q, aq, sa, B, SB, d, 4, N, K); + gemv<8, 2, 4>(q, aq, sa, B, SB, d, 8, N, K); + q.wait(); + sycl::free(w, q); + sycl::free(x, q); + sycl::free(d, q); + sycl::free(aq, q); + sycl::free(sa, q); +} +} // namespace diff --git a/ggml/src/ggml-sycl/ptq1-t2.hpp b/ggml/src/ggml-sycl/ptq1-t2.hpp new file mode 100644 index 000000000000..505d1c07c021 --- /dev/null +++ b/ggml/src/ggml-sycl/ptq1-t2.hpp @@ -0,0 +1,18 @@ +// ARC-LAB: PTQ1_0 weights repacked to the TernSYCL 2-bit layout and multiplied on XMX (s8 x s2 DPAS). +// See ptq1-t2.cpp. +#pragma once + +#include + +#include +#include + +// GGML_SYCL_PTQ1_T2: "ffn" = ffn_gate/ffn_up/ffn_down weights, "all" = every PTQ1_0 weight (tests), unset = off +bool ggml_sycl_t2_wants(const char * name, int64_t K, int64_t N); +// device bytes of the repacked weight: 2-bit codes [K/16, N] u32 + fp16 scales [K/128, N] +size_t ggml_sycl_t2_bytes(int64_t K, int64_t N); +// in place: data holds N rows of K/128 block_ptq1_0 (as uploaded) and at least ggml_sycl_t2_bytes(K, N) bytes +void ggml_sycl_t2_repack(sycl::queue & q, void * data, int64_t K, int64_t N); +// dst[m * N + n] = sum_k x[m * x_stride + k] * w[n, k], M tokens (fp32 in and out) +void ggml_sycl_t2_mul_mat(sycl::queue & q, const void * w, const float * x, int64_t x_stride, float * dst, int64_t M, + int64_t N, int64_t K); diff --git a/ggml/src/ggml-sycl/quantize.hpp b/ggml/src/ggml-sycl/quantize.hpp index b5c7a54b7924..36305314a493 100644 --- a/ggml/src/ggml-sycl/quantize.hpp +++ b/ggml/src/ggml-sycl/quantize.hpp @@ -91,6 +91,41 @@ template struct quantize_and_reorder_q8_1_soa { } }; +// ARC-LAB: q8_1 activations for the PTQ1_0 decode-once mat-vec (one lane per 128-weight block). Same values as +// quantize_and_reorder_q8_1_soa, but within a row the 16-byte chunk c of block b sits at (c * nb128 + b) * 16 and the +// half2 (d, sum) of sub-block k at kx + (k * nb128 + b) * 4, so the 16 lanes' loads of one chunk are one contiguous +// 256-byte read (the SoA layout made every such load touch 16 cache lines). Needs kx % 128 == 0. +template struct quantize_and_reorder_q8_1_ptq1_il { + __dpct_inline__ void operator()(const float * __restrict__ x, void * reordered_q8_tensor, const int kx, + const int kx_padded, const sycl::nd_item<1> & it) const { + auto subgroup_id = it.get_group(0); + auto wi_id = it.get_local_id(0); + + sycl::vec quantized_values; + float d = 0.0f; + float sum = 0.0f; + quantize_q8_1_impl(x, quantized_values, d, sum, it); + + const int num_blocks_per_row = kx / QK8_1; + const int nb128 = kx / 128; + const int row = subgroup_id / num_blocks_per_row; + const int col = subgroup_id % num_blocks_per_row; // q8_1 block within the row + const auto row_offset = (size_t) row * (kx_padded / QK8_1) * sizeof(block_q8_1); + const int b = col / 4; // 128-block + const int w = 32 * (col % 4) + wi_id * ElementsPerWI; + const int c = w / 16; + const int byte = w % 16; + + auto quant_ptr = (int8_t *) ((char *) reordered_q8_tensor + row_offset + (size_t) (c * nb128 + b) * 16 + byte); + *reinterpret_cast *>(quant_ptr) = quantized_values; + + auto ds_ptr = (sycl::half2 *) ((char *) reordered_q8_tensor + row_offset + kx) + ((col % 4) * nb128 + b); + if (wi_id == 0) { + *ds_ptr = sycl::half2(sycl::half(d), sycl::half(sum)); + } + } +}; + template struct quantize_q8_1 { __dpct_inline__ void operator()(const float * __restrict__ x, void * q8_tensor, const int kx, const int kx_padded, const sycl::nd_item<1> & it) const { diff --git a/ggml/src/ggml-sycl/quants.hpp b/ggml/src/ggml-sycl/quants.hpp index a26a6ce6e6d7..eb98039dc243 100644 --- a/ggml/src/ggml-sycl/quants.hpp +++ b/ggml/src/ggml-sycl/quants.hpp @@ -58,6 +58,45 @@ template <> struct block_q_t { static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; } }; +template <> struct block_q_t { + struct traits { + static constexpr uint32_t qk = QK_PQ2_0; + static constexpr uint32_t qi = QK_PQ2_0 / 16; // 16 two-bit quants per int + static constexpr uint32_t qr = 4; + static constexpr uint32_t vdr_mmvq = 4; + }; + + static constexpr std::pair get_block_offset(const int block_index, const int /* nblocks */) { + return { block_index * (QK_PQ2_0 / 4), 0 }; + } + + static constexpr std::pair get_d_offset(int nrows, int ncols, const int block_index) { + return { (ncols / 4 * nrows) + block_index * sizeof(ggml_half), 0 }; + } + + static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; } +}; + +template <> struct block_q_t { + struct traits { + static constexpr uint32_t qk = QK_PTQ1_0; + static constexpr uint32_t qi = 4; + static constexpr uint32_t qr = 1; + static constexpr uint32_t vdr_mmvq = 4; // one lane takes a whole block + }; + + // first: qs offset, second: qh offset + static constexpr std::pair get_block_offset(const int block_index, const int nblocks) { + return { block_index * 24, nblocks * 24 + block_index * 2 }; + } + + static constexpr std::pair get_d_offset(int nrows, int ncols, const int block_index) { + return { (ncols / QK_PTQ1_0 * nrows) * 26 + block_index * sizeof(ggml_half), 0 }; + } + + static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; } +}; + template <> struct block_q_t { struct traits { static constexpr uint32_t qk = QK_K; diff --git a/ggml/src/ggml-sycl/ternsycl/LICENSE.md b/ggml/src/ggml-sycl/ternsycl/LICENSE.md new file mode 100644 index 000000000000..37e99b93421a --- /dev/null +++ b/ggml/src/ggml-sycl/ternsycl/LICENSE.md @@ -0,0 +1,29 @@ +# BSD 3-Clause License + +Copyright (c) 2026, Intel Corporation +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/ggml/src/ggml-sycl/ternsycl/NOTICE b/ggml/src/ggml-sycl/ternsycl/NOTICE new file mode 100644 index 000000000000..44128779c2e6 --- /dev/null +++ b/ggml/src/ggml-sycl/ternsycl/NOTICE @@ -0,0 +1,8 @@ +TernSYCL +Copyright (c) 2026, Intel Corporation. BSD 3-Clause License, see LICENSE.md. + +The SYCL kernels are ports of the TernOCL OpenCL kernels +(https://github.com/libxsmm/TernOCL), which port Intel XeTLA kernels +(int2_fp16_upcvt_xmx_xe.hpp, int2_fp16_dpas_xmx_xe.hpp, +int2_bf16_dpas_xmx_xe.hpp) and follow their data layouts, dequantization and +quantization schemes. diff --git a/ggml/src/ggml-sycl/ternsycl/epilogue_dev.hpp b/ggml/src/ggml-sycl/ternsycl/epilogue_dev.hpp new file mode 100644 index 000000000000..c85661cbda23 --- /dev/null +++ b/ggml/src/ggml-sycl/ternsycl/epilogue_dev.hpp @@ -0,0 +1,71 @@ +// Fused epilogues on the fp32 accumulator, numbered like the xetla vLLM plugin +// and OpenVINO integration (same as TernOCL's epilogue.clh): +// postop 0 none, 1 silu(acc) * other (SwiGLU gate), 2 acc + other (residual), +// 3 acc + bias[n], 4 sigmoid(acc) +// f32: C (and bias) in fp32 instead of DT (lm_head logits) +// Unlike the OpenCL -D build, postop and f32 are uniform kernel arguments, so +// one kernel binary serves every epilogue; they only branch at the store. +#pragma once + +#include "xe2.hpp" + +struct Epi { + const unsigned short *other; // DT [M, N] + const void *bias; // DT or fp32 [N] + int postop; + int f32; +}; + +template struct epi_dev { + using D = xe2::dt; + + // xetla_sigmoid; the clamp is a multiply (IGC miscompiles the ?: form in OpenCL) + static float sigmoid(float x) { + return sycl::native::recip(sycl::native::exp(-x) + 1.0f) * (float)(x > -10.0f); + } + static float apply(float v, float e, int p) { + if (p == 1) return v * sigmoid(v) * e; + if (p == 2 || p == 3) return v + e; + if (p == 4) return sigmoid(v); + return v; + } + // epilogue operand of element i = m * N + n + static float in(const Epi &e, size_t i, int n) { + if (e.postop == 1 || e.postop == 2) return D::tof(e.other[i]); + if (e.postop == 3) + return e.f32 ? ((const float *)e.bias)[n] : D::tof(((const unsigned short *)e.bias)[n]); + return 0.0f; + } + static void store(void *C, const Epi &e, size_t i, int n, float v) { + v = apply(v, in(e, i, n), e.postop); + if (e.f32) ((float *)C)[i] = v; + else ((unsigned short *)C)[i] = D::from(v); + } + + // epilogue + 2D block store of one 8 x 16 tile at (m0, n0); out of range + // rows and columns are zero-filled on read and clipped on write. + // P / F >= 0 fix postop / f32 at compile time (-1: read them from e). + template + static void store8x16(void *C, const Epi &e, int M, int N, int m0, int n0, xe2::float8 v) { + using namespace xe2; + const int postop = P >= 0 ? P : e.postop; + const bool f32 = F >= 0 ? F : e.f32; + if (postop == 1 || postop == 2) { + const ushort8 t = rd_16b_8r16(surf(e.other, N * 2, M, N * 2), n0, m0); + for (int r = 0; r < 8; ++r) v[r] = apply(v[r], D::tof(t[r]), postop); + } else if (postop == 3) { + const float b = f32 ? sycl::bit_cast(rd_32b_1r16(surf(e.bias, N * 4, 1, N * 4), n0, 0)) + : D::tof(rd_16b_1r16(surf(e.bias, N * 2, 1, N * 2), n0, 0)); + for (int r = 0; r < 8; ++r) v[r] += b; + } else if (postop == 4) { + for (int r = 0; r < 8; ++r) v[r] = sigmoid(v[r]); + } + if (f32) { + wr_32b_8r16(surf(C, N * 4, M, N * 4), n0, m0, __builtin_bit_cast(uint8, v)); + } else { + ushort8 o; + for (int r = 0; r < 8; ++r) o[r] = D::from(v[r]); + wr_16b_8r16(surf(C, N * 2, M, N * 2), n0, m0, o); + } + } +}; diff --git a/ggml/src/ggml-sycl/ternsycl/int2_int8_dpas.hpp b/ggml/src/ggml-sycl/ternsycl/int2_int8_dpas.hpp new file mode 100644 index 000000000000..ac3c05d59878 --- /dev/null +++ b/ggml/src/ggml-sycl/ternsycl/int2_int8_dpas.hpp @@ -0,0 +1,435 @@ +// SYCL port of TernOCL's int2_int8_dpas.cl, itself a port of the xetla int2 x +// int8 DPAS GEMM/GEMV (int2_fp16_dpas_xmx_xe.hpp, int2_bf16_dpas_xmx_xe.hpp): +// +// C[m,n] = DT( epi( sum_g float(sum_{k in g} q[m,k] * code(B[k,n])) +// * (SB[g,n] * (1 / SA[g,m])) ) ) +// q[m,k] = sat_int8_rtz(A[m,k] * SA[g,m]), SA[g,m] = DT(127 / absmax_g(A[m,:])) +// +// A : DT [M, K] row-major SA : DT [K/128, LDSA(M)] +// B : uint32 [K/16, N], 2-bit two's-complement codes of K rows +// 16*kp .. 16*kp+15 of column n (row 16*kp+j at bits 2j..2j+1) +// SB : DT [K/128, N] C : DT (or fp32) [M, N] +// Aq : int8 [M, K] (QMODE 0 only) +// +// B never gets unpacked: the DPAS takes it natively (s8 x s2, K = 32); per +// lane (= column) the int2 B operand is two consecutive words of the layout. +// int32 accumulation within a 128-group, fp32 rescale per group. +// +// QMODE (A quantization): +// 0 upfront : quant_a writes SA and int8 Aq; the GEMM reads int8 A +// 1 xetla : quant_a writes SA only (xetla external_scale_a_calc=1); the +// GEMM quantizes the DT A tile on the fly (large-M: xetla's instruction +// sequence as inline vISA, quant8x32) +#pragma once + +#include "epilogue_dev.hpp" + +#include + +namespace int8dpas { + +using namespace xe2; +namespace syclex = sycl::ext::oneapi::experimental; +namespace intelex = sycl::ext::intel::experimental; + +constexpr int GS = 128; +// SA row pitch, padded so SA is a valid 2D block surface (pitch % 64 B == 0) +inline int ldsa(int M) { return (M + 31) & ~31; } +constexpr float EPS = 1.1920928955078125e-07f; // FLT_EPSILON, xetla absmax init + +// two DT (low, high half of h2) -> two int8 packed in a short; clamp + RTZ +// convert lowers to one mov.sat per element +template inline short q2(unsigned h2, float sa) { + float f0, f1; + if constexpr (BF16) { + f0 = sycl::bit_cast(h2 << 16); + f1 = sycl::bit_cast(h2 & 0xffff0000u); + } else { + f0 = dt::tof((unsigned short)h2); + f1 = dt::tof((unsigned short)(h2 >> 16)); + } + const signed char c0 = (signed char)sycl::clamp(f0 * sa, -128.0f, 127.0f); + const signed char c1 = (signed char)sycl::clamp(f1 * sa, -128.0f, 127.0f); + return (short)((unsigned char)c0 | ((unsigned)(unsigned char)c1 << 8)); +} + +// xetla's elemwise_scale_{fp16,bf16}_to_int8 on one 8-row x 32-K block, on the +// row-contiguous registers (SIMT code would split and re-pair every pair). +// a: row r = a[r] (lane l holds K 2l, 2l+1); lane r of sal = scale of row r. +// SIMD32 NoMask like xetla: call only from convergent code. +#define TS_QROW(r, g, orow, ocol) \ + "mul (M1_NM, 32) T(" #g ",0)<1> T(" #g ",0)<1;1,0> %2(0," #r ")<0;1,0>\n" \ + "mov.sat (M1_NM, 32) TB(" #g ",0)<4> T(" #g ",0)<1;1,0>\n" \ + "mov (M1_NM, 32) QB(" #orow "," #ocol ")<1> TB(" #g ",0)<4;1,0>\n" +#define TS_QUANT(A2F) \ + "{\n" \ + ".decl AH v_type=G type=hf num_elts=256 align=GRF alias=<%1,0>\n" \ + ".decl AW v_type=G type=uw num_elts=256 align=GRF alias=<%1,0>\n" \ + ".decl QB v_type=G type=b num_elts=256 align=GRF alias=<%0,0>\n" \ + ".decl T v_type=G type=f num_elts=256 align=GRF\n" \ + ".decl TD v_type=G type=ud num_elts=256 align=GRF alias=\n" \ + ".decl TB v_type=G type=b num_elts=1024 align=GRF alias=\n" \ + A2F(0, 0) TS_QROW(0, 0, 0, 0) A2F(1, 2) TS_QROW(1, 2, 0, 32) \ + A2F(2, 4) TS_QROW(2, 4, 1, 0) A2F(3, 6) TS_QROW(3, 6, 1, 32) \ + A2F(4, 8) TS_QROW(4, 8, 2, 0) A2F(5, 10) TS_QROW(5, 10, 2, 32) \ + A2F(6, 12) TS_QROW(6, 12, 3, 0) A2F(7, 14) TS_QROW(7, 14, 3, 32) \ + "}\n" +#define TS_A2F_BF16(r, g) "shl (M1_NM, 32) TD(" #g ",0)<1> AW(" #r ",0)<1;1,0> 0x10:uw\n" +#define TS_A2F_FP16(r, g) "mov (M1_NM, 32) T(" #g ",0)<1> AH(" #r ",0)<1;1,0>\n" + +template inline short8 quant8x32(uint8 a, float sal) { + short8 q; +#ifdef __SYCL_DEVICE_ONLY__ + if constexpr (BF16) __asm__ volatile(TS_QUANT(TS_A2F_BF16) : "=rw"(q) : "rw"(a), "rw"(sal)); + else __asm__ volatile(TS_QUANT(TS_A2F_FP16) : "=rw"(q) : "rw"(a), "rw"(sal)); +#endif + return q; +} + +// --------------------------------------------------------------------------- +// Pre-kernel: one sub-group per (row, 128-group). SA always, Aq if WRITE_Q. +template +struct QuantA { + const unsigned short *A; + unsigned short *SA; + signed char *Aq; + int M, K; + + void operator()(sycl::nd_item<2> it) const { + using D = dt; + const auto sg = it.get_sub_group(); + const int lane = sg.get_local_linear_id(); + const int g = (int)it.get_global_id(1) / 16; + const int m = (int)it.get_global_id(0); + if (g >= K / GS || m >= M) return; + const size_t off = (size_t)m * K + g * GS + 8 * lane; + const ushort8 h = *(const ushort8 *)(A + off); + float a[8], mx = 0.0f; +#pragma unroll + for (int i = 0; i < 8; ++i) { + a[i] = D::tof(h[i]); + mx = sycl::fmax(mx, sycl::fabs(a[i])); + } + mx = sycl::reduce_over_group(sg, mx, sycl::maximum()); + const unsigned short sh = D::from(127.0f / sycl::fmax(mx, EPS)); + if (lane == 0) SA[(size_t)g * ldsa(M) + m] = sh; + if constexpr (WRITE_Q) { + const float s = D::tof(sh); + unsigned long long q = 0; +#pragma unroll + for (int i = 0; i < 8; ++i) + q |= (unsigned long long)(unsigned char)(signed char)sycl::clamp(a[i] * s, -128.0f, 127.0f) + << (8 * i); + *(unsigned long long *)(Aq + off) = q; + } + } + + auto get(syclex::properties_tag) const { + return syclex::properties{syclex::sub_group_size<16>, syclex::work_group_size<1, 16>}; + } +}; + +template struct rows; +template <> struct rows<1> { using a_t = short; using ia_t = int; using fa_t = float; }; +template <> struct rows<2> { using a_t = short2; using ia_t = int2; using fa_t = float2; }; +template <> struct rows<4> { using a_t = short4; using ia_t = int4; using fa_t = float4; }; +template <> struct rows<8> { using a_t = short8; using ia_t = int8; using fa_t = float8; }; + +template inline auto el(const V &v, int r) { + if constexpr (SGM == 1) return v; + else return v[r]; +} +template inline void set_el(V &v, int r, T x) { + if constexpr (SGM == 1) v = x; + else v[r] = x; +} + +// --------------------------------------------------------------------------- +// GEMV / small M. A sub-group owns 16 columns and SGM rows and walks its K +// slice in 128-steps: one 2D block read of 8 B words (4 DPAS of K = 32), one +// SB row, SGM A rows (quantized in registers for QMODE 1). +// SGM rows per sub-group (1, 2, 4, 8) NSG_N sub-groups along N +// LS K-slices per column block, reduced through SLM +// U 128-steps whose loads are issued before any compute +// ARC-LAB: NS = activation scales per 128-step (1 = TernSYCL; 4 = one per 32, like q8_1; QMODE 0 only, SA [K/32, ldsa]) +template +struct Gemv { + const unsigned short *A; + const signed char *Aq; + const unsigned short *SA; + const unsigned *B; + const unsigned short *SB; + void *C; + Epi epi; + int M, N, K; + + using a_t = typename rows::a_t; + using ia_t = typename rows::ia_t; + using fa_t = typename rows::fa_t; + using D = dt; + static constexpr int WG = 16 * NSG_N * LS; + + // SGM x 128 A tile of step s: aq[c] = K 32c..32c+31, inv[r] = 1 / SA of row r + void load_a(int m0, int s, a_t *aq, float *inv) const { + const int lda = ldsa(M); +#pragma unroll + for (int r = 0; r < SGM; ++r) { + const bool ok = m0 + r < M; + if constexpr (QMODE == 0) { + const ushort4 v = ok ? intel_sub_group_block_read_us4( + gptr((const unsigned short *)(Aq + (size_t)(m0 + r) * K + s * GS))) : ushort4{}; +#pragma unroll + for (int c = 0; c < 4; ++c) set_el(aq[c], r, (short)v[c]); +#pragma unroll + for (int cs = 0; cs < NS; ++cs) + inv[cs * SGM + r] = ok ? sycl::native::recip(D::tof(SA[(size_t)(s * NS + cs) * lda + m0 + r])) : 0.0f; + } else { + const uint4 v = ok ? intel_sub_group_block_read4( + gptr((const unsigned *)(A + (size_t)(m0 + r) * K + s * GS))) : uint4{}; + const float sa = ok ? D::tof(SA[(size_t)s * lda + m0 + r]) : 0.0f; +#pragma unroll + for (int c = 0; c < 4; ++c) set_el(aq[c], r, q2(v[c], sa)); + inv[r] = ok ? sycl::native::recip(sa) : 0.0f; + } + } + } + + static fa_t step(fa_t acc, const uint8 &w, float sb, const a_t *aq, const float *inv) { + if constexpr (NS == 4) { +#pragma unroll + for (int c = 0; c < 4; ++c) { + ia_t ia = 0; + ia = intel_sub_group_i8_i2_matrix_mad_k32(aq[c], int2{(int)w[2 * c], (int)w[2 * c + 1]}, ia); +#pragma unroll + for (int r = 0; r < SGM; ++r) + set_el(acc, r, el(acc, r) + (float)el(ia, r) * (sb * inv[c * SGM + r])); + } + return acc; + } + ia_t ia = 0; +#pragma unroll + for (int c = 0; c < 4; ++c) + ia = intel_sub_group_i8_i2_matrix_mad_k32(aq[c], int2{(int)w[2 * c], (int)w[2 * c + 1]}, ia); +#pragma unroll + for (int r = 0; r < SGM; ++r) + set_el(acc, r, el(acc, r) + (float)el(ia, r) * (sb * inv[r])); + return acc; + } + + void operator()(sycl::nd_item<2> it) const { + const auto sgp = it.get_sub_group(); + const int lane = sgp.get_local_linear_id(); + const int sg = sgp.get_group_linear_id(); + const int sgn = sg % NSG_N, sgk = sg / NSG_N; + const int n0 = ((int)it.get_group(1) * NSG_N + sgn) * 16; + const int m0 = (int)it.get_group(0) * SGM; + + const int nsteps = K / GS; + const int per = (nsteps + LS - 1) / LS; + const int s_begin = sgk * per; + const int s_end = sycl::min(nsteps, s_begin + per); + const surf sbs(B, N * 4, K / 16, N * 4); + + fa_t acc = 0.0f; + if (n0 < N) { + int s = s_begin; + for (; s + U <= s_end; s += U) { + uint8 w[U]; + float sb[U]; + a_t aq[U][4]; + float inv[U][NS * SGM]; +#pragma unroll + for (int u = 0; u < U; ++u) { + w[u] = rd_32b_8r16(sbs, n0, (s + u) * 8); + sb[u] = D::tof(intel_sub_group_block_read_us(gptr(SB + (size_t)(s + u) * N + n0))); + load_a(m0, s + u, aq[u], inv[u]); + } +#pragma unroll + for (int u = 0; u < U; ++u) acc = step(acc, w[u], sb[u], aq[u], inv[u]); + } + for (; s < s_end; ++s) { + a_t aq[4]; + float inv[NS * SGM]; + const uint8 w = rd_32b_8r16(sbs, n0, s * 8); + const float sb = D::tof(intel_sub_group_block_read_us(gptr(SB + (size_t)s * N + n0))); + load_a(m0, s, aq, inv); + acc = step(acc, w, sb, aq, inv); + } + } + + if constexpr (LS > 1) { + float *red = *sycl::ext::oneapi::group_local_memory_for_overwrite( + it.get_group()); + if (sgk > 0) { + float *dst = red + (((sgk - 1) * NSG_N + sgn) * SGM) * 16; +#pragma unroll + for (int r = 0; r < SGM; ++r) dst[r * 16 + lane] = el(acc, r); + } + barrier_local(); + if (sgk > 0) return; + for (int j = 0; j < LS - 1; ++j) { + const float *src = red + ((j * NSG_N + sgn) * SGM) * 16; +#pragma unroll + for (int r = 0; r < SGM; ++r) set_el(acc, r, el(acc, r) + src[r * 16 + lane]); + } + } + + if (n0 >= N) return; +#pragma unroll + for (int r = 0; r < SGM; ++r) + if (m0 + r < M) + epi_dev::store(C, epi, (size_t)(m0 + r) * N + n0 + lane, n0 + lane, el(acc, r)); + } + + auto get(syclex::properties_tag) const { + return syclex::properties{syclex::sub_group_size<16>, syclex::work_group_size<1, WG>}; + } +}; + +// --------------------------------------------------------------------------- +// Large-M GEMM. A sub-group computes an MT_M x MT_N tile (MT_M % 8 == 0, +// MT_N % 16 == 0); a work-group is WG_M x WG_N sub-groups. Per 128-group: +// B (8 words per 16-column block) and SB are loaded once and reused for all +// MT_M/8 row blocks; each 8 x 128 A block is loaded (and for QMODE 1 +// quantized) once and reused for all MT_N/16 column blocks. 2D block I/O +// zero-fills out-of-range reads and clips writes, so any M works. 256 GRF. +// The epilogue is a template parameter here (as in the OpenCL -D build): a +// runtime-selected one costs ~3% in this kernel. +template +struct GemmMT { + const unsigned short *A; + const signed char *Aq; + const unsigned short *SA; + const unsigned *B; + const unsigned short *SB; + void *C; + Epi epi; + int M, N, K; + + using D = dt; + static constexpr int MB = MT_M / 8, NB = MT_N / 16, WG = 16 * WG_M * WG_N; + + void operator()(sycl::nd_item<2> it) const { + const auto sgp = it.get_sub_group(); + const int sg = sgp.get_group_linear_id(); + const int m0 = ((int)it.get_group(0) * WG_M + sg / WG_N) * MT_M; + const int n0 = ((int)it.get_group(1) * WG_N + sg % WG_N) * MT_N; + const surf sbs(B, N * 4, K / 16, N * 4), ssb(SB, N * 2, K / GS, N * 2); + // lane l gets SA[s, m + l] (pad columns past M are junk) + const surf ssa(SA, ldsa(M) * 2, K / GS, ldsa(M) * 2); + + float8 acc[MB][NB]; +#pragma unroll + for (int i = 0; i < MB; ++i) +#pragma unroll + for (int j = 0; j < NB; ++j) acc[i][j] = 0.0f; + + for (int s = 0; s < K / GS; ++s) { + uint8 w[NB]; + float sb[NB]; +#pragma unroll + for (int j = 0; j < NB; ++j) w[j] = rd_32b_8r16(sbs, n0 + 16 * j, s * 8); + // all SB loads first, then convert: converting each (bf16: via acc0) before the + // next load let IGC reuse one load register and serialize the loads + ushort2 sbr[(NB + 1) / 2]; +#pragma unroll + for (int j = 0; j < NB; j += 2) + sbr[j / 2] = j + 1 < NB ? rd_16b_1r16x2(ssb, n0 + 16 * j, s) + : ushort2{rd_16b_1r16(ssb, n0 + 16 * j, s), 0}; +#pragma unroll + for (int j = 0; j < NB; ++j) sb[j] = D::tof(sbr[j / 2][j % 2]); +#pragma unroll + for (int i = 0; i < MB; ++i) { + const int mr = m0 + 8 * i; + short8 aq[4]; + float inv[NS][8]; + // ARC-LAB: plain load; the 2D block read of SA gave junk past lane 0 in the llama.cpp JIT build + const float sal = D::tof(SA[(size_t)s * NS * ldsa(M) + mr + (int)sgp.get_local_linear_id()]); + if constexpr (QMODE == 0) { + // ARC-LAB: sub-group block reads per row (as Gemv::load_a); the 2D block read of Aq gave NaN from + // row 1 on in the llama.cpp JIT build +#pragma unroll + for (int r = 0; r < 8; ++r) { + const bool ok = mr + r < M; + const ushort4 v = ok ? intel_sub_group_block_read_us4( + gptr((const unsigned short *)(Aq + (size_t)(mr + r) * K + s * GS))) : ushort4{}; +#pragma unroll + for (int c = 0; c < 4; ++c) aq[c][r] = (short)v[c]; + } + } else { + const surf sa(A, K * 2, M, K * 2); + // all loads before the (volatile, so unreorderable) quant asm + uint8 a[4]; +#pragma unroll + for (int c = 0; c < 4; ++c) a[c] = rd_32b_8r16(sa, s * GS / 2 + 16 * c, mr); +#pragma unroll + for (int c = 0; c < 4; ++c) aq[c] = quant8x32(a[c], sal); + } + // rows >= M get inf here, but their int32 dot is 0 and the store clips them +#pragma unroll + for (int r = 0; r < 8; ++r) inv[0][r] = sycl::native::recip(sycl::group_broadcast(sgp, sal, r)); + if constexpr (NS == 4) { +#pragma unroll + for (int cs = 1; cs < 4; ++cs) { + const float salc = D::tof(SA[(size_t)(s * NS + cs) * ldsa(M) + mr + (int)sgp.get_local_linear_id()]); +#pragma unroll + for (int r = 0; r < 8; ++r) inv[cs][r] = sycl::native::recip(sycl::group_broadcast(sgp, salc, r)); + } + } +#pragma unroll + for (int j = 0; j < NB; ++j) { + if constexpr (NS == 4) { +#pragma unroll + for (int c = 0; c < 4; ++c) { + int8 ia = 0; + ia = intel_sub_group_i8_i2_matrix_mad_k32(aq[c], + int2{(int)w[j][2 * c], (int)w[j][2 * c + 1]}, ia); + const float8 fi = __builtin_convertvector(ia, float8); +#pragma unroll + for (int r = 0; r < 8; ++r) acc[i][j][r] += fi[r] * (sb[j] * inv[c][r]); + } + continue; + } + int8 ia = 0; +#pragma unroll + for (int c = 0; c < 4; ++c) + ia = intel_sub_group_i8_i2_matrix_mad_k32(aq[c], + int2{(int)w[j][2 * c], (int)w[j][2 * c + 1]}, ia); + // whole-vector convert (OpenCL's convert_float8): per-element casts + // are emitted through one scratch register and stall every mad + const float8 fi = __builtin_convertvector(ia, float8); +#pragma unroll + for (int r = 0; r < 8; ++r) acc[i][j][r] += fi[r] * (sb[j] * inv[0][r]); + } + } + } + + if constexpr (POSTOP == 0 && F32) { + // ARC-LAB: plain stores; the 2D block write misplaced rows (junk rows past M landed on real ones) in the + // llama.cpp JIT build + const int lane = (int)sgp.get_local_linear_id(); +#pragma unroll + for (int i = 0; i < MB; ++i) +#pragma unroll + for (int j = 0; j < NB; ++j) +#pragma unroll + for (int r = 0; r < 8; ++r) { + const int m = m0 + 8 * i + r, n = n0 + 16 * j + lane; + if (m < M && n < N) ((float *)C)[(size_t)m * N + n] = acc[i][j][r]; + } + } else { +#pragma unroll + for (int i = 0; i < MB; ++i) +#pragma unroll + for (int j = 0; j < NB; ++j) + epi_dev::template store8x16(C, epi, M, N, m0 + 8 * i, n0 + 16 * j, acc[i][j]); + } + } + + auto get(syclex::properties_tag) const { + return syclex::properties{syclex::sub_group_size<16>, syclex::work_group_size<1, WG>, + intelex::grf_size<256>}; + } +}; + +} // namespace int8dpas diff --git a/ggml/src/ggml-sycl/ternsycl/xe2.hpp b/ggml/src/ggml-sycl/ternsycl/xe2.hpp new file mode 100644 index 000000000000..93adf81c675c --- /dev/null +++ b/ggml/src/ggml-sycl/ternsycl/xe2.hpp @@ -0,0 +1,163 @@ +// Xe2 sub-group intrinsics for SYCL device code: the same IGC builtins the +// OpenCL kernels of TernOCL use (DPAS, 2D block I/O, sub-group block reads), +// declared with the OpenCL mangling / IGC builtin names so that IGC resolves +// them from the SPIR-V exactly as for OpenCL C. +#pragma once + +#include + +#include + +namespace xe2 { + +#define XE2_VEC(T, n, name) typedef T name __attribute__((ext_vector_type(n))) +XE2_VEC(short, 2, short2); +XE2_VEC(short, 4, short4); +XE2_VEC(short, 8, short8); +XE2_VEC(unsigned short, 2, ushort2); +XE2_VEC(unsigned short, 4, ushort4); +XE2_VEC(unsigned short, 8, ushort8); +XE2_VEC(unsigned short, 16, ushort16); +XE2_VEC(int, 2, int2); +XE2_VEC(int, 4, int4); +XE2_VEC(int, 8, int8); +XE2_VEC(unsigned, 2, uint2); +XE2_VEC(unsigned, 3, uint3); +XE2_VEC(unsigned, 4, uint4); +XE2_VEC(unsigned, 8, uint8); +XE2_VEC(float, 2, float2); +XE2_VEC(float, 4, float4); +XE2_VEC(float, 8, float8); +#undef XE2_VEC + +#define XE2_GLOBAL __attribute__((opencl_global)) + +} // namespace xe2 + +#ifdef __SYCL_DEVICE_ONLY__ +#define XE2_OCL(decl) SYCL_EXTERNAL decl +#define XE2_IB(decl) extern "C" SYCL_EXTERNAL decl +#else +#define XE2_OCL(decl) inline decl { __builtin_unreachable(); } +#define XE2_IB(decl) inline decl { __builtin_unreachable(); } +#endif + +// DPAS, SIMD16: repeat count = rows of A (1, 2, 4, 8) +#define XE2_MAD(name, A, B, C) XE2_OCL(C name(A a, B b, C acc)); +#define XE2_MAD_ALL(name) \ + XE2_MAD(name, short, xe2::int8, float) \ + XE2_MAD(name, xe2::short2, xe2::int8, xe2::float2) \ + XE2_MAD(name, xe2::short4, xe2::int8, xe2::float4) \ + XE2_MAD(name, xe2::short8, xe2::int8, xe2::float8) +XE2_MAD_ALL(intel_sub_group_f16_f16_matrix_mad_k16) +XE2_MAD_ALL(intel_sub_group_bf16_bf16_matrix_mad_k16) +#undef XE2_MAD_ALL +// s8 x s2, K = 32 +XE2_MAD(intel_sub_group_i8_i2_matrix_mad_k32, short, xe2::int2, int) +XE2_MAD(intel_sub_group_i8_i2_matrix_mad_k32, xe2::short2, xe2::int2, xe2::int2) +XE2_MAD(intel_sub_group_i8_i2_matrix_mad_k32, xe2::short4, xe2::int2, xe2::int4) +XE2_MAD(intel_sub_group_i8_i2_matrix_mad_k32, xe2::short8, xe2::int2, xe2::int8) +#undef XE2_MAD + +// sub-group block reads (16 lanes, consecutive elements per lane) +XE2_OCL(unsigned intel_sub_group_block_read(const XE2_GLOBAL unsigned *p)); +XE2_OCL(xe2::uint4 intel_sub_group_block_read4(const XE2_GLOBAL unsigned *p)); +XE2_OCL(unsigned short intel_sub_group_block_read_us(const XE2_GLOBAL unsigned short *p)); +XE2_OCL(xe2::ushort4 intel_sub_group_block_read_us4(const XE2_GLOBAL unsigned short *p)); +XE2_OCL(xe2::ushort8 intel_sub_group_block_read_us8(const XE2_GLOBAL unsigned short *p)); + +// 2D block I/O: (base, width - 1, height - 1, pitch - 1) in bytes, coord (x +// in elements, y in rows). mRkCvV = R rows x C columns x V blocks. +#define XE2_2D_RD(ret, sfx) \ + XE2_IB(ret __builtin_IB_subgroup_block_read_flat_##sfx(long base, int w, int h, int p, xe2::int2 c)); +XE2_2D_RD(unsigned, u32_m1k16v1) +XE2_2D_RD(xe2::uint2, u32_m2k16v1) +XE2_2D_RD(xe2::uint8, u32_m8k16v1) +XE2_2D_RD(unsigned short, u16_m1k16v1) +XE2_2D_RD(xe2::ushort2, u16_m1k16v2) +XE2_2D_RD(xe2::ushort8, u16_m8k16v1) +XE2_2D_RD(xe2::ushort16, u16_m8k16v2) +#undef XE2_2D_RD +XE2_IB(void __builtin_IB_subgroup_block_write_flat_u16_m8k16v1(long base, int w, int h, int p, + xe2::int2 c, xe2::ushort8 v)); +XE2_IB(void __builtin_IB_subgroup_block_write_flat_u32_m8k16v1(long base, int w, int h, int p, + xe2::int2 c, xe2::uint8 v)); +// cache control 0 = default +XE2_IB(void __builtin_IB_subgroup_block_read_prefetch_u32_m8k16v1(long base, int w, int h, int p, + xe2::int2 c, int cc)); +// per-lane LSC load with cache control (IGC LSC_LDCC: 4 = L1 cached, L3 cached) +XE2_IB(xe2::uint3 __builtin_IB_lsc_load_global_uint3(const XE2_GLOBAL xe2::uint3 *p, int imm_off, int cc)); + +namespace xe2 { + +// 2D surface: width and pitch in bytes, height in rows +struct surf { + long base; + int w, h, p; + surf(const void *b, int width_bytes, int height, int pitch_bytes) + : base((long)b), w(width_bytes - 1), h(height - 1), p(pitch_bytes - 1) {} +}; + +inline unsigned rd_32b_1r16(const surf &s, int x, int y) { + return __builtin_IB_subgroup_block_read_flat_u32_m1k16v1(s.base, s.w, s.h, s.p, int2{x, y}); +} +inline uint2 rd_32b_2r16(const surf &s, int x, int y) { + return __builtin_IB_subgroup_block_read_flat_u32_m2k16v1(s.base, s.w, s.h, s.p, int2{x, y}); +} +inline uint8 rd_32b_8r16(const surf &s, int x, int y) { + return __builtin_IB_subgroup_block_read_flat_u32_m8k16v1(s.base, s.w, s.h, s.p, int2{x, y}); +} +inline unsigned short rd_16b_1r16(const surf &s, int x, int y) { + return __builtin_IB_subgroup_block_read_flat_u16_m1k16v1(s.base, s.w, s.h, s.p, int2{x, y}); +} +inline ushort2 rd_16b_1r16x2(const surf &s, int x, int y) { + return __builtin_IB_subgroup_block_read_flat_u16_m1k16v2(s.base, s.w, s.h, s.p, int2{x, y}); +} +inline ushort8 rd_16b_8r16(const surf &s, int x, int y) { + return __builtin_IB_subgroup_block_read_flat_u16_m8k16v1(s.base, s.w, s.h, s.p, int2{x, y}); +} +// two adjacent 8x16 blocks, block-major: [0..7] = columns x..x+15, [8..15] = x+16..x+31 +inline ushort16 rd_16b_8r16x2(const surf &s, int x, int y) { + return __builtin_IB_subgroup_block_read_flat_u16_m8k16v2(s.base, s.w, s.h, s.p, int2{x, y}); +} +inline void wr_16b_8r16(const surf &s, int x, int y, ushort8 v) { + __builtin_IB_subgroup_block_write_flat_u16_m8k16v1(s.base, s.w, s.h, s.p, int2{x, y}, v); +} +inline void wr_32b_8r16(const surf &s, int x, int y, uint8 v) { + __builtin_IB_subgroup_block_write_flat_u32_m8k16v1(s.base, s.w, s.h, s.p, int2{x, y}, v); +} +inline void pf_32b_8r16(const surf &s, int x, int y) { + __builtin_IB_subgroup_block_read_prefetch_u32_m8k16v1(s.base, s.w, s.h, s.p, int2{x, y}, 0); +} + +template inline const XE2_GLOBAL T *gptr(const T *p) { + return (const XE2_GLOBAL T *)p; +} + +// work-group barrier with a local-memory fence only (OpenCL barrier(CLK_LOCAL_MEM_FENCE)) +inline void barrier_local() { +#ifdef __SYCL_DEVICE_ONLY__ + __spirv_ControlBarrier(__spv::Scope::Workgroup, __spv::Scope::Workgroup, + __spv::MemorySemanticsMask::AcquireRelease | __spv::MemorySemanticsMask::WorkgroupMemory); +#endif +} + +// 16-bit activation dtype: raw bits in, fp32 out, round-to-nearest-even back +template struct dt; +template <> struct dt { + static float tof(unsigned short h) { return (float)sycl::bit_cast(h); } + static unsigned short from(float f) { return sycl::bit_cast((sycl::half)f); } +}; +template <> struct dt { + static float tof(unsigned short h) { return sycl::bit_cast((unsigned)h << 16); } + static unsigned short from(float f) { + return sycl::bit_cast(sycl::ext::oneapi::bfloat16(f)); + } +}; + +template inline C mad_k16(A a, int8 b, C acc) { + if constexpr (BF16) return intel_sub_group_bf16_bf16_matrix_mad_k16(a, b, acc); + else return intel_sub_group_f16_f16_matrix_mad_k16(a, b, acc); +} + +} // namespace xe2 diff --git a/ggml/src/ggml-sycl/vecdotq.hpp b/ggml/src/ggml-sycl/vecdotq.hpp index dcfded35b2a0..17efc56f5467 100644 --- a/ggml/src/ggml-sycl/vecdotq.hpp +++ b/ggml/src/ggml-sycl/vecdotq.hpp @@ -85,6 +85,99 @@ static __dpct_inline__ int get_int_from_uint8_aligned( (const int*)(x8 + sizeof(int) * i32)); // assume at least 4 byte alignment } +// PTQ1_0 block x 128 int8 activations. qs is 4-byte aligned. Byte b holds 5 trits, read with +// w = v * 3, trit = w >> 8, v = w & 0xFF. Two bytes sit in 16-bit lanes so one multiply reads two. +// Lanes take bytes {0,2} and {1,3} so the four trits come out in activation order. +// sumi[k] gets sum(trit * x) over q8_1 block k, with trit in {0,1,2}. +// part 0 does qs ints 0..2, part 1 does qs ints 3..5 and qh; nparts 1 does all. +static __dpct_inline__ void ptq1_0_dot_block(const uint8_t * qs, const uint32_t qh, const int8_t * x, int * sumi, + const int part = 0, const int nparts = 1) { + const int g0 = nparts == 1 ? 0 : 3 * part; + const int g1 = nparts == 1 ? 6 : g0 + 3; +#pragma unroll + for (int g = g0; g < g1; ++g) { + const uint32_t packed = get_int_from_uint8_aligned(qs, g); + const uint32_t v_lo = packed & 0x00FF00FF; + const uint32_t v_hi = (packed >> 8) & 0x00FF00FF; +#pragma unroll + for (int t = 0; t < 5; ++t) { + // trit t of byte b is ((b * 3^t) mod 256) * 3 >> 8; no chain between trits + constexpr uint32_t pow3[5] = { 1, 3, 9, 27, 81 }; + const uint32_t w_lo = ((v_lo * pow3[t]) & 0x00FF00FF) * 3; + const uint32_t w_hi = ((v_hi * pow3[t]) & 0x00FF00FF) * 3; + const int q = (int) (((w_lo >> 8) & 0x00FF00FF) | (w_hi & 0xFF00FF00)); + const int e = g < 4 ? t * 16 + 4 * g : 80 + t * 8 + 4 * (g - 4); + sumi[e >> 5] = dpct::dp4a(q, get_int_from_int8_aligned(x, e / 4), sumi[e >> 5]); + } + } + if (nparts == 1 || part == 1) { + uint32_t v = qh; +#pragma unroll + for (int t = 0; t < 4; t += 2) { + const uint32_t w0 = v * 3; + v = w0 & 0x00FF00FF; + const uint32_t w1 = v * 3; + v = w1 & 0x00FF00FF; + const int q = (int) (((w0 >> 8) & 0xFF) | ((w0 >> 16) & 0xFF00) | ((w1 << 8) & 0xFF0000) | (w1 & 0xFF000000)); + sumi[3] = dpct::dp4a(q, get_int_from_int8_aligned(x, (120 + 2 * t) / 4), sumi[3]); + } + } +} + +// trit - 1 is the weight: subtract the q8_1 sum once per block via ds.y (part 0 only) +static __dpct_inline__ float ptq1_0_finish(const int * sumi, const sycl::half2 * ds, const float d, const int part = 0) { + float acc = 0.0f; +#pragma unroll + for (int k = 0; k < 4; ++k) { + const sycl::float2 dsf = ds[k].convert(); + acc += sumi[k] * dsf.x() - (part == 0 ? dsf.y() : 0.0f); + } + return d * acc; +} + +// ARC-LAB: PTQ1_0 decode-once helpers for multi-column mat-vec. Word w of qw holds trits (as 0..2 bytes) for elements +// 4w .. 4w+3, the same values ptq1_0_dot_block feeds to dp4a, so a block is decoded once and dotted with every column. +static __dpct_inline__ void ptq1_0_decode_block(const uint8_t * qs, const uint32_t qh, int * qw) { + // ARC-LAB: one trit level at a time, x3 as shift + add (a 32-bit integer multiply is two instructions on Xe; + // the old form did two per half-word per level: v * 3^t, then * 3) +#pragma unroll + for (int g = 0; g < 6; ++g) { + const uint32_t packed = get_int_from_uint8_aligned(qs, g); + uint32_t v_lo = packed & 0x00FF00FF; + uint32_t v_hi = (packed >> 8) & 0x00FF00FF; +#pragma unroll + for (int t = 0; t < 5; ++t) { + const uint32_t w_lo = v_lo + (v_lo << 1); + const uint32_t w_hi = v_hi + (v_hi << 1); + v_lo = w_lo & 0x00FF00FF; + v_hi = w_hi & 0x00FF00FF; + const int e = g < 4 ? t * 16 + 4 * g : 80 + t * 8 + 4 * (g - 4); + qw[e / 4] = (int) (((w_lo >> 8) & 0x00FF00FF) | (w_hi & 0xFF00FF00)); + } + } + uint32_t v = qh; +#pragma unroll + for (int t = 0; t < 4; t += 2) { + const uint32_t w0 = v + (v << 1); + v = w0 & 0x00FF00FF; + const uint32_t w1 = v + (v << 1); + v = w1 & 0x00FF00FF; + qw[(120 + 2 * t) / 4] = (int) (((w0 >> 8) & 0xFF) | ((w0 >> 16) & 0xFF00) | ((w1 << 8) & 0xFF0000) | (w1 & 0xFF000000)); + } +} + +// dot of a decoded block with 128 q8 activations, read 16 bytes at a time (sumi: 4 groups of 32 elements) +static __dpct_inline__ void ptq1_0_dot_decoded(const int * qw, const int8_t * x, int * sumi) { +#pragma unroll + for (int c = 0; c < 8; ++c) { + const sycl::int4 xv = ((const sycl::int4 *) x)[c]; +#pragma unroll + for (int u = 0; u < 4; ++u) { + sumi[c >> 1] = dpct::dp4a(qw[4 * c + u], xv[u], sumi[c >> 1]); + } + } +} + static __dpct_inline__ int byte_sub_4(const int a, const int b) { const uint32_t ua = static_cast(a); const uint32_t ub = static_cast(b); @@ -521,6 +614,69 @@ template <> struct reorder_vec_dot_q_sycl { }; }; +template <> struct reorder_vec_dot_q_sycl { + static constexpr ggml_type gtype = GGML_TYPE_PQ2_0; + + using pq2_0_block = ggml_sycl_reordered::block_q_t; + using pq2_0_traits = typename pq2_0_block::traits; + + // Each step takes 2 ints of quants = one q8_1 block of 32 values. Value is (q - 1) * d. + __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair ibx_offset, + const std::pair d_offset, const int8_t * q8_1_quant_ptr, + const sycl::half2 * q8_1_ds, const int & iqs) { + const uint8_t * qs = static_cast(vbq) + ibx_offset.first; + const ggml_half d = *(reinterpret_cast(static_cast(vbq) + d_offset.first)); + + float sum = 0.0f; +#pragma unroll + for (int b = 0; b < (int) pq2_0_traits::vdr_mmvq / 2; ++b) { + const int m = iqs + 2 * b; + int sumi = 0; +#pragma unroll + for (int h = 0; h < 2; ++h) { + const int v = get_int_from_uint8_aligned(qs, m + h); +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int q = (v >> (8 * j)) & 0xFF; + const int vi = (q & 3) | ((q >> 2 & 3) << 8) | ((q >> 4 & 3) << 16) | ((q >> 6 & 3) << 24); + sumi = dpct::dp4a(vi, get_int_from_int8_aligned(q8_1_quant_ptr, 4 * (m + h) + j), sumi); + } + } + const sycl::float2 ds8f = q8_1_ds[m / 2].convert(); + sum += sumi * ds8f.x() - ds8f.y(); + } + return d * sum; + }; +}; + +template <> struct reorder_vec_dot_q_sycl { + static constexpr ggml_type gtype = GGML_TYPE_PTQ1_0; + + __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair ibx_offset, + const std::pair d_offset, const int8_t * q8_1_quant_ptr, + const sycl::half2 * q8_1_ds, const int & iqs) { + const uint8_t * base = static_cast(vbq); + const uint8_t * qh = base + ibx_offset.second; + const ggml_half d = *(reinterpret_cast(base + d_offset.first)); + + GGML_UNUSED(iqs); + // wide loads: 128 activations as 8 x 16 B, 24 quant bytes as 3 x 8 B + int8_t x[QK_PTQ1_0]; +#pragma unroll + for (int j = 0; j < QK_PTQ1_0 / 16; ++j) { + *(sycl::int4 *) (x + 16 * j) = ((const sycl::int4 *) q8_1_quant_ptr)[j]; + } + uint8_t qs[24]; +#pragma unroll + for (int j = 0; j < 3; ++j) { + *(sycl::uint2 *) (qs + 8 * j) = ((const sycl::uint2 *) (base + ibx_offset.first))[j]; + } + int sumi[4] = { 0, 0, 0, 0 }; + ptq1_0_dot_block(qs, (uint32_t) qh[0] | ((uint32_t) qh[1] << 16), x, sumi); + return ptq1_0_finish(sumi, q8_1_ds, d); + }; +}; + template <> struct reorder_vec_dot_q_sycl { static constexpr ggml_type gtype = GGML_TYPE_Q8_0; @@ -1102,6 +1258,56 @@ vec_dot_q2_0_q8_1(const void *__restrict__ vbq, return sum0 + sum1; } +#define VDR_PQ2_0_Q8_1_MMVQ 1 +#define VDR_PTQ1_0_Q8_1_MMVQ 4 // one call takes a whole 128 block + + +// Same codec as Q2_0 at group 128: each int of qs holds 16 codes, i.e. half of one q8_1 block. +static __dpct_inline__ float +vec_dot_pq2_0_q8_1(const void *__restrict__ vbq, + const block_q8_1 *__restrict__ bq8_1, const int &iqs) { + + const block_pq2_0 * bq = (const block_pq2_0 *) vbq; + + float sum = 0.0f; +#pragma unroll + for (int h = 0; h < 2; ++h) { + const int k = iqs + h * QI_PQ2_0; + const int v = get_int_from_uint8(bq->qs, k); + const block_q8_1 & y = bq8_1[k / 2]; + const int base = 4 * (k % 2); + + int u[4]; +#pragma unroll + for (int j = 0; j < 4; ++j) { + u[j] = get_int_from_int8_aligned(y.qs, base + j); + } + sum += vec_dot_q2_0_q8_1_impl(&v, u, bq->d, y.ds); + } + return sum; +} + +static __dpct_inline__ float +vec_dot_ptq1_0_q8_1(const void *__restrict__ vbq, + const block_q8_1 *__restrict__ bq8_1, const int &iqs) { + GGML_UNUSED(iqs); + const block_ptq1_0 * bq = (const block_ptq1_0 *) vbq; + + int8_t x[QK_PTQ1_0]; + sycl::half2 ds[4]; +#pragma unroll + for (int k = 0; k < 4; ++k) { +#pragma unroll + for (int j = 0; j < QK8_1 / 4; ++j) { + *(int *) (x + k * QK8_1 + 4 * j) = get_int_from_int8_aligned(bq8_1[k].qs, j); + } + ds[k] = bq8_1[k].ds; + } + int sumi[4] = { 0, 0, 0, 0 }; + ptq1_0_dot_block(bq->qs, (uint32_t) bq->qh[0] | ((uint32_t) bq->qh[1] << 16), x, sumi); + return ptq1_0_finish(sumi, ds, bq->d); +} + static __dpct_inline__ float vec_dot_q4_1_q8_1(const void *__restrict__ vbq, const block_q8_1 *__restrict__ bq8_1, const int &iqs) { diff --git a/ggml/src/ggml-sycl/w8a8.cpp b/ggml/src/ggml-sycl/w8a8.cpp new file mode 100644 index 000000000000..38ced70f1814 --- /dev/null +++ b/ggml/src/ggml-sycl/w8a8.cpp @@ -0,0 +1,146 @@ +#include "w8a8.hpp" + +#if GGML_SYCL_DNNL +#include "dequantize.hpp" +#include "gemm.hpp" + +// per row: D = max over groups of d, ratio q_g = round(127 * d_g / D); one work-group (16 lanes) per row +static void w8a8_row_scales(ggml_type type, bool reordered, const void * w, float * row_scale, int8_t * ratio, + int64_t nrows, int64_t K, dpct::queue_ptr stream) { + const int64_t ng = K / 128; + const int64_t nb = nrows * ng; + const uint8_t * base = (const uint8_t *) w; + const bool ptq = type == GGML_TYPE_PTQ1_0; + stream->parallel_for(sycl::nd_range<1>(nrows * 16, 16), [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(16)]] { + const int64_t r = it.get_group(0); + const int lane = it.get_local_id(0); + auto dget = [&](int64_t g) -> float { + const int64_t ib = r * ng + g; + if (ptq) { + return reordered ? (float) ((const sycl::half *) (base + nb * 26))[ib] : (float) ((const block_ptq1_0 *) w)[ib].d; + } + return reordered ? (float) ((const sycl::half *) (base + nb * 32))[ib] : (float) ((const block_pq2_0 *) w)[ib].d; + }; + float m = 0.0f; + for (int64_t g = lane; g < ng; g += 16) { + m = sycl::fmax(m, sycl::fabs(dget(g))); + } + m = sycl::reduce_over_group(it.get_sub_group(), m, sycl::maximum()); + for (int64_t g = lane; g < ng; g += 16) { + ratio[r * ng + g] = m > 0 ? (int8_t) sycl::round(127.0f * dget(g) / m) : 0; + } + if (lane == 0) { + row_scale[r] = m / 127.0f; + } + }); +} + +// int8 weights = (code value) * ratio of its group; 8 elements per work item +static void w8a8_weights(ggml_type type, bool reordered, const void * w, const int8_t * ratio, int8_t * w8, + int64_t nrows, int64_t K, dpct::queue_ptr stream) { + const int64_t nb = nrows * K / 128; + const uint8_t * base = (const uint8_t *) w; + const bool ptq = type == GGML_TYPE_PTQ1_0; + stream->parallel_for(sycl::range<1>(nb * 16), [=](sycl::id<1> id) { + const int64_t ib = id[0] / 16; + const int c = (int) (id[0] % 16); + int v[8]; + if (ptq) { + const uint8_t * qs; + const uint8_t * qh; + if (reordered) { + qs = base + ib * 24; + qh = base + nb * 24 + ib * 2; + } else { + qs = ((const block_ptq1_0 *) w)[ib].qs; + qh = ((const block_ptq1_0 *) w)[ib].qh; + } + constexpr uint32_t pow3[5] = { 1, 3, 9, 27, 81 }; + if (c < 15) { + // 8 source bytes in one load: qs[0..7], qs[8..15] (c < 10) or qs[16..23] + const uint32_t p = pow3[c < 10 ? c / 2 : c - 10]; + const int off = c < 10 ? (c % 2) * 8 : 16; + const uint64_t src = reordered ? *(const uint64_t *) (qs + off) : 0; +#pragma unroll + for (int i = 0; i < 8; ++i) { + const uint32_t byte = reordered ? (uint32_t) (src >> (8 * i)) & 0xFF : qs[off + i]; + v[i] = (int) ((((byte * p) & 0xFF) * 3) >> 8) - 1; + } + } else { +#pragma unroll + for (int i = 0; i < 8; ++i) v[i] = (int) ((((qh[i % 2] * pow3[i / 2]) & 0xFF) * 3) >> 8) - 1; + } + } else { + const uint8_t * qs = reordered ? base + ib * 32 : ((const block_pq2_0 *) w)[ib].qs; + const uint32_t two = reordered ? *(const uint16_t *) (qs + 2 * c) : (uint32_t) qs[2 * c] | ((uint32_t) qs[2 * c + 1] << 8); +#pragma unroll + for (int i = 0; i < 8; ++i) v[i] = (int) ((two >> (2 * i)) & 3) - 1; + } + const int r8 = ratio[ib]; + uint64_t out = 0; +#pragma unroll + for (int i = 0; i < 8; ++i) out |= (uint64_t) (uint8_t) (int8_t) (v[i] * r8) << (8 * i); + ((uint64_t *) w8)[id[0]] = out; + }); +} + +// per token: s = max|x| / 127, x8 = round(x / s); one 256-item work-group per token row, 16-byte loads +static void w8a8_acts(const float * x, int8_t * x8, float * xs, int64_t ntok, int64_t K, dpct::queue_ptr stream) { + constexpr int WG = 256; + GGML_ASSERT(K % 4 == 0); + stream->parallel_for(sycl::nd_range<1>(ntok * WG, WG), [=](sycl::nd_item<1> it) { + const int64_t t = it.get_group(0); + const int lid = it.get_local_id(0); + const int64_t K4 = K / 4; + const sycl::float4 * row4 = (const sycl::float4 *) (x + t * K); + float m = 0.0f; + for (int64_t k4 = lid; k4 < K4; k4 += WG) { + const sycl::float4 v = sycl::fabs(row4[k4]); + m = sycl::fmax(m, sycl::fmax(sycl::fmax(v.x(), v.y()), sycl::fmax(v.z(), v.w()))); + } + m = sycl::reduce_over_group(it.get_group(), m, sycl::maximum()); + const float inv = m > 0 ? 127.0f / m : 0.0f; + sycl::char4 * out4 = (sycl::char4 *) (x8 + t * K); + for (int64_t k4 = lid; k4 < K4; k4 += WG) { + const sycl::float4 v = sycl::round(row4[k4] * inv); + out4[k4] = sycl::char4((int8_t) v.x(), (int8_t) v.y(), (int8_t) v.z(), (int8_t) v.w()); + } + if (lid == 0) { + xs[t] = m / 127.0f; + } + }); +} + +bool ggml_sycl_w8a8_mul_mat(ggml_backend_sycl_context & ctx, ggml_type type, bool reordered, const void * w, + const float * x, float * dst, int64_t nrows, int64_t ncols_x, int64_t K, + dpct::queue_ptr stream) { + if ((type != GGML_TYPE_PQ2_0 && type != GGML_TYPE_PTQ1_0) || K % 128 != 0) { + return false; + } + ggml_sycl_pool_alloc ratio(ctx.pool(), nrows * K / 128); + ggml_sycl_pool_alloc wscale(ctx.pool(), nrows); + ggml_sycl_pool_alloc w8(ctx.pool(), nrows * K); + ggml_sycl_pool_alloc x8(ctx.pool(), ncols_x * K); + ggml_sycl_pool_alloc xscale(ctx.pool(), ncols_x); + + w8a8_row_scales(type, reordered, w, wscale.get(), ratio.get(), nrows, K, stream); + w8a8_weights(type, reordered, w, ratio.get(), w8.get(), nrows, K, stream); + w8a8_acts(x, x8.get(), xscale.get(), ncols_x, K, stream); + DnnlGemmWrapper::gemm_s8(ctx, (int) nrows, (int) ncols_x, (int) K, x8.get(), w8.get(), dst, stream); + + const float * ws = wscale.get(); + const float * xs = xscale.get(); + stream->parallel_for(sycl::range<1>(ncols_x * nrows), [=](sycl::id<1> i) { + dst[i] *= xs[i[0] / nrows] * ws[i[0] % nrows]; + }); + return true; +} + +#else + +bool ggml_sycl_w8a8_mul_mat(ggml_backend_sycl_context &, ggml_type, bool, const void *, const float *, float *, int64_t, + int64_t, int64_t, dpct::queue_ptr) { + return false; +} + +#endif diff --git a/ggml/src/ggml-sycl/w8a8.hpp b/ggml/src/ggml-sycl/w8a8.hpp new file mode 100644 index 000000000000..e6a6f67f5d74 --- /dev/null +++ b/ggml/src/ggml-sycl/w8a8.hpp @@ -0,0 +1,10 @@ +#pragma once +#include "common.hpp" + +// Prompt-sized mat-mul for PQ2_0 / PTQ1_0 weights on int8 XMX via oneDNN s8 x s8. +// Per-128-group weight scales are requantized to a per-row scale (int8 weight = code * round(127 * d_g / max_g d)), +// activations are quantized to int8 per token; both scales are applied in one pass over the output. +// Returns false when not applicable (caller falls back). +bool ggml_sycl_w8a8_mul_mat(ggml_backend_sycl_context & ctx, ggml_type type, bool reordered, const void * w, + const float * x, float * dst, int64_t nrows, int64_t ncols_x, int64_t K, + dpct::queue_ptr stream); diff --git a/src/llama-context.cpp b/src/llama-context.cpp index cb6cf1f2854a..77eb959cae7e 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -678,6 +678,10 @@ void llama_context::resolve_fused_ops(const llama_memory_context_i * mctx, uint3 LLAMA_LOG_INFO("%s: resolving fused Gated Delta Net support:\n", func); resolve(llm_fused_op_gdn_ar_probe, cparams.fused_gdn_ar); resolve(llm_fused_op_gdn_ch_probe, cparams.fused_gdn_ch); + if (const char * e = getenv("LLAMA_FUSED_GDN_CH"); e && atoi(e) == 0) { + cparams.fused_gdn_ch = false; // A/B: use the graph-level chunked delta net for prompt batches + LLAMA_LOG_INFO("%s: fused GDN (chunked) disabled by LLAMA_FUSED_GDN_CH=0\n", func); + } cparams.auto_fgdn = false; } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 6f6d7b4b5538..603e0522f5a1 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -9476,6 +9476,14 @@ static std::vector> make_test_cases_eval() { } } + // ARC-LAB: ternary int8 coopmat GEMM path (Vulkan, GGML_VK_TERN_I8_MIN_N): rows % 128 == 0, several N incl. partial tiles + for (ggml_type t : {GGML_TYPE_PTQ1_0, GGML_TYPE_PQ2_0}) { + for (int n : {9, 64, 100}) { + for (int k : {1024, 5120}) { + test_cases.emplace_back(new test_mul_mat(t, GGML_TYPE_F32, 256, n, k, {1, 1}, {1, 1})); + } + } + } // PTQ1_0 / PQ2_0 integer-dot mat-vec: Bonsai-2 shapes, odd row counts (row tail), batches and multi-column B for (int64_t n : {1, 2, 3, 4, 5, 6, 7, 8}) { for (int64_t k : {1024, 5120, 6144, 17408}) { @@ -10324,6 +10332,23 @@ static std::vector> make_test_cases_eval() { } // mixed quant and Q1_0 test cases + // ARC-LAB: q4_0-cache decode kernel (SYCL fattn-dec): Bonsai 27B shape (D 256, 4 KV heads x GQA 6), 1-4 query tokens + for (int64_t kv : {256, 512, 4096, 16384}) { + for (int64_t nb : {1, 2, 3, 4, 5, 8}) { + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)); + } + } + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 1024, 1, false, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)); + // ARC-LAB: Intel prompt FA (Vulkan fa_pf_gqa): prompt batches at the Bonsai shape, f16 and q4_0 (f16 scratch), both layouts + for (ggml_type t : {GGML_TYPE_F16, GGML_TYPE_Q4_0}) { + for (int64_t nb : {16, 37, 64}) { + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 512, nb, true, false, 0, 0, GGML_PREC_F32, t, t, {0, 2, 1, 3})); + } + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 1024, 64, true, false, 0, 0, GGML_PREC_F32, t, t)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 2}, 512, 20, true, false, 0, 0, GGML_PREC_F32, t, t, {0, 2, 1, 3})); + } + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 512, 64, false, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 2}, 512, 4, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)); test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q8_0, GGML_TYPE_Q4_0)); test_cases.emplace_back(new test_flash_attn_ext(64, 64, 4, {1, 1}, 128, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext(72, 72, 4, {1, 1}, 96, 2, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0)); @@ -10479,6 +10504,19 @@ static std::vector> make_test_cases_eval() { } test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 1, 1)); + // ARC-LAB: chunked prefill path (SYCL, S = 128, one sequence, >= 64 tokens): full / partial chunks, GQA x3, + // snapshot hand-off (K > 1: last K tokens via the sequential kernel), permuted q/k + for (int nt : {64, 100, 1024}) { + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, nt, 1, 3)); + } + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 100, 1, 3, false, false, 4)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 1024, 1, 3, false, false, 17)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 100, 1, 3, true)); + // ARC-LAB: token-blocked sequential kernel (SYCL, S = 128, >= 16 tokens): several sequences, snapshots, short batches + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 64, 2, 3)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 64, 2, 3, false, false, 4)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 37, 3, 3, true)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 16, 2, 1, false, false, 3)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 16, 1, 1)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 16, 1, 1, 1, true, true)); // raw gates (sigmoid / softplus folded into the op): decode, prefill, rows mode @@ -10574,12 +10612,20 @@ static std::vector> make_test_cases_eval() { // Test cases for performance evaluation: should be representative of real-world use cases static std::vector> make_test_cases_perf() { std::vector> test_cases; + + // ARC-LAB: Bonsai 2 GDN layer shape (16 k-heads x 128, 48 v-heads): prompt batch and decode + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 1024, 1, 3)); + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 1, 1, 3)); // bandwidth comparison at Bonsai-2 shapes for (ggml_type t : {GGML_TYPE_PTQ1_0, GGML_TYPE_PQ2_0, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q2_K, GGML_TYPE_TQ2_0}) { test_cases.emplace_back(new test_mul_mat(t, GGML_TYPE_F32, 17408, 1, 5120, {1, 1}, {1, 1})); test_cases.emplace_back(new test_mul_mat(t, GGML_TYPE_F32, 5120, 1, 17408, {1, 1}, {1, 1})); test_cases.emplace_back(new test_mul_mat(t, GGML_TYPE_F32, 10240, 1, 5120, {1, 1}, {1, 1})); } + // ARC-LAB: prompt-sized GEMM ceilings at the Bonsai FFN shape - ternary vs f16 and tuned quants + for (ggml_type t : {GGML_TYPE_PTQ1_0, GGML_TYPE_PQ2_0, GGML_TYPE_F16, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0}) { + test_cases.emplace_back(new test_mul_mat(t, GGML_TYPE_F32, 17408, 512, 5120, {1, 1}, {1, 1})); + } // batched decode (several sequences per step) through the mat-vec path for (ggml_type t : {GGML_TYPE_PTQ1_0, GGML_TYPE_PQ2_0, GGML_TYPE_Q4_0}) { for (int n : {2, 3, 4, 8}) { @@ -10796,10 +10842,15 @@ static std::vector> make_test_cases_perf() { } // Ternary Bonsai 2 27B (qwen35): the bf16 gated-delta-net gate projections - for (int bs : {1, 2, 3, 4, 8}) { + for (int bs : {1, 2, 3, 4, 8, 96, 512, 2048}) { test_cases.emplace_back(new test_mul_mat(GGML_TYPE_BF16, GGML_TYPE_F32, 48, bs, 5120, {1, 1}, {1, 1})); // ssm_alpha, ssm_beta } + // ARC-LAB: Bonsai 2 27B fused FFN gate/up + SwiGLU (PTQ1_0) at decode / MTP-verify batch sizes + for (int bs : {1, 4}) { + test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_PTQ1_0, GGML_GLU_OP_SWIGLU, 17408, bs, 5120, + false, 1, 1, false, false, true, false, {1, 1})); + } // Ternary Bonsai 2 27B (qwen35, PTQ1_0) projections at speculative-decoding batch sizes for (int bs : {1, 2, 3, 4, 8}) { for (ggml_type type_a : {GGML_TYPE_PTQ1_0, GGML_TYPE_Q4_0}) { @@ -10857,6 +10908,17 @@ static std::vector> make_test_cases_perf() { // Qwen3-VL-8B https://github.com/ggml-org/llama.cpp/issues/17012 test_cases.emplace_back(new test_flash_attn_ext(72, 72, 16, {1, 1}, 5776, 5776, false, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + for (int64_t kv : {16384, 49152}) { // ARC-LAB: Bonsai 27B decode / MTP verify on a q4_0 cache + for (int64_t nb : {1, 4, 5, 8}) { + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)); + } + } + for (ggml_type t : {GGML_TYPE_Q4_0, GGML_TYPE_F16}) { // ARC-LAB: Bonsai 27B prompt batch at 16K history + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 16384, 512, true, false, 0, 0, GGML_PREC_F32, t, t)); + for (int64_t kv : {512, 16384}) { // real KV-cache layout (heads interleaved per token) + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, kv, 512, true, false, 0, 0, GGML_PREC_F32, t, t, {0, 2, 1, 3})); + } + } test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 4, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); test_cases.emplace_back(new test_flash_attn_ext(64, 64, 8, {8, 1}, 7680, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)); @@ -10997,6 +11059,10 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 64, 1, 1)); // smaller model test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 1, 1, 1, false, true)); // KDA // PP: n_seq_tokens=64,256 (prompt processing) + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 1024, 1, 3, false, false, 4)); // ARC-LAB: live server keeps snapshots (K > 1) + for (int nt : {16, 32, 64, 128}) { // ARC-LAB: Bonsai shape (48 v-heads), draft-verify batch sizes + test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, nt, 1, 3)); + } test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 64, 1)); // PP-64 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 256, 1)); // PP-256 test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 128, 512, 1)); // PP-512 From e0b174d9040ce8507c73a47715188da523d7a1bf Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sun, 27 Sep 2026 16:48:34 +1000 Subject: [PATCH 02/65] sycl: PQ2_0 weights on the TernSYCL XMX path too (GGML_SYCL_PTQ1_T2) PQ2_0 (2-bit codes 00=-1 01=0 10=+1, fp16 scale per 128) is the same size as the 2-bit XMX layout, so it is remapped in place. Code 3 (+2) has no 2-bit two's-complement form: the repack reports it and the matmul aborts with a message. test-backend-ops MUL_MAT type_a=pq2_0 with GGML_SYCL_PTQ1_T2=all: 84/84; ptq1 94/94. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 5056bde56e6f335451f89213b190afbcab532abd on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/ggml-sycl.cpp | 9 +++-- ggml/src/ggml-sycl/ptq1-t2.cpp | 62 ++++++++++++++++++++------------ ggml/src/ggml-sycl/ptq1-t2.hpp | 5 +-- 3 files changed, 48 insertions(+), 28 deletions(-) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index e6e141bed967..02976b01a6f4 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -1018,7 +1018,7 @@ static size_t ggml_backend_sycl_buffer_type_get_alloc_size(ggml_backend_buffer_t } } // ARC-LAB: room for the in-place TernSYCL 2-bit repack (GGML_SYCL_PTQ1_T2) - if (tensor->type == GGML_TYPE_PTQ1_0 && tensor->ne[2] == 1 && tensor->ne[3] == 1 && + if ((tensor->type == GGML_TYPE_PTQ1_0 || tensor->type == GGML_TYPE_PQ2_0) && tensor->ne[2] == 1 && tensor->ne[3] == 1 && ggml_sycl_t2_wants(tensor->name, tensor->ne[0], tensor->ne[1])) { size = std::max(size, ggml_sycl_t2_bytes(tensor->ne[0], tensor->ne[1])); } @@ -4812,7 +4812,7 @@ static void ggml_sycl_mul_mat_bf16_small(ggml_backend_sycl_context & ctx, const // ARC-LAB: PTQ1_0 weights chosen by GGML_SYCL_PTQ1_T2 are repacked in place (their buffer was sized for it) on first use // and from then on only this path may read them. static bool ggml_sycl_t2_tensor(const ggml_tensor * w) { - if (w->type != GGML_TYPE_PTQ1_0 || w->ne[2] != 1 || w->ne[3] != 1 || !w->extra) { + if ((w->type != GGML_TYPE_PTQ1_0 && w->type != GGML_TYPE_PQ2_0) || w->ne[2] != 1 || w->ne[3] != 1 || !w->extra) { return false; } const auto * extra = static_cast(w->extra); @@ -4832,7 +4832,10 @@ static bool ggml_sycl_t2_mul_mat_try(ggml_backend_sycl_context & ctx, const ggml GGML_ASSERT(src1->ne[3] == 1 && (src1->ne[2] == 1 || src1->nb[2] == src1->ne[1] * src1->nb[1])); dpct::queue_ptr stream = ctx.stream(); if (!extra->optimized_feature.t2) { - ggml_sycl_t2_repack(*stream, src0->data, K, N); + if (!ggml_sycl_t2_repack(*stream, src0->data, K, N, src0->type == GGML_TYPE_PQ2_0)) { + GGML_ABORT("PQ2_0 tensor %s uses code 3 (+2), which the 2-bit XMX layout cannot hold: unset GGML_SYCL_PTQ1_T2", + src0->name); + } extra->optimized_feature.t2 = true; } // tokens: ne[1] x ne[2] rows of K floats (row stride nb[1]; planes back to back) diff --git a/ggml/src/ggml-sycl/ptq1-t2.cpp b/ggml/src/ggml-sycl/ptq1-t2.cpp index 0df2e5f50fb4..f784aee979db 100644 --- a/ggml/src/ggml-sycl/ptq1-t2.cpp +++ b/ggml/src/ggml-sycl/ptq1-t2.cpp @@ -80,54 +80,70 @@ namespace { void t2_precompile(sycl::queue & q); } -void ggml_sycl_t2_repack(sycl::queue & q, void * data, int64_t K, int64_t N) { +bool ggml_sycl_t2_repack(sycl::queue & q, void * data, int64_t K, int64_t N, bool pq2) { static std::once_flag compiled; std::call_once(compiled, [&] { t2_precompile(q); }); const int64_t nb = K / QK; - const size_t src_bytes = (size_t) N * nb * BLK_BYTES; + const int64_t blk_bytes = pq2 ? 34 : BLK_BYTES; // block_pq2_0: fp16 d, then 32 bytes of 2-bit codes + const size_t src_bytes = (size_t) N * nb * blk_bytes; uint8_t * tmp = (uint8_t *) sycl::malloc_device(src_bytes, q); if (!tmp) { fprintf(stderr, "%s: no device memory for a %zu-byte repack buffer\n", __func__, src_bytes); abort(); } q.memcpy(tmp, data, src_bytes).wait(); - uint32_t * B = (uint32_t *) data; - uint16_t * SB = (uint16_t *) ((char *) data + (size_t) (K / 16) * N * 4); + uint32_t * B = (uint32_t *) data; + uint16_t * SB = (uint16_t *) ((char *) data + (size_t) (K / 16) * N * 4); + int * bad = sycl::malloc_device(1, q); + q.memset(bad, 0, sizeof(int)).wait(); q.parallel_for(sycl::range<2>((size_t) (K / 16), (size_t) N), [=](sycl::item<2> it) { const int64_t kp = it[0], n = it[1]; const int64_t g = kp / 8; - const uint8_t * b = tmp + (n * nb + g) * BLK_BYTES; + const uint8_t * b = tmp + (n * nb + g) * blk_bytes; const int e0 = (int) (kp % 8) * 16; uint32_t w = 0; for (int j = 0; j < 16; ++j) { const int e = e0 + j; - uint32_t v; - int lvl; - if (e < 80) { - v = b[e % 16]; - lvl = e / 16; - } else if (e < 120) { - v = b[16 + (e - 80) % 8]; - lvl = (e - 80) / 8; + uint32_t digit; // weight = digit - 1 + if (pq2) { + digit = (b[2 + e / 4] >> (2 * (e % 4))) & 3u; + if (digit == 3u) { + *bad = 1; + digit = 2u; + } } else { - v = b[24 + (e - 120) % 2]; - lvl = (e - 120) / 2; + uint32_t v; + int lvl; + if (e < 80) { + v = b[e % 16]; + lvl = e / 16; + } else if (e < 120) { + v = b[16 + (e - 80) % 8]; + lvl = (e - 80) / 8; + } else { + v = b[24 + (e - 120) % 2]; + lvl = (e - 120) / 2; + } + digit = 0; + for (int t = 0; t <= lvl; ++t) { // base-3 fixed-point digits, as ptq1_0_decode_block + const uint32_t x = v * 3; + digit = x >> 8; + v = x & 0xFF; + } } - uint32_t digit = 0; - for (int t = 0; t <= lvl; ++t) { // base-3 fixed-point digits, as ptq1_0_decode_block - const uint32_t x = v * 3; - digit = x >> 8; - v = x & 0xFF; - } - const uint32_t code = digit == 2 ? 1u : (digit == 1 ? 0u : 3u); // weight = digit - 1, 2-bit two's complement + const uint32_t code = digit == 2 ? 1u : (digit == 1 ? 0u : 3u); // 2-bit two's complement of digit - 1 w |= code << (2 * j); } B[kp * N + n] = w; if (kp % 8 == 0) { - SB[g * N + n] = (uint16_t) (b[26] | (b[27] << 8)); + SB[g * N + n] = pq2 ? (uint16_t) (b[0] | (b[1] << 8)) : (uint16_t) (b[26] | (b[27] << 8)); } }).wait(); + int bad_h = 0; + q.memcpy(&bad_h, bad, sizeof(int)).wait(); + sycl::free(bad, q); sycl::free(tmp, q); + return bad_h == 0; } namespace { diff --git a/ggml/src/ggml-sycl/ptq1-t2.hpp b/ggml/src/ggml-sycl/ptq1-t2.hpp index 505d1c07c021..49ef2db1e705 100644 --- a/ggml/src/ggml-sycl/ptq1-t2.hpp +++ b/ggml/src/ggml-sycl/ptq1-t2.hpp @@ -11,8 +11,9 @@ bool ggml_sycl_t2_wants(const char * name, int64_t K, int64_t N); // device bytes of the repacked weight: 2-bit codes [K/16, N] u32 + fp16 scales [K/128, N] size_t ggml_sycl_t2_bytes(int64_t K, int64_t N); -// in place: data holds N rows of K/128 block_ptq1_0 (as uploaded) and at least ggml_sycl_t2_bytes(K, N) bytes -void ggml_sycl_t2_repack(sycl::queue & q, void * data, int64_t K, int64_t N); +// in place: data holds N rows of K/128 block_ptq1_0 (pq2 = false) or block_pq2_0 (pq2 = true) as uploaded, and at least +// ggml_sycl_t2_bytes(K, N) bytes. PQ2_0 code 3 (+2) has no 2-bit two's-complement form: returns false if one is found. +bool ggml_sycl_t2_repack(sycl::queue & q, void * data, int64_t K, int64_t N, bool pq2 = false); // dst[m * N + n] = sum_k x[m * x_stride + k] * w[n, k], M tokens (fp32 in and out) void ggml_sycl_t2_mul_mat(sycl::queue & q, const void * w, const float * x, int64_t x_stride, float * dst, int64_t M, int64_t N, int64_t K); From 6836929de6b6d998e55a801eb98055ae2b767179 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sun, 27 Sep 2026 16:48:34 +1000 Subject: [PATCH 03/65] sycl: PTQ1 T2 path only on Xe2+ GPUs, fall back with a warning elsewhere The T2 kernels use 16-lane int8 x int2 DPAS builtins that only Xe2 and newer drivers provide; on an Arc 140T (Xe-LPG+) GGML_SYCL_PTQ1_T2=all failed with 'No kernel named ... was found'. Now checked once against the device architecture. GGML_SYCL_PTQ1_T2_ANYGPU=1 skips the check. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 731c1a80064de23746b35a589f7654c41ae8c2a4 on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/ptq1-t2.cpp | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-sycl/ptq1-t2.cpp b/ggml/src/ggml-sycl/ptq1-t2.cpp index f784aee979db..389f51ff1f3d 100644 --- a/ggml/src/ggml-sycl/ptq1-t2.cpp +++ b/ggml/src/ggml-sycl/ptq1-t2.cpp @@ -4,6 +4,7 @@ // feed the s8 x s2 DPAS directly. B580 standalone, 2 GiB rotating weights: 17408x5120 m=1 56 us, m=4 61 us (vs 70 / // ~100 in-model for PTQ1_0). The weight grows 31%; activations become int8 with one fp16 scale per 128 (q8_1 is per 32). #include "ptq1-t2.hpp" +#include "common.hpp" #include "ternsycl/int2_int8_dpas.hpp" @@ -61,9 +62,29 @@ void * scratch_get(sycl::queue & q, size_t bytes) { } // namespace +// the kernels use 16-lane DPAS (Xe2 and newer); Xe-LPG(+) / Alchemist XMX is 8 lanes and has no such kernels, so fall back +// to the regular path with one warning instead of failing at the first launch. GGML_SYCL_PTQ1_T2_ANYGPU=1 skips the check. +static bool t2_device_ok() { + static const bool ok = [] { + if (getenv("GGML_SYCL_PTQ1_T2_ANYGPU")) { + return true; + } + const auto & hw = ggml_sycl_info().devices[ggml_sycl_get_device()].hw_info; + const bool xe2 = hw.arch == gpu_arch::intel_gpu_bmg_g21 || hw.arch == gpu_arch::intel_gpu_bmg_g31 || + hw.arch == gpu_arch::intel_gpu_lnl_m || hw.arch == gpu_arch::intel_gpu_ptl_h || + hw.arch == gpu_arch::intel_gpu_ptl_u || hw.arch == gpu_arch::intel_gpu_wcl; + if (!xe2) { + GGML_LOG_WARN("%s: GGML_SYCL_PTQ1_T2 needs an Xe2 or newer GPU (Arc B-series, Lunar Lake, Panther Lake); " + "%s is not one, using the regular path\n", __func__, hw.name.c_str()); + } + return xe2; + }(); + return ok; +} + bool ggml_sycl_t2_wants(const char * name, int64_t K, int64_t N) { const int mode = t2_mode(); - if (mode == 0 || K % QK != 0 || N % 16 != 0) { + if (mode == 0 || K % QK != 0 || N % 16 != 0 || !t2_device_ok()) { return false; } if (mode == 2) { From 48ffc2e982ff8ae56284ab8c19cf6790726edf10 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sun, 27 Sep 2026 22:13:52 +1000 Subject: [PATCH 04/65] sycl/ternsycl: 64-bit surface base address on Windows The 2D block I/O builtins took the surface base as 'long', which is 32 bits in SYCL device code on Windows (the device follows the host's LLP64 data model), so the weight loads would have used truncated addresses. 'long long' is 64 bits on both; identical code on Linux. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 2f0cf6e8a872cd78b7add26fe5ace2811b54fd52 on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/ternsycl/xe2.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ggml/src/ggml-sycl/ternsycl/xe2.hpp b/ggml/src/ggml-sycl/ternsycl/xe2.hpp index 93adf81c675c..ad284042975c 100644 --- a/ggml/src/ggml-sycl/ternsycl/xe2.hpp +++ b/ggml/src/ggml-sycl/ternsycl/xe2.hpp @@ -69,7 +69,7 @@ XE2_OCL(xe2::ushort8 intel_sub_group_block_read_us8(const XE2_GLOBAL unsigned sh // 2D block I/O: (base, width - 1, height - 1, pitch - 1) in bytes, coord (x // in elements, y in rows). mRkCvV = R rows x C columns x V blocks. #define XE2_2D_RD(ret, sfx) \ - XE2_IB(ret __builtin_IB_subgroup_block_read_flat_##sfx(long base, int w, int h, int p, xe2::int2 c)); + XE2_IB(ret __builtin_IB_subgroup_block_read_flat_##sfx(long long base, int w, int h, int p, xe2::int2 c)); XE2_2D_RD(unsigned, u32_m1k16v1) XE2_2D_RD(xe2::uint2, u32_m2k16v1) XE2_2D_RD(xe2::uint8, u32_m8k16v1) @@ -78,12 +78,12 @@ XE2_2D_RD(xe2::ushort2, u16_m1k16v2) XE2_2D_RD(xe2::ushort8, u16_m8k16v1) XE2_2D_RD(xe2::ushort16, u16_m8k16v2) #undef XE2_2D_RD -XE2_IB(void __builtin_IB_subgroup_block_write_flat_u16_m8k16v1(long base, int w, int h, int p, +XE2_IB(void __builtin_IB_subgroup_block_write_flat_u16_m8k16v1(long long base, int w, int h, int p, xe2::int2 c, xe2::ushort8 v)); -XE2_IB(void __builtin_IB_subgroup_block_write_flat_u32_m8k16v1(long base, int w, int h, int p, +XE2_IB(void __builtin_IB_subgroup_block_write_flat_u32_m8k16v1(long long base, int w, int h, int p, xe2::int2 c, xe2::uint8 v)); // cache control 0 = default -XE2_IB(void __builtin_IB_subgroup_block_read_prefetch_u32_m8k16v1(long base, int w, int h, int p, +XE2_IB(void __builtin_IB_subgroup_block_read_prefetch_u32_m8k16v1(long long base, int w, int h, int p, xe2::int2 c, int cc)); // per-lane LSC load with cache control (IGC LSC_LDCC: 4 = L1 cached, L3 cached) XE2_IB(xe2::uint3 __builtin_IB_lsc_load_global_uint3(const XE2_GLOBAL xe2::uint3 *p, int imm_off, int cc)); @@ -92,10 +92,10 @@ namespace xe2 { // 2D surface: width and pitch in bytes, height in rows struct surf { - long base; + long long base; int w, h, p; surf(const void *b, int width_bytes, int height, int pitch_bytes) - : base((long)b), w(width_bytes - 1), h(height - 1), p(pitch_bytes - 1) {} + : base((long long)b), w(width_bytes - 1), h(height - 1), p(pitch_bytes - 1) {} }; inline unsigned rd_32b_1r16(const surf &s, int x, int y) { From c8e3bfcd4cc1c628b9c2ddcafc25bb09aaf2411b Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Mon, 28 Sep 2026 08:14:04 +1000 Subject: [PATCH 05/65] sycl: GGML_SYCL_PTQ1_T2_TILE lab knob for the TernSYCL large-M tile (default stays best) B580 pp1024 (ub 1024, all weights on XMX): 8x128/8x2 939 t/s (default), 8x128/16x1 949, others 480-857. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 244c4d44c5d90a2ad77c04924ec2722c26f921a4 on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/ptq1-t2.cpp | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-sycl/ptq1-t2.cpp b/ggml/src/ggml-sycl/ptq1-t2.cpp index 389f51ff1f3d..6807a57c9f52 100644 --- a/ggml/src/ggml-sycl/ptq1-t2.cpp +++ b/ggml/src/ggml-sycl/ptq1-t2.cpp @@ -228,10 +228,9 @@ void gemv(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const uint32_ Kern{ nullptr, (const signed char *) Aq, SA, B, SB, C, epi, M, N, K }); } -template -void gemm(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const uint32_t * B, const uint16_t * SB, float * C, - int M, int N, int K) { - constexpr int MT_M = 8, MT_N = 128, WG_M = 8, WG_N = 2; +template +void gemm_tile(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const uint32_t * B, const uint16_t * SB, float * C, + int M, int N, int K) { const size_t tm = MT_M * WG_M, tn = MT_N * WG_N; const Epi epi{ nullptr, nullptr, 0, 1 }; const sycl::range<2> local(1, 16 * WG_M * WG_N); @@ -241,6 +240,26 @@ void gemm(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const uint32_ nullptr, (const signed char *) Aq, SA, B, SB, C, epi, M, N, K }); } +// ARC-LAB lab knob GGML_SYCL_PTQ1_T2_TILE = index into TernSYCL's large-M tile table (mt_m, mt_n, wg_m, wg_n): +// 0 {8,128,8,2} (default) 1 {8,128,4,2} 2 {8,128,4,4} 3 {8,128,16,1} 4 {8,128,2,4} 5 {16,64,4,2} 6 {16,64,8,2} +// 7 {8,64,8,2} 8 {32,32,4,2} +template +void gemm(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const uint32_t * B, const uint16_t * SB, float * C, + int M, int N, int K) { + static const int tile = getenv("GGML_SYCL_PTQ1_T2_TILE") ? atoi(getenv("GGML_SYCL_PTQ1_T2_TILE")) : 0; + switch (tile) { + case 1: gemm_tile(q, Aq, SA, B, SB, C, M, N, K); break; + case 2: gemm_tile(q, Aq, SA, B, SB, C, M, N, K); break; + case 3: gemm_tile(q, Aq, SA, B, SB, C, M, N, K); break; + case 4: gemm_tile(q, Aq, SA, B, SB, C, M, N, K); break; + case 5: gemm_tile(q, Aq, SA, B, SB, C, M, N, K); break; + case 6: gemm_tile(q, Aq, SA, B, SB, C, M, N, K); break; + case 7: gemm_tile(q, Aq, SA, B, SB, C, M, N, K); break; + case 8: gemm_tile(q, Aq, SA, B, SB, C, M, N, K); break; + default: gemm_tile(q, Aq, SA, B, SB, C, M, N, K); break; + } +} + } // namespace namespace { From af00847d5193ec181d713229cf06b7ccb19d653d Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Mon, 28 Sep 2026 08:14:05 +1000 Subject: [PATCH 06/65] sycl: two-pass top-k for long rows (GGML_SYCL_TOPK_OLD=1 restores the one-group kernel) The one-work-group top-k took ~550 us per MTP draft token on the B580 (248K-entry vocabulary). Now up to 256 groups per row with a pairwise local-memory merge, then one group merges the winners; ties keep the lower index. test-backend-ops TOP_K 325/325. 200K entries: k=1 241 -> 9.6 us, k=10 647 -> 71 us. Live settings: fresh +1-2%, rename/edit within noise (the speculative step is not purely GPU-bound). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit afe4c82d51f614ef8c97d536661a998c256c51ab on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/ggml-sycl.cpp | 143 +++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 02976b01a6f4..35b118c642ba 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -2971,6 +2971,138 @@ inline void ggml_sycl_op_argsort(ggml_backend_sycl_context & ctx, ggml_tensor * main_stream, ctx.device, ctx.pool()); } +// ARC-LAB: two-pass top-k for long rows (the MTP draft picks its tokens from the 248K-entry vocabulary: the one-group +// kernel above took ~550 us per call on the B580, one work-group on one Xe core plus a serial merge by one thread). +// Pass 1: up to 256 groups per row, each thread keeps a sorted top-k of a strided slice in registers, then the group +// merges its 256 lists pairwise in local memory (log2 steps of an O(k) merge). Pass 2: one group per row merges the +// per-group winners the same way. Ties keep the lower index. +namespace topk2 { +constexpr int WG = 256; + +template +static inline void insert(float (&v)[K], int (&ix)[K], const int k, const float val, const int idx) { + if (!(val > v[k - 1] || (val == v[k - 1] && idx < ix[k - 1] && ix[k - 1] >= 0))) { + return; + } + int pos = k - 1; + while (pos > 0 && (val > v[pos - 1] || (val == v[pos - 1] && idx < ix[pos - 1]))) { + v[pos] = v[pos - 1]; + ix[pos] = ix[pos - 1]; + pos--; + } + v[pos] = val; + ix[pos] = idx; +} + +// merge the sorted lists of threads [0, WG) in sv/si (stride k) down to thread 0; every thread calls it +template +static inline void tree_merge(const sycl::nd_item<2> & it, float * sv, int * si, const int k, const int tid) { + for (int s = WG / 2; s > 0; s >>= 1) { + float mv[K]; + int mi[K]; + if (tid < s) { + const float * av = sv + tid * k; + const int * ai = si + tid * k; + const float * bv = sv + (tid + s) * k; + const int * bi = si + (tid + s) * k; + int a = 0, b = 0; + for (int o = 0; o < k; ++o) { + const bool take_a = av[a] > bv[b] || (av[a] == bv[b] && (bi[b] < 0 || (ai[a] >= 0 && ai[a] < bi[b]))); + if (take_a) { + mv[o] = av[a]; + mi[o] = ai[a]; + ++a; + } else { + mv[o] = bv[b]; + mi[o] = bi[b]; + ++b; + } + } + } + it.barrier(sycl::access::fence_space::local_space); + if (tid < s) { + for (int o = 0; o < k; ++o) { + sv[tid * k + o] = mv[o]; + si[tid * k + o] = mi[o]; + } + } + it.barrier(sycl::access::fence_space::local_space); + } +} + +template +static void run(const float * src, int32_t * dst, const int64_t ncols, const int64_t nrows, const int k, + ggml_backend_sycl_context & ctx, dpct::queue_ptr q) { + const int nblk = (int) std::min(256, (ncols + 2047) / 2048); + ggml_sycl_pool_alloc cv(ctx.pool(), (size_t) nrows * nblk * k); + ggml_sycl_pool_alloc ci(ctx.pool(), (size_t) nrows * nblk * k); + float * cvp = cv.get(); + int * cip = ci.get(); + q->submit([&](sycl::handler & cgh) { + sycl::local_accessor sv(sycl::range<1>(WG * k), cgh); + sycl::local_accessor si(sycl::range<1>(WG * k), cgh); + cgh.parallel_for(sycl::nd_range<2>({ (size_t) nrows, (size_t) nblk * WG }, { 1, WG }), [=](sycl::nd_item<2> it) { + const int64_t row = it.get_group(0); + const int blk = it.get_group(1); + const int tid = it.get_local_id(1); + const float * r = src + row * ncols; + float v[K]; + int ix[K]; + for (int i = 0; i < k; ++i) { + v[i] = -FLT_MAX; + ix[i] = -1; + } + for (int64_t c = (int64_t) blk * WG + tid; c < ncols; c += (int64_t) nblk * WG) { + insert(v, ix, k, r[c], (int) c); + } + float * svp = sv.template get_multi_ptr().get(); + int * sip = si.template get_multi_ptr().get(); + for (int i = 0; i < k; ++i) { + svp[tid * k + i] = v[i]; + sip[tid * k + i] = ix[i]; + } + it.barrier(sycl::access::fence_space::local_space); + tree_merge(it, svp, sip, k, tid); + if (tid < k) { + cvp[(row * nblk + blk) * k + tid] = svp[tid]; + cip[(row * nblk + blk) * k + tid] = sip[tid]; + } + }); + }); + q->submit([&](sycl::handler & cgh) { + sycl::local_accessor sv(sycl::range<1>(WG * k), cgh); + sycl::local_accessor si(sycl::range<1>(WG * k), cgh); + cgh.parallel_for(sycl::nd_range<2>({ (size_t) nrows, (size_t) WG }, { 1, WG }), [=](sycl::nd_item<2> it) { + const int64_t row = it.get_group(0); + const int tid = it.get_local_id(1); + float v[K]; + int ix[K]; + for (int i = 0; i < k; ++i) { + v[i] = -FLT_MAX; + ix[i] = -1; + } + for (int c = tid; c < nblk * k; c += WG) { + const int id = cip[row * nblk * k + c]; + if (id >= 0) { + insert(v, ix, k, cvp[row * nblk * k + c], id); + } + } + float * svp = sv.template get_multi_ptr().get(); + int * sip = si.template get_multi_ptr().get(); + for (int i = 0; i < k; ++i) { + svp[tid * k + i] = v[i]; + sip[tid * k + i] = ix[i]; + } + it.barrier(sycl::access::fence_space::local_space); + tree_merge(it, svp, sip, k, tid); + if (tid < k) { + dst[row * k + tid] = sip[tid]; + } + }); + }); +} +} // namespace topk2 + static void ggml_sycl_op_top_k(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { const ggml_tensor * src0 = dst->src[0]; @@ -2992,6 +3124,17 @@ static void ggml_sycl_op_top_k(ggml_backend_sycl_context & ctx, ggml_tensor * ds GGML_ASSERT(k > 0 && k <= 32); GGML_ASSERT(k <= ncols); + static const bool old_topk = getenv("GGML_SYCL_TOPK_OLD") != nullptr; // ARC-LAB A/B switch + if (!old_topk && ncols >= 8192) { + if (k <= 8) { + topk2::run<8>(src0_dd, dst_dd, ncols, nrows, k, ctx, main_stream); + } else if (k <= 16) { + topk2::run<16>(src0_dd, dst_dd, ncols, nrows, k, ctx, main_stream); + } else { + topk2::run<32>(src0_dd, dst_dd, ncols, nrows, k, ctx, main_stream); + } + return; + } top_k_f32_sycl(src0_dd, dst_dd, ncols, nrows, k, main_stream); } From cd9381252efb11bd0f1e0a63fc67da724c6d32d2 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Mon, 28 Sep 2026 08:14:05 +1000 Subject: [PATCH 07/65] sycl: q4_0 decode attention on XMX via s8 x s4 DPAS (opt-in GGML_SYCL_FA_DEC_DPAS) Q.K uses intel_sub_group_i8_i4_matrix_mad_k32 (IGC provides it on Xe2 though the public extension only lists i4_i4_k64): the raw 16-byte q4_0 block of a key XOR 0x88888888 is the B operand, one key per lane, against int8 Q with one RNE scale per row and block (q4_0 byte order absorbed by pairing dims l and l+16 in A). P.V computes O^T = V^T P^T on f16 DPAS so each lane dequantizes its own key's V block from 5 dword loads. Token chunks of 8 let one kernel serve 1..32 (or =N) query tokens, so n-gram verify batches stop falling back to oneDNN + a full-KV f16 conversion. Per layer (Bonsai 27B shape, real cache layout) old -> new, us: 16K 1 tok 266 -> 123, 4 tok 457 -> 244, 16 tok 2217 -> 776; 48K 1 tok 768 -> 433, 4 tok 1295 -> 671, 16 tok 6629 -> 1915. FLASH_ATTN_EXT decode cases incl. 12/16/17/32 tokens and the permuted layout: 57/57. GGML_SYCL_FA_DEC_DPAS_PROBE skips phases. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit e5a4d8a8f7df182dd85d6d8b01e2d3c1692975b8 on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/fattn-dec.cpp | 316 ++++++++++++++++++++++++++++++- tests/test-backend-ops.cpp | 12 ++ 2 files changed, 324 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-sycl/fattn-dec.cpp b/ggml/src/ggml-sycl/fattn-dec.cpp index 72c90d05c3d9..11278051a5c0 100644 --- a/ggml/src/ggml-sycl/fattn-dec.cpp +++ b/ggml/src/ggml-sycl/fattn-dec.cpp @@ -681,11 +681,303 @@ static void fattn_dec_q4_0_xmx(const char * Q, const char * K, const char * V, c }); } + +// ARC-LAB DPAS kernel (GGML_SYCL_FA_DEC_DPAS=1): the q4_0 cache feeds the XMX units without any dequantization pass. +// Q.K: s8 x s4 DPAS (intel_sub_group_i8_i4_matrix_mad_k32, not in the public extension spec but provided by IGC on Xe2). +// A = Q rows as int8 (one RNE scale per row per 32-dim block, computed once per call), B = the raw 16 bytes of a key's +// q4_0 block XOR 0x88888888 (u4 n+8 -> s4 n), one key per lane. q4_0 byte i holds dims i and i+16, so A's k = 2i+h +// is dim i + 16h: lane l's A short = (Q[l], Q[l + 16]). S += int result * dq[row] * dk[key]. +// P.V: f16 DPAS (k16) with P from SLM (A layout) and V dequantized in registers by the subgroup that owns 16 head dims. +// Query tokens beyond NQ are handled by extra work-groups (grid dim 0 = sequence x token chunk), each reading the KV again. +namespace { +typedef short dp_short8 __attribute__((ext_vector_type(8))); +typedef int dp_int4 __attribute__((ext_vector_type(4))); +typedef int dp_int8 __attribute__((ext_vector_type(8))); +typedef float dp_float8 __attribute__((ext_vector_type(8))); +typedef unsigned dp_uint4 __attribute__((ext_vector_type(4))); +} +#ifdef __SYCL_DEVICE_ONLY__ +SYCL_EXTERNAL dp_int8 intel_sub_group_i8_i4_matrix_mad_k32(dp_short8 a, dp_int4 b, dp_int8 acc); +SYCL_EXTERNAL dp_float8 intel_sub_group_f16_f16_matrix_mad_k16(dp_short8 a, dp_int8 b, dp_float8 acc); +#else +inline dp_int8 intel_sub_group_i8_i4_matrix_mad_k32(dp_short8, dp_int4, dp_int8) { __builtin_unreachable(); } +inline dp_float8 intel_sub_group_f16_f16_matrix_mad_k16(dp_short8, dp_int8, dp_float8) { __builtin_unreachable(); } +#endif + +template +static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, const char * mask, float * dst, + float * parts, sycl::float2 * meta, float scale, int ne01, int ne02, int ne11, + int nkvh, int ne03, int64_t nb01, int64_t nb02, int64_t nb03, int64_t nb11, + int64_t nb12, int64_t nb13, int64_t nb21, int64_t nb22, int64_t nb23, int64_t nb31, + int64_t nb33, int ne33, int nsplit, int chunk, dpct::queue_ptr stream) { + constexpr int R = G * NQ; // query rows per KV head and token chunk + constexpr int RT = (R + 7) / 8; // 8-row DPAS tiles + constexpr int RP = RT * 8; + constexpr int TK = 16 * KG; // keys per tile (KG groups of 16, one key per lane) + constexpr int NB = DEC_D / QK4_0; // 8 blocks per row + constexpr int NSG = DEC_WG / 16; // 16 subgroups + constexpr int NCH = (RT + TC - 1) / TC; // row-tile chunks per key group in the Q.K phase + constexpr int RW = DEC_D / QK4_0 * (int) sizeof(block_q4_0) / 4; // 36 dwords per q4_0 row + constexpr int R16 = (R + 15) / 16 * 16; // P.V computes O^T = V^T P^T: rows are the 16-wide N side + constexpr int RT16 = R16 / 16; + constexpr int PS = TK + 16; // f16 row stride of P ([row][key]); 32-byte aligned rows, fewer bank conflicts + static_assert(DEC_D / 16 == NSG && RW == 36, "shape"); + const int nqc = (ne01 + NQ - 1) / NQ; // token chunks + static const int probe_env = getenv("GGML_SYCL_FA_DEC_DPAS_PROBE") ? atoi(getenv("GGML_SYCL_FA_DEC_DPAS_PROBE")) : 0; + const int probe = probe_env; // phase probe: bit0 skip Q.K, bit1 skip P.V, bit2 skip softmax + + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor sQ8(sycl::range<1>(RT * NB * 16), cgh); // [tile][block][lane] -> 8 rows + sycl::local_accessor sDQ(sycl::range<1>(RT * NB), cgh); // [tile][block] -> 8 rows + sycl::local_accessor sS(sycl::range<1>(RP * TK), cgh); // [row][key] + sycl::local_accessor sP(sycl::range<1>(R16 * PS / 16), cgh); // P f16 [row][key], rows of PS halves + sycl::local_accessor sM(sycl::range<1>(R16), cgh); + sycl::local_accessor sL(sycl::range<1>(R16), cgh); + sycl::local_accessor sA(sycl::range<1>(R16), cgh); + cgh.parallel_for( + sycl::nd_range<3>(sycl::range<3>((size_t) ne03 * nqc, nkvh, (size_t) nsplit * DEC_WG), sycl::range<3>(1, 1, DEC_WG)), + [=](sycl::nd_item<3> it) [[sycl::reqd_sub_group_size(16)]] { + const int tid = it.get_local_id(2); + const int split = it.get_group(2); + const int kvh = it.get_group(1); + const int seq = it.get_group(0) / nqc; + const int c0 = (it.get_group(0) % nqc) * NQ; // first query token of this chunk + auto sg = it.get_sub_group(); + const int w = tid / 16; + const int lane = tid % 16; + short * sQs = (short *) &sQ8[0]; + short * sPs = (short *) &sP[0]; + + // Q -> int8 per (row, block), RNE; padded rows / tokens past ne01 are zero + for (int e = tid; e < RP * NB; e += DEC_WG) { + const int r = e / NB, b = e % NB, c = c0 + r / G, g = r % G; + float x[QK4_0]; + float amax = 0.0f; + const bool real = r < R && c < ne01; + const float * qr = (const float *) (Q + seq * nb03 + (int64_t) c * nb01 + (int64_t) (kvh * G + g) * nb02) + b * QK4_0; +#pragma unroll + for (int i = 0; i < QK4_0; ++i) { + x[i] = real ? qr[i] * scale : 0.0f; + amax = sycl::fmax(amax, sycl::fabs(x[i])); + } + const float dq = amax / 127.0f; + const float iq = amax > 0.0f ? 127.0f / amax : 0.0f; + const int t = r / 8, m = r % 8; + ((float *) &sDQ[0])[(t * NB + b) * 8 + m] = dq; +#pragma unroll + for (int l = 0; l < 16; ++l) { + const int lo = (int) sycl::rint(x[l] * iq), hi = (int) sycl::rint(x[l + 16] * iq); + sQs[((t * NB + b) * 16 + l) * 8 + m] = (short) ((lo & 0xFF) | ((hi & 0xFF) << 8)); + } + } + for (int e = tid; e < R16 * PS; e += DEC_WG) { + sPs[e] = 0; // padded rows keep P = 0 + } + if (tid < R16) { + sM[tid] = DEC_MINF; + sL[tid] = 0.0f; + sA[tid] = 1.0f; + } + it.barrier(sycl::access::fence_space::local_space); + + const int k_begin = split * chunk; + const int k_end = sycl::min(k_begin + chunk, ne11); + const char * Kh = K + seq * nb13 + kvh * nb12; + const char * Vh = V + seq * nb23 + kvh * nb22; + const sycl::half * mrow = mask ? (const sycl::half *) (mask + (seq % ne33) * nb33) : nullptr; + + // O^T accumulators: subgroup w owns head dims 16 w .. +15 (two 8-dim tiles), lane = query row + dp_float8 o[RT16][2]; +#pragma unroll + for (int t = 0; t < RT16; ++t) { + o[t][0] = 0.0f; + o[t][1] = 0.0f; + } + // those dims are the (w % 2) nibbles of qs bytes 0..15 of V block w / 2 + const int vb = w / 2, vsh = (w % 2) * 4; + + for (int t0 = k_begin; t0 < k_end; t0 += TK) { + // ---- S = Q K^T: job = (key group, chunk of TC row tiles) + for (int job = w; job < ((probe & 1) ? 0 : KG * NCH); job += NSG) { + const int kg = job % KG, ch = job / KG; + const int j = t0 + kg * 16 + lane; + const bool jv = j < k_end; + const dp_uint4 * kr = (const dp_uint4 *) (Kh + (int64_t) (jv ? j : k_begin) * nb11); + uint32_t row[RW]; +#pragma unroll + for (int i = 0; i < RW / 4; ++i) { + const dp_uint4 v4 = kr[i]; + row[4 * i] = v4[0]; row[4 * i + 1] = v4[1]; row[4 * i + 2] = v4[2]; row[4 * i + 3] = v4[3]; + } + dp_float8 s[TC]; +#pragma unroll + for (int tt = 0; tt < TC; ++tt) { + s[tt] = 0.0f; + } +#pragma unroll + for (int b = 0; b < NB; ++b) { + // block b = bytes 18 b .. 18 b + 17: fp16 scale, then 16 bytes of nibbles + const int o0 = 18 * b; // even b: dword aligned; odd b: 2 bytes in + uint32_t sc, q0, q1, q2, q3; + if ((o0 & 3) == 0) { + const int wd = o0 / 4; + sc = row[wd] & 0xFFFFu; + q0 = (row[wd] >> 16) | (row[wd + 1] << 16); + q1 = (row[wd + 1] >> 16) | (row[wd + 2] << 16); + q2 = (row[wd + 2] >> 16) | (row[wd + 3] << 16); + q3 = (row[wd + 3] >> 16) | (row[wd + 4] << 16); + } else { + const int wd = o0 / 4; // scale in the high half of row[wd] + sc = row[wd] >> 16; + q0 = row[wd + 1]; q1 = row[wd + 2]; q2 = row[wd + 3]; q3 = row[wd + 4]; + } + const float dk = static_cast(sycl::bit_cast((uint16_t) sc)); + const dp_int4 bv = { (int) (q0 ^ 0x88888888u), (int) (q1 ^ 0x88888888u), + (int) (q2 ^ 0x88888888u), (int) (q3 ^ 0x88888888u) }; +#pragma unroll + for (int tt = 0; tt < TC; ++tt) { + const int t = ch * TC + tt; + if (t < RT) { + const dp_int8 ia = intel_sub_group_i8_i4_matrix_mad_k32(sQ8[(t * NB + b) * 16 + lane], bv, (dp_int8) (0)); + const dp_float8 sc8 = sDQ[t * NB + b] * dk; // one broadcast vector load per tile and block + s[tt] += __builtin_convertvector(ia, dp_float8) * sc8; + } + } + } +#pragma unroll + for (int tt = 0; tt < TC; ++tt) { + const int t = ch * TC + tt; + if (t < RT) { +#pragma unroll + for (int m = 0; m < 8; ++m) { + const int r = t * 8 + m, c = c0 + r / G; + if (r < R) { + const bool ok = jv && c < ne01; + const float mv = (ok && mrow) ? static_cast(mrow[((int64_t) c * nb31) / (int64_t) sizeof(sycl::half) + j]) : 0.0f; + sS[r * TK + kg * 16 + lane] = ok ? s[tt][m] + mv : -INFINITY; + } + } + } + } + } + it.barrier(sycl::access::fence_space::local_space); + + // ---- online softmax per row -> P (f16, DPAS A layout) + for (int r = w; r < ((probe & 4) ? 0 : R); r += NSG) { + float sv[KG]; + float mt = -INFINITY; +#pragma unroll + for (int i = 0; i < KG; ++i) { + sv[i] = sS[r * TK + i * 16 + lane]; + mt = sycl::fmax(mt, sv[i]); + } + mt = sycl::reduce_over_group(sg, mt, sycl::maximum()); + const float m_old = sM[r]; + const float m_new = sycl::fmax(m_old, mt); + float lt = 0.0f; +#pragma unroll + for (int i = 0; i < KG; ++i) { + const float pv = sycl::native::exp(sv[i] - m_new); + sPs[r * PS + i * 16 + lane] = (short) sycl::bit_cast(sycl::half(pv)); + lt += pv; + } + lt = sycl::reduce_over_group(sg, lt, sycl::plus()); + if (lane == 0) { + const float a = sycl::native::exp(m_old - m_new); + sA[r] = a; + sL[r] = sL[r] * a + lt; + sM[r] = m_new; + } + } + it.barrier(sycl::access::fence_space::local_space); + + // ---- O^T = diag(alpha) O^T + V^T P^T: A = V^T (8 dims x 16 keys, lane = key), B = P^T (16 keys x 16 rows) +#pragma unroll + for (int t = 0; t < RT16; ++t) { + const float a = sA[t * 16 + lane]; + o[t][0] *= a; + o[t][1] *= a; + } +#pragma unroll 1 + for (int kg = 0; kg < ((probe & 2) ? 0 : KG); ++kg) { + const int j = t0 + kg * 16 + lane; + const bool jv = j < k_end; + const uint32_t * vr = (const uint32_t *) (Vh + (int64_t) (jv ? j : k_begin) * nb21); + const int o0 = 18 * vb, wd = o0 / 4; // block start; even block: dword aligned, odd: 2 bytes in + const uint32_t w0 = vr[wd], w1 = vr[wd + 1], w2 = vr[wd + 2], w3 = vr[wd + 3], w4 = vr[wd + 4]; + uint32_t sc, q[4]; + if ((o0 & 3) == 0) { + sc = w0 & 0xFFFFu; + q[0] = (w0 >> 16) | (w1 << 16); q[1] = (w1 >> 16) | (w2 << 16); + q[2] = (w2 >> 16) | (w3 << 16); q[3] = (w3 >> 16) | (w4 << 16); + } else { + sc = w0 >> 16; + q[0] = w1; q[1] = w2; q[2] = w3; q[3] = w4; + } + const float dv = jv ? static_cast(sycl::bit_cast((uint16_t) sc)) : 0.0f; + dp_short8 va[2]; +#pragma unroll + for (int x = 0; x < 2; ++x) { +#pragma unroll + for (int m = 0; m < 8; ++m) { + const int byte = x * 8 + m; // dim 16 w + byte + const int nib = (q[byte / 4] >> (8 * (byte % 4) + vsh)) & 0xF; + va[x][m] = (short) sycl::bit_cast(sycl::half((float) (nib - 8) * dv)); + } + } +#pragma unroll + for (int t = 0; t < RT16; ++t) { + const dp_int8 pb = sP[((t * 16 + lane) * PS + kg * 16) / 16]; + o[t][0] = intel_sub_group_f16_f16_matrix_mad_k16(va[0], pb, o[t][0]); + o[t][1] = intel_sub_group_f16_f16_matrix_mad_k16(va[1], pb, o[t][1]); + } + } + it.barrier(sycl::access::fence_space::local_space); + } + + // write: lane = row r = 16 t + lane -> (token c, head kvh * G + g); dims 16 w + 8 x + m +#pragma unroll + for (int t = 0; t < RT16; ++t) { + const int r = t * 16 + lane, c = c0 + r / G; + if (r < R && c < ne01) { + const int64_t jdu = ((int64_t) seq * ne01 + c) * ne02 + kvh * G + r % G; + const float l = sL[r]; +#pragma unroll + for (int x = 0; x < 2; ++x) { +#pragma unroll + for (int m = 0; m < 8; ++m) { + const int d = w * 16 + x * 8 + m; + if (nsplit == 1) { + dst[jdu * DEC_D + d] = l > 0.0f ? o[t][x][m] / l : 0.0f; + } else { + parts[(jdu * nsplit + split) * DEC_D + d] = o[t][x][m]; + } + } + } + } + } + if (nsplit > 1 && tid < R && c0 + tid / G < ne01) { + const int64_t jdu = ((int64_t) seq * ne01 + c0 + tid / G) * ne02 + kvh * G + tid % G; + meta[jdu * nsplit + split] = sycl::float2(sM[tid], sL[tid]); + } + }); + }); +} + +static int fattn_dec_dpas_maxq() { // 0 = DPAS kernel off; else it serves 1 .. maxq query tokens + static const int v = [] { + const char * e = getenv("GGML_SYCL_FA_DEC_DPAS"); + return e ? atoi(e) : 0; + }(); + return v == 1 ? 32 : v; // =1 -> default cap +} + bool ggml_sycl_flash_attn_ext_dec_supported(const ggml_tensor * dst) { static const bool off = getenv("GGML_SYCL_FA_DEC_OFF") != nullptr; if (off) { return false; } + const int maxq = fattn_dec_dpas_maxq(); const ggml_tensor * Q = dst->src[0]; const ggml_tensor * K = dst->src[1]; const ggml_tensor * V = dst->src[2]; @@ -697,13 +989,14 @@ bool ggml_sycl_flash_attn_ext_dec_supported(const ggml_tensor * dst) { return K->type == GGML_TYPE_Q4_0 && V->type == GGML_TYPE_Q4_0 && Q->type == GGML_TYPE_F32 && K->ne[0] == DEC_D && V->ne[0] == DEC_D && Q->ne[0] == DEC_D && Q->ne[2] == 6 * K->ne[2] && V->ne[2] == K->ne[2] && - Q->ne[1] >= 1 && Q->ne[1] <= 4 && Q->ne[3] == K->ne[3] && V->ne[3] == K->ne[3] && // 5-8: TILE is faster (v3<6,8> 1172-1240 vs 1049 us @16K) + Q->ne[1] >= 1 && Q->ne[1] <= (maxq ? maxq : 4) && Q->ne[3] == K->ne[3] && V->ne[3] == K->ne[3] && // 5-8: TILE is faster (v3<6,8> 1172-1240 vs 1049 us @16K) !sinks && max_bias == 0.0f && softcap == 0.0f && (!mask || (mask->type == GGML_TYPE_F16 && mask->ne[2] == 1)) && Q->nb[0] == sizeof(float) && K->nb[0] == ggml_type_size(K->type) && V->nb[0] == ggml_type_size(V->type) && K->nb[1] % 4 == 0 && K->nb[2] % 4 == 0 && K->nb[3] % 4 == 0 && ((uintptr_t) K->data) % 4 == 0 && V->nb[1] % 4 == 0 && V->nb[2] % 4 == 0 && V->nb[3] % 4 == 0 && ((uintptr_t) V->data) % 4 == 0 && - Q->nb[1] % 16 == 0 && Q->nb[2] % 16 == 0 && Q->nb[3] % 16 == 0 && ((uintptr_t) Q->data) % 16 == 0; + Q->nb[1] % 16 == 0 && Q->nb[2] % 16 == 0 && Q->nb[3] % 16 == 0 && ((uintptr_t) Q->data) % 16 == 0 && + (!maxq || (K->nb[1] % 16 == 0 && K->nb[2] % 16 == 0 && K->nb[3] % 16 == 0 && ((uintptr_t) K->data) % 16 == 0)); } void ggml_sycl_flash_attn_ext_dec(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { @@ -716,8 +1009,10 @@ void ggml_sycl_flash_attn_ext_dec(ggml_backend_sycl_context & ctx, ggml_tensor * memcpy(&scale, (const float *) dst->op_params + 0, sizeof(float)); const int ne01 = Q->ne[1]; + const bool dpas = fattn_dec_dpas_maxq() > 0; const int nq = ne01 <= 1 ? 1 : ne01 <= 2 ? 2 : ne01 <= 4 ? 4 : 8; - const int tk = nq == 8 ? 32 : nq == 4 ? 64 : 128; // keys per tile (must match the instantiations below) + // keys per tile (must match the instantiations below); DPAS: 16 x KG + const int tk = dpas ? (nq == 1 ? 256 : nq == 2 ? 128 : nq == 4 ? 256 : 128) : nq == 8 ? 32 : nq == 4 ? 64 : 128; const int ne11 = K->ne[1]; // slices: enough work-groups to fill the GPU (~64 slices per KV head), at least one tile each @@ -740,11 +1035,24 @@ void ggml_sycl_flash_attn_ext_dec(ggml_backend_sycl_context & ctx, ggml_tensor * const int64_t nb33 = mask ? mask->nb[3] : 0; const int ne33 = mask ? (int) mask->ne[3] : 1; +#define FATTN_DEC_DPAS(NQ_, KG_, TC_) fattn_dec_q4_0_dpas<6, NQ_, KG_, TC_>((const char *) Q->data, (const char *) K->data, \ + (const char *) V->data, mdata, (float *) dst->data, parts.get(), meta.get(), scale, ne01, (int) Q->ne[2], ne11, \ + (int) K->ne[2], (int) Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], V->nb[1], V->nb[2], \ + V->nb[3], nb31, nb33, ne33, nsplit, chunk, stream) + if (dpas) { + switch (nq) { + case 1: FATTN_DEC_DPAS(1, 16, 1); break; + case 2: FATTN_DEC_DPAS(2, 8, 1); break; + case 4: FATTN_DEC_DPAS(4, 16, 3); break; + default: FATTN_DEC_DPAS(8, 8, 3); break; // 8-token chunks, one work-group row per chunk + } + } +#undef FATTN_DEC_DPAS #define FATTN_DEC_CALL(NQ_) fattn_dec_q4_0<6, NQ_, (NQ_ == 4 ? 64 : 128), (NQ_ != 4)>( /* PIPE: see v4 notes */(const char *) Q->data, (const char *) K->data, (const char *) V->data, \ mdata, (float *) dst->data, parts.get(), meta.get(), scale, ne01, (int) Q->ne[2], ne11, (int) K->ne[2], \ (int) Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], V->nb[1], V->nb[2], V->nb[3], nb31, \ nb33, ne33, nsplit, chunk, stream) - switch (nq) { + if (!dpas) switch (nq) { case 1: FATTN_DEC_CALL(1); break; case 2: FATTN_DEC_CALL(2); break; case 8: // 5-8 tokens (MTP depth 4+, short n-gram drafts): v3 kernel, 48 rows, 32-key tiles, ~59 KB SLM diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 603e0522f5a1..2d269079232e 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10339,6 +10339,15 @@ static std::vector> make_test_cases_eval() { } } test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 1024, 1, false, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)); + // ARC-LAB: DPAS decode kernel (GGML_SYCL_FA_DEC_DPAS): longer verify batches (token chunks) and the real cache layout + for (int64_t kv : {512, 4096}) { + for (int64_t nb : {12, 16, 17, 32}) { + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)); + } + for (int64_t nb : {1, 3, 4, 7, 16}) { + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, {0, 2, 1, 3})); + } + } // ARC-LAB: Intel prompt FA (Vulkan fa_pf_gqa): prompt batches at the Bonsai shape, f16 and q4_0 (f16 scratch), both layouts for (ggml_type t : {GGML_TYPE_F16, GGML_TYPE_Q4_0}) { for (int64_t nb : {16, 37, 64}) { @@ -10912,6 +10921,9 @@ static std::vector> make_test_cases_perf() { for (int64_t nb : {1, 4, 5, 8}) { test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)); } + for (int64_t nb : {1, 4, 16, 32}) { // real cache layout (DPAS kernel targets) + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, {0, 2, 1, 3})); + } } for (ggml_type t : {GGML_TYPE_Q4_0, GGML_TYPE_F16}) { // ARC-LAB: Bonsai 27B prompt batch at 16K history test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 16384, 512, true, false, 0, 0, GGML_PREC_F32, t, t)); From fa64caa57fc983cdde72de80086d1f13b16356a0 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Mon, 28 Sep 2026 08:14:05 +1000 Subject: [PATCH 08/65] sycl: MKL flash-attention softmax, one work-group per query row mkl_fa_online_softmax_chunk gave each query row to one work-item that walked the whole KV chunk twice, so neighbouring work-items read addresses chunk_size floats apart. Above the oneDNN KV cap this kernel was 72% of generation time (Bonsai 27B, 115K-token context: verify batches > 32 tokens and every new prompt). Now 256 work-items per row with coalesced reads and group reductions; same math. Bonsai 27B, live flags, 114.6K-token code context: new code 17.6 -> 44.0 t/s, edit 10.8 -> 27.5 t/s, identical output; short prompts unchanged. FLASH_ATTN_EXT with the MKL path forced (ONEDNN_MAX_KV=512): 82/84, the same two pre-existing prompt-batch failures as before. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit a122b2b8c814570438d42fb4ce87b21f638ed57e on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/fattn-mkl.cpp | 82 +++++++++++++------------------- 1 file changed, 34 insertions(+), 48 deletions(-) diff --git a/ggml/src/ggml-sycl/fattn-mkl.cpp b/ggml/src/ggml-sycl/fattn-mkl.cpp index 2d164a0840fd..d06ed4530c0e 100644 --- a/ggml/src/ggml-sycl/fattn-mkl.cpp +++ b/ggml/src/ggml-sycl/fattn-mkl.cpp @@ -126,79 +126,65 @@ static void mkl_fa_online_softmax_chunk( int64_t mask_row_stride, int mask_n_heads, float logit_softcap, int64_t wg_size) { - const int64_t wg = ((q_rows + wg_size - 1) / wg_size) * wg_size; - + // ARC-LAB: one work-group per query row (was one work-item per row walking the whole chunk: neighbouring + // work-items read addresses chunk_size floats apart, 72% of generation time at 115K context). Same math. + GGML_UNUSED(wg_size); + constexpr int SWG = 256; stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for(sycl::nd_range<1>(wg, wg_size), + cgh.parallel_for(sycl::nd_range<1>((size_t) q_rows * SWG, SWG), [=](sycl::nd_item<1> item) { - int jc_rel = item.get_global_id(0); - if (jc_rel >= q_rows) return; - int jc_abs = q0 + jc_rel; + const int jc_rel = item.get_group(0); + const int tid = item.get_local_id(0); + const int jc_abs = q0 + jc_rel; + auto grp = item.get_group(); const int gqa_group = jc_abs / n_queries; const int q_row = jc_abs % n_queries; - // Score buffers are tile-local (relative index). - const float * __restrict KQ_row = KQ_f32 - + jc_rel * (int64_t)chunk_size; - // Persistent accumulator is full-sized (absolute index). - float * __restrict vkq = VKQ_accum - + jc_abs * (int64_t)DV; + const float * __restrict KQ_row = KQ_f32 + jc_rel * (int64_t)chunk_size; + float * __restrict vkq = VKQ_accum + jc_abs * (int64_t)DV; const sycl::half * mask_h = nullptr; - int64_t m_stride = 0; if (mask_data) { - int m_head = (mask_n_heads > 1) - ? (kvh_head + gqa_group) : 0; - mask_h = mask_data + (int64_t)m_head * mask_head_stride; - m_stride = mask_row_stride; + const int m_head = (mask_n_heads > 1) ? (kvh_head + gqa_group) : 0; + mask_h = mask_data + (int64_t)m_head * mask_head_stride + (int64_t)q_row * mask_row_stride + chunk_start; } - - // Row-wise local maximum (softcap before mask) - float local_max = -1e30f; - for (int i = 0; i < chunk_size; i++) { + auto score = [&](int i) { float s = KQ_row[i]; if (logit_softcap != 0.0f) { s = logit_softcap * sycl::tanh(s); } if (mask_h) { - s += (float)mask_h[q_row * m_stride - + (chunk_start + i)]; + s += (float)mask_h[i]; } - if (s > local_max) local_max = s; - } + return s; + }; - // Rescale previous accumulator by exp(old_max - new_max) - float old_max = KQ_max[jc_abs]; - float new_max = (old_max > local_max) ? old_max : local_max; - float rescale = (old_max < -1e29f) ? 1.0f - : sycl::native::exp(old_max - new_max); + const float old_max = KQ_max[jc_abs]; // read by every item before item 0 updates it below + float local_max = -1e30f; + for (int i = tid; i < chunk_size; i += SWG) { + local_max = sycl::fmax(local_max, score(i)); + } + local_max = sycl::reduce_over_group(grp, local_max, sycl::maximum()); - for (int v = 0; v < DV; v++) { + const float new_max = (old_max > local_max) ? old_max : local_max; + const float rescale = (old_max < -1e29f) ? 1.0f : sycl::native::exp(old_max - new_max); + for (int v = tid; v < DV; v += SWG) { vkq[v] *= rescale; } - // Softmax and write S_f16 (tile-local index) + sycl::half * __restrict S_row = S_f16 + jc_rel * (int64_t)chunk_size; float local_sum = 0.0f; - sycl::half * __restrict S_row = S_f16 - + jc_rel * (int64_t)chunk_size; - - for (int i = 0; i < chunk_size; i++) { - float s = KQ_row[i]; - if (logit_softcap != 0.0f) { - s = logit_softcap * sycl::tanh(s); - } - if (mask_h) { - s += (float)mask_h[q_row * m_stride - + (chunk_start + i)]; - } - float val = sycl::native::exp(s - new_max); + for (int i = tid; i < chunk_size; i += SWG) { + const float val = sycl::native::exp(score(i) - new_max); S_row[i] = sycl::half(val); local_sum += val; } - - KQ_sum[jc_abs] = KQ_sum[jc_abs] * rescale + local_sum; - KQ_max[jc_abs] = new_max; + local_sum = sycl::reduce_over_group(grp, local_sum, sycl::plus()); + if (tid == 0) { + KQ_sum[jc_abs] = KQ_sum[jc_abs] * rescale + local_sum; + KQ_max[jc_abs] = new_max; + } }); }); } From bf5feb2bbd5c87a0082dae7733515c6ab6b43e0a Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Mon, 28 Sep 2026 08:14:05 +1000 Subject: [PATCH 09/65] sycl: GGML_SYCL_MMVQ_CHUNK_MAX, largest batch run as MMVQ column chunks Quantized mul_mat with 33+ columns fell through to dequantize-all-of-src0 + GEMM per call. For speculative-verify sized batches that is ruinous: Gemma 4 12B q4_0 with n-gram drafts of 33-64 tokens spent ~94% of GPU time in dequantize_row_q4_0_sycl_reorder. Default stays 4 x MMVQ_MAX_BATCH_SIZE (32); with 128, Gemma 4 12B + QAT assistant (ngram n-max 64): rename 79 -> 186 t/s, edit 57 -> 136. Bonsai 27B unchanged within noise. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit e8bd53439d5283999ee3202f59ccb79e0d878924 on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/ggml-sycl.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 35b118c642ba..d08c13b4b16f 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -4988,6 +4988,14 @@ static bool ggml_sycl_t2_mul_mat_try(ggml_backend_sycl_context & ctx, const ggml return true; } +// ARC-LAB: largest batch run as MMVQ column chunks instead of dequantize-all + GEMM (GGML_SYCL_MMVQ_CHUNK_MAX, default 32). +// A chunk re-reads the quantized weights; the GEMM path writes and reads the whole matrix as f16 per call, so chunks win +// for speculative-verify sized batches (Gemma 4 12B q4_0: n-gram drafts of 33+ tokens were ~94% dequantization). +static int64_t mmvq_chunk_max() { + static const int64_t v = getenv("GGML_SYCL_MMVQ_CHUNK_MAX") ? atoll(getenv("GGML_SYCL_MMVQ_CHUNK_MAX")) : 4 * MMVQ_MAX_BATCH_SIZE; + return v; +} + static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) { scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2); @@ -5037,7 +5045,7 @@ static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor if (!split && ggml_is_quantized(src0->type) && ggml_is_contiguous(src0) && src0->ne[2] == 1 && src0->ne[3] == 1 && src1->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F32 && src1->ne[2] == 1 && src1->ne[3] == 1 && src1->ne[1] > MMVQ_MAX_BATCH_SIZE && src1->nb[0] == sizeof(float) && - (src1->ne[1] <= 4 * MMVQ_MAX_BATCH_SIZE || (size_t) src0->ne[0] * src0->ne[1] * sizeof(float) > (1ull << 30))) { + (src1->ne[1] <= mmvq_chunk_max() || (size_t) src0->ne[0] * src0->ne[1] * sizeof(float) > (1ull << 30))) { for (int64_t c0 = 0; c0 < src1->ne[1]; c0 += MMVQ_MAX_BATCH_SIZE) { const int64_t nc = std::min(MMVQ_MAX_BATCH_SIZE, src1->ne[1] - c0); ggml_tensor src1_c = *src1; From 82d8626bf1cf4808bede27872b7d2b06da2c0b0f Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Mon, 28 Sep 2026 08:14:05 +1000 Subject: [PATCH 10/65] sycl: oneDNN / MKL flash attention only for packed interleaved K/V Reported from an Arc 140T and reproduced on a B580: FLASH_ATTN_EXT with quantized K/V in the permuted layout returned ERR ~1.2 for every KV length from 1024 up (ggml-org#27769). The failing test builds the cache view with a gap between tokens (2x the heads allocated, half viewed); the oneDNN SDPA and MKL dequant paths take the token stride from the head count. llama.cpp's own cache view is always packed, so real models were unaffected (Bonsai 27B perplexity at 4K: oneDNN 8.7273, MKL 8.7311, TILE 8.7330, and still 8.7273 with this guard). Interleaved K/V with a gap now falls back to TILE. FLASH_ATTN_EXT 4137/4137 (was 2 failing), plus a new q4_0 real-layout sweep (kv 1024..8192, nb 33/64/512) 29/29. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 226166a466753be0518e6192a0e81feda0017db6 on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/fattn-onednn.cpp | 9 +++++++++ ggml/src/ggml-sycl/fattn.cpp | 5 +++++ tests/test-backend-ops.cpp | 6 ++++++ 3 files changed, 20 insertions(+) diff --git a/ggml/src/ggml-sycl/fattn-onednn.cpp b/ggml/src/ggml-sycl/fattn-onednn.cpp index a501295192fb..caea33f23636 100644 --- a/ggml/src/ggml-sycl/fattn-onednn.cpp +++ b/ggml/src/ggml-sycl/fattn-onednn.cpp @@ -27,6 +27,15 @@ bool ggml_sycl_flash_attn_ext_onednn_supported(const ggml_tensor * dst) { const ggml_tensor * mask = dst->src[3]; const ggml_tensor * sinks = dst->src[4]; + // ARC-LAB: heads interleaved per token (llama.cpp's KV-cache view, nb[2] < nb[1]) must be packed: the SDPA / dequant + // paths derive the token stride from the head count. A gap between tokens (test-backend-ops kv_view + permute) read + // the wrong rows (ERR ~1.2); the real cache is always packed. See ggml-org#27769. + for (const ggml_tensor * t : {K, V}) { + if (t->nb[2] < t->nb[1] && t->nb[1] != t->nb[2] * t->ne[2]) { + return false; + } + } + // F16 KV: native SDPA at any KV length. // Non-F16: dequant to F16 then SDPA at prefill lengths. Only the // standard quantized KV cache types (Q4_0-Q8_0) and F32 are accepted diff --git a/ggml/src/ggml-sycl/fattn.cpp b/ggml/src/ggml-sycl/fattn.cpp index 9ccac7fa548e..b52e8a505975 100644 --- a/ggml/src/ggml-sycl/fattn.cpp +++ b/ggml/src/ggml-sycl/fattn.cpp @@ -172,6 +172,11 @@ static best_fattn_kernel ggml_sycl_get_best_fattn_kernel(const int device, const // (nb1 == H * ne0*2). Only pathological test strides like nb1=32 or // nb1=75 for ne0=40 fall through to TILE. bool kv_strides_ok = true; + for (const ggml_tensor * t : {K, V}) { // ARC-LAB: interleaved heads must be packed (see fattn-onednn.cpp) + if (t->nb[2] < t->nb[1] && t->nb[1] != t->nb[2] * t->ne[2]) { + kv_strides_ok = false; + } + } for (const ggml_tensor * t : {K, V}) { if (t->type == GGML_TYPE_F16 && t->nb[1] % (t->ne[0] * 2) != 0) { kv_strides_ok = false; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 2d269079232e..e0ef672b4012 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10348,6 +10348,12 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, {0, 2, 1, 3})); } } + // ARC-LAB: oneDNN / MKL prompt-FA correctness sweep in the real cache layout (a report showed kv=1025 / 4096 failing) + for (int64_t kv : {1024, 1025, 2048, 3072, 4095, 4096, 4097, 6144, 8192}) { + for (int64_t nb : {33, 64, 512}) { + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, {0, 2, 1, 3})); + } + } // ARC-LAB: Intel prompt FA (Vulkan fa_pf_gqa): prompt batches at the Bonsai shape, f16 and q4_0 (f16 scratch), both layouts for (ggml_type t : {GGML_TYPE_F16, GGML_TYPE_Q4_0}) { for (int64_t nb : {16, 37, 64}) { From 4210fca01cc2bc5f9b73afbbba2a396b7975a6fd Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Mon, 28 Sep 2026 08:29:38 +1000 Subject: [PATCH 11/65] sycl: XMX decode attention only on Xe2+, fall back with a warning elsewhere GGML_SYCL_FA_DEC_DPAS uses the 16-lane int8 x int4 DPAS builtin that only Xe2 and newer provide; on Alchemist / Xe-LPG it would fail at the first launch (like the T2 path did before 9c25b48). Now checked once against the device architecture, sharing ggml_sycl_device_is_xe2() with the T2 gate; GGML_SYCL_FA_DEC_DPAS_ANYGPU=1 skips the check. B580 unchanged: 4-token verify at 48K 671 us (1295 without). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit d4410763a512e558027d6255e28b2a788c823ba3 on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/fattn-dec.cpp | 11 ++++++++++- ggml/src/ggml-sycl/ptq1-t2.cpp | 11 ++++++++--- ggml/src/ggml-sycl/ptq1-t2.hpp | 2 ++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-sycl/fattn-dec.cpp b/ggml/src/ggml-sycl/fattn-dec.cpp index 11278051a5c0..1b4addf6117d 100644 --- a/ggml/src/ggml-sycl/fattn-dec.cpp +++ b/ggml/src/ggml-sycl/fattn-dec.cpp @@ -15,6 +15,7 @@ #include "common.hpp" #include "fattn-common.hpp" #include "fattn-dec.hpp" +#include "ptq1-t2.hpp" #include namespace { @@ -967,7 +968,15 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, static int fattn_dec_dpas_maxq() { // 0 = DPAS kernel off; else it serves 1 .. maxq query tokens static const int v = [] { const char * e = getenv("GGML_SYCL_FA_DEC_DPAS"); - return e ? atoi(e) : 0; + int n = e ? atoi(e) : 0; + // the kernel needs Xe2's 16-lane int8 x int4 DPAS; elsewhere fall back instead of failing at the first launch + if (n && !getenv("GGML_SYCL_FA_DEC_DPAS_ANYGPU") && !ggml_sycl_device_is_xe2()) { + GGML_LOG_WARN("%s: GGML_SYCL_FA_DEC_DPAS needs an Xe2 or newer GPU (Arc B-series, Lunar Lake, Panther Lake); " + "%s is not one, using the regular decode attention\n", __func__, + ggml_sycl_info().devices[ggml_sycl_get_device()].hw_info.name.c_str()); + n = 0; + } + return n; }(); return v == 1 ? 32 : v; // =1 -> default cap } diff --git a/ggml/src/ggml-sycl/ptq1-t2.cpp b/ggml/src/ggml-sycl/ptq1-t2.cpp index 6807a57c9f52..f7383ec5c502 100644 --- a/ggml/src/ggml-sycl/ptq1-t2.cpp +++ b/ggml/src/ggml-sycl/ptq1-t2.cpp @@ -64,15 +64,20 @@ void * scratch_get(sycl::queue & q, size_t bytes) { // the kernels use 16-lane DPAS (Xe2 and newer); Xe-LPG(+) / Alchemist XMX is 8 lanes and has no such kernels, so fall back // to the regular path with one warning instead of failing at the first launch. GGML_SYCL_PTQ1_T2_ANYGPU=1 skips the check. +bool ggml_sycl_device_is_xe2() { + const auto & hw = ggml_sycl_info().devices[ggml_sycl_get_device()].hw_info; + return hw.arch == gpu_arch::intel_gpu_bmg_g21 || hw.arch == gpu_arch::intel_gpu_bmg_g31 || + hw.arch == gpu_arch::intel_gpu_lnl_m || hw.arch == gpu_arch::intel_gpu_ptl_h || + hw.arch == gpu_arch::intel_gpu_ptl_u || hw.arch == gpu_arch::intel_gpu_wcl; +} + static bool t2_device_ok() { static const bool ok = [] { if (getenv("GGML_SYCL_PTQ1_T2_ANYGPU")) { return true; } const auto & hw = ggml_sycl_info().devices[ggml_sycl_get_device()].hw_info; - const bool xe2 = hw.arch == gpu_arch::intel_gpu_bmg_g21 || hw.arch == gpu_arch::intel_gpu_bmg_g31 || - hw.arch == gpu_arch::intel_gpu_lnl_m || hw.arch == gpu_arch::intel_gpu_ptl_h || - hw.arch == gpu_arch::intel_gpu_ptl_u || hw.arch == gpu_arch::intel_gpu_wcl; + const bool xe2 = ggml_sycl_device_is_xe2(); if (!xe2) { GGML_LOG_WARN("%s: GGML_SYCL_PTQ1_T2 needs an Xe2 or newer GPU (Arc B-series, Lunar Lake, Panther Lake); " "%s is not one, using the regular path\n", __func__, hw.name.c_str()); diff --git a/ggml/src/ggml-sycl/ptq1-t2.hpp b/ggml/src/ggml-sycl/ptq1-t2.hpp index 49ef2db1e705..0ee23de97c13 100644 --- a/ggml/src/ggml-sycl/ptq1-t2.hpp +++ b/ggml/src/ggml-sycl/ptq1-t2.hpp @@ -9,6 +9,8 @@ // GGML_SYCL_PTQ1_T2: "ffn" = ffn_gate/ffn_up/ffn_down weights, "all" = every PTQ1_0 weight (tests), unset = off bool ggml_sycl_t2_wants(const char * name, int64_t K, int64_t N); +// Xe2 or newer (16-lane DPAS incl. the int8 x int2 / int8 x int4 builtins): Arc B-series, Lunar Lake, Panther Lake +bool ggml_sycl_device_is_xe2(); // device bytes of the repacked weight: 2-bit codes [K/16, N] u32 + fp16 scales [K/128, N] size_t ggml_sycl_t2_bytes(int64_t K, int64_t N); // in place: data holds N rows of K/128 block_ptq1_0 (pq2 = false) or block_pq2_0 (pq2 = true) as uploaded, and at least From 03f06b0899516f73a26a92345f0ca9f16c81d911 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Mon, 28 Sep 2026 09:08:23 +1000 Subject: [PATCH 12/65] sycl: XMX q4_0 decode attention for Gemma 4 (head 512 x GQA 16 global, head 256 x GQA 2 sliding) The s8 x s4 DPAS decode kernel takes the head size as a template parameter (K read block by block, P.V subgroups own D/16 dims) and serves two more shapes when GGML_SYCL_FA_DEC_DPAS is on: Gemma 4's global layers (512, 1 KV x 16 Q heads, token chunks of 2) and sliding layers (256, 2 per KV head, chunks of 8). Splits target ~256 work-groups in all (a single KV head used to leave the GPU idle); combine step templated on D. Global-layer attention at 48K, us: 1 token 1114 (f16 TILE) -> 236, 4 tokens 2011 -> 635. Gemma 4 12B QAT + assistant, q4_0 KV, 128K context: short 92 -> 100 t/s new code; at 48K new code 47 -> 65, edit 44 -> 57 (f16 KV was 55 / 47). Bonsai shape unchanged (4-token verify at 48K 638 us). FLASH_ATTN_EXT: Gemma global 20/20, sliding 10/10, Bonsai 36/36. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit c759ddb2b75bd1660e883ca2d1c5d6b536b27ce7 on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/fattn-dec.cpp | 159 +++++++++++++++++++------------ tests/test-backend-ops.cpp | 18 ++++ 2 files changed, 116 insertions(+), 61 deletions(-) diff --git a/ggml/src/ggml-sycl/fattn-dec.cpp b/ggml/src/ggml-sycl/fattn-dec.cpp index 1b4addf6117d..964cf6a99327 100644 --- a/ggml/src/ggml-sycl/fattn-dec.cpp +++ b/ggml/src/ggml-sycl/fattn-dec.cpp @@ -705,7 +705,7 @@ inline dp_int8 intel_sub_group_i8_i4_matrix_mad_k32(dp_short8, dp_int4, dp_int inline dp_float8 intel_sub_group_f16_f16_matrix_mad_k16(dp_short8, dp_int8, dp_float8) { __builtin_unreachable(); } #endif -template +template static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, const char * mask, float * dst, float * parts, sycl::float2 * meta, float scale, int ne01, int ne02, int ne11, int nkvh, int ne03, int64_t nb01, int64_t nb02, int64_t nb03, int64_t nb11, @@ -715,14 +715,15 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, constexpr int RT = (R + 7) / 8; // 8-row DPAS tiles constexpr int RP = RT * 8; constexpr int TK = 16 * KG; // keys per tile (KG groups of 16, one key per lane) - constexpr int NB = DEC_D / QK4_0; // 8 blocks per row + constexpr int NB = D / QK4_0; // q4_0 blocks per row (8 at D 256, 16 at D 512) constexpr int NSG = DEC_WG / 16; // 16 subgroups constexpr int NCH = (RT + TC - 1) / TC; // row-tile chunks per key group in the Q.K phase - constexpr int RW = DEC_D / QK4_0 * (int) sizeof(block_q4_0) / 4; // 36 dwords per q4_0 row + constexpr int DPS = D / NSG; // head dims per subgroup in P.V (16 at D 256, 32 at D 512): all in one block + constexpr int XT = DPS / 8; // 8-dim output tiles per subgroup constexpr int R16 = (R + 15) / 16 * 16; // P.V computes O^T = V^T P^T: rows are the 16-wide N side constexpr int RT16 = R16 / 16; constexpr int PS = TK + 16; // f16 row stride of P ([row][key]); 32-byte aligned rows, fewer bank conflicts - static_assert(DEC_D / 16 == NSG && RW == 36, "shape"); + static_assert(D % 256 == 0 && DPS % 16 == 0 && DPS <= QK4_0, "shape"); const int nqc = (ne01 + NQ - 1) / NQ; // token chunks static const int probe_env = getenv("GGML_SYCL_FA_DEC_DPAS_PROBE") ? atoi(getenv("GGML_SYCL_FA_DEC_DPAS_PROBE")) : 0; const int probe = probe_env; // phase probe: bit0 skip Q.K, bit1 skip P.V, bit2 skip softmax @@ -787,15 +788,17 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, const char * Vh = V + seq * nb23 + kvh * nb22; const sycl::half * mrow = mask ? (const sycl::half *) (mask + (seq % ne33) * nb33) : nullptr; - // O^T accumulators: subgroup w owns head dims 16 w .. +15 (two 8-dim tiles), lane = query row - dp_float8 o[RT16][2]; + // O^T accumulators: subgroup w owns head dims DPS w .. +DPS-1 (XT 8-dim tiles), lane = query row + dp_float8 o[RT16][XT]; #pragma unroll for (int t = 0; t < RT16; ++t) { - o[t][0] = 0.0f; - o[t][1] = 0.0f; +#pragma unroll + for (int x = 0; x < XT; ++x) { + o[t][x] = 0.0f; + } } - // those dims are the (w % 2) nibbles of qs bytes 0..15 of V block w / 2 - const int vb = w / 2, vsh = (w % 2) * 4; + // those dims lie in V block (DPS w) / 32; dim d of the block = nibble (d / 16) of qs byte d % 16 + const int vb = (DPS * w) / QK4_0, vd0 = (DPS * w) % QK4_0; for (int t0 = k_begin; t0 < k_end; t0 += TK) { // ---- S = Q K^T: job = (key group, chunk of TC row tiles) @@ -803,13 +806,7 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, const int kg = job % KG, ch = job / KG; const int j = t0 + kg * 16 + lane; const bool jv = j < k_end; - const dp_uint4 * kr = (const dp_uint4 *) (Kh + (int64_t) (jv ? j : k_begin) * nb11); - uint32_t row[RW]; -#pragma unroll - for (int i = 0; i < RW / 4; ++i) { - const dp_uint4 v4 = kr[i]; - row[4 * i] = v4[0]; row[4 * i + 1] = v4[1]; row[4 * i + 2] = v4[2]; row[4 * i + 3] = v4[3]; - } + const uint32_t * kr = (const uint32_t *) (Kh + (int64_t) (jv ? j : k_begin) * nb11); dp_float8 s[TC]; #pragma unroll for (int tt = 0; tt < TC; ++tt) { @@ -819,18 +816,18 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, for (int b = 0; b < NB; ++b) { // block b = bytes 18 b .. 18 b + 17: fp16 scale, then 16 bytes of nibbles const int o0 = 18 * b; // even b: dword aligned; odd b: 2 bytes in + const int wd = o0 / 4; // the block's 18 bytes lie in these 5 dwords + const uint32_t k0 = kr[wd], k1 = kr[wd + 1], k2 = kr[wd + 2], k3 = kr[wd + 3], k4 = kr[wd + 4]; uint32_t sc, q0, q1, q2, q3; if ((o0 & 3) == 0) { - const int wd = o0 / 4; - sc = row[wd] & 0xFFFFu; - q0 = (row[wd] >> 16) | (row[wd + 1] << 16); - q1 = (row[wd + 1] >> 16) | (row[wd + 2] << 16); - q2 = (row[wd + 2] >> 16) | (row[wd + 3] << 16); - q3 = (row[wd + 3] >> 16) | (row[wd + 4] << 16); + sc = k0 & 0xFFFFu; + q0 = (k0 >> 16) | (k1 << 16); + q1 = (k1 >> 16) | (k2 << 16); + q2 = (k2 >> 16) | (k3 << 16); + q3 = (k3 >> 16) | (k4 << 16); } else { - const int wd = o0 / 4; // scale in the high half of row[wd] - sc = row[wd] >> 16; - q0 = row[wd + 1]; q1 = row[wd + 2]; q2 = row[wd + 3]; q3 = row[wd + 4]; + sc = k0 >> 16; // scale in the high half of k0 + q0 = k1; q1 = k2; q2 = k3; q3 = k4; } const float dk = static_cast(sycl::bit_cast((uint16_t) sc)); const dp_int4 bv = { (int) (q0 ^ 0x88888888u), (int) (q1 ^ 0x88888888u), @@ -896,8 +893,10 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, #pragma unroll for (int t = 0; t < RT16; ++t) { const float a = sA[t * 16 + lane]; - o[t][0] *= a; - o[t][1] *= a; +#pragma unroll + for (int x = 0; x < XT; ++x) { + o[t][x] *= a; + } } #pragma unroll 1 for (int kg = 0; kg < ((probe & 2) ? 0 : KG); ++kg) { @@ -916,21 +915,24 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, q[0] = w1; q[1] = w2; q[2] = w3; q[3] = w4; } const float dv = jv ? static_cast(sycl::bit_cast((uint16_t) sc)) : 0.0f; - dp_short8 va[2]; + dp_short8 va[XT]; #pragma unroll - for (int x = 0; x < 2; ++x) { + for (int x = 0; x < XT; ++x) { #pragma unroll for (int m = 0; m < 8; ++m) { - const int byte = x * 8 + m; // dim 16 w + byte - const int nib = (q[byte / 4] >> (8 * (byte % 4) + vsh)) & 0xF; + const int d = vd0 + x * 8 + m; // dim within block vb + const int byte = d % 16, sh = (d / 16) * 4; + const int nib = (q[byte / 4] >> (8 * (byte % 4) + sh)) & 0xF; va[x][m] = (short) sycl::bit_cast(sycl::half((float) (nib - 8) * dv)); } } #pragma unroll for (int t = 0; t < RT16; ++t) { const dp_int8 pb = sP[((t * 16 + lane) * PS + kg * 16) / 16]; - o[t][0] = intel_sub_group_f16_f16_matrix_mad_k16(va[0], pb, o[t][0]); - o[t][1] = intel_sub_group_f16_f16_matrix_mad_k16(va[1], pb, o[t][1]); +#pragma unroll + for (int x = 0; x < XT; ++x) { + o[t][x] = intel_sub_group_f16_f16_matrix_mad_k16(va[x], pb, o[t][x]); + } } } it.barrier(sycl::access::fence_space::local_space); @@ -944,14 +946,14 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, const int64_t jdu = ((int64_t) seq * ne01 + c) * ne02 + kvh * G + r % G; const float l = sL[r]; #pragma unroll - for (int x = 0; x < 2; ++x) { + for (int x = 0; x < XT; ++x) { #pragma unroll for (int m = 0; m < 8; ++m) { - const int d = w * 16 + x * 8 + m; + const int d = w * DPS + x * 8 + m; if (nsplit == 1) { - dst[jdu * DEC_D + d] = l > 0.0f ? o[t][x][m] / l : 0.0f; + dst[jdu * D + d] = l > 0.0f ? o[t][x][m] / l : 0.0f; } else { - parts[(jdu * nsplit + split) * DEC_D + d] = o[t][x][m]; + parts[(jdu * nsplit + split) * D + d] = o[t][x][m]; } } } @@ -996,8 +998,10 @@ bool ggml_sycl_flash_attn_ext_dec_supported(const ggml_tensor * dst) { memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float)); memcpy(&softcap, (const float *) dst->op_params + 2, sizeof(float)); return K->type == GGML_TYPE_Q4_0 && V->type == GGML_TYPE_Q4_0 && Q->type == GGML_TYPE_F32 && - K->ne[0] == DEC_D && V->ne[0] == DEC_D && Q->ne[0] == DEC_D && - Q->ne[2] == 6 * K->ne[2] && V->ne[2] == K->ne[2] && + // Bonsai 2 27B (head 256, 6 query heads per KV head); with the DPAS kernel also Gemma 4's global layers (512, 16) + ((K->ne[0] == DEC_D && Q->ne[2] == 6 * K->ne[2]) || (maxq && K->ne[0] == 512 && Q->ne[2] == 16 * K->ne[2]) || + (maxq && K->ne[0] == DEC_D && Q->ne[2] == 2 * K->ne[2])) && // Gemma 4 sliding layers (256, 2) + V->ne[0] == K->ne[0] && Q->ne[0] == K->ne[0] && V->ne[2] == K->ne[2] && Q->ne[1] >= 1 && Q->ne[1] <= (maxq ? maxq : 4) && Q->ne[3] == K->ne[3] && V->ne[3] == K->ne[3] && // 5-8: TILE is faster (v3<6,8> 1172-1240 vs 1049 us @16K) !sinks && max_bias == 0.0f && softcap == 0.0f && (!mask || (mask->type == GGML_TYPE_F16 && mask->ne[2] == 1)) && @@ -1019,13 +1023,17 @@ void ggml_sycl_flash_attn_ext_dec(ggml_backend_sycl_context & ctx, ggml_tensor * const int ne01 = Q->ne[1]; const bool dpas = fattn_dec_dpas_maxq() > 0; - const int nq = ne01 <= 1 ? 1 : ne01 <= 2 ? 2 : ne01 <= 4 ? 4 : 8; + const bool d512 = K->ne[0] == 512; // Gemma 4 global layers: DPAS kernel only, token chunks of 2 (16 heads x 2 rows) + const bool g2 = !d512 && Q->ne[2] == 2 * K->ne[2]; // Gemma 4 sliding layers: DPAS kernel only, token chunks of 8 + const int nq = d512 ? (ne01 <= 1 ? 1 : 2) : g2 ? (ne01 <= 1 ? 1 : 8) : ne01 <= 1 ? 1 : ne01 <= 2 ? 2 : ne01 <= 4 ? 4 : 8; // keys per tile (must match the instantiations below); DPAS: 16 x KG - const int tk = dpas ? (nq == 1 ? 256 : nq == 2 ? 128 : nq == 4 ? 256 : 128) : nq == 8 ? 32 : nq == 4 ? 64 : 128; + const int tk = (d512 || g2) ? (nq == 1 ? 256 : 128) + : dpas ? (nq == 1 ? 256 : nq == 2 ? 128 : nq == 4 ? 256 : 128) : nq == 8 ? 32 : nq == 4 ? 64 : 128; const int ne11 = K->ne[1]; - // slices: enough work-groups to fill the GPU (~64 slices per KV head), at least one tile each - static const int target = getenv("GGML_SYCL_FA_DEC_SPLITS") ? atoi(getenv("GGML_SYCL_FA_DEC_SPLITS")) : 64; + // slices: enough work-groups to fill the GPU (~64 slices per KV head, ~256 in all), at least one tile each + static const int target_env = getenv("GGML_SYCL_FA_DEC_SPLITS") ? atoi(getenv("GGML_SYCL_FA_DEC_SPLITS")) : 0; + const int target = target_env ? target_env : std::max(64, 256 / std::max(1, (int) K->ne[2])); const int ntiles = (ne11 + tk - 1) / tk; int nsplit = std::max(1, std::min(target, ntiles)); const int chunk = ((ntiles + nsplit - 1) / nsplit) * tk; @@ -1044,11 +1052,33 @@ void ggml_sycl_flash_attn_ext_dec(ggml_backend_sycl_context & ctx, ggml_tensor * const int64_t nb33 = mask ? mask->nb[3] : 0; const int ne33 = mask ? (int) mask->ne[3] : 1; -#define FATTN_DEC_DPAS(NQ_, KG_, TC_) fattn_dec_q4_0_dpas<6, NQ_, KG_, TC_>((const char *) Q->data, (const char *) K->data, \ +#define FATTN_DEC_DPAS(NQ_, KG_, TC_) fattn_dec_q4_0_dpas((const char *) Q->data, (const char *) K->data, \ (const char *) V->data, mdata, (float *) dst->data, parts.get(), meta.get(), scale, ne01, (int) Q->ne[2], ne11, \ (int) K->ne[2], (int) Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], V->nb[1], V->nb[2], \ V->nb[3], nb31, nb33, ne33, nsplit, chunk, stream) - if (dpas) { + if (d512) { +#define FATTN_DEC_DPAS512(NQ_, KG_, TC_) fattn_dec_q4_0_dpas<512, 16, NQ_, KG_, TC_>((const char *) Q->data, (const char *) K->data, \ + (const char *) V->data, mdata, (float *) dst->data, parts.get(), meta.get(), scale, ne01, (int) Q->ne[2], ne11, \ + (int) K->ne[2], (int) Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], V->nb[1], V->nb[2], \ + V->nb[3], nb31, nb33, ne33, nsplit, chunk, stream) + if (nq == 1) { + FATTN_DEC_DPAS512(1, 16, 2); // 16 rows = 2 tiles, one chunk of 2 per key group + } else { + FATTN_DEC_DPAS512(2, 8, 2); // 32 rows = 4 tiles, 2 chunks x 8 key groups + } +#undef FATTN_DEC_DPAS512 + } else if (g2) { +#define FATTN_DEC_DPASG2(NQ_, KG_, TC_) fattn_dec_q4_0_dpas((const char *) Q->data, (const char *) K->data, \ + (const char *) V->data, mdata, (float *) dst->data, parts.get(), meta.get(), scale, ne01, (int) Q->ne[2], ne11, \ + (int) K->ne[2], (int) Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], V->nb[1], V->nb[2], \ + V->nb[3], nb31, nb33, ne33, nsplit, chunk, stream) + if (nq == 1) { + FATTN_DEC_DPASG2(1, 16, 1); // 2 rows (1 tile) + } else { + FATTN_DEC_DPASG2(8, 8, 1); // 16 rows = 2 tiles, 8 key groups x 2 + } +#undef FATTN_DEC_DPASG2 + } else if (dpas) { switch (nq) { case 1: FATTN_DEC_DPAS(1, 16, 1); break; case 2: FATTN_DEC_DPAS(2, 8, 1); break; @@ -1061,7 +1091,7 @@ void ggml_sycl_flash_attn_ext_dec(ggml_backend_sycl_context & ctx, ggml_tensor * mdata, (float *) dst->data, parts.get(), meta.get(), scale, ne01, (int) Q->ne[2], ne11, (int) K->ne[2], \ (int) Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], V->nb[1], V->nb[2], V->nb[3], nb31, \ nb33, ne33, nsplit, chunk, stream) - if (!dpas) switch (nq) { + if (!dpas && !d512 && !g2) switch (nq) { case 1: FATTN_DEC_CALL(1); break; case 2: FATTN_DEC_CALL(2); break; case 8: // 5-8 tokens (MTP depth 4+, short n-gram drafts): v3 kernel, 48 rows, 32-key tiles, ~59 KB SLM @@ -1093,19 +1123,26 @@ void ggml_sycl_flash_attn_ext_dec(ggml_backend_sycl_context & ctx, ggml_tensor * #undef FATTN_DEC_CALL if (nsplit > 1) { - const int D = DEC_D; - const sycl::range<3> grid(Q->ne[3], Q->ne[2], (size_t) ne01 * D); - const size_t nbytes_shared = nsplit * sizeof(sycl::float2); - float * parts_p = parts.get(); - sycl::float2 * meta_p = meta.get(); - float * dst_p = (float *) dst->data; - stream->submit([&](sycl::handler & cgh) { - sycl::local_accessor lm(sycl::range<1>(nbytes_shared), cgh); - cgh.parallel_for(sycl::nd_range<3>(grid, sycl::range<3>(1, 1, D)), - [=](sycl::nd_item<3>) [[sycl::reqd_sub_group_size(16)]] { - flash_attn_combine_results(parts_p, meta_p, dst_p, nsplit, - lm.get_multi_ptr().get()); - }); - }); + auto combine = [&](auto dk) { + constexpr int D = decltype(dk)::value; + const sycl::range<3> grid(Q->ne[3], Q->ne[2], (size_t) ne01 * D); + const size_t nbytes_shared = nsplit * sizeof(sycl::float2); + float * parts_p = parts.get(); + sycl::float2 * meta_p = meta.get(); + float * dst_p = (float *) dst->data; + stream->submit([&](sycl::handler & cgh) { + sycl::local_accessor lm(sycl::range<1>(nbytes_shared), cgh); + cgh.parallel_for(sycl::nd_range<3>(grid, sycl::range<3>(1, 1, D)), + [=](sycl::nd_item<3>) [[sycl::reqd_sub_group_size(16)]] { + flash_attn_combine_results(parts_p, meta_p, dst_p, nsplit, + lm.get_multi_ptr().get()); + }); + }); + }; + if (d512) { + combine(std::integral_constant{}); + } else { + combine(std::integral_constant{}); + } } } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index e0ef672b4012..df65d5dc6230 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10348,6 +10348,19 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, {0, 2, 1, 3})); } } + // ARC-LAB: Gemma 4 global layers on the DPAS decode kernel (head 512, 1 KV head x 16 query heads, q4_0 cache) + for (int64_t kv : {512, 4096}) { + for (int64_t nb : {1, 2, 3, 4, 8}) { + test_cases.emplace_back(new test_flash_attn_ext(512, 512, 1, {16, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)); + test_cases.emplace_back(new test_flash_attn_ext(512, 512, 1, {16, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, {0, 2, 1, 3})); + } + } + // ARC-LAB: Gemma 4 sliding layers on the DPAS decode kernel (head 256, 8 KV heads x 2 query heads, q4_0 cache) + for (int64_t kv : {512, 1024}) { + for (int64_t nb : {1, 2, 5, 8, 12}) { + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 8, {2, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, {0, 2, 1, 3})); + } + } // ARC-LAB: oneDNN / MKL prompt-FA correctness sweep in the real cache layout (a report showed kv=1025 / 4096 failing) for (int64_t kv : {1024, 1025, 2048, 3072, 4095, 4096, 4097, 6144, 8192}) { for (int64_t nb : {33, 64, 512}) { @@ -10927,6 +10940,11 @@ static std::vector> make_test_cases_perf() { for (int64_t nb : {1, 4, 5, 8}) { test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)); } + for (int64_t nb : {1, 4}) { // ARC-LAB: Gemma 4 global layers, q4_0 vs f16 cache + for (ggml_type t : {GGML_TYPE_Q4_0, GGML_TYPE_F16}) { + test_cases.emplace_back(new test_flash_attn_ext(512, 512, 1, {16, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, t, t, {0, 2, 1, 3})); + } + } for (int64_t nb : {1, 4, 16, 32}) { // real cache layout (DPAS kernel targets) test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, {0, 2, 1, 3})); } From 76067c4fa8697401e60edae6d9af416e5e745e35 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Mon, 28 Sep 2026 11:53:11 +1000 Subject: [PATCH 13/65] sycl: reordered q4_0 small-batch GEMM on XMX via s8 x s4 DPAS (opt-in GGML_SYCL_Q4_0_DPAS) Verify batches of 5..256 columns on q4_0 weights used MMVQ column chunks (weights re-read per 8 columns) or dequantize-everything + GEMM. The new kernel streams each weight row once for up to 64 columns (chunks of 64 beyond): one lane per weight row, B = the row's 16 q4_0 bytes XOR 0x88888888 through intel_sub_group_i8_i4_matrix_mad_k32, A = activations quantized once per call to int8 (per row and 32-block, elements l / l+16 paired). Xe2 only (ggml_sycl_device_is_xe2). GGML_SYCL_Q4_0_DPAS_MIN / _MAX (5 / 256). 15360x3840 q4_0: 8 columns 150 -> 81 us, 16 columns 292 -> 89 us. Gemma 4 12B QAT + assistant: rename 188 -> 275 t/s, edit 138 -> 168 (ngram n-max 96), new code unchanged, rename/edit outputs byte-identical. MUL_MAT q4_0 Gemma shapes 1..256 columns: 20/20. Also Gemma 4 q4_0 mat-mul perf cases. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 03594d53c908bc0d73af7916e4c756261d3aa6af on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/ggml-sycl.cpp | 17 +++++ ggml/src/ggml-sycl/mmvq.cpp | 117 +++++++++++++++++++++++++++++++ ggml/src/ggml-sycl/mmvq.hpp | 3 + tests/test-backend-ops.cpp | 12 ++++ 4 files changed, 149 insertions(+) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index d08c13b4b16f..aa5761b5aa12 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -5040,6 +5040,23 @@ static void ggml_sycl_mul_mat(ggml_backend_sycl_context & ctx, const ggml_tensor min_compute_capability = ggml_sycl_info().devices[ctx.device].cc; } + // ARC-LAB: reordered q4_0 with 5..64 columns on XMX (weights streamed once for all columns), GGML_SYCL_Q4_0_DPAS=1 + { + static const bool q4d_on = getenv("GGML_SYCL_Q4_0_DPAS") && atoi(getenv("GGML_SYCL_Q4_0_DPAS")) != 0 && + ggml_sycl_device_is_xe2(); + static const int q4d_min = getenv("GGML_SYCL_Q4_0_DPAS_MIN") ? atoi(getenv("GGML_SYCL_Q4_0_DPAS_MIN")) : 5; + static const int q4d_max = getenv("GGML_SYCL_Q4_0_DPAS_MAX") ? atoi(getenv("GGML_SYCL_Q4_0_DPAS_MAX")) : 256; + const auto * ex = static_cast(src0->extra); + if (q4d_on && !split && src0->type == GGML_TYPE_Q4_0 && ex && ex->optimized_feature.reorder && + ggml_is_contiguous(src0) && src0->ne[2] == 1 && src0->ne[3] == 1 && src1->type == GGML_TYPE_F32 && + dst->type == GGML_TYPE_F32 && src1->ne[2] == 1 && src1->ne[3] == 1 && src1->nb[0] == sizeof(float) && + src1->ne[1] >= q4d_min && src1->ne[1] <= q4d_max && ggml_is_contiguous(dst) && + ggml_sycl_q4_0_dpas_gemm(ctx, src0->data, (const float *) src1->data, src1->nb[1] / (int64_t) sizeof(float), + (float *) dst->data, (int) src0->ne[1], (int) src0->ne[0], (int) src1->ne[1], ctx.stream())) { + return; + } + } + // Small batches (e.g. speculative verify) on quantized weights: run MMVQ on chunks of columns // instead of dequantizing all of src0 to fp32 (the output matrix is ~5 GB as fp32). if (!split && ggml_is_quantized(src0->type) && ggml_is_contiguous(src0) && src0->ne[2] == 1 && src0->ne[3] == 1 && diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cpp index 4e96f068d24c..0644c622e4a7 100644 --- a/ggml/src/ggml-sycl/mmvq.cpp +++ b/ggml/src/ggml-sycl/mmvq.cpp @@ -4016,3 +4016,120 @@ bool ggml_sycl_mul_mat_vec_q_glu_reorder(enum ggml_type src0_type, enum ggml_glu return false; } } + +// ARC-LAB: reordered q4_0 x f32 for 5..64 columns (speculative verify / n-gram drafts) on XMX via the s8 x s4 DPAS builtin +// (intel_sub_group_i8_i4_matrix_mad_k32, Xe2). The MMVQ column kernels re-read the weights per 8 columns and the dequantize +// + GEMM path converts the whole matrix; here each weight row streams once for all columns. One lane per weight row: B = +// the row's 16 q4_0 bytes of a block XOR 0x88888888 (u4 n+8 -> s4 n), A = 8 activation rows as int8 (per row and 32-block +// RNE scale; q4_0 byte i holds elements i and i+16, so A's k = 2i+h pairs elements l and l+16). GGML_SYCL_Q4_0_DPAS=1. +namespace { +typedef short q4d_short8 __attribute__((ext_vector_type(8))); +typedef int q4d_int4 __attribute__((ext_vector_type(4))); +typedef int q4d_int8 __attribute__((ext_vector_type(8))); +typedef float q4d_float8 __attribute__((ext_vector_type(8))); +typedef unsigned q4d_uint4 __attribute__((ext_vector_type(4))); +} +#ifdef __SYCL_DEVICE_ONLY__ +SYCL_EXTERNAL q4d_int8 intel_sub_group_i8_i4_matrix_mad_k32(q4d_short8 a, q4d_int4 b, q4d_int8 acc); +#else +inline q4d_int8 intel_sub_group_i8_i4_matrix_mad_k32(q4d_short8, q4d_int4, q4d_int8) { __builtin_unreachable(); } +#endif + +template +static void q4_0_dpas_gemm_launch(const void * vx, const q4d_short8 * xq, const q4d_float8 * xd, float * dst, const int nrows, + const int ncols, const int ntok, dpct::queue_ptr stream) { + const int NB = ncols / QK4_0; + constexpr int WG = 256; + const size_t nsg = (size_t) nrows / 16; + const size_t gsz = (nsg + WG / 16 - 1) / (WG / 16) * WG; + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<1>(gsz, WG), [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(16)]] { + const auto sg = it.get_sub_group(); + const int lane = sg.get_local_linear_id(); + const int sgi = (int) (it.get_group(0) * (WG / 16) + sg.get_group_linear_id()); + if (sgi * 16 >= nrows) { + return; + } + const int n = sgi * 16 + lane; // this lane's weight row + const uint8_t * qs = (const uint8_t *) vx + (size_t) n * NB * (QK4_0 / 2); + const sycl::half * dw = (const sycl::half *) ((const uint8_t *) vx + (size_t) ncols / 2 * nrows) + (size_t) n * NB; + q4d_float8 acc[MT]; +#pragma unroll + for (int t = 0; t < MT; ++t) { + acc[t] = 0.0f; + } + for (int b = 0; b < NB; ++b) { + const q4d_uint4 q = *(const q4d_uint4 *) (qs + b * (QK4_0 / 2)); + const q4d_int4 bv = { (int) (q[0] ^ 0x88888888u), (int) (q[1] ^ 0x88888888u), + (int) (q[2] ^ 0x88888888u), (int) (q[3] ^ 0x88888888u) }; + const float d = static_cast(dw[b]); +#pragma unroll + for (int t = 0; t < MT; ++t) { + const q4d_int8 ia = intel_sub_group_i8_i4_matrix_mad_k32(xq[((size_t) t * NB + b) * 16 + lane], bv, (q4d_int8) (0)); + acc[t] += __builtin_convertvector(ia, q4d_float8) * (xd[(size_t) t * NB + b] * d); + } + } +#pragma unroll + for (int t = 0; t < MT; ++t) { +#pragma unroll + for (int m = 0; m < 8; ++m) { + const int tok = t * 8 + m; + if (tok < ntok) { + dst[(size_t) tok * nrows + n] = acc[t][m]; + } + } + } + }); + }); +} + +bool ggml_sycl_q4_0_dpas_gemm(ggml_backend_sycl_context & ctx, const void * vx, const float * x, int64_t x_stride, + float * dst, const int nrows, const int ncols, const int ntok, dpct::queue_ptr stream) { + if (ntok < 1 || nrows % 16 != 0 || ncols % QK4_0 != 0) { + return false; + } + if (ntok > 64) { // longer n-gram verify batches: chunks of 64 columns, the weights stream once per chunk + for (int c0 = 0; c0 < ntok; c0 += 64) { + if (!ggml_sycl_q4_0_dpas_gemm(ctx, vx, x + (size_t) c0 * x_stride, x_stride, dst + (size_t) c0 * nrows, nrows, + ncols, std::min(64, ntok - c0), stream)) { + return false; + } + } + return true; + } + const int MT = (ntok + 7) / 8, NB = ncols / QK4_0; + ggml_sycl_pool_alloc xq(ctx.pool(), (size_t) MT * NB * 16); + ggml_sycl_pool_alloc xd(ctx.pool(), (size_t) MT * NB); + q4d_short8 * xq_p = xq.get(); + q4d_float8 * xd_p = xd.get(); + // activations -> int8 A layout: [tile][block][lane] -> 8 rows, one scale per row and block; padded rows are zero + stream->parallel_for(sycl::range<1>((size_t) MT * 8 * NB), [=](sycl::item<1> it) { + const int e = (int) it[0], m = e / NB, b = e % NB, t = m / 8, mm = m % 8; + float v[QK4_0]; + float amax = 0.0f; +#pragma unroll + for (int i = 0; i < QK4_0; ++i) { + v[i] = m < ntok ? x[(size_t) m * x_stride + b * QK4_0 + i] : 0.0f; + amax = sycl::fmax(amax, sycl::fabs(v[i])); + } + const float iq = amax > 0.0f ? 127.0f / amax : 0.0f; + ((float *) &xd_p[(size_t) t * NB + b])[mm] = amax / 127.0f; + short * row = (short *) &xq_p[((size_t) t * NB + b) * 16]; +#pragma unroll + for (int l = 0; l < 16; ++l) { + const int lo = (int) sycl::rint(v[l] * iq), hi = (int) sycl::rint(v[l + 16] * iq); + row[l * 8 + mm] = (short) ((lo & 0xFF) | ((hi & 0xFF) << 8)); + } + }); + switch (MT) { + case 1: q4_0_dpas_gemm_launch<1>(vx, xq_p, xd_p, dst, nrows, ncols, ntok, stream); break; + case 2: q4_0_dpas_gemm_launch<2>(vx, xq_p, xd_p, dst, nrows, ncols, ntok, stream); break; + case 3: q4_0_dpas_gemm_launch<3>(vx, xq_p, xd_p, dst, nrows, ncols, ntok, stream); break; + case 4: q4_0_dpas_gemm_launch<4>(vx, xq_p, xd_p, dst, nrows, ncols, ntok, stream); break; + case 5: q4_0_dpas_gemm_launch<5>(vx, xq_p, xd_p, dst, nrows, ncols, ntok, stream); break; + case 6: q4_0_dpas_gemm_launch<6>(vx, xq_p, xd_p, dst, nrows, ncols, ntok, stream); break; + case 7: q4_0_dpas_gemm_launch<7>(vx, xq_p, xd_p, dst, nrows, ncols, ntok, stream); break; + default: q4_0_dpas_gemm_launch<8>(vx, xq_p, xd_p, dst, nrows, ncols, ntok, stream); break; + } + return true; +} diff --git a/ggml/src/ggml-sycl/mmvq.hpp b/ggml/src/ggml-sycl/mmvq.hpp index c3f04bbbc1d0..1dd14b46e0af 100644 --- a/ggml/src/ggml-sycl/mmvq.hpp +++ b/ggml/src/ggml-sycl/mmvq.hpp @@ -84,4 +84,7 @@ bool ggml_sycl_ptq1_0_mul_mat_vec_multi_ncols(const void * const * vx, float * c const int nm, const void * vy, const int ncols, const int ncols_dst, const int stride_col_y_bytes, dpct::queue_ptr stream); +// ARC-LAB: reordered q4_0 x f32, 1..64 columns, weights streamed once on XMX (s8 x s4 DPAS, Xe2) +bool ggml_sycl_q4_0_dpas_gemm(ggml_backend_sycl_context & ctx, const void * vx, const float * x, int64_t x_stride, + float * dst, int nrows, int ncols, int ntok, dpct::queue_ptr stream); #endif // GGML_SYCL_MMVQ_HPP diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index df65d5dc6230..7b582296be53 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10355,6 +10355,12 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_flash_attn_ext(512, 512, 1, {16, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, {0, 2, 1, 3})); } } + // ARC-LAB: reordered q4_0 small-batch XMX GEMM (GGML_SYCL_Q4_0_DPAS): Gemma 4 shapes, 5..64 columns + for (auto mk : std::vector>{{4096, 3840}, {3840, 15360}}) { + for (int64_t n : {1, 5, 8, 9, 16, 33, 64, 65, 129, 256}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, mk[0], n, mk[1], {1, 1}, {1, 1})); + } + } // ARC-LAB: Gemma 4 sliding layers on the DPAS decode kernel (head 256, 8 KV heads x 2 query heads, q4_0 cache) for (int64_t kv : {512, 1024}) { for (int64_t nb : {1, 2, 5, 8, 12}) { @@ -10936,6 +10942,12 @@ static std::vector> make_test_cases_perf() { // Qwen3-VL-8B https://github.com/ggml-org/llama.cpp/issues/17012 test_cases.emplace_back(new test_flash_attn_ext(72, 72, 16, {1, 1}, 5776, 5776, false, false, 0, 0, GGML_PREC_F32, GGML_TYPE_F16, GGML_TYPE_F16)); + // ARC-LAB: Gemma 4 12B QAT q4_0 weight shapes at decode / verify batch sizes (q4_0 weight-kernel work) + for (auto mk : std::vector>{{15360, 3840}, {3840, 15360}, {4096, 3840}, {2048, 3840}, {3840, 4096}}) { + for (int64_t n : {1, 2, 4, 8, 16}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q4_0, GGML_TYPE_F32, mk[0], n, mk[1], {1, 1}, {1, 1})); + } + } for (int64_t kv : {16384, 49152}) { // ARC-LAB: Bonsai 27B decode / MTP verify on a q4_0 cache for (int64_t nb : {1, 4, 5, 8}) { test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0)); From 22d436f077875c08f4e35c8befc0c42bf6326144 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Mon, 28 Sep 2026 11:53:12 +1000 Subject: [PATCH 14/65] sycl: large T2 prompt batches through oneDNN's int8 GEMM (GGML_SYCL_T2_W8A8_MIN) Before the TernSYCL path went live, PTQ1_0 prompts ran on the W8A8 oneDNN GEMM (pp512 1013); GemmMT is better for verify-sized batches but slower on full prompt batches. New ggml_sycl_w8a8_mul_mat_t2 converts the repacked 2-bit layout to int8 (per-row scale, group ratios) and uses the same gemm_s8. The conversion is a fixed cost per call, so it only pays from ~1024 columns: Bonsai 27B pp1024 (ub 1024) 941 -> 1062, pp512 901 -> 818 (so MIN=1024 on the live config); 48.6K-token prompt read 67.7 -> 61.6 s (719 -> 789 t/s). PPL c4096 8.7273 -> 8.7359 (+0.1%, int8 rounding as the old W8A8 path). Off by default (0). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit a68382d688c2fbba780a91b85dd4d902199e9601 on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/ggml-sycl.cpp | 6 +++ ggml/src/ggml-sycl/w8a8.cpp | 70 ++++++++++++++++++++++++++++++++ ggml/src/ggml-sycl/w8a8.hpp | 4 ++ 3 files changed, 80 insertions(+) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index aa5761b5aa12..273ea4dd4ed0 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -4983,6 +4983,12 @@ static bool ggml_sycl_t2_mul_mat_try(ggml_backend_sycl_context & ctx, const ggml } // tokens: ne[1] x ne[2] rows of K floats (row stride nb[1]; planes back to back) const int64_t M = src1->ne[1] * src1->ne[2]; + // ARC-LAB: large prompt batches through oneDNN's int8 GEMM (faster than GemmMT there), GGML_SYCL_T2_W8A8_MIN (0 = off) + static const int64_t w8a8_min = getenv("GGML_SYCL_T2_W8A8_MIN") ? atoll(getenv("GGML_SYCL_T2_W8A8_MIN")) : 0; + if (w8a8_min > 0 && M >= w8a8_min && g_ggml_sycl_enable_dnn && src1->nb[1] == (size_t) K * sizeof(float) && + ggml_sycl_w8a8_mul_mat_t2(ctx, src0->data, (const float *) src1->data, K, (float *) dst->data, N, M, K, stream)) { + return true; + } ggml_sycl_t2_mul_mat(*stream, src0->data, (const float *) src1->data, src1->nb[1] / sizeof(float), (float *) dst->data, M, N, K); return true; diff --git a/ggml/src/ggml-sycl/w8a8.cpp b/ggml/src/ggml-sycl/w8a8.cpp index 38ced70f1814..e6a714af8889 100644 --- a/ggml/src/ggml-sycl/w8a8.cpp +++ b/ggml/src/ggml-sycl/w8a8.cpp @@ -136,6 +136,71 @@ bool ggml_sycl_w8a8_mul_mat(ggml_backend_sycl_context & ctx, ggml_type type, boo return true; } +// ARC-LAB: the same int8 GEMM from the TernSYCL 2-bit layout (PTQ1_0 / PQ2_0 repacked in place by ggml_sycl_t2_repack): +// codes [K/16][N] u32 (16 two-bit two's-complement weights per word, k = 16 w + j at bits 2 j), scales [K/128][N] fp16. +// Large prompt batches: oneDNN's int8 GEMM beat the TernSYCL GemmMT for them before T2 went live (pp512 1013 vs ~900). +bool ggml_sycl_w8a8_mul_mat_t2(ggml_backend_sycl_context & ctx, const void * w, const float * x, int64_t x_stride, + float * dst, int64_t nrows, int64_t ntok, int64_t K, dpct::queue_ptr stream) { + if (K % 128 != 0) { + return false; + } + const int64_t N = nrows, ng = K / 128; + const uint32_t * B = (const uint32_t *) w; + const uint16_t * SB = (const uint16_t *) ((const char *) w + (size_t) (K / 16) * N * 4); + ggml_sycl_pool_alloc ratio(ctx.pool(), N * ng); + ggml_sycl_pool_alloc wscale(ctx.pool(), N); + ggml_sycl_pool_alloc w8(ctx.pool(), N * K); + ggml_sycl_pool_alloc x8(ctx.pool(), ntok * K); + ggml_sycl_pool_alloc xscale(ctx.pool(), ntok); + int8_t * ratio_p = ratio.get(); + float * ws_p = wscale.get(); + int8_t * w8_p = w8.get(); + // per row: D = max group scale, ratio = round(127 d_g / D) + stream->parallel_for(sycl::nd_range<1>(N * 16, 16), [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(16)]] { + const int64_t r = it.get_group(0); + const int lane = it.get_local_id(0); + auto dget = [&](int64_t g) -> float { return (float) sycl::bit_cast(SB[g * N + r]); }; + float m = 0.0f; + for (int64_t g = lane; g < ng; g += 16) { + m = sycl::fmax(m, sycl::fabs(dget(g))); + } + m = sycl::reduce_over_group(it.get_sub_group(), m, sycl::maximum()); + for (int64_t g = lane; g < ng; g += 16) { + ratio_p[r * ng + g] = m > 0 ? (int8_t) sycl::round(127.0f * dget(g) / m) : 0; + } + if (lane == 0) { + ws_p[r] = m / 127.0f; + } + }); + // int8 weights [N][K]: one work item per (16-weight word, row), rows fastest (coalesced word reads) + stream->parallel_for(sycl::range<1>((size_t) (K / 16) * N), [=](sycl::id<1> id) { + const int64_t kw = id[0] / N, n = id[0] % N; + const uint32_t c = B[kw * N + n]; + const int r8 = ratio_p[n * ng + (kw * 16) / 128]; + uint32_t o[4]; +#pragma unroll + for (int q = 0; q < 4; ++q) { + uint32_t pk = 0; +#pragma unroll + for (int j = 0; j < 4; ++j) { + const int v = ((int) ((c >> (2 * (4 * q + j))) << 30)) >> 30; // sign-extend the 2-bit code + pk |= (uint32_t) (uint8_t) (int8_t) (v * r8) << (8 * j); + } + o[q] = pk; + } + uint32_t * dstw = (uint32_t *) (w8_p + n * K + kw * 16); + dstw[0] = o[0]; dstw[1] = o[1]; dstw[2] = o[2]; dstw[3] = o[3]; + }); + GGML_ASSERT(x_stride == K); + w8a8_acts(x, x8.get(), xscale.get(), ntok, K, stream); + DnnlGemmWrapper::gemm_s8(ctx, (int) N, (int) ntok, (int) K, x8.get(), w8_p, dst, stream); + const float * xs = xscale.get(); + stream->parallel_for(sycl::range<1>(ntok * N), [=](sycl::id<1> i) { + dst[i] *= xs[i[0] / N] * ws_p[i[0] % N]; + }); + return true; +} + #else bool ggml_sycl_w8a8_mul_mat(ggml_backend_sycl_context &, ggml_type, bool, const void *, const float *, float *, int64_t, @@ -143,4 +208,9 @@ bool ggml_sycl_w8a8_mul_mat(ggml_backend_sycl_context &, ggml_type, bool, const return false; } +bool ggml_sycl_w8a8_mul_mat_t2(ggml_backend_sycl_context &, const void *, const float *, int64_t, float *, int64_t, int64_t, + int64_t, dpct::queue_ptr) { + return false; +} + #endif diff --git a/ggml/src/ggml-sycl/w8a8.hpp b/ggml/src/ggml-sycl/w8a8.hpp index e6a6f67f5d74..492512a5c0bd 100644 --- a/ggml/src/ggml-sycl/w8a8.hpp +++ b/ggml/src/ggml-sycl/w8a8.hpp @@ -8,3 +8,7 @@ bool ggml_sycl_w8a8_mul_mat(ggml_backend_sycl_context & ctx, ggml_type type, bool reordered, const void * w, const float * x, float * dst, int64_t nrows, int64_t ncols_x, int64_t K, dpct::queue_ptr stream); + +// ARC-LAB: int8 oneDNN GEMM straight from the TernSYCL 2-bit layout (large prompt batches) +bool ggml_sycl_w8a8_mul_mat_t2(ggml_backend_sycl_context & ctx, const void * w, const float * x, int64_t x_stride, + float * dst, int64_t nrows, int64_t ntok, int64_t K, dpct::queue_ptr stream); From 837ba6b83b29d30688ac897a27b79d3c31882360 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 3 Oct 2026 09:45:11 +1000 Subject: [PATCH 15/65] SYCL decode attention: SLM-staged K/V tiles for 1 query token, one-wave split count; PTQ1_0 T2 NSG knob fattn-dec (DPAS q4_0 path, D 256): single-token decode copies each 128-key K and V tile into SLM with overlapped 16-byte loads and reads lane = key from there; the split count now fills one wave of work-groups (Xe cores x work-groups per core / KV heads) instead of a fixed 64 per head. B580: 48K 1 token 433 -> 244 us, 4 tokens 640 -> 456 us; whole model plain decode +12% at 48K, +21% at 115K, same KLD and output. GGML_SYCL_FA_DEC_STAGE=0 and GGML_SYCL_FA_DEC_SPLITS=64 restore the old behaviour. ptq1-t2: GGML_SYCL_PTQ1_T2_NSG=4 (4 sub-groups for 2-8 row GEMV, +~1%, default off). Co-Authored-By: Claude (cherry picked from commit 9cfffcb06d20399b2b3ec0e757ee22781a7b9900 on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/fattn-dec.cpp | 89 ++++++++++++++++++++++++++++++-- ggml/src/ggml-sycl/ptq1-t2.cpp | 26 ++++++++-- 2 files changed, 107 insertions(+), 8 deletions(-) diff --git a/ggml/src/ggml-sycl/fattn-dec.cpp b/ggml/src/ggml-sycl/fattn-dec.cpp index 964cf6a99327..9e4e7570e4a1 100644 --- a/ggml/src/ggml-sycl/fattn-dec.cpp +++ b/ggml/src/ggml-sycl/fattn-dec.cpp @@ -705,7 +705,11 @@ inline dp_int8 intel_sub_group_i8_i4_matrix_mad_k32(dp_short8, dp_int4, dp_int inline dp_float8 intel_sub_group_f16_f16_matrix_mad_k16(dp_short8, dp_int8, dp_float8) { __builtin_unreachable(); } #endif -template +// ST = 1 (GGML_SYCL_FA_DEC_STAGE, D 256 only): each key tile of K, then of V, is first copied into SLM with coalesced loads +// (consecutive work-items read consecutive dwords of a key row) and the lane = key reads come from there (row stride 37 +// dwords: conflict-free). Without it every lane reads its own key straight from memory: 16 cache lines per load +// instruction, ~90-110 GB/s per phase at 48K. The V copy is issued before the softmax and shares its barrier. +template static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, const char * mask, float * dst, float * parts, sycl::float2 * meta, float scale, int ne01, int ne02, int ne11, int nkvh, int ne03, int64_t nb01, int64_t nb02, int64_t nb03, int64_t nb11, @@ -724,9 +728,12 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, constexpr int RT16 = R16 / 16; constexpr int PS = TK + 16; // f16 row stride of P ([row][key]); 32-byte aligned rows, fewer bank conflicts static_assert(D % 256 == 0 && DPS % 16 == 0 && DPS <= QK4_0, "shape"); + constexpr int RW = NB * 18 / 4; // dwords per q4_0 key row (36 at D 256) + constexpr int SW = RW + 1; // SLM row stride (odd: no bank conflicts on lane = key reads) + static_assert(ST == 0 || (D == 256 && RW % 4 == 0), "staging is sized for D 256"); const int nqc = (ne01 + NQ - 1) / NQ; // token chunks static const int probe_env = getenv("GGML_SYCL_FA_DEC_DPAS_PROBE") ? atoi(getenv("GGML_SYCL_FA_DEC_DPAS_PROBE")) : 0; - const int probe = probe_env; // phase probe: bit0 skip Q.K, bit1 skip P.V, bit2 skip softmax + const int probe = probe_env; // phase probe: bit0 skip Q.K, bit1 skip P.V, bit2 skip softmax, bit3 Q.K loads only (no DPAS / scale math) stream->submit([&](sycl::handler & cgh) { sycl::local_accessor sQ8(sycl::range<1>(RT * NB * 16), cgh); // [tile][block][lane] -> 8 rows @@ -736,6 +743,7 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, sycl::local_accessor sM(sycl::range<1>(R16), cgh); sycl::local_accessor sL(sycl::range<1>(R16), cgh); sycl::local_accessor sA(sycl::range<1>(R16), cgh); + sycl::local_accessor sKV(sycl::range<1>(ST ? TK * SW : 1), cgh); // staged K or V tile cgh.parallel_for( sycl::nd_range<3>(sycl::range<3>((size_t) ne03 * nqc, nkvh, (size_t) nsplit * DEC_WG), sycl::range<3>(1, 1, DEC_WG)), [=](sycl::nd_item<3> it) [[sycl::reqd_sub_group_size(16)]] { @@ -800,7 +808,35 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, // those dims lie in V block (DPS w) / 32; dim d of the block = nibble (d / 16) of qs byte d % 16 const int vb = (DPS * w) / QK4_0, vd0 = (DPS * w) % QK4_0; + // coalesced copy of the tile's key rows (keys past k_end repeat key k_begin, as the direct path) + // 16-byte loads (a 144-byte row = 9, rows 16-byte aligned), all issued before any SLM store so they overlap + auto stage = [&](const char * base, int64_t nb, int t0) { + constexpr int RV = RW / 4, NV = TK * RV, PER = (NV + DEC_WG - 1) / DEC_WG; + sycl::uint4 v[PER]; +#pragma unroll + for (int i = 0; i < PER; ++i) { + const int e = tid + i * DEC_WG; + if (e < NV) { + const int key = e / RV, q = e - key * RV; + const int j = t0 + key < k_end ? t0 + key : k_begin; + v[i] = ((const sycl::uint4 *) (base + (int64_t) j * nb))[q]; + } + } +#pragma unroll + for (int i = 0; i < PER; ++i) { + const int e = tid + i * DEC_WG; + if (e < NV) { + const int key = e / RV, q = e - key * RV, o = key * SW + 4 * q; + sKV[o] = v[i][0]; sKV[o + 1] = v[i][1]; sKV[o + 2] = v[i][2]; sKV[o + 3] = v[i][3]; + } + } + }; + for (int t0 = k_begin; t0 < k_end; t0 += TK) { + if constexpr (ST) { + stage(Kh, nb11, t0); + it.barrier(sycl::access::fence_space::local_space); + } // ---- S = Q K^T: job = (key group, chunk of TC row tiles) for (int job = w; job < ((probe & 1) ? 0 : KG * NCH); job += NSG) { const int kg = job % KG, ch = job / KG; @@ -817,7 +853,13 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, // block b = bytes 18 b .. 18 b + 17: fp16 scale, then 16 bytes of nibbles const int o0 = 18 * b; // even b: dword aligned; odd b: 2 bytes in const int wd = o0 / 4; // the block's 18 bytes lie in these 5 dwords - const uint32_t k0 = kr[wd], k1 = kr[wd + 1], k2 = kr[wd + 2], k3 = kr[wd + 3], k4 = kr[wd + 4]; + uint32_t k0, k1, k2, k3, k4; + if constexpr (ST) { + const int sb = (kg * 16 + lane) * SW + wd; + k0 = sKV[sb]; k1 = sKV[sb + 1]; k2 = sKV[sb + 2]; k3 = sKV[sb + 3]; k4 = sKV[sb + 4]; + } else { + k0 = kr[wd]; k1 = kr[wd + 1]; k2 = kr[wd + 2]; k3 = kr[wd + 3]; k4 = kr[wd + 4]; + } uint32_t sc, q0, q1, q2, q3; if ((o0 & 3) == 0) { sc = k0 & 0xFFFFu; @@ -832,6 +874,10 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, const float dk = static_cast(sycl::bit_cast((uint16_t) sc)); const dp_int4 bv = { (int) (q0 ^ 0x88888888u), (int) (q1 ^ 0x88888888u), (int) (q2 ^ 0x88888888u), (int) (q3 ^ 0x88888888u) }; + if (probe & 8) { // keep the loads live, skip the math + s[0][0] += (float) (int) ((q0 ^ q1 ^ q2 ^ q3 ^ sc) & 1u); + continue; + } #pragma unroll for (int tt = 0; tt < TC; ++tt) { const int t = ch * TC + tt; @@ -859,6 +905,9 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, } } it.barrier(sycl::access::fence_space::local_space); + if constexpr (ST) { + stage(Vh, nb21, t0); // K is done; the softmax below does not touch sKV + } // ---- online softmax per row -> P (f16, DPAS A layout) for (int r = w; r < ((probe & 4) ? 0 : R); r += NSG) { @@ -904,7 +953,13 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, const bool jv = j < k_end; const uint32_t * vr = (const uint32_t *) (Vh + (int64_t) (jv ? j : k_begin) * nb21); const int o0 = 18 * vb, wd = o0 / 4; // block start; even block: dword aligned, odd: 2 bytes in - const uint32_t w0 = vr[wd], w1 = vr[wd + 1], w2 = vr[wd + 2], w3 = vr[wd + 3], w4 = vr[wd + 4]; + uint32_t w0, w1, w2, w3, w4; + if constexpr (ST) { + const int sb = (kg * 16 + lane) * SW + wd; + w0 = sKV[sb]; w1 = sKV[sb + 1]; w2 = sKV[sb + 2]; w3 = sKV[sb + 3]; w4 = sKV[sb + 4]; + } else { + w0 = vr[wd]; w1 = vr[wd + 1]; w2 = vr[wd + 2]; w3 = vr[wd + 3]; w4 = vr[wd + 4]; + } uint32_t sc, q[4]; if ((o0 & 3) == 0) { sc = w0 & 0xFFFFu; @@ -1027,13 +1082,25 @@ void ggml_sycl_flash_attn_ext_dec(ggml_backend_sycl_context & ctx, ggml_tensor * const bool g2 = !d512 && Q->ne[2] == 2 * K->ne[2]; // Gemma 4 sliding layers: DPAS kernel only, token chunks of 8 const int nq = d512 ? (ne01 <= 1 ? 1 : 2) : g2 ? (ne01 <= 1 ? 1 : 8) : ne01 <= 1 ? 1 : ne01 <= 2 ? 2 : ne01 <= 4 ? 4 : 8; // keys per tile (must match the instantiations below); DPAS: 16 x KG + // SLM-staged K/V tiles (128 keys) for single-token decode: 48K 432 -> 247 us with the split count below. For 2+ tokens + // the extra SLM halves the work-groups per core and it loses (4 tok 48K 458 unstaged vs ~593 staged). GGML_SYCL_FA_DEC_STAGE=0 off. + static const bool stage_env = !(getenv("GGML_SYCL_FA_DEC_STAGE") && atoi(getenv("GGML_SYCL_FA_DEC_STAGE")) == 0); + const bool stage = dpas && stage_env && !d512 && !g2 && nq == 1; const int tk = (d512 || g2) ? (nq == 1 ? 256 : 128) + : stage ? 128 : dpas ? (nq == 1 ? 256 : nq == 2 ? 128 : nq == 4 ? 256 : 128) : nq == 8 ? 32 : nq == 4 ? 64 : 128; const int ne11 = K->ne[1]; // slices: enough work-groups to fill the GPU (~64 slices per KV head, ~256 in all), at least one tile each static const int target_env = getenv("GGML_SYCL_FA_DEC_SPLITS") ? atoi(getenv("GGML_SYCL_FA_DEC_SPLITS")) : 0; - const int target = target_env ? target_env : std::max(64, 256 / std::max(1, (int) K->ne[2])); + // DPAS (D 256, 6 rows): one wave of work-groups - Xe cores x the work-groups that fit per core (SLM-bound: 4 for 1-2 + // query tokens, 2 for 4-8) - over the KV heads. A fixed 64 per head ran 3-6 waves with a mostly idle last one: B580 + // 48K 1 tok 432 -> 302 us (247 staged), 4 tok 648 -> 458 us. Xe2: 8 EUs per core, nsm = EUs / 16. + const int xe_cores = std::max(1, ggml_sycl_info().devices[ctx.device].nsm * 2); + const int wave = xe_cores * (nq <= 2 ? 4 : 2); + const int target = target_env ? target_env + : (dpas && !d512 && !g2) ? std::max(1, wave / std::max(1, (int) K->ne[2])) + : std::max(64, 256 / std::max(1, (int) K->ne[2])); const int ntiles = (ne11 + tk - 1) / tk; int nsplit = std::max(1, std::min(target, ntiles)); const int chunk = ((ntiles + nsplit - 1) / nsplit) * tk; @@ -1078,6 +1145,18 @@ void ggml_sycl_flash_attn_ext_dec(ggml_backend_sycl_context & ctx, ggml_tensor * FATTN_DEC_DPASG2(8, 8, 1); // 16 rows = 2 tiles, 8 key groups x 2 } #undef FATTN_DEC_DPASG2 + } else if (stage) { +#define FATTN_DEC_DPAS_ST(NQ_, KG_, TC_) fattn_dec_q4_0_dpas((const char *) Q->data, (const char *) K->data, \ + (const char *) V->data, mdata, (float *) dst->data, parts.get(), meta.get(), scale, ne01, (int) Q->ne[2], ne11, \ + (int) K->ne[2], (int) Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], V->nb[1], V->nb[2], \ + V->nb[3], nb31, nb33, ne33, nsplit, chunk, stream) + switch (nq) { + case 1: FATTN_DEC_DPAS_ST(1, 8, 1); break; + case 2: FATTN_DEC_DPAS_ST(2, 8, 1); break; + case 4: FATTN_DEC_DPAS_ST(4, 8, 3); break; + default: FATTN_DEC_DPAS_ST(8, 8, 3); break; + } +#undef FATTN_DEC_DPAS_ST } else if (dpas) { switch (nq) { case 1: FATTN_DEC_DPAS(1, 16, 1); break; diff --git a/ggml/src/ggml-sycl/ptq1-t2.cpp b/ggml/src/ggml-sycl/ptq1-t2.cpp index f7383ec5c502..2d6d65c49c36 100644 --- a/ggml/src/ggml-sycl/ptq1-t2.cpp +++ b/ggml/src/ggml-sycl/ptq1-t2.cpp @@ -221,11 +221,18 @@ void quant_a(sycl::queue & q, const float * x, int64_t x_stride, int64_t M, int6 }); } -template +// GGML_SYCL_PTQ1_T2_NSG: sub-groups per GEMV work-group for 2..8 rows (verify batches); 2 (default) or 4. The B580 tile +// sweep (TernSYCL main, Bonsai 27B shapes, M=4, qmode 0) had 4 fastest on 5 of 6 shapes (+0-8%); M=1 keeps 2. +int t2_nsg() { + static const int v = getenv("GGML_SYCL_PTQ1_T2_NSG") && atoi(getenv("GGML_SYCL_PTQ1_T2_NSG")) == 4 ? 4 : 2; + return v; +} + +template void gemv(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const uint32_t * B, const uint16_t * SB, float * C, int M, int N, int K) { - using Kern = int8dpas::Gemv; - const size_t wgn = 16 * 2; + using Kern = int8dpas::Gemv; + const size_t wgn = 16 * NSG; const Epi epi{ nullptr, nullptr, 0, 1 }; const sycl::range<2> local(1, Kern::WG); const sycl::range<2> global((M + SGM - 1) / SGM, (N + wgn - 1) / wgn * Kern::WG); @@ -275,6 +282,14 @@ void t2_dispatch(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const gemm(q, Aq, SA, B, SB, dst, m, n, k); } else if (m == 1) { n <= 8192 ? gemv<1, 4, NS>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<1, 2, NS>(q, Aq, SA, B, SB, dst, m, n, k); + } else if (t2_nsg() == 4) { + if (m == 2) { + n <= 8192 ? gemv<2, 4, NS, 4>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<2, 2, NS, 4>(q, Aq, SA, B, SB, dst, m, n, k); + } else if (m <= 4) { + n <= 8192 ? gemv<4, 4, NS, 4>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<4, 2, NS, 4>(q, Aq, SA, B, SB, dst, m, n, k); + } else { + n <= 8192 ? gemv<8, 4, NS, 4>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<8, 2, NS, 4>(q, Aq, SA, B, SB, dst, m, n, k); + } } else if (m == 2) { n <= 8192 ? gemv<2, 4, NS>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<2, 2, NS>(q, Aq, SA, B, SB, dst, m, n, k); } else if (m <= 4) { @@ -345,6 +360,11 @@ void t2_precompile(sycl::queue & q) { gemv<2, 2, 4>(q, aq, sa, B, SB, d, 2, N, K); gemv<4, 2, 4>(q, aq, sa, B, SB, d, 4, N, K); gemv<8, 2, 4>(q, aq, sa, B, SB, d, 8, N, K); + if (t2_nsg() == 4) { // the N > 8192 GEMV tiles with 4 sub-groups (t2_dispatch above covered the N <= 8192 ones) + gemv<2, 2, 1, 4>(q, aq, sa, B, SB, d, 2, N, K); + gemv<4, 2, 1, 4>(q, aq, sa, B, SB, d, 4, N, K); + gemv<8, 2, 1, 4>(q, aq, sa, B, SB, d, 8, N, K); + } q.wait(); sycl::free(w, q); sycl::free(x, q); From 0f16f1dfc8939b06001053bc567bd36075db96c3 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 3 Oct 2026 12:17:23 +1000 Subject: [PATCH 16/65] SYCL: s8 x s4 DPAS as inline vISA instead of the intel_sub_group_i8_i4_matrix_mad_k32 builtin The builtin is not in the public subgroup matrix extension list. On a Lunar Lake (Arc 140V) driver that does not provide it, loading the device image failed with "No device image found for external symbol _Z36intel_sub_group_i8_i4_matrix_mad_k32" and every model crashed at the first decode. xe2-dpas.hpp emits the dpas.s4.s8 instruction directly (as libxsmm/TernSYCL's xe2.hpp helpers), so the symbol is gone from libggml-sycl. Used by the q4_0 decode attention and the q4_0 small-batch GEMM. B580: FA 4167/4167, MUL_MAT q4_0 (GGML_SYCL_Q4_0_DPAS=1) 59/59; whole model output byte-identical to the builtin at short / 48K / 115K context, plain and MTP. Single-token attention kernel ~6% slower (48K 244 -> 259 us), whole model within 1-2%. Co-Authored-By: Claude (cherry picked from commit f31a4f611d1b91a7bb591e5f8d1071f85293138f on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/fattn-dec.cpp | 7 +++---- ggml/src/ggml-sycl/mmvq.cpp | 12 ++++-------- ggml/src/ggml-sycl/xe2-dpas.hpp | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 12 deletions(-) create mode 100644 ggml/src/ggml-sycl/xe2-dpas.hpp diff --git a/ggml/src/ggml-sycl/fattn-dec.cpp b/ggml/src/ggml-sycl/fattn-dec.cpp index 9e4e7570e4a1..4740c6cc943c 100644 --- a/ggml/src/ggml-sycl/fattn-dec.cpp +++ b/ggml/src/ggml-sycl/fattn-dec.cpp @@ -10,6 +10,7 @@ // tile or slice gets zero weight, never NaN); thread d accumulates P.V for head dim d of every row; // - slices are merged by flash_attn_combine_results (same partial layout as the vec kernel). +#include "xe2-dpas.hpp" #include #include "dpct/helper.hpp" #include "common.hpp" @@ -684,7 +685,7 @@ static void fattn_dec_q4_0_xmx(const char * Q, const char * K, const char * V, c // ARC-LAB DPAS kernel (GGML_SYCL_FA_DEC_DPAS=1): the q4_0 cache feeds the XMX units without any dequantization pass. -// Q.K: s8 x s4 DPAS (intel_sub_group_i8_i4_matrix_mad_k32, not in the public extension spec but provided by IGC on Xe2). +// Q.K: s8 x s4 DPAS (inline vISA, xe2-dpas.hpp; was the IGC builtin intel_sub_group_i8_i4_matrix_mad_k32). // A = Q rows as int8 (one RNE scale per row per 32-dim block, computed once per call), B = the raw 16 bytes of a key's // q4_0 block XOR 0x88888888 (u4 n+8 -> s4 n), one key per lane. q4_0 byte i holds dims i and i+16, so A's k = 2i+h // is dim i + 16h: lane l's A short = (Q[l], Q[l + 16]). S += int result * dq[row] * dk[key]. @@ -698,10 +699,8 @@ typedef float dp_float8 __attribute__((ext_vector_type(8))); typedef unsigned dp_uint4 __attribute__((ext_vector_type(4))); } #ifdef __SYCL_DEVICE_ONLY__ -SYCL_EXTERNAL dp_int8 intel_sub_group_i8_i4_matrix_mad_k32(dp_short8 a, dp_int4 b, dp_int8 acc); SYCL_EXTERNAL dp_float8 intel_sub_group_f16_f16_matrix_mad_k16(dp_short8 a, dp_int8 b, dp_float8 acc); #else -inline dp_int8 intel_sub_group_i8_i4_matrix_mad_k32(dp_short8, dp_int4, dp_int8) { __builtin_unreachable(); } inline dp_float8 intel_sub_group_f16_f16_matrix_mad_k16(dp_short8, dp_int8, dp_float8) { __builtin_unreachable(); } #endif @@ -882,7 +881,7 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, for (int tt = 0; tt < TC; ++tt) { const int t = ch * TC + tt; if (t < RT) { - const dp_int8 ia = intel_sub_group_i8_i4_matrix_mad_k32(sQ8[(t * NB + b) * 16 + lane], bv, (dp_int8) (0)); + const dp_int8 ia = xe2dp::dpas_s4s8_r8(sQ8[(t * NB + b) * 16 + lane], bv); const dp_float8 sc8 = sDQ[t * NB + b] * dk; // one broadcast vector load per tile and block s[tt] += __builtin_convertvector(ia, dp_float8) * sc8; } diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cpp index 0644c622e4a7..ef9ff069ccf4 100644 --- a/ggml/src/ggml-sycl/mmvq.cpp +++ b/ggml/src/ggml-sycl/mmvq.cpp @@ -1,3 +1,4 @@ +#include "xe2-dpas.hpp" #include "mmvq.hpp" #include "ggml.h" @@ -4017,8 +4018,8 @@ bool ggml_sycl_mul_mat_vec_q_glu_reorder(enum ggml_type src0_type, enum ggml_glu } } -// ARC-LAB: reordered q4_0 x f32 for 5..64 columns (speculative verify / n-gram drafts) on XMX via the s8 x s4 DPAS builtin -// (intel_sub_group_i8_i4_matrix_mad_k32, Xe2). The MMVQ column kernels re-read the weights per 8 columns and the dequantize +// ARC-LAB: reordered q4_0 x f32 for 5..64 columns (speculative verify / n-gram drafts) on XMX via s8 x s4 DPAS +// (inline vISA, xe2-dpas.hpp; Xe2). The MMVQ column kernels re-read the weights per 8 columns and the dequantize // + GEMM path converts the whole matrix; here each weight row streams once for all columns. One lane per weight row: B = // the row's 16 q4_0 bytes of a block XOR 0x88888888 (u4 n+8 -> s4 n), A = 8 activation rows as int8 (per row and 32-block // RNE scale; q4_0 byte i holds elements i and i+16, so A's k = 2i+h pairs elements l and l+16). GGML_SYCL_Q4_0_DPAS=1. @@ -4029,11 +4030,6 @@ typedef int q4d_int8 __attribute__((ext_vector_type(8))); typedef float q4d_float8 __attribute__((ext_vector_type(8))); typedef unsigned q4d_uint4 __attribute__((ext_vector_type(4))); } -#ifdef __SYCL_DEVICE_ONLY__ -SYCL_EXTERNAL q4d_int8 intel_sub_group_i8_i4_matrix_mad_k32(q4d_short8 a, q4d_int4 b, q4d_int8 acc); -#else -inline q4d_int8 intel_sub_group_i8_i4_matrix_mad_k32(q4d_short8, q4d_int4, q4d_int8) { __builtin_unreachable(); } -#endif template static void q4_0_dpas_gemm_launch(const void * vx, const q4d_short8 * xq, const q4d_float8 * xd, float * dst, const int nrows, @@ -4065,7 +4061,7 @@ static void q4_0_dpas_gemm_launch(const void * vx, const q4d_short8 * xq, const const float d = static_cast(dw[b]); #pragma unroll for (int t = 0; t < MT; ++t) { - const q4d_int8 ia = intel_sub_group_i8_i4_matrix_mad_k32(xq[((size_t) t * NB + b) * 16 + lane], bv, (q4d_int8) (0)); + const q4d_int8 ia = xe2dp::dpas_s4s8_r8(xq[((size_t) t * NB + b) * 16 + lane], bv); acc[t] += __builtin_convertvector(ia, q4d_float8) * (xd[(size_t) t * NB + b] * d); } } diff --git a/ggml/src/ggml-sycl/xe2-dpas.hpp b/ggml/src/ggml-sycl/xe2-dpas.hpp new file mode 100644 index 000000000000..8c61f0e74113 --- /dev/null +++ b/ggml/src/ggml-sycl/xe2-dpas.hpp @@ -0,0 +1,32 @@ +// ARC-LAB: Xe2 s8 x s4 DPAS as inline vISA (as libxsmm/TernSYCL's xe2.hpp dpas helpers). It replaces the IGC builtin +// intel_sub_group_i8_i4_matrix_mad_k32, which is not in the public extension list: drivers that don't provide it (e.g. Lunar +// Lake, 2026-10) failed to load the whole device image ("No device image found for external symbol"), crashing every model. +// SIMD16, systolic depth 8, repeat count 8: A = 8 rows of K32 s8 (one short = 2 s8 per lane per row), B = K32 x 16 s4 (one +// int4 = 16 bytes of nibbles per lane / column), zero accumulator. Same result as the builtin with acc = 0. +#pragma once + +#include + +namespace xe2dp { +typedef short s8x8 __attribute__((ext_vector_type(8))); +typedef int s4x4 __attribute__((ext_vector_type(4))); +typedef int i32x8 __attribute__((ext_vector_type(8))); + +inline i32x8 dpas_s4s8_r8(s8x8 a, s4x4 b) { + i32x8 d; +#ifdef __SYCL_DEVICE_ONLY__ + __asm__("{\n" + ".decl DB v_type=G type=ud num_elts=64 align=GRF alias=<%1,0>\n" + ".decl DA v_type=G type=ud num_elts=64 align=GRF alias=<%2,0>\n" + "dpas.s4.s8.8.8 (M1, 16) %0.0 %%null.0 DB.0 DA(0,0)\n" + "}\n" + : "=rw"(d) + : "rw"(b), "rw"(a)); +#else + d = 0; + (void) a; + (void) b; +#endif + return d; +} +} // namespace xe2dp From 9d262b55df740aba961ac2e6b1259c58c035c2df Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 3 Oct 2026 14:10:39 +1000 Subject: [PATCH 17/65] SYCL: no IGC builtins left (TernSYCL main, f16 DPAS via vISA), device code split per kernel - ternsycl/: libxsmm/TernSYCL main (11484da) + the long long address fix of TernSYCL#3. Inline vISA only, so the undocumented builtins the old copy declared (intel_sub_group_i8_i2_matrix_mad_k32, __builtin_IB_* 2D block I/O) are gone; it also drops our three JIT workarounds. The per-32 activation scale variant (off by default) is removed. - ptq1-t2.cpp is compiled with -fp-model=precise: under icpx's default fast model the TernSYCL GemmMT returns NaN (upstream's own driver fails validation built that way). - fattn-dec: the f16 P.V DPAS also goes through inline vISA (xe2-dpas.hpp dpas_hf_r8). - CMake: -fsycl-device-code-split=per_kernel (GGML_SYCL_DEVICE_CODE_SPLIT_PER_KERNEL, default ON): the Xe2 XMX kernels are only JIT-compiled when launched, so GPUs without DPAS (Xe-LP, Gen9) don't compile them at all. - test-backend-ops: FA cases for 64-256 query tokens and ~115K history. B580: libggml-sycl has 0 matrix_mad / __builtin_IB references (was f16, i8_i2, 9 IB). FA 4167/4167, MUL_MAT q4_0 59/59, ptq1_0 T2 94/94. KLD (ub 1 / 4 / 512) identical to the previous build. Whole model output identical; MTP 48K 73.6 t/s, 115K 59.6 (was 69.6 / 57.2), plain unchanged. Startup unchanged (first request 4 s). Co-Authored-By: Claude (cherry picked from commit 2e8a0e6754977ff401a886b141e4299d6bda4e52 on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/CMakeLists.txt | 12 + ggml/src/ggml-sycl/fattn-dec.cpp | 9 +- ggml/src/ggml-sycl/ptq1-t2.cpp | 90 ++--- .../src/ggml-sycl/ternsycl/int2_int8_dpas.hpp | 167 ++++---- ggml/src/ggml-sycl/ternsycl/xe2.hpp | 361 +++++++++++++----- ggml/src/ggml-sycl/xe2-dpas.hpp | 18 + tests/test-backend-ops.cpp | 5 +- 7 files changed, 405 insertions(+), 257 deletions(-) diff --git a/ggml/src/ggml-sycl/CMakeLists.txt b/ggml/src/ggml-sycl/CMakeLists.txt index bd4dc57101af..56e3c8b8525a 100644 --- a/ggml/src/ggml-sycl/CMakeLists.txt +++ b/ggml/src/ggml-sycl/CMakeLists.txt @@ -105,6 +105,18 @@ endif() target_compile_options(ggml-sycl PRIVATE "-Wno-narrowing") +# ARC-LAB: one device image per kernel. The Xe2 XMX kernels (DPAS) then only get JIT-compiled when they are first launched, +# so a GPU without DPAS (Xe-LP, Gen9) never compiles them; with one image per source file, compiling any kernel of +# fattn-dec / mmvq / ptq1-t2 built the DPAS ones too and the shader compiler crashed (UHD 630). OFF = compiler default. +# ARC-LAB: the TernSYCL kernels (ptq1-t2.cpp) need IEEE semantics: under icpx's default -fp-model=fast their GemmMT returns +# NaN / inf (upstream's own driver fails validation built that way, passes with precise, which their Makefile uses). +set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/ptq1-t2.cpp PROPERTIES COMPILE_OPTIONS "-fp-model=precise") +option(GGML_SYCL_DEVICE_CODE_SPLIT_PER_KERNEL "ggml: SYCL device code split per kernel" ON) +if (GGML_SYCL_DEVICE_CODE_SPLIT_PER_KERNEL) + target_compile_options(ggml-sycl PRIVATE "-fsycl-device-code-split=per_kernel") + target_link_options(ggml-sycl PRIVATE "-fsycl-device-code-split=per_kernel") +endif() + message(STATUS "GGML_SYCL_SUPPORT_LEVEL_ZERO_API ${GGML_SYCL_SUPPORT_LEVEL_ZERO_API}") if (GGML_SYCL_SUPPORT_LEVEL_ZERO_API) # Link against Level Zero loader for direct device memory allocation. diff --git a/ggml/src/ggml-sycl/fattn-dec.cpp b/ggml/src/ggml-sycl/fattn-dec.cpp index 4740c6cc943c..8ad0dabca4a4 100644 --- a/ggml/src/ggml-sycl/fattn-dec.cpp +++ b/ggml/src/ggml-sycl/fattn-dec.cpp @@ -689,7 +689,7 @@ static void fattn_dec_q4_0_xmx(const char * Q, const char * K, const char * V, c // A = Q rows as int8 (one RNE scale per row per 32-dim block, computed once per call), B = the raw 16 bytes of a key's // q4_0 block XOR 0x88888888 (u4 n+8 -> s4 n), one key per lane. q4_0 byte i holds dims i and i+16, so A's k = 2i+h // is dim i + 16h: lane l's A short = (Q[l], Q[l + 16]). S += int result * dq[row] * dk[key]. -// P.V: f16 DPAS (k16) with P from SLM (A layout) and V dequantized in registers by the subgroup that owns 16 head dims. +// P.V: f16 DPAS (k16, inline vISA) with P from SLM (A layout) and V dequantized in registers by the subgroup that owns 16 head dims. // Query tokens beyond NQ are handled by extra work-groups (grid dim 0 = sequence x token chunk), each reading the KV again. namespace { typedef short dp_short8 __attribute__((ext_vector_type(8))); @@ -698,11 +698,6 @@ typedef int dp_int8 __attribute__((ext_vector_type(8))); typedef float dp_float8 __attribute__((ext_vector_type(8))); typedef unsigned dp_uint4 __attribute__((ext_vector_type(4))); } -#ifdef __SYCL_DEVICE_ONLY__ -SYCL_EXTERNAL dp_float8 intel_sub_group_f16_f16_matrix_mad_k16(dp_short8 a, dp_int8 b, dp_float8 acc); -#else -inline dp_float8 intel_sub_group_f16_f16_matrix_mad_k16(dp_short8, dp_int8, dp_float8) { __builtin_unreachable(); } -#endif // ST = 1 (GGML_SYCL_FA_DEC_STAGE, D 256 only): each key tile of K, then of V, is first copied into SLM with coalesced loads // (consecutive work-items read consecutive dwords of a key row) and the lane = key reads come from there (row stride 37 @@ -985,7 +980,7 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, const dp_int8 pb = sP[((t * 16 + lane) * PS + kg * 16) / 16]; #pragma unroll for (int x = 0; x < XT; ++x) { - o[t][x] = intel_sub_group_f16_f16_matrix_mad_k16(va[x], pb, o[t][x]); + o[t][x] = xe2dp::dpas_hf_r8(va[x], pb, o[t][x]); } } } diff --git a/ggml/src/ggml-sycl/ptq1-t2.cpp b/ggml/src/ggml-sycl/ptq1-t2.cpp index 2d6d65c49c36..7ab10a566791 100644 --- a/ggml/src/ggml-sycl/ptq1-t2.cpp +++ b/ggml/src/ggml-sycl/ptq1-t2.cpp @@ -1,5 +1,5 @@ // ARC-LAB: PTQ1_0 on XMX through the TernSYCL int2 x int8 DPAS kernels (ternsycl/, BSD 3-Clause, libxsmm/TernSYCL -// @5e162d8). The base-3 PTQ1_0 packing (1.625 + 0.125 bits/weight) has to be decoded on the ALUs before any DPAS, which +// @11484da + the long long address fix of libxsmm/TernSYCL#3: inline vISA only, no IGC builtins). The base-3 PTQ1_0 packing (1.625 + 0.125 bits/weight) has to be decoded on the ALUs before any DPAS, which // makes the 4-column verify mat-vec compute-bound (~190 GB/s on the B580); 2-bit two's-complement codes (2.125 bits) // feed the s8 x s2 DPAS directly. B580 standalone, 2 GiB rotating weights: 17408x5120 m=1 56 us, m=4 61 us (vs 70 / // ~100 in-model for PTQ1_0). The weight grows 31%; activations become int8 with one fp16 scale per 128 (q8_1 is per 32). @@ -174,13 +174,6 @@ bool ggml_sycl_t2_repack(sycl::queue & q, void * data, int64_t K, int64_t N, boo namespace { -// GGML_SYCL_PTQ1_T2_S32: activation scales per 32 (q8_1 granularity) instead of per 128: 1 = GEMV (<= 8 tokens) only, -// 2 = GEMV + GEMM; 0 = TernSYCL's per 128 -int t2_s32() { - static const int v = getenv("GGML_SYCL_PTQ1_T2_S32") ? atoi(getenv("GGML_SYCL_PTQ1_T2_S32")) : 0; - return v; -} - // fp32 activations -> int8 Aq [M, K] + fp16 SA [K/128, ldsa(M)] (TernSYCL QMODE 0: SA = 127 / absmax, saturate) // G = 128 (one scale per 16 lanes x 8 values) or 32 (a scale per 4 lanes); SA [K/G, ldsa(M)] void quant_a(sycl::queue & q, const float * x, int64_t x_stride, int64_t M, int64_t K, int8_t * Aq, uint16_t * SA, @@ -228,10 +221,10 @@ int t2_nsg() { return v; } -template +template void gemv(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const uint32_t * B, const uint16_t * SB, float * C, int M, int N, int K) { - using Kern = int8dpas::Gemv; + using Kern = int8dpas::Gemv; const size_t wgn = 16 * NSG; const Epi epi{ nullptr, nullptr, 0, 1 }; const sycl::range<2> local(1, Kern::WG); @@ -240,7 +233,7 @@ void gemv(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const uint32_ Kern{ nullptr, (const signed char *) Aq, SA, B, SB, C, epi, M, N, K }); } -template +template void gemm_tile(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const uint32_t * B, const uint16_t * SB, float * C, int M, int N, int K) { const size_t tm = MT_M * WG_M, tn = MT_N * WG_N; @@ -248,82 +241,75 @@ void gemm_tile(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const ui const sycl::range<2> local(1, 16 * WG_M * WG_N); const sycl::range<2> global((M + tm - 1) / tm, (N + tn - 1) / tn * local[1]); q.parallel_for(sycl::nd_range<2>(global, local), - int8dpas::GemmMT{ + int8dpas::GemmMT{ nullptr, (const signed char *) Aq, SA, B, SB, C, epi, M, N, K }); } // ARC-LAB lab knob GGML_SYCL_PTQ1_T2_TILE = index into TernSYCL's large-M tile table (mt_m, mt_n, wg_m, wg_n): // 0 {8,128,8,2} (default) 1 {8,128,4,2} 2 {8,128,4,4} 3 {8,128,16,1} 4 {8,128,2,4} 5 {16,64,4,2} 6 {16,64,8,2} // 7 {8,64,8,2} 8 {32,32,4,2} -template void gemm(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const uint32_t * B, const uint16_t * SB, float * C, int M, int N, int K) { static const int tile = getenv("GGML_SYCL_PTQ1_T2_TILE") ? atoi(getenv("GGML_SYCL_PTQ1_T2_TILE")) : 0; switch (tile) { - case 1: gemm_tile(q, Aq, SA, B, SB, C, M, N, K); break; - case 2: gemm_tile(q, Aq, SA, B, SB, C, M, N, K); break; - case 3: gemm_tile(q, Aq, SA, B, SB, C, M, N, K); break; - case 4: gemm_tile(q, Aq, SA, B, SB, C, M, N, K); break; - case 5: gemm_tile(q, Aq, SA, B, SB, C, M, N, K); break; - case 6: gemm_tile(q, Aq, SA, B, SB, C, M, N, K); break; - case 7: gemm_tile(q, Aq, SA, B, SB, C, M, N, K); break; - case 8: gemm_tile(q, Aq, SA, B, SB, C, M, N, K); break; - default: gemm_tile(q, Aq, SA, B, SB, C, M, N, K); break; + case 1: gemm_tile<8, 128, 4, 2>(q, Aq, SA, B, SB, C, M, N, K); break; + case 2: gemm_tile<8, 128, 4, 4>(q, Aq, SA, B, SB, C, M, N, K); break; + case 3: gemm_tile<8, 128, 16, 1>(q, Aq, SA, B, SB, C, M, N, K); break; + case 4: gemm_tile<8, 128, 2, 4>(q, Aq, SA, B, SB, C, M, N, K); break; + case 5: gemm_tile<16, 64, 4, 2>(q, Aq, SA, B, SB, C, M, N, K); break; + case 6: gemm_tile<16, 64, 8, 2>(q, Aq, SA, B, SB, C, M, N, K); break; + case 7: gemm_tile<8, 64, 8, 2>(q, Aq, SA, B, SB, C, M, N, K); break; + case 8: gemm_tile<32, 32, 4, 2>(q, Aq, SA, B, SB, C, M, N, K); break; + default: gemm_tile<8, 128, 8, 2>(q, Aq, SA, B, SB, C, M, N, K); break; } } } // namespace namespace { -template void t2_dispatch(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const uint32_t * B, const uint16_t * SB, float * dst, int m, int n, int k, bool use_gemm) { if (use_gemm) { - gemm(q, Aq, SA, B, SB, dst, m, n, k); + gemm(q, Aq, SA, B, SB, dst, m, n, k); } else if (m == 1) { - n <= 8192 ? gemv<1, 4, NS>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<1, 2, NS>(q, Aq, SA, B, SB, dst, m, n, k); + n <= 8192 ? gemv<1, 4>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<1, 2>(q, Aq, SA, B, SB, dst, m, n, k); } else if (t2_nsg() == 4) { if (m == 2) { - n <= 8192 ? gemv<2, 4, NS, 4>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<2, 2, NS, 4>(q, Aq, SA, B, SB, dst, m, n, k); + n <= 8192 ? gemv<2, 4, 4>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<2, 2, 4>(q, Aq, SA, B, SB, dst, m, n, k); } else if (m <= 4) { - n <= 8192 ? gemv<4, 4, NS, 4>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<4, 2, NS, 4>(q, Aq, SA, B, SB, dst, m, n, k); + n <= 8192 ? gemv<4, 4, 4>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<4, 2, 4>(q, Aq, SA, B, SB, dst, m, n, k); } else { - n <= 8192 ? gemv<8, 4, NS, 4>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<8, 2, NS, 4>(q, Aq, SA, B, SB, dst, m, n, k); + n <= 8192 ? gemv<8, 4, 4>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<8, 2, 4>(q, Aq, SA, B, SB, dst, m, n, k); } } else if (m == 2) { - n <= 8192 ? gemv<2, 4, NS>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<2, 2, NS>(q, Aq, SA, B, SB, dst, m, n, k); + n <= 8192 ? gemv<2, 4>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<2, 2>(q, Aq, SA, B, SB, dst, m, n, k); } else if (m <= 4) { - n <= 8192 ? gemv<4, 4, NS>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<4, 2, NS>(q, Aq, SA, B, SB, dst, m, n, k); + n <= 8192 ? gemv<4, 4>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<4, 2>(q, Aq, SA, B, SB, dst, m, n, k); } else { - n <= 8192 ? gemv<8, 4, NS>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<8, 2, NS>(q, Aq, SA, B, SB, dst, m, n, k); + n <= 8192 ? gemv<8, 4>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<8, 2>(q, Aq, SA, B, SB, dst, m, n, k); } } } // namespace void ggml_sycl_t2_mul_mat(sycl::queue & q, const void * w, const float * x, int64_t x_stride, float * dst, int64_t M, int64_t N, int64_t K) { - // batches > 8 (n-gram verify, prompts): GemmMT reads each weight once per 64 rows. Its SA read, A read and C write - // were switched from 2D block I/O to plain / sub-group block access (the 2D forms misbehaved in this JIT build). + // batches > 8 (n-gram verify, prompts): GemmMT reads each weight once per 64 rows (TernSYCL main: the 2D block I/O + // that misbehaved in the JIT build with the old IGC builtins validates under inline vISA, so it is upstream's kernel). static const bool gemm_off = getenv("GGML_SYCL_PTQ1_T2_GEMM_OFF") != nullptr; const bool use_gemm = M > 8 && !gemm_off; - const bool s32 = use_gemm ? t2_s32() >= 2 : t2_s32() >= 1; const int lda = int8dpas::ldsa((int) M); const size_t aq_bytes = (size_t) M * K; const size_t sa_off = (aq_bytes + 255) & ~(size_t) 255; - const size_t sa_bytes = (size_t) (K / 32) * lda * 2 + 256; + const size_t sa_bytes = (size_t) (K / QK) * lda * 2 + 256; char * s = (char *) scratch_get(q, sa_off + sa_bytes); int8_t * Aq = (int8_t *) s; uint16_t * SA = (uint16_t *) (s + sa_off); const uint32_t * B = (const uint32_t *) w; const uint16_t * SB = (const uint16_t *) ((const char *) w + (size_t) (K / 16) * N * 4); - quant_a(q, x, x_stride, M, K, Aq, SA, s32 ? 32 : QK); - if (s32) { - t2_dispatch<4>(q, Aq, SA, B, SB, dst, (int) M, (int) N, (int) K, use_gemm); - } else { - t2_dispatch<1>(q, Aq, SA, B, SB, dst, (int) M, (int) N, (int) K, use_gemm); - } + quant_a(q, x, x_stride, M, K, Aq, SA, QK); + t2_dispatch(q, Aq, SA, B, SB, dst, (int) M, (int) N, (int) K, use_gemm); } namespace { @@ -341,29 +327,17 @@ void t2_precompile(sycl::queue & q) { const uint32_t * B = (const uint32_t *) w; const uint16_t * SB = (const uint16_t *) (w + (K / 16) * N * 4); quant_a(q, x, K, M, K, aq, sa, QK); - quant_a(q, x, K, M, K, aq, sa, 32); - for (int m : { 1, 2, 4, 8, M }) { - for (int ns = 0; ns < 2; ++ns) { - // both GEMV tiles (the N > 8192 one is picked by n, so call the templates directly) - if (ns == 0) { - t2_dispatch<1>(q, aq, sa, B, SB, d, m, N, K, m > 8); - } else { - t2_dispatch<4>(q, aq, sa, B, SB, d, m, N, K, m > 8); - } - } + for (int m : { 1, 2, 4, 8, M }) { // the N <= 8192 GEMV tiles + GemmMT (the N > 8192 ones are called below) + t2_dispatch(q, aq, sa, B, SB, d, m, N, K, m > 8); } gemv<1, 2>(q, aq, sa, B, SB, d, 1, N, K); gemv<2, 2>(q, aq, sa, B, SB, d, 2, N, K); gemv<4, 2>(q, aq, sa, B, SB, d, 4, N, K); gemv<8, 2>(q, aq, sa, B, SB, d, 8, N, K); - gemv<1, 2, 4>(q, aq, sa, B, SB, d, 1, N, K); - gemv<2, 2, 4>(q, aq, sa, B, SB, d, 2, N, K); - gemv<4, 2, 4>(q, aq, sa, B, SB, d, 4, N, K); - gemv<8, 2, 4>(q, aq, sa, B, SB, d, 8, N, K); if (t2_nsg() == 4) { // the N > 8192 GEMV tiles with 4 sub-groups (t2_dispatch above covered the N <= 8192 ones) - gemv<2, 2, 1, 4>(q, aq, sa, B, SB, d, 2, N, K); - gemv<4, 2, 1, 4>(q, aq, sa, B, SB, d, 4, N, K); - gemv<8, 2, 1, 4>(q, aq, sa, B, SB, d, 8, N, K); + gemv<2, 2, 4>(q, aq, sa, B, SB, d, 2, N, K); + gemv<4, 2, 4>(q, aq, sa, B, SB, d, 4, N, K); + gemv<8, 2, 4>(q, aq, sa, B, SB, d, 8, N, K); } q.wait(); sycl::free(w, q); diff --git a/ggml/src/ggml-sycl/ternsycl/int2_int8_dpas.hpp b/ggml/src/ggml-sycl/ternsycl/int2_int8_dpas.hpp index ac3c05d59878..35e0f2d1fdde 100644 --- a/ggml/src/ggml-sycl/ternsycl/int2_int8_dpas.hpp +++ b/ggml/src/ggml-sycl/ternsycl/int2_int8_dpas.hpp @@ -151,8 +151,7 @@ template inline void set_el(V &v, int r, T x) { // SGM rows per sub-group (1, 2, 4, 8) NSG_N sub-groups along N // LS K-slices per column block, reduced through SLM // U 128-steps whose loads are issued before any compute -// ARC-LAB: NS = activation scales per 128-step (1 = TernSYCL; 4 = one per 32, like q8_1; QMODE 0 only, SA [K/32, ldsa]) -template +template struct Gemv { const unsigned short *A; const signed char *Aq; @@ -175,17 +174,16 @@ struct Gemv { #pragma unroll for (int r = 0; r < SGM; ++r) { const bool ok = m0 + r < M; + const size_t row = (size_t)sycl::min(m0 + r, M - 1) * K + s * GS; if constexpr (QMODE == 0) { - const ushort4 v = ok ? intel_sub_group_block_read_us4( - gptr((const unsigned short *)(Aq + (size_t)(m0 + r) * K + s * GS))) : ushort4{}; + const ushort4 l = sg_rd_us4((const unsigned short *)(Aq + row)); + const ushort4 v = ok ? l : ushort4{}; #pragma unroll for (int c = 0; c < 4; ++c) set_el(aq[c], r, (short)v[c]); -#pragma unroll - for (int cs = 0; cs < NS; ++cs) - inv[cs * SGM + r] = ok ? sycl::native::recip(D::tof(SA[(size_t)(s * NS + cs) * lda + m0 + r])) : 0.0f; + inv[r] = ok ? sycl::native::recip(D::tof(SA[(size_t)s * lda + m0 + r])) : 0.0f; } else { - const uint4 v = ok ? intel_sub_group_block_read4( - gptr((const unsigned *)(A + (size_t)(m0 + r) * K + s * GS))) : uint4{}; + const uint4 l = sg_rd_u4((const unsigned *)(A + row)); + const uint4 v = ok ? l : uint4{}; const float sa = ok ? D::tof(SA[(size_t)s * lda + m0 + r]) : 0.0f; #pragma unroll for (int c = 0; c < 4; ++c) set_el(aq[c], r, q2(v[c], sa)); @@ -195,21 +193,10 @@ struct Gemv { } static fa_t step(fa_t acc, const uint8 &w, float sb, const a_t *aq, const float *inv) { - if constexpr (NS == 4) { -#pragma unroll - for (int c = 0; c < 4; ++c) { - ia_t ia = 0; - ia = intel_sub_group_i8_i2_matrix_mad_k32(aq[c], int2{(int)w[2 * c], (int)w[2 * c + 1]}, ia); -#pragma unroll - for (int r = 0; r < SGM; ++r) - set_el(acc, r, el(acc, r) + (float)el(ia, r) * (sb * inv[c * SGM + r])); - } - return acc; - } - ia_t ia = 0; + ia_t ia = dpas_s2s8_z(aq[0], int2{(int)w[0], (int)w[1]}); #pragma unroll - for (int c = 0; c < 4; ++c) - ia = intel_sub_group_i8_i2_matrix_mad_k32(aq[c], int2{(int)w[2 * c], (int)w[2 * c + 1]}, ia); + for (int c = 1; c < 4; ++c) + ia = dpas_s2s8(aq[c], int2{(int)w[2 * c], (int)w[2 * c + 1]}, ia); #pragma unroll for (int r = 0; r < SGM; ++r) set_el(acc, r, el(acc, r) + (float)el(ia, r) * (sb * inv[r])); @@ -233,25 +220,27 @@ struct Gemv { fa_t acc = 0.0f; if (n0 < N) { int s = s_begin; +#pragma unroll 1 for (; s + U <= s_end; s += U) { uint8 w[U]; float sb[U]; a_t aq[U][4]; - float inv[U][NS * SGM]; + float inv[U][SGM]; #pragma unroll for (int u = 0; u < U; ++u) { w[u] = rd_32b_8r16(sbs, n0, (s + u) * 8); - sb[u] = D::tof(intel_sub_group_block_read_us(gptr(SB + (size_t)(s + u) * N + n0))); + sb[u] = D::tof(sg_rd_us(SB + (size_t)(s + u) * N + n0)); load_a(m0, s + u, aq[u], inv[u]); } #pragma unroll for (int u = 0; u < U; ++u) acc = step(acc, w[u], sb[u], aq[u], inv[u]); } + if constexpr (U > 1) for (; s < s_end; ++s) { a_t aq[4]; - float inv[NS * SGM]; + float inv[SGM]; const uint8 w = rd_32b_8r16(sbs, n0, s * 8); - const float sb = D::tof(intel_sub_group_block_read_us(gptr(SB + (size_t)s * N + n0))); + const float sb = D::tof(sg_rd_us(SB + (size_t)s * N + n0)); load_a(m0, s, aq, inv); acc = step(acc, w, sb, aq, inv); } @@ -295,7 +284,7 @@ struct Gemv { // zero-fills out-of-range reads and clips writes, so any M works. 256 GRF. // The epilogue is a template parameter here (as in the OpenCL -D build): a // runtime-selected one costs ~3% in this kernel. -template +template struct GemmMT { const unsigned short *A; const signed char *Aq; @@ -324,106 +313,86 @@ struct GemmMT { #pragma unroll for (int j = 0; j < NB; ++j) acc[i][j] = 0.0f; + // 2D payloads built once; each K step only moves y (B, SB, SA) or x (A) + unsigned pb = pl2d(sbs, n0, 0), psb2 = pl2d(ssb, n0, 0), + psb1 = pl2d(ssb, n0, 0), psa = pl2d(ssa, m0, 0), pq; + if constexpr (QMODE == 0) pq = pl2d(surf(Aq, K, M, K), 0, m0); + else pq = pl2d(surf(A, K * 2, M, K * 2), 0, m0); + for (int s = 0; s < K / GS; ++s) { + // A (and SA) loads first: they feed the quantization / unpacking that the + // first dpas waits on (the asm reads are issued in source order). Row + // block 0 before B and SB, block I + 1 at the start of block I. + pl2d_y(psa, s); + pl2d_x(pq, s * GS / 2); + unsigned short sar[MB]; + std::conditional_t ar[MB]; + auto load_a = [&](auto ii) { + constexpr int I = decltype(ii)::value; + sar[I] = rd2d(psa); + if constexpr (QMODE == 0) + static_for<2>([&](auto h) { ar[I][h] = rd2d(pq); }); + else + static_for<4>([&](auto c) { ar[I][c] = rd2d(pq); }); + }; + load_a(std::integral_constant{}); uint8 w[NB]; float sb[NB]; -#pragma unroll - for (int j = 0; j < NB; ++j) w[j] = rd_32b_8r16(sbs, n0 + 16 * j, s * 8); + pl2d_y(pb, s * 8); + static_for([&](auto j) { w[j] = rd2d(pb); }); // all SB loads first, then convert: converting each (bf16: via acc0) before the // next load let IGC reuse one load register and serialize the loads ushort2 sbr[(NB + 1) / 2]; -#pragma unroll - for (int j = 0; j < NB; j += 2) - sbr[j / 2] = j + 1 < NB ? rd_16b_1r16x2(ssb, n0 + 16 * j, s) - : ushort2{rd_16b_1r16(ssb, n0 + 16 * j, s), 0}; + pl2d_y(psb2, s); + pl2d_y(psb1, s); + static_for<(NB + 1) / 2>([&](auto h) { + constexpr int J = 2 * decltype(h)::value; + if constexpr (J + 1 < NB) sbr[h] = rd2d(psb2); + else sbr[h] = ushort2{rd2d(psb1), 0}; + }); #pragma unroll for (int j = 0; j < NB; ++j) sb[j] = D::tof(sbr[j / 2][j % 2]); -#pragma unroll - for (int i = 0; i < MB; ++i) { - const int mr = m0 + 8 * i; + static_for([&](auto ii) { + constexpr int I = decltype(ii)::value; + if constexpr (I + 1 < MB) load_a(std::integral_constant{}); short8 aq[4]; - float inv[NS][8]; - // ARC-LAB: plain load; the 2D block read of SA gave junk past lane 0 in the llama.cpp JIT build - const float sal = D::tof(SA[(size_t)s * NS * ldsa(M) + mr + (int)sgp.get_local_linear_id()]); + float inv[8]; + const float sal = D::tof(sar[I]); if constexpr (QMODE == 0) { - // ARC-LAB: sub-group block reads per row (as Gemv::load_a); the 2D block read of Aq gave NaN from - // row 1 on in the llama.cpp JIT build #pragma unroll - for (int r = 0; r < 8; ++r) { - const bool ok = mr + r < M; - const ushort4 v = ok ? intel_sub_group_block_read_us4( - gptr((const unsigned short *)(Aq + (size_t)(mr + r) * K + s * GS))) : ushort4{}; + for (int h = 0; h < 2; ++h) #pragma unroll - for (int c = 0; c < 4; ++c) aq[c][r] = (short)v[c]; - } + for (int r = 0; r < 8; ++r) { + aq[2 * h][r] = (short)ar[I][h][r]; + aq[2 * h + 1][r] = (short)ar[I][h][8 + r]; + } } else { - const surf sa(A, K * 2, M, K * 2); - // all loads before the (volatile, so unreorderable) quant asm - uint8 a[4]; -#pragma unroll - for (int c = 0; c < 4; ++c) a[c] = rd_32b_8r16(sa, s * GS / 2 + 16 * c, mr); #pragma unroll - for (int c = 0; c < 4; ++c) aq[c] = quant8x32(a[c], sal); + for (int c = 0; c < 4; ++c) aq[c] = quant8x32(ar[I][c], sal); } // rows >= M get inf here, but their int32 dot is 0 and the store clips them #pragma unroll - for (int r = 0; r < 8; ++r) inv[0][r] = sycl::native::recip(sycl::group_broadcast(sgp, sal, r)); - if constexpr (NS == 4) { -#pragma unroll - for (int cs = 1; cs < 4; ++cs) { - const float salc = D::tof(SA[(size_t)(s * NS + cs) * ldsa(M) + mr + (int)sgp.get_local_linear_id()]); -#pragma unroll - for (int r = 0; r < 8; ++r) inv[cs][r] = sycl::native::recip(sycl::group_broadcast(sgp, salc, r)); - } - } + for (int r = 0; r < 8; ++r) inv[r] = sycl::native::recip(sycl::group_broadcast(sgp, sal, r)); #pragma unroll for (int j = 0; j < NB; ++j) { - if constexpr (NS == 4) { -#pragma unroll - for (int c = 0; c < 4; ++c) { - int8 ia = 0; - ia = intel_sub_group_i8_i2_matrix_mad_k32(aq[c], - int2{(int)w[j][2 * c], (int)w[j][2 * c + 1]}, ia); - const float8 fi = __builtin_convertvector(ia, float8); + int8 ia = dpas_s2s8_z(aq[0], int2{(int)w[j][0], (int)w[j][1]}); #pragma unroll - for (int r = 0; r < 8; ++r) acc[i][j][r] += fi[r] * (sb[j] * inv[c][r]); - } - continue; - } - int8 ia = 0; -#pragma unroll - for (int c = 0; c < 4; ++c) - ia = intel_sub_group_i8_i2_matrix_mad_k32(aq[c], - int2{(int)w[j][2 * c], (int)w[j][2 * c + 1]}, ia); + for (int c = 1; c < 4; ++c) + ia = dpas_s2s8(aq[c], int2{(int)w[j][2 * c], (int)w[j][2 * c + 1]}, ia); // whole-vector convert (OpenCL's convert_float8): per-element casts // are emitted through one scratch register and stall every mad const float8 fi = __builtin_convertvector(ia, float8); #pragma unroll - for (int r = 0; r < 8; ++r) acc[i][j][r] += fi[r] * (sb[j] * inv[0][r]); + for (int r = 0; r < 8; ++r) acc[I][j][r] += fi[r] * (sb[j] * inv[r]); } - } + }); } - if constexpr (POSTOP == 0 && F32) { - // ARC-LAB: plain stores; the 2D block write misplaced rows (junk rows past M landed on real ones) in the - // llama.cpp JIT build - const int lane = (int)sgp.get_local_linear_id(); #pragma unroll - for (int i = 0; i < MB; ++i) -#pragma unroll - for (int j = 0; j < NB; ++j) -#pragma unroll - for (int r = 0; r < 8; ++r) { - const int m = m0 + 8 * i + r, n = n0 + 16 * j + lane; - if (m < M && n < N) ((float *)C)[(size_t)m * N + n] = acc[i][j][r]; - } - } else { -#pragma unroll - for (int i = 0; i < MB; ++i) + for (int i = 0; i < MB; ++i) #pragma unroll - for (int j = 0; j < NB; ++j) - epi_dev::template store8x16(C, epi, M, N, m0 + 8 * i, n0 + 16 * j, acc[i][j]); - } + for (int j = 0; j < NB; ++j) + epi_dev::template store8x16(C, epi, M, N, m0 + 8 * i, n0 + 16 * j, acc[i][j]); } auto get(syclex::properties_tag) const { diff --git a/ggml/src/ggml-sycl/ternsycl/xe2.hpp b/ggml/src/ggml-sycl/ternsycl/xe2.hpp index ad284042975c..bdeb2ac55096 100644 --- a/ggml/src/ggml-sycl/ternsycl/xe2.hpp +++ b/ggml/src/ggml-sycl/ternsycl/xe2.hpp @@ -1,10 +1,13 @@ -// Xe2 sub-group intrinsics for SYCL device code: the same IGC builtins the -// OpenCL kernels of TernOCL use (DPAS, 2D block I/O, sub-group block reads), -// declared with the OpenCL mangling / IGC builtin names so that IGC resolves -// them from the SPIR-V exactly as for OpenCL C. +// Xe2 sub-group primitives for SIMT SYCL device code, without OpenCL builtins +// or IGC-internal intrinsics: +// * DPAS, 2D block I/O, cached LSC loads: inline vISA (the instructions IGC +// emits for the OpenCL builtins; operands are the SIMD16 per-lane vectors, +// row r of a block / DPAS operand = vector component r) +// * sub-group block reads: sycl_ext_oneapi_group_load_store (group_load) #pragma once #include +#include #include @@ -18,6 +21,8 @@ XE2_VEC(unsigned short, 2, ushort2); XE2_VEC(unsigned short, 4, ushort4); XE2_VEC(unsigned short, 8, ushort8); XE2_VEC(unsigned short, 16, ushort16); +XE2_VEC(unsigned short, 32, ushort32); +XE2_VEC(unsigned short, 64, ushort64); XE2_VEC(int, 2, int2); XE2_VEC(int, 4, int4); XE2_VEC(int, 8, int8); @@ -30,67 +35,8 @@ XE2_VEC(float, 4, float4); XE2_VEC(float, 8, float8); #undef XE2_VEC -#define XE2_GLOBAL __attribute__((opencl_global)) - -} // namespace xe2 - -#ifdef __SYCL_DEVICE_ONLY__ -#define XE2_OCL(decl) SYCL_EXTERNAL decl -#define XE2_IB(decl) extern "C" SYCL_EXTERNAL decl -#else -#define XE2_OCL(decl) inline decl { __builtin_unreachable(); } -#define XE2_IB(decl) inline decl { __builtin_unreachable(); } -#endif - -// DPAS, SIMD16: repeat count = rows of A (1, 2, 4, 8) -#define XE2_MAD(name, A, B, C) XE2_OCL(C name(A a, B b, C acc)); -#define XE2_MAD_ALL(name) \ - XE2_MAD(name, short, xe2::int8, float) \ - XE2_MAD(name, xe2::short2, xe2::int8, xe2::float2) \ - XE2_MAD(name, xe2::short4, xe2::int8, xe2::float4) \ - XE2_MAD(name, xe2::short8, xe2::int8, xe2::float8) -XE2_MAD_ALL(intel_sub_group_f16_f16_matrix_mad_k16) -XE2_MAD_ALL(intel_sub_group_bf16_bf16_matrix_mad_k16) -#undef XE2_MAD_ALL -// s8 x s2, K = 32 -XE2_MAD(intel_sub_group_i8_i2_matrix_mad_k32, short, xe2::int2, int) -XE2_MAD(intel_sub_group_i8_i2_matrix_mad_k32, xe2::short2, xe2::int2, xe2::int2) -XE2_MAD(intel_sub_group_i8_i2_matrix_mad_k32, xe2::short4, xe2::int2, xe2::int4) -XE2_MAD(intel_sub_group_i8_i2_matrix_mad_k32, xe2::short8, xe2::int2, xe2::int8) -#undef XE2_MAD - -// sub-group block reads (16 lanes, consecutive elements per lane) -XE2_OCL(unsigned intel_sub_group_block_read(const XE2_GLOBAL unsigned *p)); -XE2_OCL(xe2::uint4 intel_sub_group_block_read4(const XE2_GLOBAL unsigned *p)); -XE2_OCL(unsigned short intel_sub_group_block_read_us(const XE2_GLOBAL unsigned short *p)); -XE2_OCL(xe2::ushort4 intel_sub_group_block_read_us4(const XE2_GLOBAL unsigned short *p)); -XE2_OCL(xe2::ushort8 intel_sub_group_block_read_us8(const XE2_GLOBAL unsigned short *p)); - -// 2D block I/O: (base, width - 1, height - 1, pitch - 1) in bytes, coord (x -// in elements, y in rows). mRkCvV = R rows x C columns x V blocks. -#define XE2_2D_RD(ret, sfx) \ - XE2_IB(ret __builtin_IB_subgroup_block_read_flat_##sfx(long long base, int w, int h, int p, xe2::int2 c)); -XE2_2D_RD(unsigned, u32_m1k16v1) -XE2_2D_RD(xe2::uint2, u32_m2k16v1) -XE2_2D_RD(xe2::uint8, u32_m8k16v1) -XE2_2D_RD(unsigned short, u16_m1k16v1) -XE2_2D_RD(xe2::ushort2, u16_m1k16v2) -XE2_2D_RD(xe2::ushort8, u16_m8k16v1) -XE2_2D_RD(xe2::ushort16, u16_m8k16v2) -#undef XE2_2D_RD -XE2_IB(void __builtin_IB_subgroup_block_write_flat_u16_m8k16v1(long long base, int w, int h, int p, - xe2::int2 c, xe2::ushort8 v)); -XE2_IB(void __builtin_IB_subgroup_block_write_flat_u32_m8k16v1(long long base, int w, int h, int p, - xe2::int2 c, xe2::uint8 v)); -// cache control 0 = default -XE2_IB(void __builtin_IB_subgroup_block_read_prefetch_u32_m8k16v1(long long base, int w, int h, int p, - xe2::int2 c, int cc)); -// per-lane LSC load with cache control (IGC LSC_LDCC: 4 = L1 cached, L3 cached) -XE2_IB(xe2::uint3 __builtin_IB_lsc_load_global_uint3(const XE2_GLOBAL xe2::uint3 *p, int imm_off, int cc)); - -namespace xe2 { - -// 2D surface: width and pitch in bytes, height in rows +// 2D surface: width and pitch in bytes, height in rows. Addresses are long long: +// long is 32 bits in SYCL device code on Windows (LLP64) and would truncate them struct surf { long long base; int w, h, p; @@ -98,42 +44,251 @@ struct surf { : base((long long)b), w(width_bytes - 1), h(height - 1), p(pitch_bytes - 1) {} }; -inline unsigned rd_32b_1r16(const surf &s, int x, int y) { - return __builtin_IB_subgroup_block_read_flat_u32_m1k16v1(s.base, s.w, s.h, s.p, int2{x, y}); -} -inline uint2 rd_32b_2r16(const surf &s, int x, int y) { - return __builtin_IB_subgroup_block_read_flat_u32_m2k16v1(s.base, s.w, s.h, s.p, int2{x, y}); -} -inline uint8 rd_32b_8r16(const surf &s, int x, int y) { - return __builtin_IB_subgroup_block_read_flat_u32_m8k16v1(s.base, s.w, s.h, s.p, int2{x, y}); -} +#ifdef __SYCL_DEVICE_ONLY__ +#define XE2_ASM(...) __asm__(__VA_ARGS__) +#define XE2_ASM_V(...) __asm__ volatile(__VA_ARGS__) +#else +#define XE2_ASM(...) +#define XE2_ASM_V(...) +#endif + +// 2D block I/O, lsc_load_block2d / lsc_store_block2d on an explicit surface +// flat[base, width-1, height-1, pitch-1, x (elements), y (rows)]; shape +// d.[V x]C x R: V blocks of C columns x R rows, block-major. +#define XE2_2D_RD(name, T, shape) \ + inline T name(const surf &s, int x, int y) { \ + T v; \ + XE2_ASM("{\n" \ + ".decl SB v_type=G type=q num_elts=1 align=qword alias=<%1,0>\n" \ + ".decl SW v_type=G type=d num_elts=1 align=dword alias=<%2,0>\n" \ + ".decl SH v_type=G type=d num_elts=1 align=dword alias=<%3,0>\n" \ + ".decl SP v_type=G type=d num_elts=1 align=dword alias=<%4,0>\n" \ + ".decl SX v_type=G type=d num_elts=1 align=dword alias=<%5,0>\n" \ + ".decl SY v_type=G type=d num_elts=1 align=dword alias=<%6,0>\n" \ + "lsc_load_block2d.ugm (M1, 1) %0:" shape " flat[SB,SW,SH,SP,SX,SY]\n" \ + "}\n" \ + : "=rw"(v) \ + : "rw.u"(s.base), "rw.u"(s.w), "rw.u"(s.h), "rw.u"(s.p), "rw.u"(x), "rw.u"(y)); \ + return v; \ + } +#define XE2_2D_WR(name, T, shape) \ + inline void name(const surf &s, int x, int y, T v) { \ + XE2_ASM_V("{\n" \ + ".decl SB v_type=G type=q num_elts=1 align=qword alias=<%0,0>\n" \ + ".decl SW v_type=G type=d num_elts=1 align=dword alias=<%1,0>\n" \ + ".decl SH v_type=G type=d num_elts=1 align=dword alias=<%2,0>\n" \ + ".decl SP v_type=G type=d num_elts=1 align=dword alias=<%3,0>\n" \ + ".decl SX v_type=G type=d num_elts=1 align=dword alias=<%4,0>\n" \ + ".decl SY v_type=G type=d num_elts=1 align=dword alias=<%5,0>\n" \ + "lsc_store_block2d.ugm (M1, 1) flat[SB,SW,SH,SP,SX,SY] %6:" shape "\n" \ + "}\n" \ + :: "rw.u"(s.base), "rw.u"(s.w), "rw.u"(s.h), "rw.u"(s.p), "rw.u"(x), "rw.u"(y), "rw"(v)); \ + } +XE2_2D_RD(rd_32b_1r16, unsigned, "d32.16x1nn") +XE2_2D_RD(rd_32b_2r16, uint2, "d32.16x2nn") +XE2_2D_RD(rd_32b_8r16, uint8, "d32.16x8nn") +// a 16x1 d16 block is half a GRF, but the load writes a whole (zero-padded) GRF inline unsigned short rd_16b_1r16(const surf &s, int x, int y) { - return __builtin_IB_subgroup_block_read_flat_u16_m1k16v1(s.base, s.w, s.h, s.p, int2{x, y}); + unsigned short v; + XE2_ASM("{\n" + ".decl SB v_type=G type=q num_elts=1 align=qword alias=<%1,0>\n" + ".decl SW v_type=G type=d num_elts=1 align=dword alias=<%2,0>\n" + ".decl SH v_type=G type=d num_elts=1 align=dword alias=<%3,0>\n" + ".decl SP v_type=G type=d num_elts=1 align=dword alias=<%4,0>\n" + ".decl SX v_type=G type=d num_elts=1 align=dword alias=<%5,0>\n" + ".decl SY v_type=G type=d num_elts=1 align=dword alias=<%6,0>\n" + ".decl TP v_type=G type=uw num_elts=32 align=GRF\n" + "lsc_load_block2d.ugm (M1, 1) TP:d16.16x1nn flat[SB,SW,SH,SP,SX,SY]\n" + "mov (M1, 16) %0(0,0)<1> TP(0,0)<1;1,0>\n" + "}\n" + : "=rw"(v) + : "rw.u"(s.base), "rw.u"(s.w), "rw.u"(s.h), "rw.u"(s.p), "rw.u"(x), "rw.u"(y)); + return v; } -inline ushort2 rd_16b_1r16x2(const surf &s, int x, int y) { - return __builtin_IB_subgroup_block_read_flat_u16_m1k16v2(s.base, s.w, s.h, s.p, int2{x, y}); +// two adjacent 1x16 blocks = one 1x32 block: [0] = column x + lane, [1] = x + 16 + lane +XE2_2D_RD(rd_16b_1r16x2, ushort2, "d16.32x1nn") +XE2_2D_RD(rd_16b_8r16, ushort8, "d16.16x8nn") +// two adjacent 8x16 blocks, block-major: [0..7] = columns x..x+15, [8..15] = x+16..x+31 +XE2_2D_RD(rd_16b_8r16x2, ushort16, "d16.2x16x8nn") +XE2_2D_WR(wr_16b_8r16, ushort8, "d16.16x8nn") +XE2_2D_WR(wr_32b_8r16, uint8, "d32.16x8nn") +#undef XE2_2D_RD +#undef XE2_2D_WR + +// 2D block reads from a prebuilt address payload plus immediate (DX, DY) offsets, +// as IGC emits for the builtins: one payload per surface and K step instead of +// ~6 movs per load. The vISA text is built at compile time (asm((expr))). +namespace detail { +template struct cstr { + char s[N]{}; + size_t n = 0; + constexpr size_t size() const { return n; } + constexpr const char *data() const { return s; } + constexpr void add(const char *p) { while (*p) s[n++] = *p++; } + constexpr void addi(int v) { + if (v < 0) { s[n++] = '-'; v = -v; } + char t[12]; + int k = 0; + do { t[k++] = char('0' + v % 10); v /= 10; } while (v); + while (k) s[n++] = t[--k]; + } +}; +template constexpr auto rd2d_str() { + cstr<320> c; + c.add("{\n.decl PD v_type=G type=ud num_elts=8 align=GRF alias=<%1,0>\n"); + if constexpr (SH::pad) c.add(".decl TP v_type=G type=uw num_elts=32 align=GRF\n" + "lsc_load_block2d.ugm (M1, 1) TP:"); + else c.add("lsc_load_block2d.ugm (M1, 1) %0:"); + c.add(SH::v); + c.add(" flat[PD + ("); + c.addi(DX); + c.add(","); + c.addi(DY); + c.add(")]\n"); + if constexpr (SH::pad) c.add("mov (M1, 16) %0(0,0)<1> TP(0,0)<1;1,0>\n"); + c.add("}\n"); + return c; } -inline ushort8 rd_16b_8r16(const surf &s, int x, int y) { - return __builtin_IB_subgroup_block_read_flat_u16_m8k16v1(s.base, s.w, s.h, s.p, int2{x, y}); +} // namespace detail + +// block shape: vISA type string and payload dword 7 = (V-1) << 16 | (R-1) << 8 | (C-1); +// pad: the block is half a GRF but the load writes a whole (zero-padded) GRF +struct b32_16x1 { static constexpr const char *v = "d32.16x1nn"; static constexpr int code = 0x00f; static constexpr bool pad = false; }; +struct b32_16x8 { static constexpr const char *v = "d32.16x8nn"; static constexpr int code = 0x70f; static constexpr bool pad = false; }; +struct b16_16x8 { static constexpr const char *v = "d16.16x8nn"; static constexpr int code = 0x70f; static constexpr bool pad = false; }; +struct b16_2x16x8 { static constexpr const char *v = "d16.2x16x8nn"; static constexpr int code = 0x1070f; static constexpr bool pad = false; }; +struct b16_16x16 { static constexpr const char *v = "d16.16x16nn"; static constexpr int code = 0xf0f; static constexpr bool pad = false; }; +struct b16_2x16x16 { static constexpr const char *v = "d16.2x16x16nn"; static constexpr int code = 0x10f0f; static constexpr bool pad = false; }; +struct b16_16x32 { static constexpr const char *v = "d16.16x32nn"; static constexpr int code = 0x1f0f; static constexpr bool pad = false; }; +struct b16_2x16x32 { static constexpr const char *v = "d16.2x16x32nn"; static constexpr int code = 0x11f0f; static constexpr bool pad = false; }; +struct b16_32x1 { static constexpr const char *v = "d16.32x1nn"; static constexpr int code = 0x01f; static constexpr bool pad = false; }; +struct b16_16x1 { static constexpr const char *v = "d16.16x1nn"; static constexpr int code = 0x00f; static constexpr bool pad = true; }; + +template inline unsigned pl2d(const surf &s, int x, int y) { + unsigned pl; + XE2_ASM("{\n" + ".decl PQ v_type=G type=uq num_elts=4 align=GRF alias=<%0,0>\n" + ".decl PD v_type=G type=ud num_elts=8 align=GRF alias=<%0,0>\n" + "mov (M1_NM, 1) PQ(0,0)<1> %1(0,0)<0;1,0>\n" + "mov (M1_NM, 1) PD(0,2)<1> %2(0,0)<0;1,0>\n" + "mov (M1_NM, 1) PD(0,3)<1> %3(0,0)<0;1,0>\n" + "mov (M1_NM, 1) PD(0,4)<1> %4(0,0)<0;1,0>\n" + "mov (M1_NM, 1) PD(0,5)<1> %5(0,0)<0;1,0>\n" + "mov (M1_NM, 1) PD(0,6)<1> %6(0,0)<0;1,0>\n" + "mov (M1_NM, 1) PD(0,7)<1> %7(0,0)<0;1,0>\n" + "}\n" + : "=rw"(pl) + : "rw.u"(s.base), "rw.u"(s.w), "rw.u"(s.h), "rw.u"(s.p), "rw.u"(x), "rw.u"(y), "rw.u"(SH::code)); + return pl; } -// two adjacent 8x16 blocks, block-major: [0..7] = columns x..x+15, [8..15] = x+16..x+31 -inline ushort16 rd_16b_8r16x2(const surf &s, int x, int y) { - return __builtin_IB_subgroup_block_read_flat_u16_m8k16v2(s.base, s.w, s.h, s.p, int2{x, y}); +// in-place update of the block x (dword 5) or y (dword 6) of a payload: build the +// payload once outside the K loop and move it per step (keeps one register per +// payload live across the loop, as IGC does, instead of rebuilding all 8 dwords) +template inline void pl2d_set(unsigned &pl, int v) { + static_assert(D == 5 || D == 6); + if constexpr (D == 5) + XE2_ASM("{\n.decl PD v_type=G type=ud num_elts=8 align=GRF alias=<%0,0>\n" + "mov (M1_NM, 1) PD(0,5)<1> %1(0,0)<0;1,0>\n}\n" : "+rw"(pl) : "rw.u"(v)); + else + XE2_ASM("{\n.decl PD v_type=G type=ud num_elts=8 align=GRF alias=<%0,0>\n" + "mov (M1_NM, 1) PD(0,6)<1> %1(0,0)<0;1,0>\n}\n" : "+rw"(pl) : "rw.u"(v)); } -inline void wr_16b_8r16(const surf &s, int x, int y, ushort8 v) { - __builtin_IB_subgroup_block_write_flat_u16_m8k16v1(s.base, s.w, s.h, s.p, int2{x, y}, v); +inline void pl2d_x(unsigned &pl, int x) { pl2d_set<5>(pl, x); } +inline void pl2d_y(unsigned &pl, int y) { pl2d_set<6>(pl, y); } +template inline T rd2d(unsigned pl) { + T v; + XE2_ASM((detail::rd2d_str()) : "=rw"(v) : "rw"(pl)); + return v; } -inline void wr_32b_8r16(const surf &s, int x, int y, uint8 v) { - __builtin_IB_subgroup_block_write_flat_u32_m8k16v1(s.base, s.w, s.h, s.p, int2{x, y}, v); + +// compile-time loop: f(std::integral_constant) for I = 0..N-1 +template inline void static_for(F &&f) { + [&](std::integer_sequence) { (f(std::integral_constant{}), ...); }( + std::make_integer_sequence{}); } -inline void pf_32b_8r16(const surf &s, int x, int y) { - __builtin_IB_subgroup_block_read_prefetch_u32_m8k16v1(s.base, s.w, s.h, s.p, int2{x, y}, 0); + +// Sub-group block reads, striped: element i of lane l = p[l + 16 i]; p must be +// global and 4-byte aligned (16-byte for the vector forms). One transposed LSC +// block load, whose dword order is the SIMD16 vector layout (group_load lowers +// these to per-lane gathers). The scalar form fills half a GRF but the load +// writes a whole one, so it goes through a temp. +inline unsigned short sg_rd_us(const unsigned short *p) { + unsigned short v; + XE2_ASM("{\n.decl TP v_type=G type=uw num_elts=32 align=GRF\n" + ".decl AD v_type=G type=q num_elts=1 align=GRF\n" + "mov (M1_NM, 1) AD(0,0)<1> %1(0,0)<0;1,0>\n" + "lsc_load.ugm (M1_NM, 1) TP:d32x8t flat[AD]:a64\n" + "mov (M1, 16) %0(0,0)<1> TP(0,0)<1;1,0>\n}\n" + : "=rw"(v) : "rw.u"((long long)p)); + return v; } +// the send address must start a GRF: a uniform scalar is copied into an aligned temp +#define XE2_BLK_RD(name, R, P, shape) \ + inline R name(const P *p) { \ + R v; \ + XE2_ASM("{\n.decl AD v_type=G type=q num_elts=1 align=GRF\n" \ + "mov (M1_NM, 1) AD(0,0)<1> %1(0,0)<0;1,0>\n" \ + "lsc_load.ugm (M1_NM, 1) %0:" shape " flat[AD]:a64\n}\n" \ + : "=rw"(v) : "rw.u"((long long)p)); \ + return v; \ + } +XE2_BLK_RD(sg_rd_us4, ushort4, unsigned short, "d32x32t") +XE2_BLK_RD(sg_rd_us8, ushort8, unsigned short, "d32x64t") +XE2_BLK_RD(sg_rd_u4, uint4, unsigned, "d32x64t") +#undef XE2_BLK_RD -template inline const XE2_GLOBAL T *gptr(const T *p) { - return (const XE2_GLOBAL T *)p; +// per-lane 3-dword load, L1 and L3 cached +inline uint3 ld_u3_cached(const uint3 *p) { + uint3 v; + XE2_ASM("lsc_load.ugm.ca.ca (M1, 16) %0:d32x3 flat[%1]:a64" : "=rw"(v) : "rw"((long long)p)); + return v; } +// DPAS, SIMD16, systolic depth 8. Repeat count R = rows of A: A is one +// short per row per lane (K16 fp16/bf16, or K32 s8), B the VNNI weights; +// na / nb = A / B size in dwords. +#define XE2_DPAS(name, prec, A, B, C, R, na, nb) \ + inline C name(A a, B b, C acc) { \ + XE2_ASM("{\n" \ + ".decl DB v_type=G type=ud num_elts=" #nb " align=GRF alias=<%1,0>\n" \ + ".decl DA v_type=G type=ud num_elts=" #na " align=GRF alias=<%2,0>\n" \ + "dpas." prec ".8." #R " (M1, 16) %0.0 %0.0 DB.0 DA(0,0)\n" \ + "}\n" \ + : "+rw"(acc) : "rw"(b), "rw"(a)); \ + return acc; \ + } +#define XE2_DPAS_Z(name, prec, A, B, C, R, na, nb) \ + inline C name(A a, B b) { \ + C d; \ + XE2_ASM("{\n" \ + ".decl DB v_type=G type=ud num_elts=" #nb " align=GRF alias=<%1,0>\n" \ + ".decl DA v_type=G type=ud num_elts=" #na " align=GRF alias=<%2,0>\n" \ + "dpas." prec ".8." #R " (M1, 16) %0.0 %%null.0 DB.0 DA(0,0)\n" \ + "}\n" \ + : "=rw"(d) : "rw"(b), "rw"(a)); \ + return d; \ + } +XE2_DPAS(dpas_hf, "hf.hf", short, int8, float, 1, 8, 128) +XE2_DPAS(dpas_hf, "hf.hf", short2, int8, float2, 2, 16, 128) +XE2_DPAS(dpas_hf, "hf.hf", short4, int8, float4, 4, 32, 128) +XE2_DPAS(dpas_hf, "hf.hf", short8, int8, float8, 8, 64, 128) +XE2_DPAS(dpas_bf, "bf.bf", short, int8, float, 1, 8, 128) +XE2_DPAS(dpas_bf, "bf.bf", short2, int8, float2, 2, 16, 128) +XE2_DPAS(dpas_bf, "bf.bf", short4, int8, float4, 4, 32, 128) +XE2_DPAS(dpas_bf, "bf.bf", short8, int8, float8, 8, 64, 128) +// s8 A (K32) x s2 B: 4 weights per byte of B +XE2_DPAS(dpas_s2s8, "s2.s8", short, int2, int, 1, 8, 32) +XE2_DPAS(dpas_s2s8, "s2.s8", short2, int2, int2, 2, 16, 32) +XE2_DPAS(dpas_s2s8, "s2.s8", short4, int2, int4, 4, 32, 32) +XE2_DPAS(dpas_s2s8, "s2.s8", short8, int2, int8, 8, 64, 32) +// same with a zero accumulator (src0 = null) +XE2_DPAS_Z(dpas_s2s8_z, "s2.s8", short, int2, int, 1, 8, 32) +XE2_DPAS_Z(dpas_s2s8_z, "s2.s8", short2, int2, int2, 2, 16, 32) +XE2_DPAS_Z(dpas_s2s8_z, "s2.s8", short4, int2, int4, 4, 32, 32) +XE2_DPAS_Z(dpas_s2s8_z, "s2.s8", short8, int2, int8, 8, 64, 32) +#undef XE2_DPAS +#undef XE2_DPAS_Z + // work-group barrier with a local-memory fence only (OpenCL barrier(CLK_LOCAL_MEM_FENCE)) inline void barrier_local() { #ifdef __SYCL_DEVICE_ONLY__ @@ -156,8 +311,30 @@ template <> struct dt { }; template inline C mad_k16(A a, int8 b, C acc) { - if constexpr (BF16) return intel_sub_group_bf16_bf16_matrix_mad_k16(a, b, acc); - else return intel_sub_group_f16_f16_matrix_mad_k16(a, b, acc); + if constexpr (BF16) return dpas_bf(a, b, acc); + else return dpas_hf(a, b, acc); +} + +namespace detail { +template constexpr auto dpas_at_str() { + cstr<300> c; + c.add("{\n.decl DB v_type=G type=ud num_elts=128 align=GRF alias=<%1,0>\n" + ".decl DA v_type=G type=ud num_elts="); + c.addi(NA); + c.add(" align=GRF alias=<%2,0>\ndpas."); + c.add(BF16 ? "bf.bf" : "hf.hf"); + c.add(".8.8 (M1, 16) %0.0 %0.0 DB.0 DA("); + c.addi(ROW); + c.add(",0)\n}\n"); + return c; +} +} // namespace detail + +// 8-row DPAS whose A operand starts at GRF row ROW of a larger A block (e.g. a +// 32-row x 2-block 2D read), aliased in place as IGC does for the builtins +template inline float8 mad8_at(const AV &a, int8 b, float8 acc) { + XE2_ASM((detail::dpas_at_str()) : "+rw"(acc) : "rw"(b), "rw"(a)); + return acc; } } // namespace xe2 diff --git a/ggml/src/ggml-sycl/xe2-dpas.hpp b/ggml/src/ggml-sycl/xe2-dpas.hpp index 8c61f0e74113..8d2a13bae851 100644 --- a/ggml/src/ggml-sycl/xe2-dpas.hpp +++ b/ggml/src/ggml-sycl/xe2-dpas.hpp @@ -29,4 +29,22 @@ inline i32x8 dpas_s4s8_r8(s8x8 a, s4x4 b) { #endif return d; } +// f16 x f16 -> f32, repeat count 8: A = 8 rows of K16 halves (one short per lane per row), B = K16 x 16 (VNNI: 8 dwords +// per lane), accumulate. Replaces intel_sub_group_f16_f16_matrix_mad_k16 (public, but one builtin fewer to resolve). +typedef float f32x8 __attribute__((ext_vector_type(8))); +inline f32x8 dpas_hf_r8(s8x8 a, i32x8 b, f32x8 acc) { +#ifdef __SYCL_DEVICE_ONLY__ + __asm__("{\n" + ".decl DB v_type=G type=ud num_elts=128 align=GRF alias=<%1,0>\n" + ".decl DA v_type=G type=ud num_elts=64 align=GRF alias=<%2,0>\n" + "dpas.hf.hf.8.8 (M1, 16) %0.0 %0.0 DB.0 DA(0,0)\n" + "}\n" + : "+rw"(acc) + : "rw"(b), "rw"(a)); +#else + (void) a; + (void) b; +#endif + return acc; +} } // namespace xe2dp diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 7b582296be53..c3b665b9c682 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10957,10 +10957,13 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_flash_attn_ext(512, 512, 1, {16, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, t, t, {0, 2, 1, 3})); } } - for (int64_t nb : {1, 4, 16, 32}) { // real cache layout (DPAS kernel targets) + for (int64_t nb : {1, 4, 16, 32, 64, 128, 256}) { // real cache layout (DPAS kernel targets; 64+ = long n-gram drafts) test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, {0, 2, 1, 3})); } } + for (int64_t nb : {1, 4, 64, 256}) { // ARC-LAB: ~115K history (Bonsai at 128K context), real cache layout + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 114688, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, {0, 2, 1, 3})); + } for (ggml_type t : {GGML_TYPE_Q4_0, GGML_TYPE_F16}) { // ARC-LAB: Bonsai 27B prompt batch at 16K history test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 16384, 512, true, false, 0, 0, GGML_PREC_F32, t, t)); for (int64_t kv : {512, 16384}) { // real KV-cache layout (heads interleaved per token) From 1bb8282a6adffae361a56cc5f268e06ea02cdcb0 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 3 Oct 2026 14:23:21 +1000 Subject: [PATCH 18/65] SYCL decode attention: default DPAS cap 32 -> 256 query tokens Long n-gram verify batches now stay on the q4_0 DPAS kernel instead of oneDNN's path, which converts the whole KV cache to f16 each call. B580 per layer: 48K 64 tokens 28.8 -> 4.9 ms, 115K 256 tokens 253 -> 44 ms. KLD (ub 64-256) 0.012699 vs 0.012800, same top p 92.60 vs 92.89 %, equal within error. Whole model MTP: 48K 73.2 -> 76.1 t/s, 115K 60.3 -> 65.1. GGML_SYCL_FA_DEC_DPAS=32 restores the old cap. Co-Authored-By: Claude (cherry picked from commit 064461a0b41a7e006c21c7d60039e2069e10b2d9 on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/fattn-dec.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-sycl/fattn-dec.cpp b/ggml/src/ggml-sycl/fattn-dec.cpp index 8ad0dabca4a4..d73a4e431bc6 100644 --- a/ggml/src/ggml-sycl/fattn-dec.cpp +++ b/ggml/src/ggml-sycl/fattn-dec.cpp @@ -1029,7 +1029,10 @@ static int fattn_dec_dpas_maxq() { // 0 = DPAS kernel off; else it serves 1 .. } return n; }(); - return v == 1 ? 32 : v; // =1 -> default cap + // =1 -> default cap 256: long n-gram verify batches stay on this kernel instead of oneDNN's f16 copy of the whole KV + // cache (B580, per layer: 48K 64 tok 28.8 -> 4.9 ms, 115K 256 tok 253 -> 44 ms; KLD 0.01270 vs 0.01280, same within + // error; whole model MTP 48K 73.2 -> 76.1 t/s, 115K 60.3 -> 65.1). =32 restores the old cap. + return v == 1 ? 256 : v; } bool ggml_sycl_flash_attn_ext_dec_supported(const ggml_tensor * dst) { From 1d3b194ca11fa853c2b74dae9dbb2131ff121076 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 3 Oct 2026 19:31:44 +1000 Subject: [PATCH 19/65] SYCL decode attention: 5-token query chunks for MTP verifies MTP with 4 drafts verifies 5 tokens per step, which ran the 8-token DPAS kernel with 3 empty tokens (48 rows of Q.K / softmax for 30). A 5-token chunk (128-key tiles) cuts the kernel from 637 to 502 us per layer at 48K (215 -> 175 us at 16K); whole model MTP 48K 77.4 -> 78.1 t/s, 115K 65.7 -> 67.1, output identical. GGML_SYCL_FA_DEC_NQ5=0 turns it off (=2 SLM-staged and =3 256-key tiles were slower). test-backend-ops: 5/6-token real-layout cases. Co-Authored-By: Claude (cherry picked from commit 081d86870faf08e91cd737377b86fb774c8a41c1 on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/fattn-dec.cpp | 15 +++++++++++++-- tests/test-backend-ops.cpp | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-sycl/fattn-dec.cpp b/ggml/src/ggml-sycl/fattn-dec.cpp index d73a4e431bc6..039f6ffcfcfd 100644 --- a/ggml/src/ggml-sycl/fattn-dec.cpp +++ b/ggml/src/ggml-sycl/fattn-dec.cpp @@ -1082,9 +1082,15 @@ void ggml_sycl_flash_attn_ext_dec(ggml_backend_sycl_context & ctx, ggml_tensor * // SLM-staged K/V tiles (128 keys) for single-token decode: 48K 432 -> 247 us with the split count below. For 2+ tokens // the extra SLM halves the work-groups per core and it loses (4 tok 48K 458 unstaged vs ~593 staged). GGML_SYCL_FA_DEC_STAGE=0 off. static const bool stage_env = !(getenv("GGML_SYCL_FA_DEC_STAGE") && atoi(getenv("GGML_SYCL_FA_DEC_STAGE")) == 0); - const bool stage = dpas && stage_env && !d512 && !g2 && nq == 1; + // ARC-LAB: 5-token query chunks (default, GGML_SYCL_FA_DEC_NQ5=1): MTP 4 drafts + 1 = 5-token verifies otherwise run the + // 8-token kernel with 3 empty tokens (48 rows of Q.K / softmax for 30). B580 per layer: 16K 215 -> 175 us, 48K 637 -> + // 502 us. =0 off, =2 SLM-staged (slower: 48K 579), =3 256-key tiles (48K 513). + static const int nq5_env = getenv("GGML_SYCL_FA_DEC_NQ5") ? atoi(getenv("GGML_SYCL_FA_DEC_NQ5")) : 1; + const int nq5 = (dpas && !d512 && !g2 && ne01 == 5) ? nq5_env : 0; + const bool stage = dpas && stage_env && !d512 && !g2 && (nq == 1 || nq5 == 2); const int tk = (d512 || g2) ? (nq == 1 ? 256 : 128) - : stage ? 128 + : nq5 == 3 ? 256 + : (stage || nq5) ? 128 : dpas ? (nq == 1 ? 256 : nq == 2 ? 128 : nq == 4 ? 256 : 128) : nq == 8 ? 32 : nq == 4 ? 64 : 128; const int ne11 = K->ne[1]; @@ -1142,6 +1148,10 @@ void ggml_sycl_flash_attn_ext_dec(ggml_backend_sycl_context & ctx, ggml_tensor * FATTN_DEC_DPASG2(8, 8, 1); // 16 rows = 2 tiles, 8 key groups x 2 } #undef FATTN_DEC_DPASG2 + } else if (nq5 == 1) { + FATTN_DEC_DPAS(5, 8, 2); + } else if (nq5 == 3) { + FATTN_DEC_DPAS(5, 16, 2); // 256-key tiles, as the 4-token kernel } else if (stage) { #define FATTN_DEC_DPAS_ST(NQ_, KG_, TC_) fattn_dec_q4_0_dpas((const char *) Q->data, (const char *) K->data, \ (const char *) V->data, mdata, (float *) dst->data, parts.get(), meta.get(), scale, ne01, (int) Q->ne[2], ne11, \ @@ -1149,6 +1159,7 @@ void ggml_sycl_flash_attn_ext_dec(ggml_backend_sycl_context & ctx, ggml_tensor * V->nb[3], nb31, nb33, ne33, nsplit, chunk, stream) switch (nq) { case 1: FATTN_DEC_DPAS_ST(1, 8, 1); break; + case 8: FATTN_DEC_DPAS_ST(5, 8, 2); break; // only with nq5 == 2 (stage is otherwise 1-token only) case 2: FATTN_DEC_DPAS_ST(2, 8, 1); break; case 4: FATTN_DEC_DPAS_ST(4, 8, 3); break; default: FATTN_DEC_DPAS_ST(8, 8, 3); break; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index c3b665b9c682..3ccda28b007d 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10957,7 +10957,7 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_flash_attn_ext(512, 512, 1, {16, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, t, t, {0, 2, 1, 3})); } } - for (int64_t nb : {1, 4, 16, 32, 64, 128, 256}) { // real cache layout (DPAS kernel targets; 64+ = long n-gram drafts) + for (int64_t nb : {1, 4, 5, 6, 8, 16, 32, 64, 128, 256}) { // real cache layout (DPAS kernel targets; 5 = MTP verify; 64+ = long n-gram drafts) test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, {0, 2, 1, 3})); } } From 53744cc9e81f448d561065fa56469bdd47443503 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:45:52 +1000 Subject: [PATCH 20/65] lab: runtime-switchable SYCL tuning knobs (GGML_SYCL_KNOBS_FILE); LLAMA_ARG_SPEC_NGRAM_FRESH - ggml_sycl_knob(name, def): the attention (DPAS cap, staging, 5-token chunks, split count, probe) and PTQ1_0 T2 (tile, sub-groups, GEMM off) switches are read through it. With GGML_SYCL_KNOBS_FILE set, the NAME=VALUE file is re-read at the start of a graph compute when it changes, so one server can A/B variants. Without it, each knob is read from the environment once, as before. - LLAMA_ARG_SPEC_NGRAM_FRESH=1: the n-gram drafter forgets earlier requests at the start of each one, so repeated benchmark prompts don't replay earlier answers as drafts. Co-Authored-By: Claude (cherry picked from commit 9db940800ec2e371d41cccba893b41d57cb3e287 on Torchit1/llama.cpp arc-b580) Excluded from this port: common --- ggml/src/ggml-sycl/common.hpp | 6 +++ ggml/src/ggml-sycl/fattn-dec.cpp | 20 ++++----- ggml/src/ggml-sycl/ggml-sycl.cpp | 69 ++++++++++++++++++++++++++++++++ ggml/src/ggml-sycl/ptq1-t2.cpp | 6 +-- 4 files changed, 88 insertions(+), 13 deletions(-) diff --git a/ggml/src/ggml-sycl/common.hpp b/ggml/src/ggml-sycl/common.hpp index 0ee0689da04d..a73b2261daa6 100644 --- a/ggml/src/ggml-sycl/common.hpp +++ b/ggml/src/ggml-sycl/common.hpp @@ -1044,4 +1044,10 @@ static __dpct_inline__ float ggml_sycl_ue4m3_to_fp32(uint8_t x) { return raw * 0.5f; } +// ARC-LAB lab knobs: integer tuning switches read through here can be changed while the process runs. With +// GGML_SYCL_KNOBS_FILE= set, the file (NAME=VALUE lines) is re-read at the start of a graph compute when its +// mtime changes and its values override the environment; without it, each knob is read from the environment once. +int ggml_sycl_knob(const char * name, int def); +void ggml_sycl_knobs_refresh(); + #endif // GGML_SYCL_COMMON_HPP diff --git a/ggml/src/ggml-sycl/fattn-dec.cpp b/ggml/src/ggml-sycl/fattn-dec.cpp index 039f6ffcfcfd..f23c712aceb6 100644 --- a/ggml/src/ggml-sycl/fattn-dec.cpp +++ b/ggml/src/ggml-sycl/fattn-dec.cpp @@ -726,7 +726,7 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, constexpr int SW = RW + 1; // SLM row stride (odd: no bank conflicts on lane = key reads) static_assert(ST == 0 || (D == 256 && RW % 4 == 0), "staging is sized for D 256"); const int nqc = (ne01 + NQ - 1) / NQ; // token chunks - static const int probe_env = getenv("GGML_SYCL_FA_DEC_DPAS_PROBE") ? atoi(getenv("GGML_SYCL_FA_DEC_DPAS_PROBE")) : 0; + const int probe_env = ggml_sycl_knob("GGML_SYCL_FA_DEC_DPAS_PROBE", 0); const int probe = probe_env; // phase probe: bit0 skip Q.K, bit1 skip P.V, bit2 skip softmax, bit3 Q.K loads only (no DPAS / scale math) stream->submit([&](sycl::handler & cgh) { @@ -1017,18 +1017,18 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, } static int fattn_dec_dpas_maxq() { // 0 = DPAS kernel off; else it serves 1 .. maxq query tokens - static const int v = [] { - const char * e = getenv("GGML_SYCL_FA_DEC_DPAS"); - int n = e ? atoi(e) : 0; + static const bool ok = [] { // the kernel needs Xe2's 16-lane int8 x int4 DPAS; elsewhere fall back instead of failing at the first launch - if (n && !getenv("GGML_SYCL_FA_DEC_DPAS_ANYGPU") && !ggml_sycl_device_is_xe2()) { + if (getenv("GGML_SYCL_FA_DEC_DPAS") && atoi(getenv("GGML_SYCL_FA_DEC_DPAS")) && !getenv("GGML_SYCL_FA_DEC_DPAS_ANYGPU") && + !ggml_sycl_device_is_xe2()) { GGML_LOG_WARN("%s: GGML_SYCL_FA_DEC_DPAS needs an Xe2 or newer GPU (Arc B-series, Lunar Lake, Panther Lake); " "%s is not one, using the regular decode attention\n", __func__, ggml_sycl_info().devices[ggml_sycl_get_device()].hw_info.name.c_str()); - n = 0; + return false; } - return n; + return getenv("GGML_SYCL_FA_DEC_DPAS_ANYGPU") != nullptr || ggml_sycl_device_is_xe2(); }(); + const int v = ok ? ggml_sycl_knob("GGML_SYCL_FA_DEC_DPAS", 0) : 0; // =1 -> default cap 256: long n-gram verify batches stay on this kernel instead of oneDNN's f16 copy of the whole KV // cache (B580, per layer: 48K 64 tok 28.8 -> 4.9 ms, 115K 256 tok 253 -> 44 ms; KLD 0.01270 vs 0.01280, same within // error; whole model MTP 48K 73.2 -> 76.1 t/s, 115K 60.3 -> 65.1). =32 restores the old cap. @@ -1081,11 +1081,11 @@ void ggml_sycl_flash_attn_ext_dec(ggml_backend_sycl_context & ctx, ggml_tensor * // keys per tile (must match the instantiations below); DPAS: 16 x KG // SLM-staged K/V tiles (128 keys) for single-token decode: 48K 432 -> 247 us with the split count below. For 2+ tokens // the extra SLM halves the work-groups per core and it loses (4 tok 48K 458 unstaged vs ~593 staged). GGML_SYCL_FA_DEC_STAGE=0 off. - static const bool stage_env = !(getenv("GGML_SYCL_FA_DEC_STAGE") && atoi(getenv("GGML_SYCL_FA_DEC_STAGE")) == 0); + const bool stage_env = ggml_sycl_knob("GGML_SYCL_FA_DEC_STAGE", 1) != 0; // ARC-LAB: 5-token query chunks (default, GGML_SYCL_FA_DEC_NQ5=1): MTP 4 drafts + 1 = 5-token verifies otherwise run the // 8-token kernel with 3 empty tokens (48 rows of Q.K / softmax for 30). B580 per layer: 16K 215 -> 175 us, 48K 637 -> // 502 us. =0 off, =2 SLM-staged (slower: 48K 579), =3 256-key tiles (48K 513). - static const int nq5_env = getenv("GGML_SYCL_FA_DEC_NQ5") ? atoi(getenv("GGML_SYCL_FA_DEC_NQ5")) : 1; + const int nq5_env = ggml_sycl_knob("GGML_SYCL_FA_DEC_NQ5", 1); const int nq5 = (dpas && !d512 && !g2 && ne01 == 5) ? nq5_env : 0; const bool stage = dpas && stage_env && !d512 && !g2 && (nq == 1 || nq5 == 2); const int tk = (d512 || g2) ? (nq == 1 ? 256 : 128) @@ -1095,7 +1095,7 @@ void ggml_sycl_flash_attn_ext_dec(ggml_backend_sycl_context & ctx, ggml_tensor * const int ne11 = K->ne[1]; // slices: enough work-groups to fill the GPU (~64 slices per KV head, ~256 in all), at least one tile each - static const int target_env = getenv("GGML_SYCL_FA_DEC_SPLITS") ? atoi(getenv("GGML_SYCL_FA_DEC_SPLITS")) : 0; + const int target_env = ggml_sycl_knob("GGML_SYCL_FA_DEC_SPLITS", 0); // DPAS (D 256, 6 rows): one wave of work-groups - Xe cores x the work-groups that fit per core (SLM-bound: 4 for 1-2 // query tokens, 2 for 4-8) - over the KV heads. A fixed 64 per head ran 3-6 waves with a mostly idle last one: B580 // 48K 1 tok 432 -> 302 us (247 staged), 4 tok 648 -> 458 us. Xe2: 8 EUs per core, nsm = EUs / 16. diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 273ea4dd4ed0..963892e806e5 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -10,6 +10,10 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // +#include +#include +#include +#include #include #include #include @@ -6570,8 +6574,73 @@ static bool check_graph_compatibility(ggml_cgraph * cgraph) { } #endif +namespace { +std::mutex g_knob_mtx; +std::unordered_map g_knob_file; // from GGML_SYCL_KNOBS_FILE (wins) +std::unordered_map g_knob_env; // environment, read once per name +} // namespace + +int ggml_sycl_knob(const char * name, int def) { + std::lock_guard lock(g_knob_mtx); + auto f = g_knob_file.find(name); + if (f != g_knob_file.end()) { + return f->second; + } + auto e = g_knob_env.find(name); + if (e != g_knob_env.end()) { + return e->second; + } + const char * v = getenv(name); + const int r = v ? atoi(v) : def; + g_knob_env[name] = r; + return r; +} + +void ggml_sycl_knobs_refresh() { + static const char * path = getenv("GGML_SYCL_KNOBS_FILE"); + if (!path) { + return; + } + static std::filesystem::file_time_type last_time{}; + static std::uintmax_t last_size = 0; + std::error_code ec; + const auto t = std::filesystem::last_write_time(path, ec); + if (ec) { + return; + } + const auto sz = std::filesystem::file_size(path, ec); + if (ec || (t == last_time && sz == last_size)) { + return; + } + last_time = t; + last_size = sz; + std::unordered_map m; + if (FILE * fp = fopen(path, "r")) { + char line[256]; + while (fgets(line, sizeof(line), fp)) { + char * eq = strchr(line, '='); + if (!eq || line[0] == '#') { + continue; + } + *eq = 0; + std::string k(line); + k.erase(k.find_last_not_of(" \t") + 1); + m[k] = atoi(eq + 1); + } + fclose(fp); + } + std::string desc; + for (auto & kv : m) { + desc += " " + kv.first + "=" + std::to_string(kv.second); + } + GGML_LOG_INFO("%s: knobs reloaded from %s:%s\n", __func__, path, desc.c_str()); + std::lock_guard lock(g_knob_mtx); + g_knob_file = std::move(m); +} + static ggml_status ggml_backend_sycl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { auto * sycl_ctx = static_cast(backend->context); + ggml_sycl_knobs_refresh(); #ifdef GGML_SYCL_GRAPH bool use_sycl_graph = false; diff --git a/ggml/src/ggml-sycl/ptq1-t2.cpp b/ggml/src/ggml-sycl/ptq1-t2.cpp index 7ab10a566791..a2705479d4ff 100644 --- a/ggml/src/ggml-sycl/ptq1-t2.cpp +++ b/ggml/src/ggml-sycl/ptq1-t2.cpp @@ -217,7 +217,7 @@ void quant_a(sycl::queue & q, const float * x, int64_t x_stride, int64_t M, int6 // GGML_SYCL_PTQ1_T2_NSG: sub-groups per GEMV work-group for 2..8 rows (verify batches); 2 (default) or 4. The B580 tile // sweep (TernSYCL main, Bonsai 27B shapes, M=4, qmode 0) had 4 fastest on 5 of 6 shapes (+0-8%); M=1 keeps 2. int t2_nsg() { - static const int v = getenv("GGML_SYCL_PTQ1_T2_NSG") && atoi(getenv("GGML_SYCL_PTQ1_T2_NSG")) == 4 ? 4 : 2; + const int v = ggml_sycl_knob("GGML_SYCL_PTQ1_T2_NSG", 2) == 4 ? 4 : 2; return v; } @@ -250,7 +250,7 @@ void gemm_tile(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const ui // 7 {8,64,8,2} 8 {32,32,4,2} void gemm(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const uint32_t * B, const uint16_t * SB, float * C, int M, int N, int K) { - static const int tile = getenv("GGML_SYCL_PTQ1_T2_TILE") ? atoi(getenv("GGML_SYCL_PTQ1_T2_TILE")) : 0; + const int tile = ggml_sycl_knob("GGML_SYCL_PTQ1_T2_TILE", 0); switch (tile) { case 1: gemm_tile<8, 128, 4, 2>(q, Aq, SA, B, SB, C, M, N, K); break; case 2: gemm_tile<8, 128, 4, 4>(q, Aq, SA, B, SB, C, M, N, K); break; @@ -295,7 +295,7 @@ void ggml_sycl_t2_mul_mat(sycl::queue & q, const void * w, const float * x, int6 int64_t N, int64_t K) { // batches > 8 (n-gram verify, prompts): GemmMT reads each weight once per 64 rows (TernSYCL main: the 2D block I/O // that misbehaved in the JIT build with the old IGC builtins validates under inline vISA, so it is upstream's kernel). - static const bool gemm_off = getenv("GGML_SYCL_PTQ1_T2_GEMM_OFF") != nullptr; + const bool gemm_off = ggml_sycl_knob("GGML_SYCL_PTQ1_T2_GEMM_OFF", 0) != 0; const bool use_gemm = M > 8 && !gemm_off; const int lda = int8dpas::ldsa((int) M); From 7ab9e58d7dbd78deff6fb5bca07db0d724838805 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 3 Oct 2026 22:39:57 +1000 Subject: [PATCH 21/65] sycl: T2 quantized-activation cache + fused Hadamard rotate/quantize Back-to-back T2 mat-vecs on one tensor reuse its int8 activations (GGML_SYCL_PTQ1_T2_QCACHE), and the signed 1024-point FWHT that feeds them runs 128 lanes per block and writes Aq/SA itself (GGML_SYCL_PTQ1_T2_FWHTQ). B580 Bonsai 27B plain decode +3.6-4.9%, KLD unchanged. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 254dfb2ca12ea098c5ed8ff0c21304a5f71e3e0a on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/ggml-sycl.cpp | 11 ++- ggml/src/ggml-sycl/ptq1-t2.cpp | 144 +++++++++++++++++++++++++++++-- ggml/src/ggml-sycl/ptq1-t2.hpp | 12 ++- 3 files changed, 155 insertions(+), 12 deletions(-) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 963892e806e5..29fa636e74f1 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -4994,7 +4994,7 @@ static bool ggml_sycl_t2_mul_mat_try(ggml_backend_sycl_context & ctx, const ggml return true; } ggml_sycl_t2_mul_mat(*stream, src0->data, (const float *) src1->data, src1->nb[1] / sizeof(float), (float *) dst->data, - M, N, K); + M, N, K, src1->view_src && src1->view_offs == 0 ? src1->view_src : src1); return true; } @@ -6498,6 +6498,14 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc (x->type == GGML_TYPE_F32 || x->type == GGML_TYPE_F16) && node->type == x->type && ggml_is_contiguous(x) && ggml_is_contiguous(signs) && signs->ne[0] == x->ne[0] && signs->ne[0] % mm->src[0]->ne[0] == 0; + // ARC-LAB: the rotation that feeds T2 mat-vecs also quantizes for them (they read mm through a reshape) + if (pattern_ok && x->type == GGML_TYPE_F32 && mm->src[0]->ne[0] == 1024 && x->ne[3] == 1 && + ggml_is_contiguous(mm) && + ggml_sycl_t2_fwht_quant(*sycl_ctx->stream(), (const float *) x->data, (const float *) signs->data, + (float *) mm->data, ggml_nrows(x), x->ne[0], mm)) { + i += 2; + continue; + } if (pattern_ok && ggml_sycl_op_fwht_signed(*sycl_ctx, x, signs, mm)) { i += 2; continue; @@ -6641,6 +6649,7 @@ void ggml_sycl_knobs_refresh() { static ggml_status ggml_backend_sycl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { auto * sycl_ctx = static_cast(backend->context); ggml_sycl_knobs_refresh(); + ggml_sycl_t2_new_graph(); #ifdef GGML_SYCL_GRAPH bool use_sycl_graph = false; diff --git a/ggml/src/ggml-sycl/ptq1-t2.cpp b/ggml/src/ggml-sycl/ptq1-t2.cpp index a2705479d4ff..799bf9f3ae36 100644 --- a/ggml/src/ggml-sycl/ptq1-t2.cpp +++ b/ggml/src/ggml-sycl/ptq1-t2.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include namespace { @@ -291,24 +292,149 @@ void t2_dispatch(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const } } // namespace +namespace { +// what the activation scratch holds: the tensor it was quantized from and in which graph (one queue, in order) +struct quant_key { + const void * key = nullptr; const float * x = nullptr; int64_t stride = 0, M = 0, K = 0; uint64_t graph = 0; + bool operator==(const quant_key & o) const { + return key == o.key && x == o.x && stride == o.stride && M == o.M && K == o.K && graph == o.graph; + } +}; +quant_key g_quant_have; +std::atomic g_graph{ 1 }; +} // namespace + +void ggml_sycl_t2_new_graph() { g_graph.fetch_add(1); } + +namespace { +// the activation scratch: int8 Aq [M, K], then fp16 SA [K/QK, ldsa(M)] +void scratch_aq(sycl::queue & q, int64_t M, int64_t K, int8_t *& Aq, uint16_t *& SA) { + const int lda = int8dpas::ldsa((int) M); + const size_t aq_bytes = (size_t) M * K; + const size_t sa_off = (aq_bytes + 255) & ~(size_t) 255; + const size_t sa_bytes = (size_t) (K / QK) * lda * 2 + 256; + char * s = (char *) scratch_get(q, sa_off + sa_bytes); + Aq = (int8_t *) s; + SA = (uint16_t *) (s + sa_off); +} +} // namespace + +// signed 1024-point Hadamard rotation (as fwht.cpp: dst = H (x * signs) / 32, same butterfly order, so the same floats) +// that also quantizes its output for the T2 mat-vecs which read it next: one 128-lane work-group per 1024 block where +// fwht.cpp gives a block one 16-lane sub-group (5 sub-groups in all for a decode token), and no quant_a launch after it. +bool ggml_sycl_t2_fwht_quant(sycl::queue & q, const float * x, const float * signs, float * dst, int64_t M, int64_t K, + const void * key) { + constexpr int N = 1024; + if (K % N != 0 || M <= 0 || M > 512 || !ggml_sycl_knob("GGML_SYCL_PTQ1_T2_FWHTQ", 1)) { + return false; + } + int8_t * Aq; + uint16_t * SA; + scratch_aq(q, M, K, Aq, SA); + const int lda = int8dpas::ldsa((int) M); + const int64_t nblk = K / N, rows = M * nblk; + q.submit([&](sycl::handler & cgh) { + sycl::local_accessor sl(sycl::range<1>(N), cgh); + cgh.parallel_for(sycl::nd_range<1>(rows * 128, 128), [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(16)]] { + const auto sg = it.get_sub_group(); + const int lane = sg.get_local_linear_id(); + const int sgi = sg.get_group_linear_id(); + const int64_t r = it.get_group(0); + const int64_t m = r / nblk, kb = r % nblk; + const float * src = x + r * N; + const float * sgn = signs + kb * N; + float * const s = sl.get_multi_ptr().get(); + float v[8]; + // layout 1: element lane + 16 j + 128 sgi; butterflies h = 1..8 across lanes, then 16, 32, 64 in registers +#pragma unroll + for (int j = 0; j < 8; ++j) { + const int e = lane + 16 * j + 128 * sgi; + v[j] = src[e] * (1.0f / 32.0f) * sgn[e]; + } +#pragma unroll + for (int h = 1; h < 16; h *= 2) { +#pragma unroll + for (int j = 0; j < 8; ++j) { + const float o = sycl::permute_group_by_xor(sg, v[j], h); + v[j] = (lane & h) == 0 ? v[j] + o : o - v[j]; + } + } +#pragma unroll + for (int st = 1; st < 8; st *= 2) { +#pragma unroll + for (int j = 0; j < 8; ++j) { + if ((j & st) == 0) { + const float a = v[j], b = v[j + st]; + v[j] = a + b; v[j + st] = a - b; + } + } + } + // layout 2: element t + 128 i (t = 16 sgi + lane); butterflies h = 128, 256, 512 in registers + const int t = 16 * sgi + lane; +#pragma unroll + for (int j = 0; j < 8; ++j) s[lane + 16 * j + 128 * sgi] = v[j]; + it.barrier(sycl::access::fence_space::local_space); +#pragma unroll + for (int i = 0; i < 8; ++i) v[i] = s[t + 128 * i]; +#pragma unroll + for (int st = 1; st < 8; st *= 2) { +#pragma unroll + for (int i = 0; i < 8; ++i) { + if ((i & st) == 0) { + const float a = v[i], b = v[i + st]; + v[i] = a + b; v[i + st] = a - b; + } + } + } + it.barrier(sycl::access::fence_space::local_space); +#pragma unroll + for (int i = 0; i < 8; ++i) s[t + 128 * i] = v[i]; + it.barrier(sycl::access::fence_space::local_space); + // layout 1 again: sub-group sgi holds quant group sgi (128 values, 8 per lane) -> quant_a's numbers + float mx = 0.0f; +#pragma unroll + for (int j = 0; j < 8; ++j) { + v[j] = s[lane + 16 * j + 128 * sgi]; + mx = sycl::fmax(mx, sycl::fabs(v[j])); + dst[r * N + lane + 16 * j + 128 * sgi] = v[j]; + } + mx = sycl::reduce_over_group(sg, mx, sycl::maximum()); + const sycl::half sh = sycl::half(127.0f / sycl::fmax(mx, int8dpas::EPS)); + const float scl = (float) sh; + const int64_t g = kb * (N / QK) + sgi; + if (lane == 0) { + SA[g * lda + m] = sycl::bit_cast(sh); + } + int8_t * aq = Aq + m * K + g * QK; +#pragma unroll + for (int j = 0; j < 8; ++j) { + aq[lane + 16 * j] = (int8_t) sycl::rint(sycl::clamp(v[j] * scl, -128.0f, 127.0f)); + } + }); + }); + g_quant_have = quant_key{ key, dst, K, M, K, g_graph.load() }; + return true; +} + void ggml_sycl_t2_mul_mat(sycl::queue & q, const void * w, const float * x, int64_t x_stride, float * dst, int64_t M, - int64_t N, int64_t K) { + int64_t N, int64_t K, const void * x_key) { // batches > 8 (n-gram verify, prompts): GemmMT reads each weight once per 64 rows (TernSYCL main: the 2D block I/O // that misbehaved in the JIT build with the old IGC builtins validates under inline vISA, so it is upstream's kernel). const bool gemm_off = ggml_sycl_knob("GGML_SYCL_PTQ1_T2_GEMM_OFF", 0) != 0; const bool use_gemm = M > 8 && !gemm_off; - const int lda = int8dpas::ldsa((int) M); - const size_t aq_bytes = (size_t) M * K; - const size_t sa_off = (aq_bytes + 255) & ~(size_t) 255; - const size_t sa_bytes = (size_t) (K / QK) * lda * 2 + 256; - char * s = (char *) scratch_get(q, sa_off + sa_bytes); - int8_t * Aq = (int8_t *) s; - uint16_t * SA = (uint16_t *) (s + sa_off); + int8_t * Aq; + uint16_t * SA; + scratch_aq(q, M, K, Aq, SA); const uint32_t * B = (const uint32_t *) w; const uint16_t * SB = (const uint16_t *) ((const char *) w + (size_t) (K / 16) * N * 4); - quant_a(q, x, x_stride, M, K, Aq, SA, QK); + // GGML_SYCL_PTQ1_T2_QCACHE=0: quantize for every call + const quant_key want{ x_key, x, x_stride, M, K, g_graph.load() }; + if (!x_key || !ggml_sycl_knob("GGML_SYCL_PTQ1_T2_QCACHE", 1) || !(g_quant_have == want)) { + quant_a(q, x, x_stride, M, K, Aq, SA, QK); + g_quant_have = x_key ? want : quant_key{}; + } t2_dispatch(q, Aq, SA, B, SB, dst, (int) M, (int) N, (int) K, use_gemm); } diff --git a/ggml/src/ggml-sycl/ptq1-t2.hpp b/ggml/src/ggml-sycl/ptq1-t2.hpp index 0ee23de97c13..035380536b60 100644 --- a/ggml/src/ggml-sycl/ptq1-t2.hpp +++ b/ggml/src/ggml-sycl/ptq1-t2.hpp @@ -16,6 +16,14 @@ size_t ggml_sycl_t2_bytes(int64_t K, int64_t N); // in place: data holds N rows of K/128 block_ptq1_0 (pq2 = false) or block_pq2_0 (pq2 = true) as uploaded, and at least // ggml_sycl_t2_bytes(K, N) bytes. PQ2_0 code 3 (+2) has no 2-bit two's-complement form: returns false if one is found. bool ggml_sycl_t2_repack(sycl::queue & q, void * data, int64_t K, int64_t N, bool pq2 = false); -// dst[m * N + n] = sum_k x[m * x_stride + k] * w[n, k], M tokens (fp32 in and out) +// dst[m * N + n] = sum_k x[m * x_stride + k] * w[n, k], M tokens (fp32 in and out). x_key (optional) names the +// activation within the current graph (its tensor + ggml_sycl_t2_new_graph epoch): back-to-back calls on the same x_key +// reuse the int8 activations instead of quantizing again (q/k/v, gate/up, the DeltaNet input projections). void ggml_sycl_t2_mul_mat(sycl::queue & q, const void * w, const float * x, int64_t x_stride, float * dst, int64_t M, - int64_t N, int64_t K); + int64_t N, int64_t K, const void * x_key = nullptr); +// a new graph compute starts: activations quantized for the previous one are stale +void ggml_sycl_t2_new_graph(); +// dst [M, K] = 1024-point Hadamard of (x * signs) per block / 32 (signs: K floats), quantized for the T2 mat-vecs at once; +// key = the tensor they will read it through (see x_key). false: shape not served (K % 1024, M > 512) or knob off. +bool ggml_sycl_t2_fwht_quant(sycl::queue & q, const float * x, const float * signs, float * dst, int64_t M, int64_t K, + const void * key); From 846c033dc0b11d351f8dbff6ba3c8a078bf8a740 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 3 Oct 2026 23:11:03 +1000 Subject: [PATCH 22/65] sycl: fuse residual add + RMS norm into the T2 rotate/quantize kernel ADD + RMS_NORM + MUL(weight) [+ MUL(signs) + RESHAPE + Hadamard MUL_MAT] runs as one work-group per token (GGML_SYCL_PTQ1_T2_NORMQ). Outputs only used inside the chain are not written; across tokens, outputs must not partly overlap inputs. B580 Bonsai 27B plain decode +1.9-2.6%, KLD unchanged. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 4d50f848c92d99f8d4656111003b88c6072435d1 on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/ggml-sycl.cpp | 104 +++++++++++++++++- ggml/src/ggml-sycl/ptq1-t2.cpp | 182 +++++++++++++++++++++---------- ggml/src/ggml-sycl/ptq1-t2.hpp | 5 + 3 files changed, 235 insertions(+), 56 deletions(-) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 29fa636e74f1..21c1e4215b33 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -6436,13 +6436,108 @@ static bool ggml_sycl_try_ptq1_multi(ggml_backend_sycl_context & ctx, ggml_cgrap return true; } +// ARC-LAB: see the call in ggml_backend_sycl_graph_compute_impl. Advances i past the nodes it computed. +static bool ggml_sycl_try_add_norm_rot(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int & i) { + const int n0 = i; + ggml_tensor * add = cgraph->nodes[n0]->op == GGML_OP_ADD ? cgraph->nodes[n0] : nullptr; + const int in = add ? n0 + 1 : n0; // RMS_NORM + if (in + 1 >= cgraph->n_nodes) { + return false; + } + ggml_tensor * norm = cgraph->nodes[in]; + ggml_tensor * mulw = cgraph->nodes[in + 1]; + if (norm->op != GGML_OP_RMS_NORM || mulw->op != GGML_OP_MUL || mulw->src[0] != norm) { + return false; + } + if (add && norm->src[0] != add) { + return false; + } + const ggml_tensor * x = add ? add : norm->src[0]; + const ggml_tensor * w = mulw->src[1]; + const int64_t K = x->ne[0], M = ggml_nrows(x); + auto f32c = [](const ggml_tensor * t) { return t->type == GGML_TYPE_F32 && ggml_is_contiguous(t); }; + if (!f32c(x) || !f32c(norm) || !f32c(mulw) || !f32c(w) || w->ne[0] != K || ggml_nrows(w) != 1 || x->ne[3] != 1) { + return false; + } + if (add && (!f32c(add->src[0]) || !f32c(add->src[1]) || !ggml_are_same_shape(add->src[0], add) || + !ggml_are_same_shape(add->src[1], add))) { + return false; + } + // rotation tail: MUL(signs) + RESHAPE + MUL_MAT(hint Hadamard 1024) + ggml_tensor * mm = nullptr; + const float * signs = nullptr; + if (in + 4 < cgraph->n_nodes) { + ggml_tensor * ms = cgraph->nodes[in + 2]; + ggml_tensor * rs = cgraph->nodes[in + 3]; + ggml_tensor * mt = cgraph->nodes[in + 4]; + const ggml_tensor * sg = ms->src[1]; + if (ms->op == GGML_OP_MUL && ms->src[0] == mulw && rs->op == GGML_OP_RESHAPE && rs->src[0] == ms && + mt->op == GGML_OP_MUL_MAT && mt->src[1] == rs && ggml_get_op_params_i32(mt, 1) == GGML_HINT_SRC0_IS_HADAMARD && + mt->src[0]->ne[0] == 1024 && f32c(mt) && f32c(ms) && sg && f32c(sg) && sg->ne[0] == K && ggml_nrows(sg) == 1) { + mm = mt; + signs = (const float *) sg->data; + } + } + if (!add && !mm) { + return false; // plain RMS_NORM + MUL: ggml_sycl_op_rms_norm_fused below + } + const int last = mm ? in + 4 : in + 1; + // nodes inside the chain must have no other consumers; write the ADD and the weighted norm only if used after it + std::vector outs; + if (add) outs.push_back(n0); + outs.push_back(in + 1); + if (mm) outs.push_back(last); + std::vector ops; + for (int j = n0; j <= last; ++j) ops.push_back(cgraph->nodes[j]->op); + { + std::vector idx; + for (int j = n0; j <= last; ++j) idx.push_back(j); + if (!ggml_can_fuse_subgraph_ext(cgraph, idx.data(), (int) idx.size(), ops.data(), outs.data(), (int) outs.size())) { + return false; + } + } + auto used_after = [&](int j) { + int inside = 0; + for (int k = j + 1; k <= last; ++k) { + for (int s = 0; s < GGML_MAX_SRC; ++s) inside += cgraph->nodes[k]->src[s] == cgraph->nodes[j]; + } + return (cgraph->nodes[j]->flags & GGML_TENSOR_FLAG_OUTPUT) || ggml_node_get_use_count(cgraph, j) > inside; + }; + float * add_out = add && used_after(n0) ? (float *) add->data : nullptr; + float * normed = !mm || used_after(in + 1) ? (float *) mulw->data : nullptr; + // across tokens a work-group may write where another still reads: outputs must miss the inputs or match them exactly + if (M > 1) { + const ggml_tensor * ins[2] = { add ? add->src[0] : x, add ? add->src[1] : nullptr }; + std::pair outr[3] = { { add_out, add_out ? ggml_nbytes(add) : 0 }, + { normed, normed ? ggml_nbytes(mulw) : 0 }, + { mm ? mm->data : nullptr, mm ? ggml_nbytes(mm) : 0 } }; + for (const auto & o : outr) { + for (const ggml_tensor * t : ins) { + if (!o.first || !t) continue; + const char * ob = (const char *) o.first, * tb = (const char *) t->data; + if (ob != tb && ob < tb + ggml_nbytes(t) && tb < ob + o.second) { + return false; + } + } + } + } + float eps; + memcpy(&eps, norm->op_params, sizeof(float)); + if (!ggml_sycl_t2_add_norm_rot(*ctx.stream(), add ? (const float *) add->src[0]->data : (const float *) x->data, + add ? (const float *) add->src[1]->data : nullptr, add_out, (const float *) w->data, eps, + normed, signs, mm ? (float *) mm->data : nullptr, M, K, mm)) { + return false; + } + i = last; + return true; +} + static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * sycl_ctx, ggml_cgraph * cgraph) { ggml_sycl_set_main_device(sycl_ctx->device); ggml_sycl_gdn_gather_reset(*sycl_ctx); // ARC-LAB: nodes already computed by ggml_sycl_try_ptq1_multi, and staged outputs still to copy in std::vector done(cgraph->n_nodes, 0); std::vector pending(cgraph->n_nodes, nullptr); - for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; if (done[i]) { @@ -6484,6 +6579,13 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc continue; } } + // ARC-LAB: [residual ADD +] RMS_NORM + MUL(weight) [+ MUL(signs) + RESHAPE + FWHT-hint MUL_MAT] in one kernel + // (ggml_sycl_t2_add_norm_rot): the post-attention norm of every layer and the pre-attention norm of the full + // attention layers end in the rotation that feeds the T2 mat-vecs; the DeltaNet layers' norm is followed by + // cache ops first, so there only the ADD folds in. + if ((node->op == GGML_OP_ADD || node->op == GGML_OP_RMS_NORM) && ggml_sycl_try_add_norm_rot(*sycl_ctx, cgraph, i)) { + continue; + } // Hadamard sign flip + reshape + FWHT-hint matmul: apply the signs in the transform's load if (node->op == GGML_OP_MUL && ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_MUL, GGML_OP_RESHAPE, GGML_OP_MUL_MAT }, { i + 2 })) { diff --git a/ggml/src/ggml-sycl/ptq1-t2.cpp b/ggml/src/ggml-sycl/ptq1-t2.cpp index 799bf9f3ae36..75853ca64d80 100644 --- a/ggml/src/ggml-sycl/ptq1-t2.cpp +++ b/ggml/src/ggml-sycl/ptq1-t2.cpp @@ -319,6 +319,74 @@ void scratch_aq(sycl::queue & q, int64_t M, int64_t K, int8_t *& Aq, uint16_t *& } } // namespace +namespace { +// one 1024 block of the rotation, already loaded and signed: v[j] = element lane + 16 j + 128 sgi (sgi = sub-group 0..7 of +// the block's 128 lanes), s = the block's 1024 floats of SLM. Writes the rotated block to dst (1024 floats) and its 8 +// quant groups to Aq (1024 int8) + SA (column m, groups g0..g0+7). Butterflies in fwht.cpp's order (h = 1, 2, 4 ... 512). +// Every lane of the work-group must call it (barriers). +template +inline void rot_quant_block(float (&v)[8], float * s, const Item & it, const sycl::sub_group & sg, int lane, int sgi, + float * dst, int8_t * aq, uint16_t * SA, int64_t g0, int lda, int64_t m) { +#pragma unroll + for (int h = 1; h < 16; h *= 2) { +#pragma unroll + for (int j = 0; j < 8; ++j) { + const float o = sycl::permute_group_by_xor(sg, v[j], h); + v[j] = (lane & h) == 0 ? v[j] + o : o - v[j]; + } + } +#pragma unroll + for (int st = 1; st < 8; st *= 2) { +#pragma unroll + for (int j = 0; j < 8; ++j) { + if ((j & st) == 0) { + const float a = v[j], b = v[j + st]; + v[j] = a + b; v[j + st] = a - b; + } + } + } + // layout 2: element t + 128 i (t = 16 sgi + lane); butterflies h = 128, 256, 512 in registers + const int t = 16 * sgi + lane; +#pragma unroll + for (int j = 0; j < 8; ++j) s[lane + 16 * j + 128 * sgi] = v[j]; + it.barrier(sycl::access::fence_space::local_space); +#pragma unroll + for (int i = 0; i < 8; ++i) v[i] = s[t + 128 * i]; +#pragma unroll + for (int st = 1; st < 8; st *= 2) { +#pragma unroll + for (int i = 0; i < 8; ++i) { + if ((i & st) == 0) { + const float a = v[i], b = v[i + st]; + v[i] = a + b; v[i + st] = a - b; + } + } + } + it.barrier(sycl::access::fence_space::local_space); +#pragma unroll + for (int i = 0; i < 8; ++i) s[t + 128 * i] = v[i]; + it.barrier(sycl::access::fence_space::local_space); + // layout 1 again: sub-group sgi holds quant group sgi (128 values, 8 per lane) -> quant_a's numbers + float mx = 0.0f; +#pragma unroll + for (int j = 0; j < 8; ++j) { + v[j] = s[lane + 16 * j + 128 * sgi]; + mx = sycl::fmax(mx, sycl::fabs(v[j])); + dst[lane + 16 * j + 128 * sgi] = v[j]; + } + mx = sycl::reduce_over_group(sg, mx, sycl::maximum()); + const sycl::half sh = sycl::half(127.0f / sycl::fmax(mx, int8dpas::EPS)); + const float sc = (float) sh; + if (lane == 0) { + SA[(g0 + sgi) * lda + m] = sycl::bit_cast(sh); + } +#pragma unroll + for (int j = 0; j < 8; ++j) { + aq[128 * sgi + lane + 16 * j] = (int8_t) sycl::rint(sycl::clamp(v[j] * sc, -128.0f, 127.0f)); + } +} +} // namespace + // signed 1024-point Hadamard rotation (as fwht.cpp: dst = H (x * signs) / 32, same butterfly order, so the same floats) // that also quantizes its output for the T2 mat-vecs which read it next: one 128-lane work-group per 1024 block where // fwht.cpp gives a block one 16-lane sub-group (5 sub-groups in all for a decode token), and no quant_a launch after it. @@ -343,76 +411,80 @@ bool ggml_sycl_t2_fwht_quant(sycl::queue & q, const float * x, const float * sig const int64_t m = r / nblk, kb = r % nblk; const float * src = x + r * N; const float * sgn = signs + kb * N; - float * const s = sl.get_multi_ptr().get(); float v[8]; - // layout 1: element lane + 16 j + 128 sgi; butterflies h = 1..8 across lanes, then 16, 32, 64 in registers #pragma unroll for (int j = 0; j < 8; ++j) { const int e = lane + 16 * j + 128 * sgi; v[j] = src[e] * (1.0f / 32.0f) * sgn[e]; } + rot_quant_block(v, sl.get_multi_ptr().get(), it, sg, lane, sgi, dst + r * N, + Aq + m * K + kb * N, SA, kb * (N / QK), lda, m); + }); + }); + g_quant_have = quant_key{ key, dst, K, M, K, g_graph.load() }; + return true; +} + +// residual add + RMS norm + norm weight (+ the signed rotation + quantize of ggml_sycl_t2_fwht_quant): one work-group +// per token, K / 1024 blocks of 128 lanes. Every read of a and b happens before the row's sum of squares is reduced +// (a work-group barrier), so outputs may reuse an input's memory within a token; the caller checks across tokens. +bool ggml_sycl_t2_add_norm_rot(sycl::queue & q, const float * a, const float * b, float * add_out, const float * w, + float eps, float * normed, const float * signs, float * mm, int64_t M, int64_t K, + const void * key) { + constexpr int N = 1024; + if (K % N != 0 || K > 8 * N || M <= 0 || M > 512 || !ggml_sycl_knob("GGML_SYCL_PTQ1_T2_NORMQ", 1)) { + return false; + } + const bool rot = signs != nullptr; + int8_t * Aq = nullptr; + uint16_t * SA = nullptr; + if (rot) { + scratch_aq(q, M, K, Aq, SA); + } + const int lda = int8dpas::ldsa((int) M); + const int nt = (int) (K / N) * 128; + q.submit([&](sycl::handler & cgh) { + sycl::local_accessor sl(sycl::range<1>(rot ? K : 1), cgh); + cgh.parallel_for(sycl::nd_range<1>(M * nt, nt), [=](sycl::nd_item<1> it) [[sycl::reqd_sub_group_size(16)]] { + const auto sg = it.get_sub_group(); + const int lane = sg.get_local_linear_id(); + const int sgid = sg.get_group_linear_id(); + const int kb = sgid / 8, sgi = sgid % 8; + const int64_t m = it.get_group(0); + const int64_t base = m * K + kb * N; + float v[8], ss = 0.0f; #pragma unroll - for (int h = 1; h < 16; h *= 2) { -#pragma unroll - for (int j = 0; j < 8; ++j) { - const float o = sycl::permute_group_by_xor(sg, v[j], h); - v[j] = (lane & h) == 0 ? v[j] + o : o - v[j]; - } + for (int j = 0; j < 8; ++j) { + const int64_t e = base + lane + 16 * j + 128 * sgi; + v[j] = b ? a[e] + b[e] : a[e]; + ss += v[j] * v[j]; } + ss = sycl::reduce_over_group(it.get_group(), ss, sycl::plus()); + const float scale = sycl::rsqrt(ss / K + eps); #pragma unroll - for (int st = 1; st < 8; st *= 2) { -#pragma unroll - for (int j = 0; j < 8; ++j) { - if ((j & st) == 0) { - const float a = v[j], b = v[j + st]; - v[j] = a + b; v[j + st] = a - b; - } + for (int j = 0; j < 8; ++j) { + const int c = kb * N + lane + 16 * j + 128 * sgi; + const int64_t e = m * K + c; + if (add_out) { + add_out[e] = v[j]; } - } - // layout 2: element t + 128 i (t = 16 sgi + lane); butterflies h = 128, 256, 512 in registers - const int t = 16 * sgi + lane; -#pragma unroll - for (int j = 0; j < 8; ++j) s[lane + 16 * j + 128 * sgi] = v[j]; - it.barrier(sycl::access::fence_space::local_space); -#pragma unroll - for (int i = 0; i < 8; ++i) v[i] = s[t + 128 * i]; -#pragma unroll - for (int st = 1; st < 8; st *= 2) { -#pragma unroll - for (int i = 0; i < 8; ++i) { - if ((i & st) == 0) { - const float a = v[i], b = v[i + st]; - v[i] = a + b; v[i + st] = a - b; - } + v[j] = scale * v[j] * w[c]; + if (normed) { + normed[e] = v[j]; + } + if (rot) { + v[j] = v[j] * (1.0f / 32.0f) * signs[c]; } } - it.barrier(sycl::access::fence_space::local_space); -#pragma unroll - for (int i = 0; i < 8; ++i) s[t + 128 * i] = v[i]; - it.barrier(sycl::access::fence_space::local_space); - // layout 1 again: sub-group sgi holds quant group sgi (128 values, 8 per lane) -> quant_a's numbers - float mx = 0.0f; -#pragma unroll - for (int j = 0; j < 8; ++j) { - v[j] = s[lane + 16 * j + 128 * sgi]; - mx = sycl::fmax(mx, sycl::fabs(v[j])); - dst[r * N + lane + 16 * j + 128 * sgi] = v[j]; - } - mx = sycl::reduce_over_group(sg, mx, sycl::maximum()); - const sycl::half sh = sycl::half(127.0f / sycl::fmax(mx, int8dpas::EPS)); - const float scl = (float) sh; - const int64_t g = kb * (N / QK) + sgi; - if (lane == 0) { - SA[g * lda + m] = sycl::bit_cast(sh); - } - int8_t * aq = Aq + m * K + g * QK; -#pragma unroll - for (int j = 0; j < 8; ++j) { - aq[lane + 16 * j] = (int8_t) sycl::rint(sycl::clamp(v[j] * scl, -128.0f, 127.0f)); + if (rot) { + rot_quant_block(v, sl.get_multi_ptr().get() + kb * N, it, sg, lane, sgi, + mm + base, Aq + base, SA, (int64_t) kb * (N / QK), lda, m); } }); }); - g_quant_have = quant_key{ key, dst, K, M, K, g_graph.load() }; + if (rot) { + g_quant_have = quant_key{ key, mm, K, M, K, g_graph.load() }; + } return true; } diff --git a/ggml/src/ggml-sycl/ptq1-t2.hpp b/ggml/src/ggml-sycl/ptq1-t2.hpp index 035380536b60..63b250b2228e 100644 --- a/ggml/src/ggml-sycl/ptq1-t2.hpp +++ b/ggml/src/ggml-sycl/ptq1-t2.hpp @@ -27,3 +27,8 @@ void ggml_sycl_t2_new_graph(); // key = the tensor they will read it through (see x_key). false: shape not served (K % 1024, M > 512) or knob off. bool ggml_sycl_t2_fwht_quant(sycl::queue & q, const float * x, const float * signs, float * dst, int64_t M, int64_t K, const void * key); +// [add_out =] a (+ b); normed = rms_norm(.) * w; [mm = rotation of normed * signs, quantized as above]. b, add_out, +// normed, signs (and then mm, key) may be null. One work-group per token: K % 1024 == 0, K <= 8192, M <= 512. +bool ggml_sycl_t2_add_norm_rot(sycl::queue & q, const float * a, const float * b, float * add_out, const float * w, + float eps, float * normed, const float * signs, float * mm, int64_t M, int64_t K, + const void * key); From 975a35991541a4c542dc36c7702af5efb9998f8b Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sun, 4 Oct 2026 12:17:18 +1100 Subject: [PATCH 23/65] sycl: vendored TernSYCL -> 8b0ae42, drop the -fp-model=precise override Upstream fixed the GemmMT row-scale broadcast that returned NaN under icpx's default -fp-model=fast (libxsmm/TernSYCL#1). B580: MUL_MAT ptq1_0/pq2_0 178/178 with fast math, KLD unchanged, decode speed unchanged. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 546f94c81c4d38117ba73754296497418de74118 on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/CMakeLists.txt | 3 --- ggml/src/ggml-sycl/ptq1-t2.cpp | 2 +- .../src/ggml-sycl/ternsycl/int2_int8_dpas.hpp | 25 +++++++++++-------- ggml/src/ggml-sycl/ternsycl/xe2.hpp | 10 ++++++++ 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/ggml/src/ggml-sycl/CMakeLists.txt b/ggml/src/ggml-sycl/CMakeLists.txt index 56e3c8b8525a..f60ec4c537f2 100644 --- a/ggml/src/ggml-sycl/CMakeLists.txt +++ b/ggml/src/ggml-sycl/CMakeLists.txt @@ -108,9 +108,6 @@ target_compile_options(ggml-sycl PRIVATE "-Wno-narrowing") # ARC-LAB: one device image per kernel. The Xe2 XMX kernels (DPAS) then only get JIT-compiled when they are first launched, # so a GPU without DPAS (Xe-LP, Gen9) never compiles them; with one image per source file, compiling any kernel of # fattn-dec / mmvq / ptq1-t2 built the DPAS ones too and the shader compiler crashed (UHD 630). OFF = compiler default. -# ARC-LAB: the TernSYCL kernels (ptq1-t2.cpp) need IEEE semantics: under icpx's default -fp-model=fast their GemmMT returns -# NaN / inf (upstream's own driver fails validation built that way, passes with precise, which their Makefile uses). -set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/ptq1-t2.cpp PROPERTIES COMPILE_OPTIONS "-fp-model=precise") option(GGML_SYCL_DEVICE_CODE_SPLIT_PER_KERNEL "ggml: SYCL device code split per kernel" ON) if (GGML_SYCL_DEVICE_CODE_SPLIT_PER_KERNEL) target_compile_options(ggml-sycl PRIVATE "-fsycl-device-code-split=per_kernel") diff --git a/ggml/src/ggml-sycl/ptq1-t2.cpp b/ggml/src/ggml-sycl/ptq1-t2.cpp index 75853ca64d80..1147b39fd81a 100644 --- a/ggml/src/ggml-sycl/ptq1-t2.cpp +++ b/ggml/src/ggml-sycl/ptq1-t2.cpp @@ -1,5 +1,5 @@ // ARC-LAB: PTQ1_0 on XMX through the TernSYCL int2 x int8 DPAS kernels (ternsycl/, BSD 3-Clause, libxsmm/TernSYCL -// @11484da + the long long address fix of libxsmm/TernSYCL#3: inline vISA only, no IGC builtins). The base-3 PTQ1_0 packing (1.625 + 0.125 bits/weight) has to be decoded on the ALUs before any DPAS, which +// @8b0ae42: inline vISA only, no IGC builtins; builds with icpx's default -fp-model=fast). The base-3 PTQ1_0 packing (1.625 + 0.125 bits/weight) has to be decoded on the ALUs before any DPAS, which // makes the 4-column verify mat-vec compute-bound (~190 GB/s on the B580); 2-bit two's-complement codes (2.125 bits) // feed the s8 x s2 DPAS directly. B580 standalone, 2 GiB rotating weights: 17408x5120 m=1 56 us, m=4 61 us (vs 70 / // ~100 in-model for PTQ1_0). The weight grows 31%; activations become int8 with one fp16 scale per 128 (q8_1 is per 32). diff --git a/ggml/src/ggml-sycl/ternsycl/int2_int8_dpas.hpp b/ggml/src/ggml-sycl/ternsycl/int2_int8_dpas.hpp index 35e0f2d1fdde..8e2ec35dd65c 100644 --- a/ggml/src/ggml-sycl/ternsycl/int2_int8_dpas.hpp +++ b/ggml/src/ggml-sycl/ternsycl/int2_int8_dpas.hpp @@ -37,8 +37,9 @@ constexpr int GS = 128; inline int ldsa(int M) { return (M + 31) & ~31; } constexpr float EPS = 1.1920928955078125e-07f; // FLT_EPSILON, xetla absmax init -// two DT (low, high half of h2) -> two int8 packed in a short; clamp + RTZ -// convert lowers to one mov.sat per element +// two DT (low, high half of h2) -> two int8 packed in a short, rounded to nearest +// even: + 1.5 * 2^23 rounds to nearest even into the low mantissa bits +// (|f * sa| <= 127.5), whose low byte is the int8 template inline short q2(unsigned h2, float sa) { float f0, f1; if constexpr (BF16) { @@ -48,18 +49,19 @@ template inline short q2(unsigned h2, float sa) { f0 = dt::tof((unsigned short)h2); f1 = dt::tof((unsigned short)(h2 >> 16)); } - const signed char c0 = (signed char)sycl::clamp(f0 * sa, -128.0f, 127.0f); - const signed char c1 = (signed char)sycl::clamp(f1 * sa, -128.0f, 127.0f); - return (short)((unsigned char)c0 | ((unsigned)(unsigned char)c1 << 8)); + const unsigned u0 = sycl::bit_cast(sycl::fma(f0, sa, 12582912.0f)); + const unsigned u1 = sycl::bit_cast(sycl::fma(f1, sa, 12582912.0f)); + return (short)((u0 & 0xffu) | ((u1 & 0xffu) << 8)); } // xetla's elemwise_scale_{fp16,bf16}_to_int8 on one 8-row x 32-K block, on the // row-contiguous registers (SIMT code would split and re-pair every pair). // a: row r = a[r] (lane l holds K 2l, 2l+1); lane r of sal = scale of row r. -// SIMD32 NoMask like xetla: call only from convergent code. -#define TS_QROW(r, g, orow, ocol) \ - "mul (M1_NM, 32) T(" #g ",0)<1> T(" #g ",0)<1;1,0> %2(0," #r ")<0;1,0>\n" \ - "mov.sat (M1_NM, 32) TB(" #g ",0)<4> T(" #g ",0)<1;1,0>\n" \ +// SIMD32 NoMask like xetla: call only from convergent code. a * sal + 1.5 * 2^23 +// rounds to nearest even into the low mantissa bits (|a * sal| <= 127.5); the +// low byte of each float is the int8. +#define TS_QROW(r, g, orow, ocol) \ + "mad (M1_NM, 32) T(" #g ",0)<1> T(" #g ",0)<1;1,0> %2(0," #r ")<0;1,0> 0x4b400000:f\n" \ "mov (M1_NM, 32) QB(" #orow "," #ocol ")<1> TB(" #g ",0)<4;1,0>\n" #define TS_QUANT(A2F) \ "{\n" \ @@ -118,7 +120,7 @@ struct QuantA { unsigned long long q = 0; #pragma unroll for (int i = 0; i < 8; ++i) - q |= (unsigned long long)(unsigned char)(signed char)sycl::clamp(a[i] * s, -128.0f, 127.0f) + q |= (unsigned long long)(unsigned char)(signed char)sycl::clamp(sycl::rint(a[i] * s), -128.0f, 127.0f) << (8 * i); *(unsigned long long *)(Aq + off) = q; } @@ -371,8 +373,9 @@ struct GemmMT { for (int c = 0; c < 4; ++c) aq[c] = quant8x32(ar[I][c], sal); } // rows >= M get inf here, but their int32 dot is 0 and the store clips them + const float invl = sycl::native::recip(sal); #pragma unroll - for (int r = 0; r < 8; ++r) inv[r] = sycl::native::recip(sycl::group_broadcast(sgp, sal, r)); + for (int r = 0; r < 8; ++r) inv[r] = sycl::group_broadcast(sgp, invl, r); #pragma unroll for (int j = 0; j < NB; ++j) { int8 ia = dpas_s2s8_z(aq[0], int2{(int)w[j][0], (int)w[j][1]}); diff --git a/ggml/src/ggml-sycl/ternsycl/xe2.hpp b/ggml/src/ggml-sycl/ternsycl/xe2.hpp index bdeb2ac55096..9a25467049bc 100644 --- a/ggml/src/ggml-sycl/ternsycl/xe2.hpp +++ b/ggml/src/ggml-sycl/ternsycl/xe2.hpp @@ -286,6 +286,16 @@ XE2_DPAS_Z(dpas_s2s8_z, "s2.s8", short, int2, int, 1, 8, 32) XE2_DPAS_Z(dpas_s2s8_z, "s2.s8", short2, int2, int2, 2, 16, 32) XE2_DPAS_Z(dpas_s2s8_z, "s2.s8", short4, int2, int4, 4, 32, 32) XE2_DPAS_Z(dpas_s2s8_z, "s2.s8", short8, int2, int8, 8, 64, 32) +// s8 A (K32) x s4 B: 2 weights per byte of B, 16 bytes (32 K) per lane; dword d of a lane's B holds K 8d .. 8d+7, +// low nibble first. Same result as the IGC builtin intel_sub_group_i8_i4_matrix_mad_k32 (repeat count 8). +XE2_DPAS(dpas_s4s8, "s4.s8", short, int4, int, 1, 8, 64) +XE2_DPAS(dpas_s4s8, "s4.s8", short2, int4, int2, 2, 16, 64) +XE2_DPAS(dpas_s4s8, "s4.s8", short4, int4, int4, 4, 32, 64) +XE2_DPAS(dpas_s4s8, "s4.s8", short8, int4, int8, 8, 64, 64) +XE2_DPAS_Z(dpas_s4s8_z, "s4.s8", short, int4, int, 1, 8, 64) +XE2_DPAS_Z(dpas_s4s8_z, "s4.s8", short2, int4, int2, 2, 16, 64) +XE2_DPAS_Z(dpas_s4s8_z, "s4.s8", short4, int4, int4, 4, 32, 64) +XE2_DPAS_Z(dpas_s4s8_z, "s4.s8", short8, int4, int8, 8, 64, 64) #undef XE2_DPAS #undef XE2_DPAS_Z From 1f169215ff60f054e59257350919aba8f3485ef0 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Mon, 5 Oct 2026 11:04:50 +1100 Subject: [PATCH 24/65] sycl: lab knob GGML_SYCL_T2_ZERO_WEIGHTS (zeroes T2 weights to profile non-weight cost) Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 2dbd83934f0f6b232efdf7e324bf41a353c15159 on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/ptq1-t2.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ggml/src/ggml-sycl/ptq1-t2.cpp b/ggml/src/ggml-sycl/ptq1-t2.cpp index 1147b39fd81a..21981bfad382 100644 --- a/ggml/src/ggml-sycl/ptq1-t2.cpp +++ b/ggml/src/ggml-sycl/ptq1-t2.cpp @@ -170,6 +170,12 @@ bool ggml_sycl_t2_repack(sycl::queue & q, void * data, int64_t K, int64_t N, boo q.memcpy(&bad_h, bad, sizeof(int)).wait(); sycl::free(bad, q); sycl::free(tmp, q); + // ARC-LAB experiment: GGML_SYCL_T2_ZERO_WEIGHTS=1 zeroes every T2 weight after the repack (output is garbage). The Xe2 + // memory compression then makes weight reads almost free, so a profile shows everything else a token costs (2026-10-05). + static const bool zero_w = getenv("GGML_SYCL_T2_ZERO_WEIGHTS") && atoi(getenv("GGML_SYCL_T2_ZERO_WEIGHTS")); + if (zero_w) { + q.memset(B, 0, (size_t) (K / 16) * N * 4).wait(); + } return bad_h == 0; } From 870f3c8041167c745c6556f2b2c1cc7159d49a8e Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Mon, 5 Oct 2026 11:17:15 +1100 Subject: [PATCH 25/65] sycl: lab instrumentation - host token timing (GGML_SYCL_HOST_TIMING), W8A8 threshold as a runtime knob; test shapes GGML_SYCL_HOST_TIMING=1 splits decode graphs into between / enqueue / wait host time plus per-op enqueue cost, without a tracer. test-backend-ops: Bonsai attn k/v and output-head mat-vec shapes, FA nh=1,2 at 48K. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit ec71714bcbe40949b54ceb92890ebae4c06c051b on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/ggml-sycl.cpp | 76 +++++++++++++++++++++++++++++++- tests/test-backend-ops.cpp | 6 +++ 2 files changed, 81 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 21c1e4215b33..2a9319e55388 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -4988,7 +4989,8 @@ static bool ggml_sycl_t2_mul_mat_try(ggml_backend_sycl_context & ctx, const ggml // tokens: ne[1] x ne[2] rows of K floats (row stride nb[1]; planes back to back) const int64_t M = src1->ne[1] * src1->ne[2]; // ARC-LAB: large prompt batches through oneDNN's int8 GEMM (faster than GemmMT there), GGML_SYCL_T2_W8A8_MIN (0 = off) - static const int64_t w8a8_min = getenv("GGML_SYCL_T2_W8A8_MIN") ? atoll(getenv("GGML_SYCL_T2_W8A8_MIN")) : 0; + static const int64_t w8a8_env = getenv("GGML_SYCL_T2_W8A8_MIN") ? atoll(getenv("GGML_SYCL_T2_W8A8_MIN")) : 0; + const int64_t w8a8_min = ggml_sycl_knob("GGML_SYCL_T2_W8A8_MIN", (int) w8a8_env); if (w8a8_min > 0 && M >= w8a8_min && g_ggml_sycl_enable_dnn && src1->nb[1] == (size_t) K * sizeof(float) && ggml_sycl_w8a8_mul_mat_t2(ctx, src0->data, (const float *) src1->data, K, (float *) dst->data, N, M, K, stream)) { return true; @@ -6131,11 +6133,54 @@ catch (sycl::exception const &exc) { std::exit(1); } +// ARC-LAB: GGML_SYCL_HOST_TIMING=1 -> host-side token timing without a tracer (tracers slow the host 2x and invent gaps). +// Per graph: between = host time since the previous synchronize returned (sampling, graph build, logits copy), +// enqueue = time to submit every kernel, wait = synchronize after the last submit. Plus host enqueue time per op type. +// Printed every 64 graphs with more than 500 nodes (decode graphs), then reset. +struct sycl_host_timing { + bool on = getenv("GGML_SYCL_HOST_TIMING") && atoi(getenv("GGML_SYCL_HOST_TIMING")); + using clk = std::chrono::steady_clock; + clk::time_point last_sync_end{}, t_in{}, t_enq{}; + bool in_big = false; + int graphs = 0; + double between = 0, enqueue = 0, wait = 0, waits_mid = 0; + long nodes = 0, syncs = 0; + std::map> per_op; + void report() { + if (graphs == 0) return; + fprintf(stderr, "host-timing: %d graphs, per graph: between %.3f ms, enqueue %.3f ms (%ld nodes), wait %.3f ms, other syncs %.1f\n", + graphs, between / graphs, enqueue / graphs, nodes / graphs, wait / graphs, (double) syncs / graphs); + std::vector> v; + for (auto & kv : per_op) v.push_back({ kv.second.first, kv.first }); + std::sort(v.rbegin(), v.rend()); + for (size_t i = 0; i < v.size() && i < 12; ++i) { + fprintf(stderr, "host-timing: %-22s %.3f ms/graph (%ld calls/graph)\n", v[i].second.c_str(), v[i].first / graphs, + per_op[v[i].second].second / graphs); + } + graphs = 0; between = enqueue = wait = 0; nodes = 0; syncs = 0; per_op.clear(); + } +}; +static sycl_host_timing g_ht; + static void ggml_backend_sycl_synchronize(ggml_backend_t backend) try { GGML_SYCL_DEBUG("[SYCL] call %s\n", __func__); ggml_backend_sycl_context * sycl_ctx = (ggml_backend_sycl_context *)backend->context; const queue_ptr stream = sycl_ctx->stream(sycl_ctx->device, 0); + const auto ht_t0 = sycl_host_timing::clk::now(); SYCL_CHECK(CHECK_TRY_ERROR((stream)->wait())); + if (g_ht.on) { + const auto t1 = sycl_host_timing::clk::now(); + if (g_ht.in_big) { // first synchronize after a decode graph + g_ht.wait += std::chrono::duration(t1 - ht_t0).count(); + g_ht.in_big = false; + if (++g_ht.graphs == 64) { + g_ht.report(); + } + } else { + g_ht.syncs++; + } + g_ht.last_sync_end = t1; + } GGML_UNUSED(backend); } @@ -6538,8 +6583,21 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc // ARC-LAB: nodes already computed by ggml_sycl_try_ptq1_multi, and staged outputs still to copy in std::vector done(cgraph->n_nodes, 0); std::vector pending(cgraph->n_nodes, nullptr); + const bool ht_on = g_ht.on && cgraph->n_nodes > 500; + sycl_host_timing::clk::time_point ht_prev = sycl_host_timing::clk::now(); + const char * ht_op = nullptr; for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; + if (ht_on) { + const auto now = sycl_host_timing::clk::now(); + if (ht_op) { + auto & a = g_ht.per_op[ht_op]; + a.first += std::chrono::duration(now - ht_prev).count(); + a.second++; + } + ht_prev = now; + ht_op = ggml_op_desc(node); + } if (done[i]) { if (pending[i]) { SYCL_CHECK(CHECK_TRY_ERROR(sycl_ctx->stream()->memcpy(node->data, pending[i], ggml_nbytes(node)))); @@ -6750,6 +6808,22 @@ void ggml_sycl_knobs_refresh() { static ggml_status ggml_backend_sycl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { auto * sycl_ctx = static_cast(backend->context); + g_ht.in_big = g_ht.on && cgraph->n_nodes > 500; + if (g_ht.in_big) { + g_ht.t_in = sycl_host_timing::clk::now(); + if (g_ht.last_sync_end.time_since_epoch().count()) { + g_ht.between += std::chrono::duration(g_ht.t_in - g_ht.last_sync_end).count(); + } + g_ht.nodes += cgraph->n_nodes; + } + struct ht_exit { + ~ht_exit() { + if (g_ht.in_big) { + g_ht.t_enq = sycl_host_timing::clk::now(); + g_ht.enqueue += std::chrono::duration(g_ht.t_enq - g_ht.t_in).count(); + } + } + } ht_exit_guard; ggml_sycl_knobs_refresh(); ggml_sycl_t2_new_graph(); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 3ccda28b007d..c2be2ff51ada 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10896,6 +10896,9 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_mul_mat(type_a, GGML_TYPE_F32, 12288, bs, 5120, {1, 1}, {1, 1})); // attn_q } } + // ARC-LAB: the smallest and largest Bonsai 27B mat-vecs (attn k / v, output head) + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_PTQ1_0, GGML_TYPE_F32, 1024, 1, 5120, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_PTQ1_0, GGML_TYPE_F32, 248320, 1, 5120, {1, 1}, {1, 1})); // qwen3-30b-a3b for (int bs : {1, 4, 8, 32, 64, 128, 256, 512}) { @@ -10961,6 +10964,9 @@ static std::vector> make_test_cases_perf() { test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, {0, 2, 1, 3})); } } + for (int64_t nkv : {1, 2}) { // ARC-LAB: fewer KV heads, same 6 q heads per KV head: per-head cost without the 4-head row interleave + test_cases.emplace_back(new test_flash_attn_ext(256, 256, nkv, {6, 1}, 49152, 1, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, {0, 2, 1, 3})); + } for (int64_t nb : {1, 4, 64, 256}) { // ARC-LAB: ~115K history (Bonsai at 128K context), real cache layout test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 114688, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, {0, 2, 1, 3})); } From 204450d3fdec74fba0f2d718102bb99289ccc4fe Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:00:02 +1100 Subject: [PATCH 26/65] sycl: fuse the DeltaNet conv chain for one-token decode (GGML_SYCL_CONV_CHAIN) CONCAT + conv-cache CPY + SSM_CONV + SILU + L2_NORM in one kernel per layer (one work-group per 128-channel head). Outputs are written at the CONCAT's position, so nodes in between must not touch them (checked); an output may sit exactly on the qkv input (same element per channel), and an L2 output on top of the old conv state falls back to its own L2_NORM kernel. Also GGML_SYCL_FAST_SET_TENSOR (opt-in pinned input uploads). B580 Bonsai 27B plain decode +1.3-1.8%, KLD ub1 unchanged. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit b56951ee0b9ecff7bb7e16f129b24269062bcc67 on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/ggml-sycl.cpp | 248 +++++++++++++++++++++++++++++++ 1 file changed, 248 insertions(+) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 2a9319e55388..5020cd177399 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -662,6 +662,10 @@ catch (sycl::exception const &exc) { std::exit(1); } +// ARC-LAB: per-token accounting of synchronous buffer_set_tensor calls, printed with the GGML_SYCL_HOST_TIMING report +static double g_st_ms = 0; static long g_st_calls = 0; static size_t g_st_bytes = 0; +static void ggml_sycl_note_set_tensor(size_t bytes, double ms) { g_st_ms += ms; g_st_calls++; g_st_bytes += bytes; } + static void ggml_backend_sycl_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor *tensor, const void *data, size_t offset, @@ -672,6 +676,29 @@ static void ggml_backend_sycl_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_backend_sycl_buffer_context * ctx = ( ggml_backend_sycl_buffer_context *)buffer->context; ggml_sycl_set_device(ctx->device); auto stream = &(dpct::dev_mgr::instance().get_device(ctx->device).default_queue()); + // ARC-LAB: GGML_SYCL_HOST_TIMING counts these (llama.cpp writes every per-token input through here, synchronously) + struct st_timer { + size_t bytes; std::chrono::steady_clock::time_point t0 = std::chrono::steady_clock::now(); + ~st_timer() { ggml_sycl_note_set_tensor(bytes, std::chrono::duration(std::chrono::steady_clock::now() - t0).count()); } + } st_guard{ size }; + // ARC-LAB: GGML_SYCL_FAST_SET_TENSOR=1: small writes (<= 4 MB, i.e. per-token inputs, not weights) skip the device-wide + // wait and the malloc'd bounce buffer (the mmap workaround is for weights loaded from disk): one pinned staging copy and + // one wait on this queue. Still synchronous for the caller. + const int fast_st = ggml_sycl_knob("GGML_SYCL_FAST_SET_TENSOR", 0); // 2 = keep the device-wide wait + if (size <= (4u << 20) && fast_st) { + if (fast_st == 2) { + SYCL_CHECK(CHECK_TRY_ERROR(dpct::dev_mgr::instance().get_device(ctx->device).queues_wait_and_throw())); + } + static char * pinned = nullptr; + if (!pinned) { + pinned = (char *) sycl::malloc_host(4u << 20, *stream); + } + if (pinned) { + memcpy(pinned, data, size); + SYCL_CHECK(CHECK_TRY_ERROR((*stream).memcpy((char *) tensor->data + offset, pinned, size).wait())); + return; + } + } SYCL_CHECK(CHECK_TRY_ERROR(dpct::dev_mgr::instance().get_device(ctx->device).queues_wait_and_throw())); #ifndef _WIN32 // Note: Use host buffer to save the data from mmap(), then copy to device. It's workaround for mmap() issue on PVC GPU. @@ -6061,6 +6088,19 @@ static void ggml_backend_sycl_free(ggml_backend_t backend) { delete backend; } +// ARC-LAB: pinned upload ring for ggml_backend_sycl_set_tensor_async (GGML_SYCL_PINNED_UPLOAD) +static char * g_upload_ring = nullptr; +static const size_t g_upload_ring_bytes = 32u << 20; +static size_t g_upload_off = 0; +// GGML_SYCL_HOST_TIMING=1 also prints the uploads of the first decode-sized graphs (name, bytes) +static void ggml_sycl_log_upload(const ggml_tensor * t, size_t size) { + static const bool on = getenv("GGML_SYCL_HOST_TIMING") && atoi(getenv("GGML_SYCL_HOST_TIMING")); + static int n = 0; + if (on && n < 60) { + ++n; + fprintf(stderr, "upload: %-28s %zu bytes\n", t->name, size); + } +} static void ggml_backend_sycl_set_tensor_async(ggml_backend_t backend, ggml_tensor *tensor, const void *data, size_t offset, @@ -6073,6 +6113,27 @@ static void ggml_backend_sycl_set_tensor_async(ggml_backend_t backend, GGML_ASSERT(buf->buft == ggml_backend_sycl_buffer_type(sycl_ctx->device) && "unsupported buffer type"); const queue_ptr stream = sycl_ctx->stream(sycl_ctx->device, 0); + // ARC-LAB: GGML_SYCL_PINNED_UPLOAD=1 stages small per-token inputs (token embedding, positions, masks, state ids) in a + // pinned host ring first: a copy straight from pageable memory goes through the driver's staging path. The ring + // restarts at each synchronize (the in-order queue has consumed every staged copy by then). + if (ggml_sycl_knob("GGML_SYCL_PINNED_UPLOAD", 0) && size <= g_upload_ring_bytes / 4) { + if (!g_upload_ring) { + g_upload_ring = (char *) sycl::malloc_host(g_upload_ring_bytes, *stream); + } + if (g_upload_ring) { + if (g_upload_off + size > g_upload_ring_bytes) { + stream->wait(); + g_upload_off = 0; + } + char * staged = g_upload_ring + g_upload_off; + memcpy(staged, data, size); + g_upload_off = (g_upload_off + size + 255) & ~(size_t) 255; + SYCL_CHECK(CHECK_TRY_ERROR((stream)->memcpy((char *) tensor->data + offset, staged, size))); + ggml_sycl_log_upload(tensor, size); + return; + } + } + ggml_sycl_log_upload(tensor, size); SYCL_CHECK(CHECK_TRY_ERROR( (stream)->memcpy((char *)tensor->data + offset, data, size))); } @@ -6150,6 +6211,9 @@ struct sycl_host_timing { if (graphs == 0) return; fprintf(stderr, "host-timing: %d graphs, per graph: between %.3f ms, enqueue %.3f ms (%ld nodes), wait %.3f ms, other syncs %.1f\n", graphs, between / graphs, enqueue / graphs, nodes / graphs, wait / graphs, (double) syncs / graphs); + fprintf(stderr, "host-timing: set_tensor (sync uploads) per graph: %.3f ms, %.1f calls, %.0f bytes\n", g_st_ms / graphs, + (double) g_st_calls / graphs, (double) g_st_bytes / graphs); + g_st_ms = 0; g_st_calls = 0; g_st_bytes = 0; std::vector> v; for (auto & kv : per_op) v.push_back({ kv.second.first, kv.first }); std::sort(v.rbegin(), v.rend()); @@ -6168,6 +6232,7 @@ static void ggml_backend_sycl_synchronize(ggml_backend_t backend) try { const queue_ptr stream = sycl_ctx->stream(sycl_ctx->device, 0); const auto ht_t0 = sycl_host_timing::clk::now(); SYCL_CHECK(CHECK_TRY_ERROR((stream)->wait())); + g_upload_off = 0; if (g_ht.on) { const auto t1 = sycl_host_timing::clk::now(); if (g_ht.in_big) { // first synchronize after a decode graph @@ -6577,6 +6642,186 @@ static bool ggml_sycl_try_add_norm_rot(ggml_backend_sycl_context & ctx, ggml_cgr return true; } +// ARC-LAB: DeltaNet conv chain for one token: CONCAT(conv state, new qkv) -> CPY(newest d_conv-1 columns -> conv cache) -> +// SSM_CONV -> SILU -> L2_NORM(q/k heads) in one kernel (one work-group per head of S channels), instead of 5 launches. +// The nodes are spread over ~15 graph nodes with cache ops in between; the fused outputs are written at the CONCAT's +// position, so every node computed in between must not touch their memory (checked), and they must not overlap inputs. +// GGML_SYCL_CONV_CHAIN=0 turns it off. +static bool ggml_sycl_try_conv_chain(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int i, std::vector & done) { + if (!ggml_sycl_knob("GGML_SYCL_CONV_CHAIN", 1)) { + return false; + } + ggml_tensor * cat = cgraph->nodes[i]; + if (cat->op != GGML_OP_CONCAT || cat->type != GGML_TYPE_F32 || ggml_get_op_params_i32(cat, 0) != 0) { + return false; + } + const int64_t dc = cat->ne[0], C = cat->ne[1]; + const ggml_tensor * cs = cat->src[0]; + const ggml_tensor * x = cat->src[1]; + if (cat->ne[2] != 1 || cat->ne[3] != 1 || dc < 2 || dc > 8 || !ggml_is_contiguous(cat) || cs->type != GGML_TYPE_F32 || + x->type != GGML_TYPE_F32 || cs->ne[0] != dc - 1 || cs->ne[1] != C || !ggml_is_contiguous(cs) || x->ne[0] != 1 || + x->ne[1] != C) { + return false; + } + int ic = -1, iv = -1, iconv = -1, isilu = -1, il2 = -1; + const int jmax = std::min(cgraph->n_nodes - 1, i + 48); + for (int j = i + 1; j <= jmax && il2 < 0; ++j) { + ggml_tensor * n = cgraph->nodes[j]; + if (n->op == GGML_OP_CPY && n->src[0]->op == GGML_OP_VIEW && n->src[0]->view_src == cat) { + ic = j; + } else if (n->op == GGML_OP_SSM_CONV && n->src[0] == cat) { + iconv = j; + } else if (iconv >= 0 && n->op == GGML_OP_UNARY && ggml_get_unary_op(n) == GGML_UNARY_OP_SILU && + n->src[0] == cgraph->nodes[iconv]) { + isilu = j; + } else if (isilu >= 0 && n->op == GGML_OP_L2_NORM && n->src[0]->op == GGML_OP_VIEW && + n->src[0]->view_src == cgraph->nodes[isilu]) { + il2 = j; + iv = j; + } + } + static int reasons = 0; + auto why = [&](const char * r) { + if (reasons < 3 && getenv("GGML_SYCL_CONV_CHAIN_DEBUG")) { + ++reasons; + fprintf(stderr, "conv chain: not fused at node %d (%s): %s\n", i, cat->name, r); + } + return false; + }; + if (ic < 0 || iconv < 0 || isilu < 0 || il2 < 0 || done[ic] || done[iconv] || done[isilu] || done[il2]) { + return why("chain nodes not found"); + } + ggml_tensor * cpy = cgraph->nodes[ic]; + ggml_tensor * conv = cgraph->nodes[iconv]; + ggml_tensor * silu = cgraph->nodes[isilu]; + ggml_tensor * l2 = cgraph->nodes[il2]; + const ggml_tensor * cv = cpy->src[0]; // view: columns 1..dc-1 of the concat + const ggml_tensor * dst = cpy->src[1]; // conv cache slot + const ggml_tensor * w = conv->src[1]; + const ggml_tensor * l2v = l2->src[0]; + const int64_t S = l2v->ne[0], H = l2v->ne[1]; + if (cv->ne[0] != dc - 1 || cv->ne[1] != C || cv->view_offs != sizeof(float) || cv->nb[1] != cat->nb[1] || + dst->type != GGML_TYPE_F32 || !ggml_is_contiguous(dst) || ggml_nelements(dst) != (dc - 1) * C || + w->type != GGML_TYPE_F32 || !ggml_is_contiguous(w) || w->ne[0] != dc || w->ne[1] != C || + ggml_nelements(conv) != C || !ggml_is_contiguous(conv) || !ggml_is_contiguous(silu) || silu->type != GGML_TYPE_F32 || + l2v->view_offs != 0 || l2v->nb[1] != (size_t) S * sizeof(float) || l2v->ne[2] != 1 || l2v->ne[3] != 1 || + !ggml_is_contiguous(l2) || S < 16 || S > 1024 || S % 16 != 0 || C % S != 0 || S * H > C || + (cpy->flags & GGML_TENSOR_FLAG_OUTPUT) || (conv->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return why("shapes/types"); + } + // the concat and the raw conv output must have no readers outside the chain + for (int j = i + 1; j < cgraph->n_nodes; ++j) { + const ggml_tensor * n = cgraph->nodes[j]; + for (int k = 0; k < GGML_MAX_SRC && n->src[k]; ++k) { + const ggml_tensor * sj = n->src[k]; + if ((sj == cat && j != iconv && n != cv) || (sj == conv && j != isilu) || (sj->view_src == cat && sj != cv)) { + return why("outside reader"); + } + } + if (n->view_src == cat && n != cv) { + return why("outside view"); + } + } + auto overlap = [](const void * a, size_t na, const void * b, size_t nb) { + const char * pa = (const char *) a, * pb = (const char *) b; + return na && nb && pa < pb + nb && pb < pa + na; + }; + const void * outs[3] = { silu->data, l2->data, dst->data }; + const size_t outn[3] = { ggml_nbytes(silu), ggml_nbytes(l2), ggml_nbytes(dst) }; + const ggml_tensor * ins[3] = { cs, x, w }; + // an output may sit exactly on x (the planner reuses its buffer after the concat): x and the silu / l2 outputs keep + // channel c at element c, a lane reads x[c] before writing c, and l2 is written after the work-group barrier + static const char * out_names[3] = { "silu", "l2", "cache" }; + static const char * in_names[3] = { "conv state", "x", "weights" }; + // l2 on top of the conv state (other lanes still read it): leave L2_NORM to its own kernel after this one + const bool l2_in = !overlap(outs[1], outn[1], cs->data, ggml_nbytes(cs)); + for (int o = 0; o < 3; ++o) { + if (o == 1 && !l2_in) { + continue; + } + for (int t = 0; t < 3; ++t) { + if (!overlap(outs[o], outn[o], ins[t]->data, ggml_nbytes(ins[t]))) { + continue; + } + const bool same_slot_ok = t == 1 && o < 2 && outs[o] == ins[t]->data && x->nb[1] == sizeof(float); + if (!same_slot_ok) { + static char msg[96]; + snprintf(msg, sizeof(msg), "output %s overlaps input %s", out_names[o], in_names[t]); + return why(msg); + } + } + } + // nodes computed between the concat and the last fused node must not touch the early-written outputs + for (int j = i + 1; j <= il2; ++j) { + if (j == ic || j == iconv || j == isilu || j == il2) { + continue; + } + const ggml_tensor * n = cgraph->nodes[j]; + if (ggml_sycl_is_view_or_noop(n) || (n->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { + continue; + } + for (int o = 0; o < 3; ++o) { + if (o == 1 && !l2_in) { + continue; + } + if (overlap(outs[o], outn[o], n->data, ggml_nbytes(n))) { + return why("node in between writes output memory"); + } + for (int k = 0; k < GGML_MAX_SRC && n->src[k]; ++k) { + if (overlap(outs[o], outn[o], n->src[k]->data, ggml_nbytes(n->src[k]))) { + return why("node in between reads output memory"); + } + } + } + } + float eps; + memcpy(&eps, l2->op_params, sizeof(float)); + const float * csd = (const float *) cs->data; + const char * xd = (const char *) x->data; + const size_t xnb = x->nb[1]; + const float * wd = (const float *) w->data; + float * dd = (float *) dst->data; + float * sd = (float *) silu->data; + float * ld = (float *) l2->data; + const int D = (int) dc; + const int64_t HL = l2_in ? H : 0; + ctx.stream()->parallel_for(sycl::nd_range<1>((size_t) C, (size_t) S), [=](sycl::nd_item<1> it) { + const int64_t c = it.get_global_id(0); + const int64_t g = it.get_group(0); + float v[8]; + for (int t = 0; t < D - 1; ++t) { + v[t] = csd[c * (D - 1) + t]; + } + v[D - 1] = *(const float *) (xd + c * xnb); + float sum = 0.0f; + for (int t = 0; t < D; ++t) { + sum += v[t] * wd[c * D + t]; + } + for (int t = 1; t < D; ++t) { + dd[c * (D - 1) + t - 1] = v[t]; + } + const float y = op_silu(sum); + sd[c] = y; + if (g < HL) { // uniform per work-group + const float ss = sycl::reduce_over_group(it.get_group(), y * y, sycl::plus()); + const float scale = sycl::rsqrt(sycl::fmax(ss, eps * eps)); + ld[g * S + it.get_local_id(0)] = scale * y; + } + }); + done[ic] = done[iconv] = done[isilu] = 1; + if (l2_in) { + done[il2] = 1; + } + static bool logged = false; + if (!logged) { + logged = true; + fprintf(stderr, "%s: fused DeltaNet conv chain active (C=%lld, d_conv=%lld, %lld heads of %lld for L2)\n", __func__, + (long long) C, (long long) dc, (long long) H, (long long) S); + } + GGML_UNUSED(iv); + return true; +} + static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * sycl_ctx, ggml_cgraph * cgraph) { ggml_sycl_set_main_device(sycl_ctx->device); ggml_sycl_gdn_gather_reset(*sycl_ctx); @@ -6637,6 +6882,9 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc continue; } } + if (node->op == GGML_OP_CONCAT && ggml_sycl_try_conv_chain(*sycl_ctx, cgraph, i, done)) { + continue; + } // ARC-LAB: [residual ADD +] RMS_NORM + MUL(weight) [+ MUL(signs) + RESHAPE + FWHT-hint MUL_MAT] in one kernel // (ggml_sycl_t2_add_norm_rot): the post-attention norm of every layer and the pre-attention norm of the full // attention layers end in the rotation that feeds the T2 mat-vecs; the DeltaNet layers' norm is followed by From c44f9961fb5e49856a735ceb87383f2469cf63a2 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Mon, 5 Oct 2026 13:40:21 +1100 Subject: [PATCH 27/65] sycl: fuse the DeltaNet gate block (GGML_SYCL_DN_GATES) bf16 alpha + beta mat-vecs (2 x 48 rows, same input, up to 8 tokens) with the dt-bias add, softplus, ssm_a multiply and sigmoid in one kernel: 9 graph nodes / 7 launches per layer -> 1 (2 when an output aliases the input and goes through a scratch buffer). GGML_SYCL_DN_GATES_DEBUG=1 prints why a block is not fused. B580 Bonsai 27B plain decode +0.5-1.4%, KLD unchanged. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 38c6be55fdcf3dba40b70b64cdc37478baf32f2f on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/ggml-sycl.cpp | 140 +++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 5020cd177399..da9cca40fedb 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -6822,6 +6822,143 @@ static bool ggml_sycl_try_conv_chain(ggml_backend_sycl_context & ctx, ggml_cgrap return true; } +// ARC-LAB: DeltaNet gate block in one kernel (GGML_SYCL_DN_GATES, default on): +// alpha = MUL_MAT(bf16 W_a, x) -> RESHAPE -> ADD(dt bias) -> SOFTPLUS -> MUL(ssm_a) -> RESHAPE +// beta = MUL_MAT(bf16 W_b, x) -> RESHAPE -> SIGMOID +// 9 graph nodes / 7 launches per layer for 2 x 48 outputs; here one work-group per row of both matrices, up to 8 tokens. +static bool ggml_sycl_try_dn_gates(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int & i) { + if (!ggml_sycl_knob("GGML_SYCL_DN_GATES", 1) || i + 8 >= cgraph->n_nodes) { + return false; + } + ggml_tensor * const * n = cgraph->nodes + i; + static int dbg = getenv("GGML_SYCL_DN_GATES_DEBUG") ? 6 : 0; + const bool bf16_mm = n[0]->src[0] && n[0]->src[0]->type == GGML_TYPE_BF16; + auto why = [&](const char * r) { + if (dbg > 0 && bf16_mm) { + --dbg; + fprintf(stderr, "dn gates: not fused at node %d (%s, src1 ne %lld %lld %lld): %s; next ops:", i, n[0]->name, + (long long) n[0]->src[1]->ne[0], (long long) n[0]->src[1]->ne[1], (long long) n[0]->src[1]->ne[2], r); + for (int k = 1; k < 9 && i + k < cgraph->n_nodes; ++k) fprintf(stderr, " %s", ggml_op_desc(n[k])); + fprintf(stderr, "\n"); + } + return false; + }; + const ggml_op ops[9] = { GGML_OP_MUL_MAT, GGML_OP_RESHAPE, GGML_OP_ADD, GGML_OP_UNARY, GGML_OP_MUL, + GGML_OP_RESHAPE, GGML_OP_MUL_MAT, GGML_OP_RESHAPE, GGML_OP_UNARY }; + for (int k = 0; k < 9; ++k) { + if (n[k]->op != ops[k]) { + return why("op sequence"); + } + } + if (ggml_get_unary_op(n[3]) != GGML_UNARY_OP_SOFTPLUS || ggml_get_unary_op(n[8]) != GGML_UNARY_OP_SIGMOID || + n[1]->src[0] != n[0] || n[2]->src[0] != n[1] || n[3]->src[0] != n[2] || n[4]->src[0] != n[3] || + n[5]->src[0] != n[4] || n[7]->src[0] != n[6] || n[8]->src[0] != n[7] || n[6]->src[1] != n[0]->src[1]) { + return why("edges"); + } + const ggml_tensor * wa = n[0]->src[0]; + const ggml_tensor * wb = n[6]->src[0]; + const ggml_tensor * x = n[0]->src[1]; + const ggml_tensor * bias = n[2]->src[1]; + const ggml_tensor * sa = n[4]->src[1]; + const int64_t K = wa->ne[0], R = wa->ne[1], M = x->ne[1] * x->ne[2]; + auto f32c = [](const ggml_tensor * t) { return t->type == GGML_TYPE_F32 && ggml_is_contiguous(t); }; + if (wa->type != GGML_TYPE_BF16 || wb->type != GGML_TYPE_BF16 || !ggml_is_contiguous(wa) || !ggml_is_contiguous(wb) || + wb->ne[0] != K || wb->ne[1] != R || !f32c(x) || x->ne[0] != K || x->ne[3] != 1 || M < 1 || M > 8 || K % 8 != 0 || + !f32c(bias) || ggml_nelements(bias) != R || !f32c(sa) || ggml_nelements(sa) != R || !f32c(n[4]) || !f32c(n[8]) || + ggml_nelements(n[4]) != R * M || ggml_nelements(n[8]) != R * M || !f32c(n[0]) || !f32c(n[6])) { + return why("shapes/types"); + } + { + int idx[9]; + for (int k = 0; k < 9; ++k) idx[k] = i + k; + const int outs[3] = { i + 4, i + 5, i + 8 }; + if (!ggml_can_fuse_subgraph_ext(cgraph, idx, 9, ops, outs, 3)) { + return why("outside readers"); + } + } + auto overlap = [](const ggml_tensor * a, const ggml_tensor * b) { + const char * pa = (const char *) a->data, * pb = (const char *) b->data; + return pa < pb + ggml_nbytes(b) && pb < pa + ggml_nbytes(a); + }; + // every work-group reads all of x: an output on top of an input (the planner reuses x's buffer once its last + // reader is done) goes through a scratch buffer and one small copy kernel + bool staged = false; + for (const ggml_tensor * o : { n[4], n[8] }) { + for (const ggml_tensor * in : { x, wa, wb, bias, sa }) { + staged |= overlap(o, in); + } + } + ggml_sycl_pool_alloc stage(ctx.pool()); + float * sa_out = nullptr; + if (staged) { + sa_out = stage.alloc((size_t) 2 * R * M); + } + const uint16_t * pa = (const uint16_t *) wa->data; + const uint16_t * pb = (const uint16_t *) wb->data; + const float * px = (const float *) x->data; + const float * pbias = (const float *) bias->data; + const float * psa = (const float *) sa->data; + float * oa = staged ? sa_out : (float *) n[4]->data; + float * ob = staged ? sa_out + R * M : (float *) n[8]->data; + const int Ki = (int) K, Ri = (int) R, Mi = (int) M; + constexpr int WG = 256; + ctx.stream()->parallel_for(sycl::nd_range<1>((size_t) 2 * R * WG, WG), [=](sycl::nd_item<1> it) { + const int g = it.get_group(0); + const bool isb = g >= Ri; + const int row = isb ? g - Ri : g; + const uint16_t * w = (isb ? pb : pa) + (size_t) row * Ki; + const int tid = it.get_local_id(0); + float acc[8] = { 0, 0, 0, 0, 0, 0, 0, 0 }; + for (int k = tid * 8; k < Ki; k += WG * 8) { + const sycl::vec wv = *(const sycl::vec *) (w + k); + float wf[8]; +#pragma unroll + for (int j = 0; j < 8; ++j) wf[j] = sycl::bit_cast((uint32_t) wv[j] << 16); + for (int m = 0; m < Mi; ++m) { + const float * xr = px + (size_t) m * Ki + k; +#pragma unroll + for (int j = 0; j < 8; ++j) acc[m] += wf[j] * xr[j]; + } + } + for (int m = 0; m < Mi; ++m) { + const float sum = sycl::reduce_over_group(it.get_group(), acc[m], sycl::plus()); + if (tid == 0) { + if (isb) { + ob[m * Ri + row] = 1.0f / (1.0f + sycl::exp(-sum)); + } else { + const float v = sum + pbias[row]; + const float sp = sycl::fmax(v, 0.0f) + sycl::log1p(sycl::exp(-sycl::fabs(v))); + oa[m * Ri + row] = sp * psa[row]; + } + } + } + }); + if (staged) { + float * da = (float *) n[4]->data, * db = (float *) n[8]->data; + const float * so = sa_out; + const size_t RM = (size_t) R * M; + ctx.stream()->parallel_for(sycl::range<1>(2 * RM), [=](sycl::id<1> j) { + if (j < RM) { + da[j] = so[j]; + } else { + db[j - RM] = so[j]; + } + }); + } + static bool logged = false, logged_staged = false; + if (staged && !logged_staged) { + logged_staged = true; + fprintf(stderr, "%s: fused DeltaNet gates active via scratch (%lld tokens)\n", __func__, (long long) M); + } + if (!logged) { + logged = true; + fprintf(stderr, "%s: fused DeltaNet gates active (K=%lld, 2 x %lld rows, %lld tokens)\n", __func__, (long long) K, + (long long) R, (long long) M); + } + i += 8; + return true; +} + static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * sycl_ctx, ggml_cgraph * cgraph) { ggml_sycl_set_main_device(sycl_ctx->device); ggml_sycl_gdn_gather_reset(*sycl_ctx); @@ -6882,6 +7019,9 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc continue; } } + if (node->op == GGML_OP_MUL_MAT && ggml_sycl_try_dn_gates(*sycl_ctx, cgraph, i)) { + continue; + } if (node->op == GGML_OP_CONCAT && ggml_sycl_try_conv_chain(*sycl_ctx, cgraph, i, done)) { continue; } From b7a7af12ad72efc0e198067da89a8a636bc09d53 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:05:30 +1100 Subject: [PATCH 28/65] sycl: same-input T2 mat-vecs in one launch (GGML_SYCL_T2_MULTI) q+k+v, GDN qkv+z and FFN gate+up (<= 8 tokens) share one quantization and one GEMV launch: TernSYCL's Gemv body now takes its work-group indices (run()), and a wrapper maps each group to its matrix through a single call site (three inlined copies spilled: 34.6 vs 46.3 t/s). Grouping and output staging as the PTQ1_0 merge. B580 Bonsai 27B plain decode +1.8-2.0%, KLD unchanged. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 31ae86e09811aa6a64aba7f6d56eb41e6a2822d9 on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/ggml-sycl.cpp | 100 ++++++++++++++++++ ggml/src/ggml-sycl/ptq1-t2.cpp | 87 +++++++++++++++ ggml/src/ggml-sycl/ptq1-t2.hpp | 3 + .../src/ggml-sycl/ternsycl/int2_int8_dpas.hpp | 10 +- 4 files changed, 197 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index da9cca40fedb..3c6db5c0edf7 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -6546,6 +6546,103 @@ static bool ggml_sycl_try_ptq1_multi(ggml_backend_sycl_context & ctx, ggml_cgrap return true; } +// ARC-LAB: same-input T2 mat-vecs (q+k+v, GDN qkv+z, FFN gate+up; <= 8 tokens) in one launch (GGML_SYCL_T2_MULTI, +// default on). Grouping and safety as ggml_sycl_try_ptq1_multi: a later member computed early must not overlap any node +// in between, else its result is staged in a slot and copied when the loop reaches it. +static bool ggml_sycl_t2_multi_eligible(const ggml_tensor * mm) { + if (mm->op != GGML_OP_MUL_MAT || (mm->flags & GGML_TENSOR_FLAG_COMPUTE) == 0 || !ggml_sycl_t2_tensor(mm->src[0])) { + return false; + } + const auto * extra = static_cast(mm->src[0]->extra); + const ggml_tensor * act = mm->src[1]; + const int64_t M = act->ne[1] * act->ne[2]; + return extra && extra->optimized_feature.t2 && act->type == GGML_TYPE_F32 && mm->type == GGML_TYPE_F32 && + act->nb[0] == sizeof(float) && act->ne[3] == 1 && (act->ne[2] == 1 || act->nb[2] == act->ne[1] * act->nb[1]) && + M >= 1 && M <= 8 && ggml_is_contiguous(mm) && mm->ne[0] == mm->src[0]->ne[1]; +} + +static bool ggml_sycl_try_t2_multi(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int i, std::vector & done, + std::vector & pending) { + static int log = getenv("GGML_SYCL_T2_MULTI_LOG") ? atoi(getenv("GGML_SYCL_T2_MULTI_LOG")) : 0; + ggml_tensor * mm0 = cgraph->nodes[i]; + if (!ggml_sycl_knob("GGML_SYCL_T2_MULTI", 1) || !ggml_sycl_t2_multi_eligible(mm0)) { + return false; + } + const ggml_tensor * act = mm0->src[1]; + int members[3] = { i, -1, -1 }; + bool staged[3] = { false, false, false }; + int nm = 1; + const int jend = std::min(cgraph->n_nodes, i + 512); + for (int j = i + 1; j < jend && nm < 3; ++j) { + ggml_tensor * mj = cgraph->nodes[j]; + if (done[j] || mj->op != GGML_OP_MUL_MAT || mj->src[1] != act || mj->src[0]->ne[0] != mm0->src[0]->ne[0] || + !ggml_sycl_t2_multi_eligible(mj)) { + continue; + } + bool safe = true; + for (int k = i + 1; k < j && safe; ++k) { + const ggml_tensor * nk = cgraph->nodes[k]; + bool member = false; + for (int m = 1; m < nm; ++m) { + member = member || members[m] == k; + } + if (member) { + continue; + } + safe = !ggml_sycl_tensors_overlap(mj, nk); + for (int s = 0; s < GGML_MAX_SRC && safe; ++s) { + safe = !ggml_sycl_tensors_overlap(mj, nk->src[s]); + } + } + for (int m = 0; m < nm && safe; ++m) { + safe = !ggml_sycl_tensors_overlap(mj, cgraph->nodes[members[m]]) && !ggml_sycl_tensors_overlap(mj, act); + } + if (safe || ggml_nbytes(mj) <= PTQ1_MULTI_SLOT_BYTES) { + staged[nm] = !safe; + members[nm++] = j; + } + } + if (nm < 2) { + return false; + } + const queue_ptr stream = ctx.stream(); + const void * w[3]; + float * dst[3]; + int64_t N[3]; + for (int m = 0; m < nm; ++m) { + ggml_tensor * t = cgraph->nodes[members[m]]; + w[m] = t->src[0]->data; + dst[m] = (float *) t->data; + N[m] = t->src[0]->ne[1]; + if (staged[m]) { + dst[m] = ggml_sycl_ptq1_multi_slot(stream); + if (!dst[m]) { + return false; + } + } + } + const void * key = act->view_src && act->view_offs == 0 ? act->view_src : act; + if (!ggml_sycl_t2_mul_mat_multi(*stream, nm, w, dst, N, (const float *) act->data, act->nb[1] / sizeof(float), + act->ne[1] * act->ne[2], act->ne[0], key)) { + return false; + } + if (log > 0) { + fprintf(stderr, "[t2-multi] %s", mm0->name); + for (int m = 1; m < nm; ++m) { + fprintf(stderr, " + %s (+%d%s)", cgraph->nodes[members[m]]->name, members[m] - i, staged[m] ? ", staged" : ""); + } + fprintf(stderr, "\n"); + --log; + } + for (int m = 1; m < nm; ++m) { + done[members[m]] = 1; + if (staged[m]) { + pending[members[m]] = dst[m]; + } + } + return true; +} + // ARC-LAB: see the call in ggml_backend_sycl_graph_compute_impl. Advances i past the nodes it computed. static bool ggml_sycl_try_add_norm_rot(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int & i) { const int n0 = i; @@ -7076,6 +7173,9 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc i += 2; continue; } + if (node->op == GGML_OP_MUL_MAT && ggml_sycl_try_t2_multi(*sycl_ctx, cgraph, i, done, pending)) { + continue; + } if (node->op == GGML_OP_MUL_MAT && ggml_sycl_try_ptq1_multi(*sycl_ctx, cgraph, i, done, pending)) { continue; } diff --git a/ggml/src/ggml-sycl/ptq1-t2.cpp b/ggml/src/ggml-sycl/ptq1-t2.cpp index 21981bfad382..54237074f70a 100644 --- a/ggml/src/ggml-sycl/ptq1-t2.cpp +++ b/ggml/src/ggml-sycl/ptq1-t2.cpp @@ -273,6 +273,45 @@ void gemm(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const uint32_ } // namespace +namespace { +// up to 3 weight matrices with the same activations in ONE launch: work-groups [gs[m], gs[m+1]) along dim 1 run +// matrix m (Gemv::run takes the group indices). All members use the same Gemv instantiation (same SLM, same shape). +template +struct GemvMulti { + using Kern = int8dpas::Gemv; + Kern k0, k1, k2; + int gs1, gs2; + void operator()(sycl::nd_item<2> it) const { + // one call site: three inlined copies of the body spilled (34.6 vs 46.3 t/s) + const int g0 = (int) it.get_group(0), g1 = (int) it.get_group(1); + const bool a = g1 < gs1, b = !a && g1 < gs2; + const Kern k = a ? k0 : (b ? k1 : k2); + k.run(it, g0, g1 - (a ? 0 : (b ? gs1 : gs2))); + } + auto get(int8dpas::syclex::properties_tag) const { return k0.get(int8dpas::syclex::properties_tag{}); } +}; + +template +void gemv_multi(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, int nm, const uint32_t * const * B, + const uint16_t * const * SB, float * const * C, const int * N, int M, int K) { + using GM = GemvMulti; + using Kern = typename GM::Kern; + const int wgn = 16 * NSG; + const Epi epi{ nullptr, nullptr, 0, 1 }; + int gs[4] = { 0, 0, 0, 0 }; + Kern k[3]; + for (int m = 0; m < 3; ++m) { + const int mm = m < nm ? m : 0; + k[m] = Kern{ nullptr, (const signed char *) Aq, SA, B[mm], SB[mm], C[mm], epi, M, N[mm], K }; + gs[m + 1] = gs[m] + (m < nm ? (N[m] + wgn - 1) / wgn : 0); + } + const GM f{ k[0], k[1], k[2], gs[1], nm > 2 ? gs[2] : gs[3] + 1 }; + const sycl::range<2> local(1, Kern::WG); + const sycl::range<2> global((M + SGM - 1) / SGM, (size_t) gs[nm] * Kern::WG); + q.parallel_for(sycl::nd_range<2>(global, local), f); +} +} // namespace + namespace { void t2_dispatch(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const uint32_t * B, const uint16_t * SB, float * dst, int m, int n, int k, bool use_gemm) { @@ -516,6 +555,54 @@ void ggml_sycl_t2_mul_mat(sycl::queue & q, const void * w, const float * x, int6 t2_dispatch(q, Aq, SA, B, SB, dst, (int) M, (int) N, (int) K, use_gemm); } +// several weights [K, N_m] on the same activations x [M, K] (M <= 8): one quantization (or the cached one) and one launch +bool ggml_sycl_t2_mul_mat_multi(sycl::queue & q, int nm, const void * const * w, float * const * dst, const int64_t * N, + const float * x, int64_t x_stride, int64_t M, int64_t K, const void * x_key) { + if (nm < 2 || nm > 3 || M < 1 || M > 8 || !ggml_sycl_knob("GGML_SYCL_T2_MULTI", 1)) { + return false; + } + int8_t * Aq; + uint16_t * SA; + scratch_aq(q, M, K, Aq, SA); + const quant_key want{ x_key, x, x_stride, M, K, g_graph.load() }; + if (!x_key || !ggml_sycl_knob("GGML_SYCL_PTQ1_T2_QCACHE", 1) || !(g_quant_have == want)) { + quant_a(q, x, x_stride, M, K, Aq, SA, QK); + g_quant_have = x_key ? want : quant_key{}; + } + const uint32_t * B[3]; + const uint16_t * SB[3]; + float * C[3]; + int Ni[3]; + int64_t nmax = 0; + for (int m = 0; m < nm; ++m) { + B[m] = (const uint32_t *) w[m]; + SB[m] = (const uint16_t *) ((const char *) w[m] + (size_t) (K / 16) * N[m] * 4); + C[m] = dst[m]; + Ni[m] = (int) N[m]; + nmax = std::max(nmax, N[m]); + } + const int m_ = (int) M, k_ = (int) K; + const bool ls4 = nmax <= 8192; + if (M == 1) { + ls4 ? gemv_multi<1, 4>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_) : gemv_multi<1, 2>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_); + } else if (t2_nsg() == 4) { + if (M == 2) { + ls4 ? gemv_multi<2, 4, 4>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_) : gemv_multi<2, 2, 4>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_); + } else if (M <= 4) { + ls4 ? gemv_multi<4, 4, 4>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_) : gemv_multi<4, 2, 4>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_); + } else { + ls4 ? gemv_multi<8, 4, 4>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_) : gemv_multi<8, 2, 4>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_); + } + } else if (M == 2) { + ls4 ? gemv_multi<2, 4>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_) : gemv_multi<2, 2>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_); + } else if (M <= 4) { + ls4 ? gemv_multi<4, 4>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_) : gemv_multi<4, 2>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_); + } else { + ls4 ? gemv_multi<8, 4>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_) : gemv_multi<8, 2>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_); + } + return true; +} + namespace { // run every kernel once on a tiny problem while the model loads (the first weight repack happens in the warm-up // decode): the JIT of GemmMT alone took ~35 s at the first prompt otherwise diff --git a/ggml/src/ggml-sycl/ptq1-t2.hpp b/ggml/src/ggml-sycl/ptq1-t2.hpp index 63b250b2228e..9670c80db07d 100644 --- a/ggml/src/ggml-sycl/ptq1-t2.hpp +++ b/ggml/src/ggml-sycl/ptq1-t2.hpp @@ -21,6 +21,9 @@ bool ggml_sycl_t2_repack(sycl::queue & q, void * data, int64_t K, int64_t N, b // reuse the int8 activations instead of quantizing again (q/k/v, gate/up, the DeltaNet input projections). void ggml_sycl_t2_mul_mat(sycl::queue & q, const void * w, const float * x, int64_t x_stride, float * dst, int64_t M, int64_t N, int64_t K, const void * x_key = nullptr); +// nm (2..3) weights on the same activations (M <= 8 tokens) in one launch; false = not served (caller runs them singly) +bool ggml_sycl_t2_mul_mat_multi(sycl::queue & q, int nm, const void * const * w, float * const * dst, const int64_t * N, + const float * x, int64_t x_stride, int64_t M, int64_t K, const void * x_key); // a new graph compute starts: activations quantized for the previous one are stale void ggml_sycl_t2_new_graph(); // dst [M, K] = 1024-point Hadamard of (x * signs) per block / 32 (signs: K floats), quantized for the T2 mat-vecs at once; diff --git a/ggml/src/ggml-sycl/ternsycl/int2_int8_dpas.hpp b/ggml/src/ggml-sycl/ternsycl/int2_int8_dpas.hpp index 8e2ec35dd65c..50bdd2521ded 100644 --- a/ggml/src/ggml-sycl/ternsycl/int2_int8_dpas.hpp +++ b/ggml/src/ggml-sycl/ternsycl/int2_int8_dpas.hpp @@ -205,13 +205,17 @@ struct Gemv { return acc; } - void operator()(sycl::nd_item<2> it) const { + // ARC-LAB (llama.cpp vendored copy): body takes the work-group indices as arguments so one launch can serve several + // weight matrices that share the activations (llama.cpp merges q/k/v, qkv/z and gate/up this way) + void operator()(sycl::nd_item<2> it) const { run(it, (int)it.get_group(0), (int)it.get_group(1)); } + + void run(sycl::nd_item<2> it, int g0, int g1) const { const auto sgp = it.get_sub_group(); const int lane = sgp.get_local_linear_id(); const int sg = sgp.get_group_linear_id(); const int sgn = sg % NSG_N, sgk = sg / NSG_N; - const int n0 = ((int)it.get_group(1) * NSG_N + sgn) * 16; - const int m0 = (int)it.get_group(0) * SGM; + const int n0 = (g1 * NSG_N + sgn) * 16; + const int m0 = g0 * SGM; const int nsteps = K / GS; const int per = (nsteps + LS - 1) / LS; From 6fe64405797c4064999c7910ffaf5a1a8b05d11a Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Mon, 5 Oct 2026 14:51:18 +1100 Subject: [PATCH 29/65] sycl: fold FFN SwiGLU and the DeltaNet pre-norm into the T2 rotation SWIGLU -> MUL(signs) -> RESHAPE -> Hadamard MUL_MAT: the rotate/quantize kernel computes silu(gate) * up on its load (GGML_SYCL_GLU_ROT). ADD + RMS_NORM + MUL whose rotation comes ~9 nodes later (DeltaNet layers): the rotation is computed with the norm when no node in between touches its output (GGML_SYCL_NORM_ROT_FAR). Same arithmetic, byte-identical outputs. B580 Bonsai 27B plain +0.3-0.8%, MTP rename +2%, edit +1.1%. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit cf4b155cac66470143681dad35158ce96a0a7f9f on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/ggml-sycl.cpp | 113 +++++++++++++++++++++++++++---- ggml/src/ggml-sycl/ptq1-t2.cpp | 22 ++++-- ggml/src/ggml-sycl/ptq1-t2.hpp | 3 +- 3 files changed, 120 insertions(+), 18 deletions(-) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 3c6db5c0edf7..783672f1c4da 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -6644,7 +6644,8 @@ static bool ggml_sycl_try_t2_multi(ggml_backend_sycl_context & ctx, ggml_cgraph } // ARC-LAB: see the call in ggml_backend_sycl_graph_compute_impl. Advances i past the nodes it computed. -static bool ggml_sycl_try_add_norm_rot(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int & i) { +static bool ggml_sycl_try_add_norm_rot(ggml_backend_sycl_context & ctx, ggml_cgraph * cgraph, int & i, + std::vector & done) { const int n0 = i; ggml_tensor * add = cgraph->nodes[n0]->op == GGML_OP_ADD ? cgraph->nodes[n0] : nullptr; const int in = add ? n0 + 1 : n0; // RMS_NORM @@ -6685,27 +6686,65 @@ static bool ggml_sycl_try_add_norm_rot(ggml_backend_sycl_context & ctx, ggml_cgr signs = (const float *) sg->data; } } + // DeltaNet layers: the rotation comes a few nodes later (conv-cache ops in between). Fold it in and write its output + // here, early, if no node in between touches that memory (GGML_SYCL_NORM_ROT_FAR, default on). + int ms_far = -1; + if (!mm && add && ggml_sycl_knob("GGML_SYCL_NORM_ROT_FAR", 1)) { + for (int j = in + 2; j + 2 < cgraph->n_nodes && j <= in + 32; ++j) { + ggml_tensor * ms = cgraph->nodes[j]; + if (ms->op != GGML_OP_MUL || ms->src[0] != mulw) { + continue; + } + ggml_tensor * rs = cgraph->nodes[j + 1]; + ggml_tensor * mt = cgraph->nodes[j + 2]; + const ggml_tensor * sg = ms->src[1]; + if (!done[j] && !done[j + 1] && !done[j + 2] && rs->op == GGML_OP_RESHAPE && rs->src[0] == ms && + mt->op == GGML_OP_MUL_MAT && mt->src[1] == rs && ggml_get_op_params_i32(mt, 1) == GGML_HINT_SRC0_IS_HADAMARD && + mt->src[0]->ne[0] == 1024 && f32c(mt) && f32c(ms) && sg && f32c(sg) && sg->ne[0] == K && ggml_nrows(sg) == 1) { + bool clear = true; // nodes computed between the norm and the rotation must not touch the rotation output + for (int k = in + 2; k < j && clear; ++k) { + const ggml_tensor * nk = cgraph->nodes[k]; + if (ggml_sycl_is_view_or_noop(nk) || (nk->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { + continue; + } + clear = !ggml_sycl_tensors_overlap(mt, nk); + for (int t = 0; t < GGML_MAX_SRC && clear; ++t) { + clear = !ggml_sycl_tensors_overlap(mt, nk->src[t]); + } + } + if (clear) { + ms_far = j; + mm = mt; + signs = (const float *) sg->data; + } + } + break; // only the first MUL of the norm output is a candidate + } + } if (!add && !mm) { return false; // plain RMS_NORM + MUL: ggml_sycl_op_rms_norm_fused below } - const int last = mm ? in + 4 : in + 1; + const int last = mm ? (ms_far >= 0 ? ms_far + 2 : in + 4) : in + 1; // nodes inside the chain must have no other consumers; write the ADD and the weighted norm only if used after it + std::vector idx; + for (int j = n0; j <= in + 1; ++j) idx.push_back(j); + if (mm) { + const int t0 = ms_far >= 0 ? ms_far : in + 2; + for (int j = t0; j <= t0 + 2; ++j) idx.push_back(j); + } std::vector outs; if (add) outs.push_back(n0); outs.push_back(in + 1); if (mm) outs.push_back(last); std::vector ops; - for (int j = n0; j <= last; ++j) ops.push_back(cgraph->nodes[j]->op); - { - std::vector idx; - for (int j = n0; j <= last; ++j) idx.push_back(j); - if (!ggml_can_fuse_subgraph_ext(cgraph, idx.data(), (int) idx.size(), ops.data(), outs.data(), (int) outs.size())) { - return false; - } + for (int j : idx) ops.push_back(cgraph->nodes[j]->op); + if (!ggml_can_fuse_subgraph_ext(cgraph, idx.data(), (int) idx.size(), ops.data(), outs.data(), (int) outs.size())) { + return false; } auto used_after = [&](int j) { int inside = 0; - for (int k = j + 1; k <= last; ++k) { + for (int k : idx) { + if (k <= j) continue; for (int s = 0; s < GGML_MAX_SRC; ++s) inside += cgraph->nodes[k]->src[s] == cgraph->nodes[j]; } return (cgraph->nodes[j]->flags & GGML_TENSOR_FLAG_OUTPUT) || ggml_node_get_use_count(cgraph, j) > inside; @@ -6735,7 +6774,17 @@ static bool ggml_sycl_try_add_norm_rot(ggml_backend_sycl_context & ctx, ggml_cgr normed, signs, mm ? (float *) mm->data : nullptr, M, K, mm)) { return false; } - i = last; + if (ms_far >= 0) { + done[ms_far] = done[ms_far + 1] = done[ms_far + 2] = 1; // computed here, skipped when the loop gets there + i = in + 1; + static bool logged = false; + if (!logged) { + logged = true; + fprintf(stderr, "%s: norm + rotation %d nodes apart fused\n", __func__, ms_far - (in + 1)); + } + } else { + i = last; + } return true; } @@ -7056,6 +7105,13 @@ static bool ggml_sycl_try_dn_gates(ggml_backend_sycl_context & ctx, ggml_cgraph return true; } +// ARC-LAB: a kernel whose work-group reads block r of an input and later writes block r of its output may run in place +// (same base address), never with a partial overlap +static bool ggml_sycl_same_or_disjoint(const ggml_tensor * out, const ggml_tensor * in) { + const char * po = (const char *) out->data, * pi = (const char *) in->data; + return po == pi || po + ggml_nbytes(out) <= pi || pi + ggml_nbytes(in) <= po; +} + static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * sycl_ctx, ggml_cgraph * cgraph) { ggml_sycl_set_main_device(sycl_ctx->device); ggml_sycl_gdn_gather_reset(*sycl_ctx); @@ -7126,9 +7182,42 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc // (ggml_sycl_t2_add_norm_rot): the post-attention norm of every layer and the pre-attention norm of the full // attention layers end in the rotation that feeds the T2 mat-vecs; the DeltaNet layers' norm is followed by // cache ops first, so there only the ADD folds in. - if ((node->op == GGML_OP_ADD || node->op == GGML_OP_RMS_NORM) && ggml_sycl_try_add_norm_rot(*sycl_ctx, cgraph, i)) { + if ((node->op == GGML_OP_ADD || node->op == GGML_OP_RMS_NORM) && ggml_sycl_try_add_norm_rot(*sycl_ctx, cgraph, i, done)) { continue; } + // ARC-LAB: FFN SWIGLU -> MUL(signs) -> RESHAPE -> FWHT-hint MUL_MAT: the rotation computes silu(gate) * up on its + // load (GGML_SYCL_GLU_ROT, default on) + if (node->op == GGML_OP_GLU && i + 3 < cgraph->n_nodes && ggml_sycl_knob("GGML_SYCL_GLU_ROT", 1)) { + ggml_tensor * ms = cgraph->nodes[i + 1]; + ggml_tensor * rs = cgraph->nodes[i + 2]; + ggml_tensor * mt = cgraph->nodes[i + 3]; + const ggml_tensor * g = node->src[0]; + const ggml_tensor * u = node->src[1]; + const ggml_tensor * sg = ms->src[1]; + const ggml_op gops[4] = { GGML_OP_GLU, GGML_OP_MUL, GGML_OP_RESHAPE, GGML_OP_MUL_MAT }; + const int gidx[4] = { i, i + 1, i + 2, i + 3 }; + const int gout[1] = { i + 3 }; + if (ggml_get_glu_op(node) == GGML_GLU_OP_SWIGLU && u && ggml_get_op_params_i32(node, 1) == 0 && + ms->op == GGML_OP_MUL && ms->src[0] == node && rs->op == GGML_OP_RESHAPE && rs->src[0] == ms && + mt->op == GGML_OP_MUL_MAT && mt->src[1] == rs && + ggml_get_op_params_i32(mt, 1) == GGML_HINT_SRC0_IS_HADAMARD && mt->src[0]->ne[0] == 1024 && + g->type == GGML_TYPE_F32 && u->type == GGML_TYPE_F32 && node->type == GGML_TYPE_F32 && + ggml_is_contiguous(g) && ggml_is_contiguous(u) && ggml_are_same_shape(g, u) && + ggml_are_same_shape(g, node) && g->ne[3] == 1 && sg && sg->type == GGML_TYPE_F32 && + ggml_is_contiguous(sg) && sg->ne[0] == g->ne[0] && ggml_nrows(sg) == 1 && ggml_is_contiguous(mt) && + ggml_can_fuse_subgraph_ext(cgraph, gidx, 4, gops, gout, 1) && + ggml_sycl_same_or_disjoint(mt, g) && ggml_sycl_same_or_disjoint(mt, u) && + ggml_sycl_t2_fwht_quant(*sycl_ctx->stream(), nullptr, (const float *) sg->data, (float *) mt->data, + ggml_nrows(g), g->ne[0], mt, (const float *) g->data, (const float *) u->data)) { + static bool glu_logged = false; + if (!glu_logged) { + glu_logged = true; + fprintf(stderr, "ggml_sycl: SWIGLU folded into the rotation (%s)\n", node->name); + } + i += 3; + continue; + } + } // Hadamard sign flip + reshape + FWHT-hint matmul: apply the signs in the transform's load if (node->op == GGML_OP_MUL && ggml_can_fuse_subgraph(cgraph, i, { GGML_OP_MUL, GGML_OP_RESHAPE, GGML_OP_MUL_MAT }, { i + 2 })) { diff --git a/ggml/src/ggml-sycl/ptq1-t2.cpp b/ggml/src/ggml-sycl/ptq1-t2.cpp index 54237074f70a..c183cb0db57d 100644 --- a/ggml/src/ggml-sycl/ptq1-t2.cpp +++ b/ggml/src/ggml-sycl/ptq1-t2.cpp @@ -436,7 +436,7 @@ inline void rot_quant_block(float (&v)[8], float * s, const Item & it, const syc // that also quantizes its output for the T2 mat-vecs which read it next: one 128-lane work-group per 1024 block where // fwht.cpp gives a block one 16-lane sub-group (5 sub-groups in all for a decode token), and no quant_a launch after it. bool ggml_sycl_t2_fwht_quant(sycl::queue & q, const float * x, const float * signs, float * dst, int64_t M, int64_t K, - const void * key) { + const void * key, const float * glu_g, const float * glu_u) { constexpr int N = 1024; if (K % N != 0 || M <= 0 || M > 512 || !ggml_sycl_knob("GGML_SYCL_PTQ1_T2_FWHTQ", 1)) { return false; @@ -454,13 +454,25 @@ bool ggml_sycl_t2_fwht_quant(sycl::queue & q, const float * x, const float * sig const int sgi = sg.get_group_linear_id(); const int64_t r = it.get_group(0); const int64_t m = r / nblk, kb = r % nblk; - const float * src = x + r * N; const float * sgn = signs + kb * N; float v[8]; + if (glu_g) { // SWIGLU folded in: x = silu(g) * u, as ggml_sycl_op_swiglu computes it + const float * gp = glu_g + r * N; + const float * up = glu_u + r * N; #pragma unroll - for (int j = 0; j < 8; ++j) { - const int e = lane + 16 * j + 128 * sgi; - v[j] = src[e] * (1.0f / 32.0f) * sgn[e]; + for (int j = 0; j < 8; ++j) { + const int e = lane + 16 * j + 128 * sgi; + const float g = gp[e]; + const float xv = g / (1.0f + sycl::exp(-g)) * up[e]; + v[j] = xv * (1.0f / 32.0f) * sgn[e]; + } + } else { + const float * src = x + r * N; +#pragma unroll + for (int j = 0; j < 8; ++j) { + const int e = lane + 16 * j + 128 * sgi; + v[j] = src[e] * (1.0f / 32.0f) * sgn[e]; + } } rot_quant_block(v, sl.get_multi_ptr().get(), it, sg, lane, sgi, dst + r * N, Aq + m * K + kb * N, SA, kb * (N / QK), lda, m); diff --git a/ggml/src/ggml-sycl/ptq1-t2.hpp b/ggml/src/ggml-sycl/ptq1-t2.hpp index 9670c80db07d..5ae5634e99ec 100644 --- a/ggml/src/ggml-sycl/ptq1-t2.hpp +++ b/ggml/src/ggml-sycl/ptq1-t2.hpp @@ -28,8 +28,9 @@ bool ggml_sycl_t2_mul_mat_multi(sycl::queue & q, int nm, const void * const * void ggml_sycl_t2_new_graph(); // dst [M, K] = 1024-point Hadamard of (x * signs) per block / 32 (signs: K floats), quantized for the T2 mat-vecs at once; // key = the tensor they will read it through (see x_key). false: shape not served (K % 1024, M > 512) or knob off. +// glu_g / glu_u (optional, then x is unused): x = silu(glu_g) * glu_u, both [M, K] contiguous (SWIGLU folded in) bool ggml_sycl_t2_fwht_quant(sycl::queue & q, const float * x, const float * signs, float * dst, int64_t M, int64_t K, - const void * key); + const void * key, const float * glu_g = nullptr, const float * glu_u = nullptr); // [add_out =] a (+ b); normed = rms_norm(.) * w; [mm = rotation of normed * signs, quantized as above]. b, add_out, // normed, signs (and then mm, key) may be null. One work-group per token: K % 1024 == 0, K <= 8192, M <= 512. bool ggml_sycl_t2_add_norm_rot(sycl::queue & q, const float * a, const float * b, float * add_out, const float * w, From 7a650fcae5ac8a3ddaa06f68b1f5c933da4faeb4 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Mon, 5 Oct 2026 16:09:54 +1100 Subject: [PATCH 30/65] sycl: per-batch-size GemmMT tile for T2 (small tile up to 32 rows); GEMV_MAX knob arctune sweep on the B580: tile {8,128,2,4} is 7-11% faster per pass for 12-32 rows (n-gram verify batches), the default {8,128,8,2} from 48 rows up. GGML_SYCL_PTQ1_T2_TILE now forces one tile for all sizes; GGML_SYCL_PTQ1_T2_TILE_SMALL_MAX (default 32) picks the crossover. GGML_SYCL_PTQ1_T2_GEMV_MAX (default 8) makes the GEMV/GemmMT switch a knob; the sweep confirmed 8. Outputs and KLD identical; MTP rename/edit +0.5%, 48K copy +1.3%. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit a70fe64480e3d6113d3ee463a8ed1aa8d4d503dc on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/ptq1-t2.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-sycl/ptq1-t2.cpp b/ggml/src/ggml-sycl/ptq1-t2.cpp index c183cb0db57d..18e4bc3badc5 100644 --- a/ggml/src/ggml-sycl/ptq1-t2.cpp +++ b/ggml/src/ggml-sycl/ptq1-t2.cpp @@ -257,7 +257,11 @@ void gemm_tile(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const ui // 7 {8,64,8,2} 8 {32,32,4,2} void gemm(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const uint32_t * B, const uint16_t * SB, float * C, int M, int N, int K) { - const int tile = ggml_sycl_knob("GGML_SYCL_PTQ1_T2_TILE", 0); + // per-batch-size tile (arctune sweep, B580, 2026-10-05): {8,128,2,4} is 7-11% faster for 12-32 rows (n-gram verify + // batches), {8,128,8,2} from 48 rows up (prompts). GGML_SYCL_PTQ1_T2_TILE forces one tile for every size; + // GGML_SYCL_PTQ1_T2_TILE_SMALL_MAX = largest batch on the small tile (0 = never). + const int forced = ggml_sycl_knob("GGML_SYCL_PTQ1_T2_TILE", -1); + const int tile = forced >= 0 ? forced : (M <= ggml_sycl_knob("GGML_SYCL_PTQ1_T2_TILE_SMALL_MAX", 32) ? 4 : 0); switch (tile) { case 1: gemm_tile<8, 128, 4, 2>(q, Aq, SA, B, SB, C, M, N, K); break; case 2: gemm_tile<8, 128, 4, 4>(q, Aq, SA, B, SB, C, M, N, K); break; @@ -550,7 +554,9 @@ void ggml_sycl_t2_mul_mat(sycl::queue & q, const void * w, const float * x, int6 // batches > 8 (n-gram verify, prompts): GemmMT reads each weight once per 64 rows (TernSYCL main: the 2D block I/O // that misbehaved in the JIT build with the old IGC builtins validates under inline vISA, so it is upstream's kernel). const bool gemm_off = ggml_sycl_knob("GGML_SYCL_PTQ1_T2_GEMM_OFF", 0) != 0; - const bool use_gemm = M > 8 && !gemm_off; + // GGML_SYCL_PTQ1_T2_GEMV_MAX: largest batch on the GEMV (default 8; above it GemmMT). GEMV re-reads the weights once per + // 8 rows, GemmMT once per 64 but less efficiently: `arctune sweep` measures the crossover. + const bool use_gemm = M > ggml_sycl_knob("GGML_SYCL_PTQ1_T2_GEMV_MAX", 8) && !gemm_off; int8_t * Aq; uint16_t * SA; From 3c5fdcd0dbb9fde4b4459acfb11205efe9e112b1 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Mon, 5 Oct 2026 17:20:28 +1100 Subject: [PATCH 31/65] sycl: decode attention prefetches the next phase's key rows (GGML_SYCL_FA_DEC_PF) Phase probes (GGML_SYCL_FA_DEC_DPAS_PROBE) showed Q.K and P.V serialised behind the K / V loads. Each tile now prefetches its V rows before Q.K and the next tile's K rows before P.V (sycl_ext_oneapi_prefetch, L1). Per layer at 1 token: 48K 263 -> 226 us, 115K 554 -> 496 us. Whole model: plain 48K +2.7%, 115K +4.4%; MTP 48K +4.2%, 115K +7.5%; outputs identical. FA Bonsai cases 5/5. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit dc575f4ff5ed3361ebecfc5effde32f1bb96dd52 on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/fattn-dec.cpp | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/ggml/src/ggml-sycl/fattn-dec.cpp b/ggml/src/ggml-sycl/fattn-dec.cpp index f23c712aceb6..7f02067417d3 100644 --- a/ggml/src/ggml-sycl/fattn-dec.cpp +++ b/ggml/src/ggml-sycl/fattn-dec.cpp @@ -728,6 +728,9 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, const int nqc = (ne01 + NQ - 1) / NQ; // token chunks const int probe_env = ggml_sycl_knob("GGML_SYCL_FA_DEC_DPAS_PROBE", 0); const int probe = probe_env; // phase probe: bit0 skip Q.K, bit1 skip P.V, bit2 skip softmax, bit3 Q.K loads only (no DPAS / scale math) + // ARC-LAB: GGML_SYCL_FA_DEC_PF: prefetch the next phase's key rows while this phase computes (the probes showed Q.K and + // P.V serialised behind the loads): 1 = into L1 (default), 2 = into L3, 0 = off + const int pf = ggml_sycl_knob("GGML_SYCL_FA_DEC_PF", 1); stream->submit([&](sycl::handler & cgh) { sycl::local_accessor sQ8(sycl::range<1>(RT * NB * 16), cgh); // [tile][block][lane] -> 8 rows @@ -826,11 +829,32 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, } }; + // a 144-byte q4_0 row (D 256) spans at most 3 cache lines from its start + 0 / 64 / 128 + auto prefetch_rows = [&](const char * base, int64_t nb, int t0n) { + if (pf == 0 || t0n >= k_end) { + return; + } + for (int e = tid; e < TK * 3; e += DEC_WG) { + const int j = t0n + e / 3; + if (j < k_end) { + void * pa = (void *) (base + (int64_t) j * nb + (e % 3) * 64); + if (pf == 2) { + sycl::ext::oneapi::experimental::prefetch(pa, sycl::ext::oneapi::experimental::properties{ sycl::ext::oneapi::experimental::prefetch_hint_L3 }); + } else { + sycl::ext::oneapi::experimental::prefetch(pa, sycl::ext::oneapi::experimental::properties{ sycl::ext::oneapi::experimental::prefetch_hint_L1 }); + } + } + } + }; + for (int t0 = k_begin; t0 < k_end; t0 += TK) { if constexpr (ST) { stage(Kh, nb11, t0); it.barrier(sycl::access::fence_space::local_space); } + if constexpr (D == 256) { + prefetch_rows(Vh, nb21, t0); // V of this tile is read right after Q.K + } // ---- S = Q K^T: job = (key group, chunk of TC row tiles) for (int job = w; job < ((probe & 1) ? 0 : KG * NCH); job += NSG) { const int kg = job % KG, ch = job / KG; @@ -932,6 +956,9 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, } it.barrier(sycl::access::fence_space::local_space); + if constexpr (D == 256) { + prefetch_rows(Kh, nb11, t0 + TK); // next tile's K, read after this P.V + } // ---- O^T = diag(alpha) O^T + V^T P^T: A = V^T (8 dims x 16 keys, lane = key), B = P^T (16 keys x 16 rows) #pragma unroll for (int t = 0; t < RT16; ++t) { From 1fe355ae041186c6a72dfc74ce078c01108279b5 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Mon, 5 Oct 2026 18:15:36 +1100 Subject: [PATCH 32/65] tests: arctune fusion tests for the SYCL fusions; fusion guard fixes test-backend-ops ARCLAB_GLU_ROT / ADD_NORM_ROT / CONV_CHAIN / DN_GATES / T2_MULTI build the exact node pattern each fusion matches, at Bonsai 27B shapes and 1-16 tokens, 3 seeds each, and compare every fused output with the CPU backend: 75/75 pass, each fusion confirmed firing (GGML_SYCL_FUSION_LOG, GGML_SYCL_T2_MULTI_LOG). Found while writing them: the far norm->rotation fusion now requires the signs to be a weight (they are read early); the conv chain no longer refuses a later reader of the cache slot; the same-input merge repacks weights on first use, so it also merges on the first graph. Whole model unchanged (plain short / 48K / 115K md5s identical). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit bfb2d7654a6df2ea01c9294c8ac8e9c19b394459 on Torchit1/llama.cpp arc-b580) Conflict resolved against prism: test-backend-ops keeps prism's fused SwiGLU/sign-flip FWHT cases and adds these. --- ggml/src/ggml-sycl/ggml-sycl.cpp | 34 +++++- tests/test-backend-ops.cpp | 204 +++++++++++++++++++++++++++++++ 2 files changed, 232 insertions(+), 6 deletions(-) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 783672f1c4da..fb178721a034 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -6549,11 +6549,18 @@ static bool ggml_sycl_try_ptq1_multi(ggml_backend_sycl_context & ctx, ggml_cgrap // ARC-LAB: same-input T2 mat-vecs (q+k+v, GDN qkv+z, FFN gate+up; <= 8 tokens) in one launch (GGML_SYCL_T2_MULTI, // default on). Grouping and safety as ggml_sycl_try_ptq1_multi: a later member computed early must not overlap any node // in between, else its result is staged in a slot and copied when the loop reaches it. -static bool ggml_sycl_t2_multi_eligible(const ggml_tensor * mm) { +static bool ggml_sycl_t2_multi_eligible(ggml_backend_sycl_context & ctx, const ggml_tensor * mm) { if (mm->op != GGML_OP_MUL_MAT || (mm->flags & GGML_TENSOR_FLAG_COMPUTE) == 0 || !ggml_sycl_t2_tensor(mm->src[0])) { return false; } - const auto * extra = static_cast(mm->src[0]->extra); + auto * extra = static_cast(mm->src[0]->extra); + if (extra && !extra->optimized_feature.t2 && mm->src[0]->ne[2] == 1) { // first use: repack here, as the unmerged path + const ggml_tensor * w = mm->src[0]; + if (!ggml_sycl_t2_repack(*ctx.stream(), w->data, w->ne[0], w->ne[1], w->type == GGML_TYPE_PQ2_0)) { + return false; + } + extra->optimized_feature.t2 = true; + } const ggml_tensor * act = mm->src[1]; const int64_t M = act->ne[1] * act->ne[2]; return extra && extra->optimized_feature.t2 && act->type == GGML_TYPE_F32 && mm->type == GGML_TYPE_F32 && @@ -6565,7 +6572,7 @@ static bool ggml_sycl_try_t2_multi(ggml_backend_sycl_context & ctx, ggml_cgraph std::vector & pending) { static int log = getenv("GGML_SYCL_T2_MULTI_LOG") ? atoi(getenv("GGML_SYCL_T2_MULTI_LOG")) : 0; ggml_tensor * mm0 = cgraph->nodes[i]; - if (!ggml_sycl_knob("GGML_SYCL_T2_MULTI", 1) || !ggml_sycl_t2_multi_eligible(mm0)) { + if (!ggml_sycl_knob("GGML_SYCL_T2_MULTI", 1) || !ggml_sycl_t2_multi_eligible(ctx, mm0)) { return false; } const ggml_tensor * act = mm0->src[1]; @@ -6576,7 +6583,7 @@ static bool ggml_sycl_try_t2_multi(ggml_backend_sycl_context & ctx, ggml_cgraph for (int j = i + 1; j < jend && nm < 3; ++j) { ggml_tensor * mj = cgraph->nodes[j]; if (done[j] || mj->op != GGML_OP_MUL_MAT || mj->src[1] != act || mj->src[0]->ne[0] != mm0->src[0]->ne[0] || - !ggml_sycl_t2_multi_eligible(mj)) { + !ggml_sycl_t2_multi_eligible(ctx, mj)) { continue; } bool safe = true; @@ -6700,7 +6707,8 @@ static bool ggml_sycl_try_add_norm_rot(ggml_backend_sycl_context & ctx, ggml_cgr const ggml_tensor * sg = ms->src[1]; if (!done[j] && !done[j + 1] && !done[j + 2] && rs->op == GGML_OP_RESHAPE && rs->src[0] == ms && mt->op == GGML_OP_MUL_MAT && mt->src[1] == rs && ggml_get_op_params_i32(mt, 1) == GGML_HINT_SRC0_IS_HADAMARD && - mt->src[0]->ne[0] == 1024 && f32c(mt) && f32c(ms) && sg && f32c(sg) && sg->ne[0] == K && ggml_nrows(sg) == 1) { + mt->src[0]->ne[0] == 1024 && f32c(mt) && f32c(ms) && sg && f32c(sg) && sg->ne[0] == K && ggml_nrows(sg) == 1 && + sg->op == GGML_OP_NONE) { // the signs are read early: they must be a weight, not computed in between bool clear = true; // nodes computed between the norm and the rotation must not touch the rotation output for (int k = in + 2; k < j && clear; ++k) { const ggml_tensor * nk = cgraph->nodes[k]; @@ -6774,6 +6782,11 @@ static bool ggml_sycl_try_add_norm_rot(ggml_backend_sycl_context & ctx, ggml_cgr normed, signs, mm ? (float *) mm->data : nullptr, M, K, mm)) { return false; } + static bool fired_logged = false; + if (!fired_logged && getenv("GGML_SYCL_FUSION_LOG")) { + fired_logged = true; + fprintf(stderr, "ggml_sycl_try_add_norm_rot: fused (%s%s%s)\n", add ? "add + " : "", "norm", mm ? " + rotation" : ""); + } if (ms_far >= 0) { done[ms_far] = done[ms_far + 1] = done[ms_far + 2] = 1; // computed here, skipped when the loop gets there i = in + 1; @@ -6906,15 +6919,24 @@ static bool ggml_sycl_try_conv_chain(ggml_backend_sycl_context & ctx, ggml_cgrap if (ggml_sycl_is_view_or_noop(n) || (n->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { continue; } + const int prod[3] = { isilu, il2, ic }; // where each output is produced in the original order for (int o = 0; o < 3; ++o) { if (o == 1 && !l2_in) { continue; } + if (j > prod[o] && o == 2) { + continue; // the cache slot is final once the original CPY would have run + } if (overlap(outs[o], outn[o], n->data, ggml_nbytes(n))) { return why("node in between writes output memory"); } + const ggml_tensor * own[3] = { silu, l2, cpy }; for (int k = 0; k < GGML_MAX_SRC && n->src[k]; ++k) { - if (overlap(outs[o], outn[o], n->src[k]->data, ggml_nbytes(n->src[k]))) { + const ggml_tensor * sk = n->src[k]; + if (j > prod[o] && (sk == own[o] || sk->view_src == own[o])) { + continue; // a consumer of this output, after its producer: reads the right value + } + if (overlap(outs[o], outn[o], sk->data, ggml_nbytes(sk))) { return why("node in between reads output memory"); } } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index c2be2ff51ada..e41b9224094c 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -4848,6 +4848,195 @@ struct test_fwht_signed : public test_case { } }; +// ARC-LAB fusion tests (arctune, 2026-10-05): each builds the exact node pattern one SYCL fusion matches, at several +// shapes / token counts, and compares the fused outputs against the CPU backend. The graph's final node only keeps every +// output alive; its operand order makes the graph come out in the model's node order. `seed` repeats a case with fresh +// random data. Run with GGML_SYCL_PTQ1_T2=all so the ternary weights take the T2 (XMX) path the fusions feed. +static void arclab_init_hadamard(ggml_tensor * t) { + const int64_t n = t->ne[0]; + std::vector data(n * n); + const float scale = 1.0f / sqrtf((float) n); + for (int64_t r = 0; r < n; r++) { + for (int64_t i = 0; i < n; i++) { + data[r * n + i] = (__builtin_popcountll((unsigned long long) (r & i)) % 2 == 0) ? scale : -scale; + } + } + ggml_backend_tensor_set(t, data.data(), 0, data.size() * sizeof(float)); +} +static void arclab_init_signs(ggml_tensor * t) { + std::vector data(ggml_nelements(t)); + for (size_t i = 0; i < data.size(); i++) { + data[i] = ((i * 2654435761u) >> 7) & 1 ? -1.0f : 1.0f; + } + ggml_backend_tensor_set(t, data.data(), 0, data.size() * sizeof(float)); +} +static void arclab_init(ggml_context * ctx) { + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != NULL; t = ggml_get_next_tensor(ctx, t)) { + if (t->op != GGML_OP_NONE) { + continue; + } + if (strcmp(t->name, "hadamard") == 0) { + arclab_init_hadamard(t); + } else if (strncmp(t->name, "signs", 5) == 0) { + arclab_init_signs(t); + } else { + init_tensor_uniform(t); + } + } +} +// rotated input -> T2 mat-vec, as llama_mul_mat_hadamard + build_lora_mm build it +static ggml_tensor * arclab_rot_matmul(ggml_context * ctx, ggml_tensor * cur, ggml_tensor * signs, ggml_tensor * had, + ggml_tensor * w, ggml_tensor ** mm_out) { + const int64_t K = cur->ne[0], M = cur->ne[1]; + ggml_tensor * ms = ggml_mul(ctx, cur, signs); + ggml_tensor * rs = ggml_reshape_2d(ctx, ms, had->ne[0], ggml_nelements(ms) / had->ne[0]); + ggml_tensor * mm = ggml_mul_mat(ctx, had, rs); + ggml_mul_mat_set_hint(mm, GGML_HINT_SRC0_IS_HADAMARD); + ggml_set_name(mm, "rotated"); + *mm_out = mm; + ggml_tensor * back = ggml_reshape_2d(ctx, mm, K, M); + ggml_tensor * out = ggml_mul_mat(ctx, w, back); + ggml_set_name(out, "t2_out"); + return out; +} + +// SWIGLU -> MUL(signs) -> RESHAPE -> Hadamard MUL_MAT -> T2 mat-vec (GGML_SYCL_GLU_ROT) +struct test_arclab_glu_rot : public test_case { + const int64_t K, M, N; const int seed; + std::vector outs; + test_arclab_glu_rot(int64_t K, int64_t M, int64_t N, int seed) : K(K), M(M), N(N), seed(seed) {} + std::string op_desc(ggml_tensor *) override { return "ARCLAB_GLU_ROT"; } + std::string vars() override { return VARS_TO_STR4(K, M, N, seed); } + bool run_whole_graph() override { return true; } + std::vector fusion_test_nodes() override { return outs; } + double max_nmse_err() override { return 5e-4; } + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * g = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, K, M); ggml_set_name(g, "gate"); + ggml_tensor * u = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, K, M); ggml_set_name(u, "up"); + ggml_tensor * sg = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, K); ggml_set_name(sg, "signs"); + ggml_tensor * had = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1024, 1024); ggml_set_name(had, "hadamard"); + ggml_tensor * w = ggml_new_tensor_2d(ctx, GGML_TYPE_PTQ1_0, K, N); ggml_set_name(w, "w"); + ggml_tensor * x = ggml_swiglu_split(ctx, g, u); + ggml_tensor * mm; + ggml_tensor * out = arclab_rot_matmul(ctx, x, sg, had, w, &mm); + outs = { mm, out }; + return out; + } + void initialize_tensors(ggml_context * ctx) override { arclab_init(ctx); } +}; + +// ADD -> RMS_NORM -> MUL(w) -> MUL(signs) -> RESHAPE -> Hadamard MUL_MAT -> T2 mat-vec (GGML_SYCL_PTQ1_T2_NORMQ). The +// far variant (GGML_SYCL_NORM_ROT_FAR: cache ops between norm and rotation) cannot be built here: a single-output graph +// has no node to put in between; it is covered by the whole-model checks (byte-identical outputs). +struct test_arclab_add_norm_rot : public test_case { + const int64_t K, M, N; const bool far; const int seed; + std::vector outs; + test_arclab_add_norm_rot(int64_t K, int64_t M, int64_t N, bool far, int seed) : K(K), M(M), N(N), far(far), seed(seed) {} + std::string op_desc(ggml_tensor *) override { return "ARCLAB_ADD_NORM_ROT"; } + std::string vars() override { return VARS_TO_STR5(K, M, N, far, seed); } + bool run_whole_graph() override { return true; } + std::vector fusion_test_nodes() override { return outs; } + double max_nmse_err() override { return 5e-4; } + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, K, M); ggml_set_name(a, "a"); + ggml_tensor * b = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, K, M); ggml_set_name(b, "b"); + ggml_tensor * nw = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, K); ggml_set_name(nw, "norm_w"); + ggml_tensor * sg = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, K); ggml_set_name(sg, "signs"); + ggml_tensor * had = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 1024, 1024); ggml_set_name(had, "hadamard"); + ggml_tensor * w = ggml_new_tensor_2d(ctx, GGML_TYPE_PTQ1_0, K, N); ggml_set_name(w, "w"); + ggml_tensor * sum = ggml_add(ctx, a, b); + ggml_tensor * normed = ggml_mul(ctx, ggml_rms_norm(ctx, sum, 1e-6f), nw); + ggml_tensor * mm; + ggml_tensor * t2 = arclab_rot_matmul(ctx, normed, sg, had, w, &mm); + // keep the residual and the normed rows alive after the chain (the next layer and other mat-vecs read them) + ggml_tensor * out = ggml_add(ctx, ggml_sum(ctx, t2), ggml_add(ctx, ggml_sum(ctx, normed), ggml_sum(ctx, sum))); + outs = { sum, normed, mm, t2 }; + return out; + } + void initialize_tensors(ggml_context * ctx) override { arclab_init(ctx); } +}; + +// DeltaNet conv chain for one token: CONCAT(state, qkv^T) -> CPY(newest columns -> cache) -> SSM_CONV -> SILU -> L2_NORM +// (GGML_SYCL_CONV_CHAIN) +struct test_arclab_conv_chain : public test_case { + const int64_t C, S; const int seed; + std::vector outs; + test_arclab_conv_chain(int64_t C, int64_t S, int seed) : C(C), S(S), seed(seed) {} + std::string op_desc(ggml_tensor *) override { return "ARCLAB_CONV_CHAIN"; } + std::string vars() override { return VARS_TO_STR3(C, S, seed); } + bool run_whole_graph() override { return true; } + std::vector fusion_test_nodes() override { return outs; } + double max_nmse_err() override { return 1e-6; } + ggml_tensor * build_graph(ggml_context * ctx) override { + const int64_t dc = 4, H = 2 * C / (5 * S); // q and k heads: the first 2 C / 5 channels (Bonsai: 32 heads of 128) + ggml_tensor * cs = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, dc - 1, C); ggml_set_name(cs, "conv_state"); + ggml_tensor * x = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, C, 1); ggml_set_name(x, "qkv"); + ggml_tensor * w = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, dc, C); ggml_set_name(w, "conv_w"); + ggml_tensor * cache = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, (dc - 1) * C); ggml_set_name(cache, "cache"); + ggml_tensor * cat = ggml_concat(ctx, cs, ggml_transpose(ctx, x), 0); + ggml_tensor * last = ggml_view_2d(ctx, cat, dc - 1, C, cat->nb[1], sizeof(float)); + ggml_tensor * cpy = ggml_cpy(ctx, last, cache); + ggml_tensor * conv = ggml_ssm_conv(ctx, cat, w); // the concat itself, as in the model + ggml_tensor * silu = ggml_silu(ctx, conv); + ggml_tensor * qk = ggml_view_2d(ctx, silu, S, H, S * sizeof(float), 0); + ggml_tensor * l2 = ggml_l2_norm(ctx, qk, 1e-6f); + ggml_tensor * out = ggml_add(ctx, ggml_sum(ctx, cpy), ggml_add(ctx, ggml_sum(ctx, l2), ggml_sum(ctx, silu))); + outs = { cpy, silu, l2 }; + return out; + } + void initialize_tensors(ggml_context * ctx) override { arclab_init(ctx); } +}; + +// DeltaNet gate block: bf16 alpha / beta mat-vecs + bias add + softplus + ssm_a mul + sigmoid (GGML_SYCL_DN_GATES) +struct test_arclab_dn_gates : public test_case { + const int64_t K, R, M; const int seed; + test_arclab_dn_gates(int64_t K, int64_t R, int64_t M, int seed) : K(K), R(R), M(M), seed(seed) {} + std::string op_desc(ggml_tensor *) override { return "ARCLAB_DN_GATES"; } + std::string vars() override { return VARS_TO_STR4(K, R, M, seed); } + bool run_whole_graph() override { return true; } + double max_nmse_err() override { return 5e-4; } + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * x = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, K, M); ggml_set_name(x, "x"); + ggml_tensor * wa = ggml_new_tensor_2d(ctx, GGML_TYPE_BF16, K, R); ggml_set_name(wa, "w_alpha"); + ggml_tensor * wb = ggml_new_tensor_2d(ctx, GGML_TYPE_BF16, K, R); ggml_set_name(wb, "w_beta"); + ggml_tensor * bias = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, R); ggml_set_name(bias, "dt_bias"); + ggml_tensor * sa = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, R); ggml_set_name(sa, "ssm_a"); + ggml_tensor * alpha = ggml_reshape_2d(ctx, ggml_mul_mat(ctx, wa, x), R, M); + ggml_tensor * gate = ggml_mul(ctx, ggml_softplus(ctx, ggml_add(ctx, alpha, bias)), sa); + ggml_tensor * gate3 = ggml_reshape_3d(ctx, gate, 1, R, M); + ggml_tensor * beta = ggml_sigmoid(ctx, ggml_reshape_3d(ctx, ggml_mul_mat(ctx, wb, x), 1, R, M)); + ggml_tensor * out = ggml_concat(ctx, gate3, beta, 0); + return out; + } + void initialize_tensors(ggml_context * ctx) override { arclab_init(ctx); } +}; + +// same-input T2 mat-vecs merged into one launch (GGML_SYCL_T2_MULTI): 2 or 3 ternary weights on one activation +struct test_arclab_t2_multi : public test_case { + const int64_t K, M; const std::array N; const int seed; + std::vector outs; + test_arclab_t2_multi(int64_t K, int64_t M, std::array N, int seed) : K(K), M(M), N(N), seed(seed) {} + std::string op_desc(ggml_tensor *) override { return "ARCLAB_T2_MULTI"; } + std::string vars() override { return VARS_TO_STR4(K, M, N, seed); } + bool run_whole_graph() override { return true; } + std::vector fusion_test_nodes() override { return outs; } + double max_nmse_err() override { return 5e-4; } + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * x = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, K, M); ggml_set_name(x, "x"); + outs.clear(); + ggml_tensor * acc = nullptr; + for (int m = 0; m < 3; ++m) { + if (N[m] == 0) continue; + ggml_tensor * w = ggml_new_tensor_2d(ctx, GGML_TYPE_PTQ1_0, K, N[m]); + ggml_tensor * y = ggml_mul_mat(ctx, w, x); + outs.push_back(y); + acc = acc ? ggml_add(ctx, acc, ggml_sum(ctx, y)) : ggml_sum(ctx, y); + } + return acc; + } + void initialize_tensors(ggml_context * ctx) override { arclab_init(ctx); } +}; + static void init_mul_mat_id_tensors(ggml_context * ctx, int n_mats) { std::random_device rd; std::default_random_engine rng(rd()); @@ -9427,6 +9616,21 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_fwht_signed(128, 1024, 5, GGML_TYPE_F32, true)); test_cases.emplace_back(new test_fwht_signed(2048, 8192, 3, GGML_TYPE_F32, true)); test_cases.emplace_back(new test_fwht_signed(1024, 5120, 4, GGML_TYPE_F16, true)); + // ARC-LAB fusion tests (arctune): Bonsai 27B shapes and token counts, 3 seeds each + for (int seed = 0; seed < 3; ++seed) { + for (int64_t M : {1, 4, 5, 8}) { + test_cases.emplace_back(new test_arclab_glu_rot(17408, M, 512, seed)); + test_cases.emplace_back(new test_arclab_add_norm_rot(5120, M, 512, false, seed)); + test_cases.emplace_back(new test_arclab_dn_gates(5120, 48, M, seed)); + test_cases.emplace_back(new test_arclab_t2_multi(5120, M, {10240, 6144, 0}, seed)); + test_cases.emplace_back(new test_arclab_t2_multi(5120, M, {12288, 1024, 1024}, seed)); + } + test_cases.emplace_back(new test_arclab_glu_rot(4096, 16, 256, seed)); + test_cases.emplace_back(new test_arclab_add_norm_rot(2048, 2, 256, false, seed)); + test_cases.emplace_back(new test_arclab_dn_gates(2048, 32, 2, seed)); + test_cases.emplace_back(new test_arclab_conv_chain(10240, 128, seed)); + test_cases.emplace_back(new test_arclab_conv_chain(3072, 64, seed)); + } // Block widths above the register path's reach, plus a couple below it as controls. 4096 and // 8192 exercise the shared-memory kernel; before it existed the CUDA backend declined them and // the op fell back, which cost both speed and (measurably) a little accuracy. From 844a3875196a846d192e4972819043e6e822dc1b Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 10 Oct 2026 00:15:26 +1100 Subject: [PATCH 33/65] ptq1-t2: GGML_SYCL_PTQ1_T2_TUNED per-shape GEMV tile table (plumbing, empty table) ARC-LAB: GGML_SYCL_PTQ1_T2_TUNED=1 (default 0) looks up a (N, K, rows, merged) row in t2_tuned_table() for every 1..8-row T2 GEMV (t2_dispatch and ggml_sycl_t2_mul_mat_multi) and launches TernSYCL's Gemv (TernSYCL, BSD 3-Clause) under a GRF mode and work-group order through GemvTuned (= gsw::Sweep of /lab/exp/gemv-sweep). Tiles are compiled from the T2_TUNED_KERNELS X-macro; t2_precompile warms them when the knob is on. Table and kernel list are empty until the B580 sweep results are in, so =1 changes nothing. =2 is a plumbing self-test: the default tiles through the tuned launcher (outputs must stay identical). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit a0d82ad1ea61670b98161b96fcd2d2d1cfe84edf on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/ptq1-t2.cpp | 162 +++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) diff --git a/ggml/src/ggml-sycl/ptq1-t2.cpp b/ggml/src/ggml-sycl/ptq1-t2.cpp index 18e4bc3badc5..baf81dee504e 100644 --- a/ggml/src/ggml-sycl/ptq1-t2.cpp +++ b/ggml/src/ggml-sycl/ptq1-t2.cpp @@ -13,6 +13,7 @@ #include #include #include +#include namespace { @@ -317,10 +318,162 @@ void gemv_multi(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, int nm, } // namespace namespace { +// ARC-LAB: GGML_SYCL_PTQ1_T2_TUNED=1 (default 0): per-shape GEMV tiles for 1..8 rows from the B580 sweep +// (/lab/exp/gemv-sweep, README "Mapping a winner into the fork"). A row of t2_tuned_table matches a launch by +// (N, K, rows, multi), N = the sum of the members' N for a same-input merged launch; no match = the default tiles above. +// =2 is a plumbing self-test (default tiles through the tuned launcher, outputs must not change). +// The tile is Gemv under a GRF mode (0 = no property, as Gemv; 128 / 256; 1 = grf_size_automatic) and +// a work-group order (0 = row-group outer, as the default path; 1 = the row-groups of a column block adjacent, so with +// SGM < rows the repeated weight reads hit L2). Same struct as gsw::Sweep in the sweep (sweep_kern.hpp): keep them equal. +template constexpr auto t2_tuned_props() { + namespace syclex = sycl::ext::oneapi::experimental; + namespace intelex = sycl::ext::intel::experimental; + if constexpr (GRF == 128 || GRF == 256) { + return syclex::properties{ syclex::sub_group_size<16>, syclex::work_group_size<1, WG>, intelex::grf_size }; + } else if constexpr (GRF == 1) { + return syclex::properties{ syclex::sub_group_size<16>, syclex::work_group_size<1, WG>, + intelex::grf_size_automatic }; + } else { + return syclex::properties{ syclex::sub_group_size<16>, syclex::work_group_size<1, WG> }; + } +} + +template +struct GemvTuned { + using Kern = int8dpas::Gemv; + Kern k0, k1, k2; + int gs1, gs2; // first column-group of members 1 and 2 (past the end when absent) + int R, C; // row-groups, column-groups (all members) + int ord; + void operator()(sycl::nd_item<2> it) const { + const int g = (int) it.get_group(1); + const int g0 = ord ? g % R : g / C; + const int g1 = ord ? g / R : g % C; + const bool a = g1 < gs1, b = !a && g1 < gs2; + const Kern k = a ? k0 : (b ? k1 : k2); // one call site of run (see GemvMulti) + k.run(it, g0, g1 - (a ? 0 : (b ? gs1 : gs2))); + } + auto get(int8dpas::syclex::properties_tag) const { return t2_tuned_props(); } +}; + +template +void gemv_tuned(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, int nm, const uint32_t * const * B, + const uint16_t * const * SB, float * const * C, const int * N, int M, int K, int ord) { + using T = GemvTuned; + using Kern = typename T::Kern; + const int wgn = 16 * NSG; + const Epi epi{ nullptr, nullptr, 0, 1 }; + int gs[4] = { 0, 0, 0, 0 }; + Kern k[3]; + for (int m = 0; m < 3; ++m) { + const int mm = m < nm ? m : 0; + k[m] = Kern{ nullptr, (const signed char *) Aq, SA, B[mm], SB[mm], C[mm], epi, M, N[mm], K }; + gs[m + 1] = gs[m] + (m < nm ? (N[m] + wgn - 1) / wgn : 0); + } + const int R = (M + SGM - 1) / SGM, Ct = gs[nm]; + const T f{ k[0], k[1], k[2], nm > 1 ? gs[1] : Ct + 1, nm > 2 ? gs[2] : Ct + 1, R, Ct, ord }; + q.parallel_for(sycl::nd_range<2>({ 1, (size_t) R * Ct * Kern::WG }, { 1, (size_t) Kern::WG }), f); +} + +// the compiled tiles: X(SGM, NSG, LS, U, GRF), one line per tile a table row uses (a row whose tile is not listed here is +// skipped with a warning) +#define T2_TUNED_KERNELS(X) + +struct t2_tuned_kernel { + int sgm, nsg, ls, u, grf; + void (*fn)(sycl::queue &, const int8_t *, const uint16_t *, int, const uint32_t * const *, const uint16_t * const *, + float * const *, const int *, int, int, int); +}; +#define T2_TUNED_X(a, b, c, d, e) t2_tuned_kernel{ a, b, c, d, e, gemv_tuned }, +const std::vector & t2_tuned_kernels() { + static const std::vector k = { T2_TUNED_KERNELS(T2_TUNED_X) }; + return k; +} +#undef T2_TUNED_X + +struct t2_tuned_row { + int N, K, M, multi; // launch: N (sum over members if merged), K, rows (1..8), merged launch (0/1) + int sgm, nsg, ls, u, grf, ord; // tile +}; +// rows from the sweep's summary ("fork table rows"); empty = GGML_SYCL_PTQ1_T2_TUNED=1 changes nothing +const std::vector & t2_tuned_table() { + static const std::vector t = { + // { N, K, M, multi, SGM, NSG, LS, U, GRF, ord }, + }; + return t; +} + +bool t2_tuned_on() { + return ggml_sycl_knob("GGML_SYCL_PTQ1_T2_TUNED", 0) != 0; +} + +// GGML_SYCL_PTQ1_T2_TUNED=2: plumbing self-test, every 1..8-row GEMV through gemv_tuned with the default tile (same LS, +// so outputs must be token-identical to =0) +template +void t2_tuned_selftest_ls(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, int nm, const uint32_t * const * B, + const uint16_t * const * SB, float * const * C, const int * N, int M, int K, bool ls4) { + ls4 ? gemv_tuned(q, Aq, SA, nm, B, SB, C, N, M, K, 0) : + gemv_tuned(q, Aq, SA, nm, B, SB, C, N, M, K, 0); +} + +template +void t2_tuned_selftest(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, int nm, const uint32_t * const * B, + const uint16_t * const * SB, float * const * C, const int * N, int M, int K, bool ls4) { + if (M == 1) { + t2_tuned_selftest_ls<1, NSG>(q, Aq, SA, nm, B, SB, C, N, M, K, ls4); + } else if (M == 2) { + t2_tuned_selftest_ls<2, NSG>(q, Aq, SA, nm, B, SB, C, N, M, K, ls4); + } else if (M <= 4) { + t2_tuned_selftest_ls<4, NSG>(q, Aq, SA, nm, B, SB, C, N, M, K, ls4); + } else { + t2_tuned_selftest_ls<8, NSG>(q, Aq, SA, nm, B, SB, C, N, M, K, ls4); + } +} + +// launches the tuned tile for this GEMV if the table has one; false = take the default path +bool t2_tuned(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, int nm, const uint32_t * const * B, + const uint16_t * const * SB, float * const * C, const int * N, int M, int K) { + const int mode = ggml_sycl_knob("GGML_SYCL_PTQ1_T2_TUNED", 0); + if (mode == 0) { + return false; + } + int ntot = 0, nmax = 0; + for (int m = 0; m < nm; ++m) { + ntot += N[m]; + nmax = std::max(nmax, N[m]); + } + if (mode == 2) { + // NSG as the default path: 2, or GGML_SYCL_PTQ1_T2_NSG=4 for 2..8 rows + (M > 1 && t2_nsg() == 4 ? t2_tuned_selftest<4> : t2_tuned_selftest<2>)(q, Aq, SA, nm, B, SB, C, N, M, K, + nmax <= 8192); + return true; + } + for (const t2_tuned_row & r : t2_tuned_table()) { + if (r.N != ntot || r.K != K || r.M != M || r.multi != (nm > 1)) { + continue; + } + for (const t2_tuned_kernel & k : t2_tuned_kernels()) { + if (k.sgm == r.sgm && k.nsg == r.nsg && k.ls == r.ls && k.u == r.u && k.grf == r.grf) { + k.fn(q, Aq, SA, nm, B, SB, C, N, M, K, r.ord); + return true; + } + } + static std::atomic warned{ false }; + if (!warned.exchange(true)) { + GGML_LOG_WARN("%s: tuned tile %d.%d.%d.%d.%d for N=%d K=%d M=%d is not in T2_TUNED_KERNELS, default tile\n", + __func__, r.sgm, r.nsg, r.ls, r.u, r.grf, r.N, r.K, r.M); + } + return false; + } + return false; +} + void t2_dispatch(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, const uint32_t * B, const uint16_t * SB, float * dst, int m, int n, int k, bool use_gemm) { if (use_gemm) { gemm(q, Aq, SA, B, SB, dst, m, n, k); + } else if (t2_tuned(q, Aq, SA, 1, &B, &SB, &dst, &n, m, k)) { + // tuned tile } else if (m == 1) { n <= 8192 ? gemv<1, 4>(q, Aq, SA, B, SB, dst, m, n, k) : gemv<1, 2>(q, Aq, SA, B, SB, dst, m, n, k); } else if (t2_nsg() == 4) { @@ -601,6 +754,9 @@ bool ggml_sycl_t2_mul_mat_multi(sycl::queue & q, int nm, const void * const * w, } const int m_ = (int) M, k_ = (int) K; const bool ls4 = nmax <= 8192; + if (t2_tuned(q, Aq, SA, nm, B, SB, C, Ni, m_, k_)) { + return true; + } if (M == 1) { ls4 ? gemv_multi<1, 4>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_) : gemv_multi<1, 2>(q, Aq, SA, nm, B, SB, C, Ni, m_, k_); } else if (t2_nsg() == 4) { @@ -648,6 +804,12 @@ void t2_precompile(sycl::queue & q) { gemv<4, 2, 4>(q, aq, sa, B, SB, d, 4, N, K); gemv<8, 2, 4>(q, aq, sa, B, SB, d, 8, N, K); } + if (t2_tuned_on()) { // GGML_SYCL_PTQ1_T2_TUNED: JIT the table's tiles now, not at the first verify + for (const t2_tuned_kernel & t : t2_tuned_kernels()) { + const int nn = N; + t.fn(q, aq, sa, 1, &B, &SB, &d, &nn, 8, K, 0); + } + } q.wait(); sycl::free(w, q); sycl::free(x, q); From 86134e0e44f8a51b0c5c5e1da41a60b5a56d6616 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 10 Oct 2026 00:38:36 +1100 Subject: [PATCH 34/65] ptq1-t2: GGML_SYCL_PTQ1_T2_TUNED table from the B580 GEMV sweep (bit-exact tiles) ARC-LAB: five rows from /lab/exp/gemv-sweep (JIT build, medians of 5 interleaved rounds), each with the same LS as the default tile, so the outputs are bit-identical: ffn_down n=5 0.775, gate+up n=5 0.896, output head n=5 0.882, q+k+v n=5 0.923, q+k+v n=1 0.974 (time vs the default tile). Tiles are TernSYCL's Gemv (BSD 3-Clause) under a different launch geometry and work-group order. Batch sizes 2-4 and other shapes keep the default tiles. Knob still defaults to 0. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 4bd894b1b7e47b348241ef888ad1981988f81507 on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/ptq1-t2.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-sycl/ptq1-t2.cpp b/ggml/src/ggml-sycl/ptq1-t2.cpp index baf81dee504e..80170edafa9b 100644 --- a/ggml/src/ggml-sycl/ptq1-t2.cpp +++ b/ggml/src/ggml-sycl/ptq1-t2.cpp @@ -377,7 +377,11 @@ void gemv_tuned(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, int nm, // the compiled tiles: X(SGM, NSG, LS, U, GRF), one line per tile a table row uses (a row whose tile is not listed here is // skipped with a warning) -#define T2_TUNED_KERNELS(X) +#define T2_TUNED_KERNELS(X) \ + X(4, 8, 4, 2, 0) \ + X(4, 1, 2, 2, 0) \ + X(4, 1, 2, 4, 0) \ + X(1, 2, 2, 1, 0) struct t2_tuned_kernel { int sgm, nsg, ls, u, grf; @@ -399,6 +403,13 @@ struct t2_tuned_row { const std::vector & t2_tuned_table() { static const std::vector t = { // { N, K, M, multi, SGM, NSG, LS, U, GRF, ord }, + // B580 sweep 2026-10-10 (/lab/exp/gemv-sweep jit.log, JIT build, medians of 5 interleaved rounds): bit-exact + // winners only (same LS as the default tile), > 2% faster, launches the model issues. Time vs the default tile: + { 5120, 17408, 5, 0, 4, 8, 4, 2, 0, 1 }, // ffn_down, 5-row verify 0.775 + { 34816, 5120, 5, 1, 4, 1, 2, 2, 0, 1 }, // ffn gate+up merged, 5 rows 0.896 + { 248320, 5120, 5, 0, 4, 1, 2, 4, 0, 1 }, // output head, 5 rows 0.882 + { 14336, 5120, 5, 1, 4, 1, 2, 2, 0, 1 }, // attention q+k+v merged, 5 rows 0.923 + { 14336, 5120, 1, 1, 1, 2, 2, 1, 0, 0 }, // attention q+k+v merged, 1 row 0.974 }; return t; } From 26cd716fdba133d9a6fa554079a5d266f80dce96 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 10 Oct 2026 00:49:43 +1100 Subject: [PATCH 35/65] ptq1-t2: tuned GEMV tiles on by default on a B580 (GGML_SYCL_PTQ1_T2_TUNED=0 turns them off) B580 A/B (sd2 head): MTP-only +2.9/+3.1%; live MTP + n-gram short +4.4%, 48K +3.1%, 115K flat; outputs identical. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 0729c9d366ecfb864b989986e9885a6384ef1744 on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/ptq1-t2.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-sycl/ptq1-t2.cpp b/ggml/src/ggml-sycl/ptq1-t2.cpp index 80170edafa9b..d528623b565c 100644 --- a/ggml/src/ggml-sycl/ptq1-t2.cpp +++ b/ggml/src/ggml-sycl/ptq1-t2.cpp @@ -318,7 +318,7 @@ void gemv_multi(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, int nm, } // namespace namespace { -// ARC-LAB: GGML_SYCL_PTQ1_T2_TUNED=1 (default 0): per-shape GEMV tiles for 1..8 rows from the B580 sweep +// ARC-LAB: GGML_SYCL_PTQ1_T2_TUNED=1 (default: 1 on a B580, else 0): per-shape GEMV tiles for 1..8 rows from the B580 sweep // (/lab/exp/gemv-sweep, README "Mapping a winner into the fork"). A row of t2_tuned_table matches a launch by // (N, K, rows, multi), N = the sum of the members' N for a same-input merged launch; no match = the default tiles above. // =2 is a plumbing self-test (default tiles through the tuned launcher, outputs must not change). @@ -414,8 +414,17 @@ const std::vector & t2_tuned_table() { return t; } +// the tiles were swept on a B580 (20 Xe cores): on by default there only; GGML_SYCL_PTQ1_T2_TUNED=1 / 0 forces it +static int t2_tuned_default() { + static const int d = [] { + const std::string name = dpct::dev_mgr::instance().get_device(ggml_sycl_get_device()).get_info(); + return name.find("B580") != std::string::npos ? 1 : 0; + }(); + return d; +} + bool t2_tuned_on() { - return ggml_sycl_knob("GGML_SYCL_PTQ1_T2_TUNED", 0) != 0; + return ggml_sycl_knob("GGML_SYCL_PTQ1_T2_TUNED", t2_tuned_default()) != 0; } // GGML_SYCL_PTQ1_T2_TUNED=2: plumbing self-test, every 1..8-row GEMV through gemv_tuned with the default tile (same LS, @@ -444,7 +453,7 @@ void t2_tuned_selftest(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, // launches the tuned tile for this GEMV if the table has one; false = take the default path bool t2_tuned(sycl::queue & q, const int8_t * Aq, const uint16_t * SA, int nm, const uint32_t * const * B, const uint16_t * const * SB, float * const * C, const int * N, int M, int K) { - const int mode = ggml_sycl_knob("GGML_SYCL_PTQ1_T2_TUNED", 0); + const int mode = ggml_sycl_knob("GGML_SYCL_PTQ1_T2_TUNED", t2_tuned_default()); if (mode == 0) { return false; } From 174e85dfcd1060fb3e1151f6a65af4fe83ab78cf Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:25:24 +1100 Subject: [PATCH 36/65] sycl: default to the legacy Level Zero adapter unless SYCL_UR_USE_LEVEL_ZERO_V2 is set Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 5336e12df3ec500ea5f99fcb63ca895cf6a31683 on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/ggml-sycl.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index fb178721a034..1936d116af18 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -331,6 +331,23 @@ static const char* dev2dev_int2str(int dev2dev) { } } +// ARC-LAB: use the legacy Level Zero adapter unless the user chose one. On the B580 (oneAPI 2025.3, where V2 is the +// default) plain decode measured +1-1.5% in llama-bench and MTP +0.5%, outputs identical. Set at library load, before +// any SYCL platform query; SYCL_UR_USE_LEVEL_ZERO_V2=1 in the environment keeps V2. +namespace { +struct ggml_sycl_l0_adapter_default { + ggml_sycl_l0_adapter_default() { + if (getenv("SYCL_UR_USE_LEVEL_ZERO_V2") == nullptr) { +#ifdef _WIN32 + _putenv_s("SYCL_UR_USE_LEVEL_ZERO_V2", "0"); +#else + setenv("SYCL_UR_USE_LEVEL_ZERO_V2", "0", 0); +#endif + } + } +} ggml_sycl_l0_adapter_default_init; +} + static void ggml_check_sycl() try { static bool initialized = false; From 58e4b0b0ed0734cfcad1904fbbe7185b6cfefe44 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:47:25 +1100 Subject: [PATCH 37/65] sycl: tuned Q6_K mat-vec and fused Q6_K gate+up+SWIGLU for the MTP draft head (knobs, off) The draft head's Q6_K mat-vecs are ~60% of a draft step and read at ~70% of the B580's bandwidth (op timing, 2026-10-10). - GGML_SYCL_Q6K_GEMV=1: a single-column reordered Q6_K mat-vec with the original per-row arithmetic (same lane -> quant-int mapping, blocks in order, same vec_dot expression accumulated in the same order, same sub-group reduction) but U blocks' operands loaded before use, aligned 32-bit ql/qh loads, and GGML_SYCL_Q6K_SG sub-groups per work-group. Sweep knobs GGML_SYCL_Q6K_U (1/2/4, default 2) and GGML_SYCL_Q6K_SG (4/8/16, default 8). GGML_SYCL_Q6K_CHECK=1 reruns the original kernel per launch and counts rows that differ in any bit (stderr), the in-server identity check. - GGML_SYCL_Q6K_GLU=1: Q6_K joins the reorder gate+up+GLU fusion (1..8 columns), whose dot products are the reorder mat-vec's and whose epilogue is the SWIGLU kernel's silu(gate) * up. - test-backend-ops: Q6_K mat-vec and gate+up fusion cases at the head's shapes (eval: 1, 2, 6 tokens; perf: 1..5 tokens). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 6dd4481f07f4361c2433fc3e317861d62f48efa6 on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/fusion.cpp | 7 +- ggml/src/ggml-sycl/mmvq.cpp | 193 ++++++++++++++++++++++++++++++++++ tests/test-backend-ops.cpp | 19 ++++ 3 files changed, 217 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-sycl/fusion.cpp b/ggml/src/ggml-sycl/fusion.cpp index 7f1c577e79ba..c8da4117d8c1 100644 --- a/ggml/src/ggml-sycl/fusion.cpp +++ b/ggml/src/ggml-sycl/fusion.cpp @@ -31,15 +31,18 @@ static bool ggml_sycl_should_fuse_mul_mat_glu(const ggml_tensor * gate, const gg } // types with a fused reorder GEMV; each walks whole blocks + // ARC-LAB: Q6_K (the MTP draft head's FFN) behind GGML_SYCL_Q6K_GLU=1: the generic reorder GLU kernel, whose per-row + // dot products are the unfused reorder mat-vec's and whose epilogue is the SWIGLU kernel's silu(gate) * up + const bool q6k_ok = wu->type == GGML_TYPE_Q6_K && wu->ne[0] % QK_K == 0 && ggml_sycl_knob("GGML_SYCL_Q6K_GLU", 0); const bool type_ok = (wu->type == GGML_TYPE_Q4_K && wu->ne[0] % QK_K == 0) || (wu->type == GGML_TYPE_PQ2_0 && wu->ne[0] % QK_PQ2_0 == 0) || - (wu->type == GGML_TYPE_PTQ1_0 && wu->ne[0] % QK_PTQ1_0 == 0); + (wu->type == GGML_TYPE_PTQ1_0 && wu->ne[0] % QK_PTQ1_0 == 0) || q6k_ok; if (!type_ok) { return false; } // single-token PQ2_0 decode is faster unfused (its own 2-row GEMV); fuse for 2..8 columns. ARC-LAB: PTQ1_0 has a // single-column fused kernel (pairs core, GGML_SYCL_PTQ1_GLU1=0 turns it off) - if (wu->type != GGML_TYPE_Q4_K && act->ne[1] < 2 && + if (wu->type != GGML_TYPE_Q4_K && wu->type != GGML_TYPE_Q6_K && act->ne[1] < 2 && !(wu->type == GGML_TYPE_PTQ1_0 && ggml_sycl_ptq1_0_glu_n1_enabled())) { return false; } diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cpp index ef9ff069ccf4..49e3d7e1bafe 100644 --- a/ggml/src/ggml-sycl/mmvq.cpp +++ b/ggml/src/ggml-sycl/mmvq.cpp @@ -2827,8 +2827,198 @@ static void reorder_mul_mat_vec_q5_k_q8_1_sycl_switch_ncols( } } +// ARC-LAB: Q6_K reordered single-column mat-vec with more loads in flight (GGML_SYCL_Q6K_GEMV=1; the MTP draft head's +// eh_proj / q / wo / ffn weights are Q6_K and ran at ~70% of the B580's bandwidth). Same arithmetic as +// mul_mat_vec_q_reorder>: one sub-group per row, lane l handles quant ints l and l + 16 of +// every block, blocks in order, `partial_sum += vec_dot_q6_K_q8_1_impl_mmvq_scalar(...)` per (block, int) in the same +// order, then the same sub-group reduction. What changes: +// - U blocks' operands are loaded before any of them is used, so U x 2 groups of loads are in flight per lane instead +// of the ones of one block (the loop body was latency bound); +// - ql / qh words are read as aligned 32-bit loads (the generic helper assembles them from two 16-bit loads; the +// reordered planes are 4-byte aligned, so the int is the same); +// - the work-group holds GGML_SYCL_Q6K_SG sub-groups (rows) instead of 16. +// GGML_SYCL_Q6K_U (1, 2, 4; default 2) and GGML_SYCL_Q6K_SG (4, 8, 16; default 8) are runtime knobs for a sweep. +// GGML_SYCL_Q6K_CHECK=1 also runs the original kernel and counts rows whose result differs in any bit (stderr, every +// 256 launches): the identity check for the GPU run. +namespace { +struct q6k_ops { + int vl, vh, u0, u1; + int sc0, sc1; + float d80, d81; +}; + +static __dpct_inline__ void q6k_load(const uint8_t * __restrict__ base, const int8_t * __restrict__ q8, + const sycl::half2 * __restrict__ ds, const int ql_byte, const int qh_byte, + const int sc_byte, const int iqs, q6k_ops & o) { + const int bq8_offset = 2 * QR6_K * (iqs / (QI6_K / 2)) + (iqs % (QI6_K / 2)) / (QI6_K / 4); + const int scale_offset = (QI6_K / 4) * (iqs / (QI6_K / 2)) + (iqs % (QI6_K / 2)) / (QI6_K / 8); + const int vh_shift = 2 * ((iqs % (QI6_K / 2)) / (QI6_K / 4)); + const int qh_int = (QI6_K / 4) * (iqs / (QI6_K / 2)) + iqs % (QI6_K / 4); + + o.vl = *(const int *) (base + ql_byte + 4 * iqs); + o.vh = *(const int *) (base + qh_byte + 4 * qh_int) >> vh_shift; + const int8_t * scs = (const int8_t *) (base + sc_byte) + scale_offset; + o.sc0 = scs[0]; + o.sc1 = scs[4]; + o.u0 = *(const int *) (q8 + bq8_offset * QK8_1 + 4 * (iqs % QI8_1)); + o.u1 = *(const int *) (q8 + (bq8_offset + 2) * QK8_1 + 4 * (iqs % QI8_1)); + o.d80 = ds[bq8_offset + 0][0]; + o.d81 = ds[bq8_offset + 2][0]; +} +} // namespace + +template +static void mul_mat_vec_q6_k_reorder_pf(const void * __restrict__ vx, const void * __restrict__ vy, + float * __restrict__ dst, const int ncols, const int nrows, + const sycl::nd_item<3> & nd_item) { + using block_type = ggml_sycl_reordered::block_q_t; + static_assert(QI6_K == 2 * WARP_SIZE, "two quant ints per lane per block"); + + const auto sg = nd_item.get_sub_group(); + const int row = nd_item.get_group_linear_id() * sg.get_group_linear_range() + sg.get_group_linear_id(); + if (row >= nrows) { + return; + } + const int lane = sg.get_local_linear_id(); + + const int blocks_per_row = ncols / QK_K; + const int nblocks = nrows * blocks_per_row; + const uint8_t * base = (const uint8_t *) vx; + + float partial_sum = 0.0f; + int i = 0; + for (; i + U <= blocks_per_row; i += U) { + q6k_ops o[U][2]; + float d[U]; +#pragma unroll + for (int u = 0; u < U; ++u) { + const int ibx = row * blocks_per_row + i + u; + const auto bx = block_type::get_block_offset(ibx, nblocks); + const auto doff = block_type::get_d_offset(nrows, ncols, ibx); + const int iby = (i + u) * block_type::block_to_q8_1_ratio(); + const int8_t * q8 = (const int8_t *) vy + iby * QK8_1; + const sycl::half2 * ds = (const sycl::half2 *) ((const char *) vy + ncols + iby * sizeof(sycl::half2)); + d[u] = *(const ggml_half *) (base + doff.second); +#pragma unroll + for (int e = 0; e < 2; ++e) { + q6k_load(base, q8, ds, bx.first, bx.second, doff.first, e * WARP_SIZE + lane, o[u][e]); + } + } +#pragma unroll + for (int u = 0; u < U; ++u) { +#pragma unroll + for (int e = 0; e < 2; ++e) { + partial_sum += vec_dot_q6_K_q8_1_impl_mmvq_scalar(o[u][e].vl, o[u][e].vh, o[u][e].u0, o[u][e].u1, + o[u][e].sc0, o[u][e].sc1, d[u], o[u][e].d80, + o[u][e].d81); + } + } + } + for (; i < blocks_per_row; ++i) { + const int ibx = row * blocks_per_row + i; + const auto bx = block_type::get_block_offset(ibx, nblocks); + const auto doff = block_type::get_d_offset(nrows, ncols, ibx); + const int iby = i * block_type::block_to_q8_1_ratio(); + const int8_t * q8 = (const int8_t *) vy + iby * QK8_1; + const sycl::half2 * ds = (const sycl::half2 *) ((const char *) vy + ncols + iby * sizeof(sycl::half2)); + const float d = *(const ggml_half *) (base + doff.second); +#pragma unroll + for (int e = 0; e < 2; ++e) { + q6k_ops o; + q6k_load(base, q8, ds, bx.first, bx.second, doff.first, e * WARP_SIZE + lane, o); + partial_sum += vec_dot_q6_K_q8_1_impl_mmvq_scalar(o.vl, o.vh, o.u0, o.u1, o.sc0, o.sc1, d, o.d80, o.d81); + } + } + + const float sum = sycl::reduce_over_group(sg, partial_sum, std::plus<>()); + if (sg.leader()) { + dst[row] = sum; + } +} + +template +static void launch_q6_k_reorder_pf(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, + const int n_sg, dpct::queue_ptr stream) { + const sycl::range<3> block_nums(1, 1, ceil_div(nrows, n_sg)); + const sycl::range<3> block_dims(1, 1, (size_t) n_sg * WARP_SIZE); + stream->submit([&](sycl::handler & cgh) { + cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims), + [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + mul_mat_vec_q6_k_reorder_pf(vx, vy, dst, ncols, nrows, nd_item); + }); + }); +} + +static void reorder_mul_mat_vec_q6_k_q8_1_sycl_orig(const void * vx, const void * vy, float * dst, const int ncols, + const int nrows, dpct::queue_ptr stream); + +// returns false when the tuned kernel is off (GGML_SYCL_Q6K_GEMV=0) +static bool reorder_mul_mat_vec_q6_k_q8_1_sycl_tuned(const void * vx, const void * vy, float * dst, const int ncols, + const int nrows, dpct::queue_ptr stream) { + if (!ggml_sycl_knob("GGML_SYCL_Q6K_GEMV", 0)) { + return false; + } + const int u = ggml_sycl_knob("GGML_SYCL_Q6K_U", 2); + int n_sg = ggml_sycl_knob("GGML_SYCL_Q6K_SG", 8); + if (n_sg != 4 && n_sg != 8 && n_sg != 16) { + n_sg = 8; + } + switch (u) { + case 1: launch_q6_k_reorder_pf<1>(vx, vy, dst, ncols, nrows, n_sg, stream); break; + case 4: launch_q6_k_reorder_pf<4>(vx, vy, dst, ncols, nrows, n_sg, stream); break; + default: launch_q6_k_reorder_pf<2>(vx, vy, dst, ncols, nrows, n_sg, stream); break; + } + + if (ggml_sycl_knob("GGML_SYCL_Q6K_CHECK", 0)) { + // the original kernel into a scratch row vector, then a bitwise compare (debug only: allocates and waits) + static float * ref = nullptr; + static int ref_n = 0; + static int * n_diff = nullptr; + static long launches = 0, rows = 0, diffs = 0; + if (ref_n < nrows) { + if (ref) { + sycl::free(ref, *stream); + } + ref = sycl::malloc_device(nrows, *stream); + ref_n = nrows; + } + if (!n_diff) { + n_diff = sycl::malloc_shared(1, *stream); + } + reorder_mul_mat_vec_q6_k_q8_1_sycl_orig(vx, vy, ref, ncols, nrows, stream); + *n_diff = 0; + stream->wait(); + int * nd = n_diff; + float * r = ref; + stream->parallel_for(sycl::range<1>(nrows), [=](sycl::id<1> k) { + const uint32_t a = sycl::bit_cast(dst[k]); + const uint32_t b = sycl::bit_cast(r[k]); + if (a != b) { + sycl::atomic_ref c(*nd); + c.fetch_add(1); + } + }).wait(); + launches++; + rows += nrows; + diffs += *n_diff; + if (launches % 256 == 0) { + fprintf(stderr, "q6k-check: %ld launches, %ld rows, %ld rows differ (U=%d, SG=%d)\n", launches, rows, diffs, u, + n_sg); + } + } + return true; +} + static void reorder_mul_mat_vec_q6_k_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols, const int nrows, dpct::queue_ptr stream) { + if (reorder_mul_mat_vec_q6_k_q8_1_sycl_tuned(vx, vy, dst, ncols, nrows, stream)) { + return; + } + reorder_mul_mat_vec_q6_k_q8_1_sycl_orig(vx, vy, dst, ncols, nrows, stream); +} + +static void reorder_mul_mat_vec_q6_k_q8_1_sycl_orig(const void * vx, const void * vy, float * dst, const int ncols, + const int nrows, dpct::queue_ptr stream) { GGML_ASSERT(ncols % QK_K == 0); // Round up to a whole number of subgroup-sized workgroups; out-of-range rows are skipped inside the kernel. constexpr size_t num_subgroups = WARP_SIZE; @@ -3992,6 +4182,9 @@ bool ggml_sycl_mul_mat_vec_q_glu_reorder(enum ggml_type src0_type, enum ggml_glu case GGML_TYPE_PQ2_0: return mul_mat_vec_q_glu_reorder_type(glu_op, vx, vgate, vy, dst, ncols, nrows, ncols_dst, stride_col_y_bytes, stride_col_dst, stream); + case GGML_TYPE_Q6_K: // ARC-LAB: GGML_SYCL_Q6K_GLU (fusion.cpp) + return mul_mat_vec_q_glu_reorder_type(glu_op, vx, vgate, vy, dst, ncols, nrows, ncols_dst, + stride_col_y_bytes, stride_col_dst, stream); case GGML_TYPE_PTQ1_0: if (ncols_dst == 1 && ggml_sycl_ptq1_0_glu_n1_enabled()) { static const int glu_nr = getenv("GGML_SYCL_PTQ1_GLU1_NR") ? atoi(getenv("GGML_SYCL_PTQ1_GLU1_NR")) : 2; diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index e41b9224094c..cdf8765da7e1 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10722,6 +10722,17 @@ static std::vector> make_test_cases_eval() { false, 1, 1, false, false, true, false, {1, 1})); } + // ARC-LAB: the MTP draft head's Q6_K weights at their shapes, 1..6 tokens (draft steps, chain step 0): the mat-vecs + // (GGML_SYCL_Q6K_GEMV / _U / _SG) and the FFN gate+up+SWIGLU (GGML_SYCL_Q6K_GLU=1 takes the fused kernel) + for (int n : {1, 2, 6}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 12288, n, 5120, {1, 1}, {1, 1})); // q + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 5120, n, 10240, {1, 1}, {1, 1})); // eh_proj + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 5120, n, 6144, {1, 1}, {1, 1})); // wo + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 5120, n, 17408, {1, 1}, {1, 1})); // down + test_cases.emplace_back(new test_mul_mat_vec_fusion(GGML_TYPE_Q6_K, GGML_GLU_OP_SWIGLU, n, 17408, 5120, + false, 1, 1, false, false, true, false, {1, 1})); // gate+up + } + for (auto gate : {GATING_FUNC_SOFTMAX, GATING_FUNC_SIGMOID, GATING_FUNC_SOFTMAX_WEIGHT, GATING_FUNC_SQRT_SOFTPLUS}) { for (bool with_norm : {false, true}) { for (bool bias_probs : {false, true}) { @@ -10854,6 +10865,14 @@ static std::vector> make_test_cases_perf() { // ARC-LAB: Bonsai 2 GDN layer shape (16 k-heads x 128, 48 v-heads): prompt batch and decode test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 1024, 1, 3)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 128, 1, 1, 3)); + // ARC-LAB: MTP draft head (Q6_K) mat-vecs at their shapes, 1..5 tokens: q, eh_proj, wo, ffn_down, ffn_gate / up + for (int n : {1, 2, 3, 4, 5}) { + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 12288, n, 5120, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 5120, n, 10240, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 5120, n, 6144, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 5120, n, 17408, {1, 1}, {1, 1})); + test_cases.emplace_back(new test_mul_mat(GGML_TYPE_Q6_K, GGML_TYPE_F32, 17408, n, 5120, {1, 1}, {1, 1})); + } // bandwidth comparison at Bonsai-2 shapes for (ggml_type t : {GGML_TYPE_PTQ1_0, GGML_TYPE_PQ2_0, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0, GGML_TYPE_Q2_K, GGML_TYPE_TQ2_0}) { test_cases.emplace_back(new test_mul_mat(t, GGML_TYPE_F32, 17408, 1, 5120, {1, 1}, {1, 1})); From 6cc63237f2ab4afd0a2580a68596121b90fca73b Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 10 Oct 2026 21:23:31 +1100 Subject: [PATCH 38/65] sycl fattn-dec: transposed 2D block loads for the DPAS kernel's lane = key reads (GGML_SYCL_FA_DEC_B2D, default off) xe2dp::ld2d_t32_8x16 (lsc_load_block2d d32.8x16tn): one message gives each lane 8 dwords of its own key row, replacing the per-lane dword gathers in Q.K (5 messages per 16 keys instead of 40) and P.V (1 instead of 5). Same bytes and math, 89/89 FLASH_ATTN_EXT cases. Engages only on 64-byte aligned K/V surfaces (logged once). Slower on its own at 128K (register spills): 5 tokens 1069 -> 1220 us, 16 tokens 2945 -> 3610 us. Kernel launch now goes through a named lambda. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit fb75ee9e56e167d79196078558d0c2fd727ce83a on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/fattn-dec.cpp | 88 ++++++++++++++++++++++++++++---- ggml/src/ggml-sycl/xe2-dpas.hpp | 32 ++++++++++++ 2 files changed, 109 insertions(+), 11 deletions(-) diff --git a/ggml/src/ggml-sycl/fattn-dec.cpp b/ggml/src/ggml-sycl/fattn-dec.cpp index 7f02067417d3..02a2103ac245 100644 --- a/ggml/src/ggml-sycl/fattn-dec.cpp +++ b/ggml/src/ggml-sycl/fattn-dec.cpp @@ -703,7 +703,10 @@ typedef unsigned dp_uint4 __attribute__((ext_vector_type(4))); // (consecutive work-items read consecutive dwords of a key row) and the lane = key reads come from there (row stride 37 // dwords: conflict-free). Without it every lane reads its own key straight from memory: 16 cache lines per load // instruction, ~90-110 GB/s per phase at 48K. The V copy is issued before the softmax and shares its barrier. -template +// BL bit 0 (GGML_SYCL_FA_DEC_B2D, ST = 0 only): the lane = key reads use transposed 2D block loads (xe2dp::ld2d_t32_8x16: +// 16 keys x 8 dwords per message) instead of per-lane dword gathers. Same bytes, same math. Needs 64-byte aligned surfaces. +// B580, 128K, us: on its own SLOWER (spills; 5 tok 1069 -> 1220, 16 tok 2945 -> 3610). Off by default. +template static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, const char * mask, float * dst, float * parts, sycl::float2 * meta, float scale, int ne01, int ne02, int ne11, int nkvh, int ne03, int64_t nb01, int64_t nb02, int64_t nb03, int64_t nb11, @@ -725,6 +728,7 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, constexpr int RW = NB * 18 / 4; // dwords per q4_0 key row (36 at D 256) constexpr int SW = RW + 1; // SLM row stride (odd: no bank conflicts on lane = key reads) static_assert(ST == 0 || (D == 256 && RW % 4 == 0), "staging is sized for D 256"); + static_assert((BL & 1) == 0 || (ST == 0 && D == 256), "block loads replace the per-lane reads of the D 256 unstaged path"); const int nqc = (ne01 + NQ - 1) / NQ; // token chunks const int probe_env = ggml_sycl_knob("GGML_SYCL_FA_DEC_DPAS_PROBE", 0); const int probe = probe_env; // phase probe: bit0 skip Q.K, bit1 skip P.V, bit2 skip softmax, bit3 Q.K loads only (no DPAS / scale math) @@ -741,9 +745,8 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, sycl::local_accessor sL(sycl::range<1>(R16), cgh); sycl::local_accessor sA(sycl::range<1>(R16), cgh); sycl::local_accessor sKV(sycl::range<1>(ST ? TK * SW : 1), cgh); // staged K or V tile - cgh.parallel_for( - sycl::nd_range<3>(sycl::range<3>((size_t) ne03 * nqc, nkvh, (size_t) nsplit * DEC_WG), sycl::range<3>(1, 1, DEC_WG)), - [=](sycl::nd_item<3> it) [[sycl::reqd_sub_group_size(16)]] { + const sycl::nd_range<3> ndr(sycl::range<3>((size_t) ne03 * nqc, nkvh, (size_t) nsplit * DEC_WG), sycl::range<3>(1, 1, DEC_WG)); + auto kern = [=](sycl::nd_item<3> it) [[sycl::reqd_sub_group_size(16)]] { const int tid = it.get_local_id(2); const int split = it.get_group(2); const int kvh = it.get_group(1); @@ -792,6 +795,12 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, const char * Kh = K + seq * nb13 + kvh * nb12; const char * Vh = V + seq * nb23 + kvh * nb22; const sycl::half * mrow = mask ? (const sycl::half *) (mask + (seq % ne33) * nb33) : nullptr; + // 2D surfaces (BL): rows = keys (pitch nb11 / nb21). Heads interleaved within a token row (the real cache + // layout, nb12 < nb11): surface = the sequence's rows, this head at dword x0; else the head's own rows. + const bool kil = nb12 < nb11, vil = nb22 < nb21; + const long long kbs = (long long) (K + seq * nb13 + (kil ? 0 : kvh * nb12)); + const long long vbs = (long long) (V + seq * nb23 + (vil ? 0 : kvh * nb22)); + const int kx0 = kil ? (int) (kvh * nb12 / 4) : 0, vx0 = vil ? (int) (kvh * nb22 / 4) : 0; // O^T accumulators: subgroup w owns head dims DPS w .. +DPS-1 (XT 8-dim tiles), lane = query row dp_float8 o[RT16][XT]; @@ -861,6 +870,16 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, const int j = t0 + kg * 16 + lane; const bool jv = j < k_end; const uint32_t * kr = (const uint32_t *) (Kh + (int64_t) (jv ? j : k_begin) * nb11); + // BL: the 16 keys' 36-dword rows (+4 unused) in 5 loads of 8 dwords, lane = key; issued two blocks + // ahead of their first use (all 5 up front = 40 more registers: spills) + xe2dp::u32x8 kb[(BL & 1) ? 5 : 1]; + auto kload = [&](int i) { + kb[i] = xe2dp::ld2d_t32_8x16(kbs, (int) nb11 - 1, ne11 - 1, (int) nb11 - 1, kx0 + 8 * i, t0 + kg * 16); + }; + if constexpr (BL & 1) { + kload(0); + kload(1); + } dp_float8 s[TC]; #pragma unroll for (int tt = 0; tt < TC; ++tt) { @@ -872,9 +891,17 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, const int o0 = 18 * b; // even b: dword aligned; odd b: 2 bytes in const int wd = o0 / 4; // the block's 18 bytes lie in these 5 dwords uint32_t k0, k1, k2, k3, k4; + if constexpr (BL & 1) { + if (b == 1 || b == 3 || b == 5) { // block b + 2 starts in dword chunk (b + 1) / 2 + 1 + kload((b + 1) / 2 + 1); + } + } if constexpr (ST) { const int sb = (kg * 16 + lane) * SW + wd; k0 = sKV[sb]; k1 = sKV[sb + 1]; k2 = sKV[sb + 2]; k3 = sKV[sb + 3]; k4 = sKV[sb + 4]; + } else if constexpr (BL & 1) { + k0 = kb[wd / 8][wd % 8]; k1 = kb[(wd + 1) / 8][(wd + 1) % 8]; k2 = kb[(wd + 2) / 8][(wd + 2) % 8]; + k3 = kb[(wd + 3) / 8][(wd + 3) % 8]; k4 = kb[(wd + 4) / 8][(wd + 4) % 8]; } else { k0 = kr[wd]; k1 = kr[wd + 1]; k2 = kr[wd + 2]; k3 = kr[wd + 3]; k4 = kr[wd + 4]; } @@ -968,6 +995,13 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, o[t][x] *= a; } } + // BL: this subgroup's V block of the next 16 keys is loaded one key group ahead + xe2dp::u32x8 vnext; + if constexpr (BL & 1) { + if (!(probe & 2)) { + vnext = xe2dp::ld2d_t32_8x16(vbs, (int) nb21 - 1, ne11 - 1, (int) nb21 - 1, vx0 + 18 * vb / 4, t0); + } + } #pragma unroll 1 for (int kg = 0; kg < ((probe & 2) ? 0 : KG); ++kg) { const int j = t0 + kg * 16 + lane; @@ -975,7 +1009,13 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, const uint32_t * vr = (const uint32_t *) (Vh + (int64_t) (jv ? j : k_begin) * nb21); const int o0 = 18 * vb, wd = o0 / 4; // block start; even block: dword aligned, odd: 2 bytes in uint32_t w0, w1, w2, w3, w4; - if constexpr (ST) { + if constexpr (BL & 1) { + const xe2dp::u32x8 vc = vnext; + if (kg + 1 < KG) { + vnext = xe2dp::ld2d_t32_8x16(vbs, (int) nb21 - 1, ne11 - 1, (int) nb21 - 1, vx0 + wd, t0 + (kg + 1) * 16); + } + w0 = vc[0]; w1 = vc[1]; w2 = vc[2]; w3 = vc[3]; w4 = vc[4]; + } else if constexpr (ST) { const int sb = (kg * 16 + lane) * SW + wd; w0 = sKV[sb]; w1 = sKV[sb + 1]; w2 = sKV[sb + 2]; w3 = sKV[sb + 3]; w4 = sKV[sb + 4]; } else { @@ -1039,7 +1079,8 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, const int64_t jdu = ((int64_t) seq * ne01 + c0 + tid / G) * ne02 + kvh * G + tid % G; meta[jdu * nsplit + split] = sycl::float2(sM[tid], sL[tid]); } - }); + }; + cgh.parallel_for(ndr, kern); }); } @@ -1113,8 +1154,31 @@ void ggml_sycl_flash_attn_ext_dec(ggml_backend_sycl_context & ctx, ggml_tensor * // 8-token kernel with 3 empty tokens (48 rows of Q.K / softmax for 30). B580 per layer: 16K 215 -> 175 us, 48K 637 -> // 502 us. =0 off, =2 SLM-staged (slower: 48K 579), =3 256-key tiles (48K 513). const int nq5_env = ggml_sycl_knob("GGML_SYCL_FA_DEC_NQ5", 1); + // ARC-LAB: GGML_SYCL_FA_DEC_B2D: 1 = transposed 2D block loads for the unstaged DPAS paths (2+ query tokens), 2 = also + // for 1 token instead of the SLM staging; 0 = per-lane reads. Needs 64-byte aligned K / V surfaces, else per-lane reads. + const int b2d_env = ggml_sycl_knob("GGML_SYCL_FA_DEC_B2D", 0); + auto b2d_fits = [](const ggml_tensor * t) { + const bool il = t->nb[2] < t->nb[1]; // heads interleaved in a token row + return ((uintptr_t) t->data) % 64 == 0 && (t->ne[3] == 1 || t->nb[3] % 64 == 0) && t->nb[1] % 16 == 0 && + t->nb[1] >= 64 && t->nb[1] < (1 << 24) && t->ne[1] < (1 << 24) && + (il ? (t->nb[2] % 4 == 0 && t->nb[2] * (t->ne[2] - 1) + (int64_t) (t->ne[0] / QK4_0) * 18 <= (int64_t) t->nb[1]) + : t->nb[2] % 64 == 0); + }; + const bool b2d_shape = dpas && b2d_env && K->ne[0] == DEC_D && Q->ne[2] == 6 * K->ne[2]; + const bool b2d = b2d_shape && b2d_fits(K) && b2d_fits(V); + if (b2d_shape) { + static bool logged_on = false, logged_off = false; + bool & logged = b2d ? logged_on : logged_off; + if (!logged) { + logged = true; + GGML_LOG_INFO("%s: GGML_SYCL_FA_DEC_B2D=%d %s (K data %% 64 = %d, nb1 %zu nb2 %zu)\n", __func__, b2d_env, + b2d ? "engaged" : "NOT engaged: surface not 64-byte aligned, per-lane reads", + (int) (((uintptr_t) K->data) % 64), K->nb[1], K->nb[2]); + } + } + const int bl = b2d ? 1 : 0; const int nq5 = (dpas && !d512 && !g2 && ne01 == 5) ? nq5_env : 0; - const bool stage = dpas && stage_env && !d512 && !g2 && (nq == 1 || nq5 == 2); + const bool stage = dpas && stage_env && !d512 && !g2 && (nq == 1 || nq5 == 2) && !(b2d && b2d_env >= 2 && nq == 1); const int tk = (d512 || g2) ? (nq == 1 ? 256 : 128) : nq5 == 3 ? 256 : (stage || nq5) ? 128 @@ -1149,10 +1213,11 @@ void ggml_sycl_flash_attn_ext_dec(ggml_backend_sycl_context & ctx, ggml_tensor * const int64_t nb33 = mask ? mask->nb[3] : 0; const int ne33 = mask ? (int) mask->ne[3] : 1; -#define FATTN_DEC_DPAS(NQ_, KG_, TC_) fattn_dec_q4_0_dpas((const char *) Q->data, (const char *) K->data, \ - (const char *) V->data, mdata, (float *) dst->data, parts.get(), meta.get(), scale, ne01, (int) Q->ne[2], ne11, \ - (int) K->ne[2], (int) Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], V->nb[1], V->nb[2], \ - V->nb[3], nb31, nb33, ne33, nsplit, chunk, stream) +#define FATTN_DEC_DPAS_BL(NQ_, KG_, TC_, BL_) fattn_dec_q4_0_dpas((const char *) Q->data, \ + (const char *) K->data, (const char *) V->data, mdata, (float *) dst->data, parts.get(), meta.get(), scale, ne01, \ + (int) Q->ne[2], ne11, (int) K->ne[2], (int) Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], \ + V->nb[1], V->nb[2], V->nb[3], nb31, nb33, ne33, nsplit, chunk, stream) +#define FATTN_DEC_DPAS(NQ_, KG_, TC_) do { if (bl) { FATTN_DEC_DPAS_BL(NQ_, KG_, TC_, 1); } else { FATTN_DEC_DPAS_BL(NQ_, KG_, TC_, 0); } } while (0) if (d512) { #define FATTN_DEC_DPAS512(NQ_, KG_, TC_) fattn_dec_q4_0_dpas<512, 16, NQ_, KG_, TC_>((const char *) Q->data, (const char *) K->data, \ (const char *) V->data, mdata, (float *) dst->data, parts.get(), meta.get(), scale, ne01, (int) Q->ne[2], ne11, \ @@ -1201,6 +1266,7 @@ void ggml_sycl_flash_attn_ext_dec(ggml_backend_sycl_context & ctx, ggml_tensor * } } #undef FATTN_DEC_DPAS +#undef FATTN_DEC_DPAS_BL #define FATTN_DEC_CALL(NQ_) fattn_dec_q4_0<6, NQ_, (NQ_ == 4 ? 64 : 128), (NQ_ != 4)>( /* PIPE: see v4 notes */(const char *) Q->data, (const char *) K->data, (const char *) V->data, \ mdata, (float *) dst->data, parts.get(), meta.get(), scale, ne01, (int) Q->ne[2], ne11, (int) K->ne[2], \ (int) Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], V->nb[1], V->nb[2], V->nb[3], nb31, \ diff --git a/ggml/src/ggml-sycl/xe2-dpas.hpp b/ggml/src/ggml-sycl/xe2-dpas.hpp index 8d2a13bae851..c814bd6da7db 100644 --- a/ggml/src/ggml-sycl/xe2-dpas.hpp +++ b/ggml/src/ggml-sycl/xe2-dpas.hpp @@ -47,4 +47,36 @@ inline f32x8 dpas_hf_r8(s8x8 a, i32x8 b, f32x8 acc) { #endif return acc; } +// ARC-LAB: transposed 2D block load (lsc_load_block2d, d32, 8 dwords x 16 rows): lane l receives row y + l, component c = +// dword x + c of that row. Surface: base (64-byte aligned), width / height / pitch minus one (bytes, rows, bytes; pitch a +// multiple of 16); rows or dwords outside the surface read as zero. One message replaces 8 per-lane dword gathers. +// The arguments must be uniform over the subgroup; lane 0's values are used (copied into scalars here, so IGC need not +// prove uniformity: "rw.u" operands failed its inline-asm validation after a spill recompile). +typedef unsigned u32x8 __attribute__((ext_vector_type(8))); +inline u32x8 ld2d_t32_8x16(long long base, int wm1, int hm1, int pm1, int x, int y) { + u32x8 v; +#ifdef __SYCL_DEVICE_ONLY__ + __asm__("{\n" + ".decl SB v_type=G type=q num_elts=1 align=qword\n" + ".decl SW v_type=G type=d num_elts=1 align=dword\n" + ".decl SH v_type=G type=d num_elts=1 align=dword\n" + ".decl SP v_type=G type=d num_elts=1 align=dword\n" + ".decl SX v_type=G type=d num_elts=1 align=dword\n" + ".decl SY v_type=G type=d num_elts=1 align=dword\n" + "mov (M1_NM, 1) SB(0,0)<1> %1(0,0)<0;1,0>\n" + "mov (M1_NM, 1) SW(0,0)<1> %2(0,0)<0;1,0>\n" + "mov (M1_NM, 1) SH(0,0)<1> %3(0,0)<0;1,0>\n" + "mov (M1_NM, 1) SP(0,0)<1> %4(0,0)<0;1,0>\n" + "mov (M1_NM, 1) SX(0,0)<1> %5(0,0)<0;1,0>\n" + "mov (M1_NM, 1) SY(0,0)<1> %6(0,0)<0;1,0>\n" + "lsc_load_block2d.ugm (M1, 1) %0:d32.8x16tn flat[SB,SW,SH,SP,SX,SY]\n" + "}\n" + : "=rw"(v) + : "rw"(base), "rw"(wm1), "rw"(hm1), "rw"(pm1), "rw"(x), "rw"(y)); +#else + v = 0; + (void) base; (void) wm1; (void) hm1; (void) pm1; (void) x; (void) y; +#endif + return v; +} } // namespace xe2dp From 82aad281f8bac3568caeefcd6d742917519e1aa0 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 10 Oct 2026 21:41:28 +1100 Subject: [PATCH 39/65] sycl fattn-dec: 256-register file for the DPAS decode kernel at 3+ query tokens (GGML_SYCL_FA_DEC_GRF, default 256) The 3+ token launch is one wave of 2 work-groups x 16 subgroups per Xe core = 32 threads, which is what the 256-register mode allows, so it costs no occupancy and removes the spills. 1-2 tokens keep the default register file (their wave puts 4 work-groups on a core). GGML_SYCL_FA_DEC_GRF=0 = off. B580, us per layer at 128K (real cache layout) 5 / 8 / 12 / 16 tokens: 1073 / 1434 / 2920 / 2939 -> 1004 / 1240 / 2465 / 2445; 1 token unchanged. Whole model (arctune x3, MTP + n-gram, sd3 head): 48K 106.1 -> 108.2 t/s (+2.0%), 115K 86.8 -> 90.6 (+4.4%), output md5 identical (d09ea3 / 291b0b). FLASH_ATTN_EXT q4_0: 480/480. Perf cases: kv 16K/64K/128K x 1/5/8/12/16 tokens. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 1eb23072230925134f76a2a104337671494c5aef on Torchit1/llama.cpp arc-b580) Also carries the FLASH_ATTN_EXT perf cases this commit widens, first added by 1fa26f1561a42d41de7d574511fe8a7aa2d63e5d (Jesse Symons), which is not picked separately. --- ggml/src/ggml-sycl/fattn-dec.cpp | 28 +++++++++++++++++++++++++--- tests/test-backend-ops.cpp | 5 +++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-sycl/fattn-dec.cpp b/ggml/src/ggml-sycl/fattn-dec.cpp index 02a2103ac245..479470671acf 100644 --- a/ggml/src/ggml-sycl/fattn-dec.cpp +++ b/ggml/src/ggml-sycl/fattn-dec.cpp @@ -12,6 +12,7 @@ #include "xe2-dpas.hpp" #include +#include #include "dpct/helper.hpp" #include "common.hpp" #include "fattn-common.hpp" @@ -703,9 +704,24 @@ typedef unsigned dp_uint4 __attribute__((ext_vector_type(4))); // (consecutive work-items read consecutive dwords of a key row) and the lane = key reads come from there (row stride 37 // dwords: conflict-free). Without it every lane reads its own key straight from memory: 16 cache lines per load // instruction, ~90-110 GB/s per phase at 48K. The V copy is issued before the softmax and shares its barrier. +// ARC-LAB: launches a kernel lambda with the 256-register file (half the threads per EU; the 3+ token one-wave launch +// already runs only 2 work-groups x 16 subgroups = 32 threads per Xe core, which is what 256-register mode allows) +template struct fattn_dec_grf256 { + F f; + void operator()(sycl::nd_item<3> it) const { f(it); } + auto get(sycl::ext::oneapi::experimental::properties_tag) const { + return sycl::ext::oneapi::experimental::properties{ sycl::ext::oneapi::experimental::sub_group_size<16>, + sycl::ext::intel::experimental::grf_size<256> }; + } +}; + // BL bit 0 (GGML_SYCL_FA_DEC_B2D, ST = 0 only): the lane = key reads use transposed 2D block loads (xe2dp::ld2d_t32_8x16: // 16 keys x 8 dwords per message) instead of per-lane dword gathers. Same bytes, same math. Needs 64-byte aligned surfaces. // B580, 128K, us: on its own SLOWER (spills; 5 tok 1069 -> 1220, 16 tok 2945 -> 3610). Off by default. +// BL bit 1: launch with the 256-register file (GGML_SYCL_FA_DEC_GRF, default 256 = on for 3+ query tokens; 0 = off). Not +// for 1-2 tokens: their one-wave launch puts 4 work-groups on a core, twice what 256-register mode can hold. +// B580, us per layer, 128K 5 / 8 / 16 tokens: 1070 / 1419 / 2840 -> 988-1006 / 1250 / 2450-2507; whole model (arctune x3, +// MTP + n-gram, sd3 head) 48K 105.2 -> 107.7 t/s, 115K 86.6 -> 90.5, outputs identical. template static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, const char * mask, float * dst, float * parts, sycl::float2 * meta, float scale, int ne01, int ne02, int ne11, @@ -1080,7 +1096,11 @@ static void fattn_dec_q4_0_dpas(const char * Q, const char * K, const char * V, meta[jdu * nsplit + split] = sycl::float2(sM[tid], sL[tid]); } }; - cgh.parallel_for(ndr, kern); + if constexpr ((BL & 2) != 0) { + cgh.parallel_for(ndr, fattn_dec_grf256{ kern }); + } else { + cgh.parallel_for(ndr, kern); + } }); } @@ -1176,7 +1196,7 @@ void ggml_sycl_flash_attn_ext_dec(ggml_backend_sycl_context & ctx, ggml_tensor * (int) (((uintptr_t) K->data) % 64), K->nb[1], K->nb[2]); } } - const int bl = b2d ? 1 : 0; + const int bl = (b2d ? 1 : 0) | ((ggml_sycl_knob("GGML_SYCL_FA_DEC_GRF", 256) == 256 && nq >= 4) ? 2 : 0); const int nq5 = (dpas && !d512 && !g2 && ne01 == 5) ? nq5_env : 0; const bool stage = dpas && stage_env && !d512 && !g2 && (nq == 1 || nq5 == 2) && !(b2d && b2d_env >= 2 && nq == 1); const int tk = (d512 || g2) ? (nq == 1 ? 256 : 128) @@ -1217,7 +1237,9 @@ void ggml_sycl_flash_attn_ext_dec(ggml_backend_sycl_context & ctx, ggml_tensor * (const char *) K->data, (const char *) V->data, mdata, (float *) dst->data, parts.get(), meta.get(), scale, ne01, \ (int) Q->ne[2], ne11, (int) K->ne[2], (int) Q->ne[3], Q->nb[1], Q->nb[2], Q->nb[3], K->nb[1], K->nb[2], K->nb[3], \ V->nb[1], V->nb[2], V->nb[3], nb31, nb33, ne33, nsplit, chunk, stream) -#define FATTN_DEC_DPAS(NQ_, KG_, TC_) do { if (bl) { FATTN_DEC_DPAS_BL(NQ_, KG_, TC_, 1); } else { FATTN_DEC_DPAS_BL(NQ_, KG_, TC_, 0); } } while (0) +#define FATTN_DEC_DPAS(NQ_, KG_, TC_) do { switch (bl) { \ + case 3: FATTN_DEC_DPAS_BL(NQ_, KG_, TC_, 3); break; case 2: FATTN_DEC_DPAS_BL(NQ_, KG_, TC_, 2); break; \ + case 1: FATTN_DEC_DPAS_BL(NQ_, KG_, TC_, 1); break; default: FATTN_DEC_DPAS_BL(NQ_, KG_, TC_, 0); break; } } while (0) if (d512) { #define FATTN_DEC_DPAS512(NQ_, KG_, TC_) fattn_dec_q4_0_dpas<512, 16, NQ_, KG_, TC_>((const char *) Q->data, (const char *) K->data, \ (const char *) V->data, mdata, (float *) dst->data, parts.get(), meta.get(), scale, ne01, (int) Q->ne[2], ne11, \ diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index cdf8765da7e1..327a63de1ed2 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -11193,6 +11193,11 @@ static std::vector> make_test_cases_perf() { for (int64_t nb : {1, 4, 64, 256}) { // ARC-LAB: ~115K history (Bonsai at 128K context), real cache layout test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 114688, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, {0, 2, 1, 3})); } + for (int64_t kv : {16384, 65536, 131072}) { // ARC-LAB: Bonsai verify shapes (1 token, MTP 5, n-gram 8-16), real cache layout + for (int64_t nb : {1, 5, 8, 12, 16}) { + test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, kv, nb, true, false, 0, 0, GGML_PREC_F32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0, {0, 2, 1, 3})); + } + } for (ggml_type t : {GGML_TYPE_Q4_0, GGML_TYPE_F16}) { // ARC-LAB: Bonsai 27B prompt batch at 16K history test_cases.emplace_back(new test_flash_attn_ext(256, 256, 4, {6, 1}, 16384, 512, true, false, 0, 0, GGML_PREC_F32, t, t)); for (int64_t kv : {512, 16384}) { // real KV-cache layout (heads interleaved per token) From b7c8d8ccb9e7203f17eb1eba15334d00a2d6bc35 Mon Sep 17 00:00:00 2001 From: bri-prism <288398250+bri-prism@users.noreply.github.com> Date: Tue, 13 Oct 2026 13:26:59 -0700 Subject: [PATCH 40/65] sycl: one in-place layout owner for PQ2_0/PTQ1_0 weights The port's MMVQ reorder (opt_for_reorder, also called by the fused gate/up path) and the PQ2_0 XMX rewrite (#294) both change these weights in place, and neither checked the other's flag. With GGML_SYCL_PTQ1_T2 unset a weight could get both layouts, so 27B PTQ1_0 perplexity went to 139615 on the B390 while every op test passed. On 16-wide DPAS devices the XMX layout now owns PQ2_0/PTQ1_0 and the reorder skips them; the fused paths that need the reorder decline and fall back to the unfused XMX path. The XMX path also refuses a weight that is already reordered or T2-repacked. --- ggml/src/ggml-sycl/ggml-sycl.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 1936d116af18..72f56a6e075d 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -4841,6 +4841,13 @@ static void opt_for_reorder(ggml_backend_sycl_context * ctx, const ggml_tensor * return; // Skip permutations and already reordered tensors } + // PQ2_0/PTQ1_0 on 16-wide DPAS devices belong to the XMX layout (ggml_sycl_pq2_xmx_use), which rewrites the same + // bytes in place; reordering them as well would leave one path reading the other's layout + if ((src0->type == GGML_TYPE_PQ2_0 || src0->type == GGML_TYPE_PTQ1_0) && + (extra->optimized_feature.xmx_pq2 || (!ggml_sycl_xmx_disabled() && ggml_sycl_device_has_dpas16(ctx->device)))) { + return; + } + switch (mm_algorithm) { case mul_mat_algo::DMMV: if (!ggml_sycl_supports_reorder_dmmv(src0->type)) { @@ -4914,6 +4921,10 @@ static bool ggml_sycl_pq2_xmx_use(ggml_backend_sycl_context & ctx, const ggml_te if (extra && extra->optimized_feature.xmx_pq2) { return true; } + // already in another in-place layout (MMVQ reorder or the TernSYCL 2-bit repack): rewriting it would corrupt it + if (extra && (extra->optimized_feature.reorder || extra->optimized_feature.t2)) { + return false; + } if (!g_ggml_sycl_enable_optimize || ggml_sycl_xmx_disabled() || !ggml_sycl_device_has_dpas16(ctx.device)) { return false; } From 888d75e8b9967fde787e607444669a3b8031f88c Mon Sep 17 00:00:00 2001 From: bri-prism <288398250+bri-prism@users.noreply.github.com> Date: Tue, 13 Oct 2026 13:30:30 -0700 Subject: [PATCH 41/65] sycl: drop PQ2_0/PTQ1_0 duplicates left by the arc-b580 merge Keep prism's dequantize, vec_dot and MMVQ dispatch; keep the port's reorder-aware convert/MMVQ cases. Prism (#235/#294) and the port both added PQ2_0/PTQ1_0 support, so the merged tree had duplicate dequant functions, switch cases and two MMVQ implementations with the same names and did not compile. Deletions only, found and built on the B390 (oneAPI 2025.3.2). --- ggml/src/ggml-sycl/convert.cpp | 8 -- ggml/src/ggml-sycl/dequantize.hpp | 32 ------- ggml/src/ggml-sycl/ggml-sycl.cpp | 2 - ggml/src/ggml-sycl/mmvq.cpp | 148 ------------------------------ ggml/src/ggml-sycl/vecdotq.hpp | 50 ---------- 5 files changed, 240 deletions(-) diff --git a/ggml/src/ggml-sycl/convert.cpp b/ggml/src/ggml-sycl/convert.cpp index b87c50f9eca7..f30625fe8b11 100644 --- a/ggml/src/ggml-sycl/convert.cpp +++ b/ggml/src/ggml-sycl/convert.cpp @@ -686,10 +686,6 @@ to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst) { switch (type) { case GGML_TYPE_Q1_0: return dequantize_block_sycl; - case GGML_TYPE_PTQ1_0: - return dequantize_block_sycl; - case GGML_TYPE_PQ2_0: - return dequantize_block_sycl; case GGML_TYPE_Q2_0: return dequantize_block_sycl; case GGML_TYPE_PTQ1_0: @@ -792,10 +788,6 @@ to_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type, ggml_tensor *dst) { switch (type) { case GGML_TYPE_Q1_0: return dequantize_block_sycl; - case GGML_TYPE_PTQ1_0: - return dequantize_block_sycl; - case GGML_TYPE_PQ2_0: - return dequantize_block_sycl; case GGML_TYPE_Q2_0: return dequantize_block_sycl; case GGML_TYPE_PTQ1_0: diff --git a/ggml/src/ggml-sycl/dequantize.hpp b/ggml/src/ggml-sycl/dequantize.hpp index 90d2ac1f89d8..076380eb05a9 100644 --- a/ggml/src/ggml-sycl/dequantize.hpp +++ b/ggml/src/ggml-sycl/dequantize.hpp @@ -47,28 +47,6 @@ static __dpct_inline__ void dequantize_q2_0(const void *vx, const int64_t ib, #endif // GGML_SYCL_F16 } -static __dpct_inline__ void dequantize_pq2_0(const void *vx, const int64_t ib, - const int iqs, dfloat2 &v) { - const block_pq2_0 * x = (const block_pq2_0 *) vx; - - const dfloat d = x[ib].d; - - const int byte_idx = iqs / 4; - const int shift = (iqs % 4) * 2; - const uint8_t vui = x[ib].qs[byte_idx]; - - v.x() = (vui >> shift) & 3; - v.y() = (vui >> (shift + 2)) & 3; - -#ifdef GGML_SYCL_F16 - v.s0() = ((dfloat)v.s0() - 1.0f) * d; - v.s1() = ((dfloat)v.s1() - 1.0f) * d; -#else - v.x() = ((dfloat)v.x() - 1.0f) * d; - v.y() = ((dfloat)v.y() - 1.0f) * d; -#endif // GGML_SYCL_F16 -} - // Trit of element e (0..127) in a PTQ1_0 block: 16 bytes x 5, 8 bytes x 5, then 2 qh bytes x 4. static __dpct_inline__ int ptq1_0_trit(const uint8_t * qs, const uint8_t * qh, const int e) { uint32_t v; @@ -89,16 +67,6 @@ static __dpct_inline__ int ptq1_0_trit(const uint8_t * qs, const uint8_t * qh, c return (int) ((v * 3) >> 8); } -static __dpct_inline__ void dequantize_ptq1_0(const void *vx, const int64_t ib, - const int iqs, dfloat2 &v) { - const block_ptq1_0 * x = (const block_ptq1_0 *) vx; - - const dfloat d = x[ib].d; - - v.x() = (dfloat) (ptq1_0_trit(x[ib].qs, x[ib].qh, iqs + 0) - 1) * d; - v.y() = (dfloat) (ptq1_0_trit(x[ib].qs, x[ib].qh, iqs + 1) - 1) * d; -} - static __dpct_inline__ void dequantize_q4_0(const void *vx, const int64_t ib, const int iqs, dfloat2 &v) { const block_q4_0 * x = (const block_q4_0 *) vx; diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 72f56a6e075d..963c50806142 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -641,8 +641,6 @@ ggml_backend_sycl_buffer_init_tensor(ggml_backend_buffer_t buffer, switch (tensor->type) { case GGML_TYPE_Q4_0: case GGML_TYPE_Q8_0: - case GGML_TYPE_PQ2_0: - case GGML_TYPE_PTQ1_0: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: case GGML_TYPE_Q4_K: diff --git a/ggml/src/ggml-sycl/mmvq.cpp b/ggml/src/ggml-sycl/mmvq.cpp index 49e3d7e1bafe..7aaef60032ac 100644 --- a/ggml/src/ggml-sycl/mmvq.cpp +++ b/ggml/src/ggml-sycl/mmvq.cpp @@ -2205,126 +2205,6 @@ static void mul_mat_vec_q2_0_q8_1_sycl_switch_ncols( } } -static void mul_mat_vec_pq2_0_q8_1_sycl(const void * vx, const void * vy, - float * dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_PQ2_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); -} - -template -static void mul_mat_vec_pq2_0_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_PQ2_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols( - vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); - }); - }); -} - -static void mul_mat_vec_pq2_0_q8_1_sycl_switch_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, const int ncols_dst, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_pq2_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_pq2_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_pq2_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_pq2_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_pq2_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_pq2_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_pq2_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_pq2_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for PQ2_0 multi-col MMVQ", ncols_dst); - } -} - -static void mul_mat_vec_ptq1_0_q8_1_sycl(const void * vx, const void * vy, - float * dst, const int ncols, - const int nrows, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_PTQ1_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q( - vx, vy, dst, ncols, nrows, item_ct1); - }); - }); -} - -template -static void mul_mat_vec_ptq1_0_q8_1_sycl_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - GGML_ASSERT(ncols % QK_PTQ1_0 == 0); - const int block_num_y = (nrows + GGML_SYCL_MMV_Y - 1) / GGML_SYCL_MMV_Y; - const sycl::range<3> block_nums(1, 1, block_num_y); - const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, WARP_SIZE); - - stream->submit([&](sycl::handler & cgh) { - cgh.parallel_for( - sycl::nd_range<3>(block_nums * block_dims, block_dims), - [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { - mul_mat_vec_q_ncols( - vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, item_ct1); - }); - }); -} - -static void mul_mat_vec_ptq1_0_q8_1_sycl_switch_ncols( - const void * vx, const void * vy, float * dst, - const int ncols, const int nrows, const int ncols_dst, - const int stride_col_y, const int stride_col_dst, - dpct::queue_ptr stream) { - switch (ncols_dst) { - case 1: mul_mat_vec_ptq1_0_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break; - case 2: mul_mat_vec_ptq1_0_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 3: mul_mat_vec_ptq1_0_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 4: mul_mat_vec_ptq1_0_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 5: mul_mat_vec_ptq1_0_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 6: mul_mat_vec_ptq1_0_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 7: mul_mat_vec_ptq1_0_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - case 8: mul_mat_vec_ptq1_0_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y, stride_col_dst, stream); break; - default: GGML_ABORT("unsupported ncols_dst=%d for PTQ1_0 multi-col MMVQ", ncols_dst); - } -} - static void mul_mat_vec_q2_K_q8_1_sycl(const void *vx, const void *vy, float *dst, const int ncols, const int nrows, @@ -3518,34 +3398,6 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens mul_mat_vec_q1_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); } break; - case GGML_TYPE_PTQ1_0: - if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { - const int stride_col_y = src1_padded_col_size / QK8_1; - const int stride_col_dst = dst->ne[0]; - GGML_SYCL_DEBUG("Calling mul_mat_vec_ptq1_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols); - mul_mat_vec_ptq1_0_q8_1_sycl_switch_ncols( - src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, - src1_ncols, stride_col_y, stride_col_dst, stream); - return; - } else if (i == 0 || src1_ncols == 1) { - GGML_SYCL_DEBUG("Calling mul_mat_vec_ptq1_0_q8_1_sycl\n"); - mul_mat_vec_ptq1_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); - } - break; - case GGML_TYPE_PQ2_0: - if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { - const int stride_col_y = src1_padded_col_size / QK8_1; - const int stride_col_dst = dst->ne[0]; - GGML_SYCL_DEBUG("Calling mul_mat_vec_pq2_0_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols); - mul_mat_vec_pq2_0_q8_1_sycl_switch_ncols( - src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff, - src1_ncols, stride_col_y, stride_col_dst, stream); - return; - } else if (i == 0 || src1_ncols == 1) { - GGML_SYCL_DEBUG("Calling mul_mat_vec_pq2_0_q8_1_sycl\n"); - mul_mat_vec_pq2_0_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream); - } - break; case GGML_TYPE_Q2_0: if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) { const int stride_col_y = src1_padded_col_size / QK8_1; diff --git a/ggml/src/ggml-sycl/vecdotq.hpp b/ggml/src/ggml-sycl/vecdotq.hpp index 17efc56f5467..4700506556ab 100644 --- a/ggml/src/ggml-sycl/vecdotq.hpp +++ b/ggml/src/ggml-sycl/vecdotq.hpp @@ -1258,56 +1258,6 @@ vec_dot_q2_0_q8_1(const void *__restrict__ vbq, return sum0 + sum1; } -#define VDR_PQ2_0_Q8_1_MMVQ 1 -#define VDR_PTQ1_0_Q8_1_MMVQ 4 // one call takes a whole 128 block - - -// Same codec as Q2_0 at group 128: each int of qs holds 16 codes, i.e. half of one q8_1 block. -static __dpct_inline__ float -vec_dot_pq2_0_q8_1(const void *__restrict__ vbq, - const block_q8_1 *__restrict__ bq8_1, const int &iqs) { - - const block_pq2_0 * bq = (const block_pq2_0 *) vbq; - - float sum = 0.0f; -#pragma unroll - for (int h = 0; h < 2; ++h) { - const int k = iqs + h * QI_PQ2_0; - const int v = get_int_from_uint8(bq->qs, k); - const block_q8_1 & y = bq8_1[k / 2]; - const int base = 4 * (k % 2); - - int u[4]; -#pragma unroll - for (int j = 0; j < 4; ++j) { - u[j] = get_int_from_int8_aligned(y.qs, base + j); - } - sum += vec_dot_q2_0_q8_1_impl(&v, u, bq->d, y.ds); - } - return sum; -} - -static __dpct_inline__ float -vec_dot_ptq1_0_q8_1(const void *__restrict__ vbq, - const block_q8_1 *__restrict__ bq8_1, const int &iqs) { - GGML_UNUSED(iqs); - const block_ptq1_0 * bq = (const block_ptq1_0 *) vbq; - - int8_t x[QK_PTQ1_0]; - sycl::half2 ds[4]; -#pragma unroll - for (int k = 0; k < 4; ++k) { -#pragma unroll - for (int j = 0; j < QK8_1 / 4; ++j) { - *(int *) (x + k * QK8_1 + 4 * j) = get_int_from_int8_aligned(bq8_1[k].qs, j); - } - ds[k] = bq8_1[k].ds; - } - int sumi[4] = { 0, 0, 0, 0 }; - ptq1_0_dot_block(bq->qs, (uint32_t) bq->qh[0] | ((uint32_t) bq->qh[1] << 16), x, sumi); - return ptq1_0_finish(sumi, ds, bq->d); -} - static __dpct_inline__ float vec_dot_q4_1_q8_1(const void *__restrict__ vbq, const block_q8_1 *__restrict__ bq8_1, const int &iqs) { From 79da29e2b8070b2b0a31f692c1a015155da04cb7 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Mon, 28 Sep 2026 08:14:05 +1000 Subject: [PATCH 42/65] qwen35 MTP: optional trimmed draft head (top-K frequent tokens, FR-Spec style) blk..nextn.draft_head (rows of output.weight for the K most frequent tokens plus all special tokens) and draft_head_ids (I32). When present, the MTP draft computes logits for those K rows only and scatters them into a full-vocab row of -inf with set_rows, so everything downstream is unchanged and the target model still verifies every draft. The head is listed under prism.hadamard.weight_names (same input-side rotation as output.weight). LLAMA_MTP_DRAFT_HEAD_OFF=1 uses the full head. Loader: GGML_OP_SET_ROWS placement check so the I32 id tensor lands in device memory. Bonsai 27B, K = 32768 (99.6% token coverage of a 4M-token code/prose corpus): the draft head reads 7.6x fewer bytes; fresh code temp 0 90.3 -> 94.8-95.8 t/s with identical output, temp 0.6 neutral within noise. K = 16384 is worse (92.1, 76 vs 81). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit e75a9943ca47e8c6db6fc1781fcfc4de8dc14367 on Torchit1/llama.cpp arc-b580) --- src/llama-arch.cpp | 4 ++++ src/llama-arch.h | 2 ++ src/llama-model-loader.cpp | 7 +++++++ src/llama-model.cpp | 1 + src/llama-model.h | 2 ++ src/models/qwen35.cpp | 23 ++++++++++++++++++++++- 6 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 4c3fcb2575a4..2119098b0420 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -562,6 +562,8 @@ static const std::map LLM_TENSOR_NAMES = { { LLM_TENSOR_NEXTN_HNORM, "blk.%d.nextn.hnorm" }, { LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "blk.%d.nextn.shared_head_head" }, { LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "blk.%d.nextn.shared_head_norm" }, + { LLM_TENSOR_NEXTN_DRAFT_HEAD, "blk.%d.nextn.draft_head" }, + { LLM_TENSOR_NEXTN_DRAFT_HEAD_IDS, "blk.%d.nextn.draft_head_ids" }, { LLM_TENSOR_ATTN_SUB_NORM, "blk.%d.attn_sub_norm" }, { LLM_TENSOR_FFN_SUB_NORM, "blk.%d.ffn_sub_norm" }, { LLM_TENSOR_DEC_OUTPUT_NORM, "dec.output_norm" }, @@ -957,6 +959,8 @@ static const std::map LLM_TENSOR_INFOS = { { LLM_TENSOR_NEXTN_HNORM, { LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL } }, { LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, { LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT } }, { LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, { LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL } }, + { LLM_TENSOR_NEXTN_DRAFT_HEAD, { LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT } }, + { LLM_TENSOR_NEXTN_DRAFT_HEAD_IDS, { LLM_TENSOR_LAYER_REPEATING, GGML_OP_SET_ROWS } }, // dspark drafter extras. fc / markov factors are matmuls; the two norms are // elementwise muls. All output-side (non-repeating) so the loader does not // require a per-block index. diff --git a/src/llama-arch.h b/src/llama-arch.h index f292e497e5ec..bdd58a7667d1 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -676,6 +676,8 @@ enum llm_tensor { LLM_TENSOR_NEXTN_HNORM, LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, + LLM_TENSOR_NEXTN_DRAFT_HEAD, // ARC-LAB: MTP LM head over the top-K frequent tokens only + LLM_TENSOR_NEXTN_DRAFT_HEAD_IDS, // ARC-LAB: vocab ids of its rows LLM_TENSOR_MASKED_EMBD_CENTROIDS, LLM_TENSOR_MASKED_EMBD_ORDERING, LLM_TENSOR_FC, diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 835d172d33f7..23d74c30a396 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -948,6 +948,13 @@ static bool weight_buft_supported(const llama_hparams & hparams, ggml_tensor * w ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 512); op_tensor = ggml_get_rows(ctx, w, b); } break; + case GGML_OP_SET_ROWS: + { + // ARC-LAB: w = I32 row indices (MTP draft-head vocab ids), scattered into a full logits row + ggml_tensor * dst = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 1, w->ne[0], 1); + ggml_tensor * src = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 1, w->ne[0], 1); + op_tensor = ggml_set_rows(ctx, dst, src, w); + } break; case GGML_OP_MUL_MAT: { ggml_tensor * b = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, w->ne[0], 512, w->ne[2], w->ne[3]); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index c2ac6a50b316..a31e031fb1c3 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1291,6 +1291,7 @@ void llama_model_base::load_hparams(llama_model_loader & ml) { "ffn_gate_exps", "ffn_up_exps", "ffn_down_exps", "ffn_gate_up_exps", "ffn_gate_shexp", "ffn_up_shexp", "ffn_down_shexp", "ssm_out", + "nextn.draft_head", // ARC-LAB: row subset of output.weight, same input-side rotation }; if (name == "output.weight") { return true; // the output head is built through build_lora_mm in every arch diff --git a/src/llama-model.h b/src/llama-model.h index 8834271a101a..d93326b86458 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -223,6 +223,8 @@ struct llama_layer_nextn { struct ggml_tensor * enorm = nullptr; struct ggml_tensor * hnorm = nullptr; struct ggml_tensor * shared_head_head = nullptr; + struct ggml_tensor * draft_head = nullptr; // ARC-LAB: [n_embd, K] rows of the LM head for top-K tokens + struct ggml_tensor * draft_head_ids = nullptr; // ARC-LAB: I32 [K] vocab id of each row struct ggml_tensor * shared_head_head_s = nullptr; struct ggml_tensor * shared_head_head_in_s = nullptr; struct ggml_tensor * shared_head_norm = nullptr; diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp index 48a2cd158f35..72fc50a942ce 100644 --- a/src/models/qwen35.cpp +++ b/src/models/qwen35.cpp @@ -117,6 +117,12 @@ void llama_model_qwen35::load_arch_tensors(llama_model_loader & ml) { layer.nextn.embed_tokens = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS, "weight", il), { n_embd, n_vocab }, mtp_flags|TENSOR_NOT_REQUIRED); layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", il), { n_embd, n_vocab }, mtp_flags|TENSOR_NOT_REQUIRED); layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", il), { n_embd }, mtp_flags|TENSOR_NOT_REQUIRED); + // ARC-LAB: optional trimmed draft head (FR-Spec style): the draft only proposes the K most frequent tokens + if (const ggml_tensor * dh = ml.get_tensor_meta(tn(LLM_TENSOR_NEXTN_DRAFT_HEAD, "weight", il).str().c_str())) { + const int64_t n_draft_vocab = dh->ne[1]; + layer.nextn.draft_head = create_tensor(tn(LLM_TENSOR_NEXTN_DRAFT_HEAD, "weight", il), { n_embd, n_draft_vocab }, mtp_flags|TENSOR_NOT_REQUIRED); + layer.nextn.draft_head_ids = create_tensor(tn(LLM_TENSOR_NEXTN_DRAFT_HEAD_IDS, "weight", il), { n_draft_vocab }, mtp_flags|TENSOR_NOT_REQUIRED); + } }; for (int i = 0; i < n_layer; ++i) { @@ -777,7 +783,22 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr ggml_tensor * head_w = layer.nextn.shared_head_head ? layer.nextn.shared_head_head : model.output; ggml_tensor * head_s = layer.nextn.shared_head_head ? layer.nextn.shared_head_head_s : model.output_s; GGML_ASSERT(head_w && "QWEN35 MTP: missing LM head (nextn.shared_head_head or model.output)"); - cur = build_lora_mm(head_w, cur, head_s); + static const bool draft_head_off = getenv("LLAMA_MTP_DRAFT_HEAD_OFF") != nullptr; + if (layer.nextn.draft_head && layer.nextn.draft_head_ids && !layer.nextn.shared_head_head && !draft_head_off) { + // ARC-LAB: logits for the top-K frequent tokens only, scattered into a full-vocab row of -inf; the target + // model still verifies every draft, so this changes only which tokens can be drafted, not the output + const int64_t n_k = layer.nextn.draft_head->ne[1]; + const int64_t n_out = cur->ne[1]; + const int64_t n_voc = head_w->ne[1]; + ggml_tensor * lk = build_lora_mm(layer.nextn.draft_head, cur, nullptr); // [K, n_out] + ggml_tensor * full = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, 1, n_voc, n_out); + full = ggml_fill(ctx0, full, -INFINITY); + full = ggml_set_rows(ctx0, full, ggml_reshape_3d(ctx0, lk, 1, n_k, n_out), + ggml_reshape_2d(ctx0, layer.nextn.draft_head_ids, n_k, 1)); + cur = ggml_reshape_2d(ctx0, full, n_voc, n_out); + } else { + cur = build_lora_mm(head_w, cur, head_s); + } cb(cur, "result_output", -1); res->t_logits = cur; From 1a5c67d6191acdac5e4f400dacbf1cbf5d211cf0 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 3 Oct 2026 15:50:28 +1000 Subject: [PATCH 43/65] speculative ngram-mod: adaptive draft length (start 64, double on full accept, halve on rejection) A copy ends where the model stops but the context goes on, so with --spec-ngram-mod-n-max 256 a short copy drafted ~200 tokens past its end; at 48K context verifying those cost ~0.3 s each and made copying slower than writing new code. Drafts are now cut to n_cur (starts at 64, doubles after a fully accepted draft up to n_max, halves after a rejected one). B580, Bonsai 27B, MTP + ngram-mod, same outputs: 48K short copy 38.6 -> 57.3 t/s, 48K 1262-token copy 161.6 -> 172.9, 48K new code 78.5 -> 79.4, short rename 428 -> 411, edit 305 -> 299. LLAMA_ARG_SPEC_NGRAM_ADAPT=0 restores the fixed length; =N sets the starting length. Co-Authored-By: Claude (cherry picked from commit e576a2879f5a1e1c7b1c04e871e880a337824926 on Torchit1/llama.cpp arc-b580) --- common/speculative.cpp | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/common/speculative.cpp b/common/speculative.cpp index 23843abe0139..f8acb3792bfb 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -2984,10 +2984,24 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl { // consecutive accept rounds with low acceptance fraction (< 0.5) int n_low = 0; + + // ARC-LAB adaptive draft length: drafts are cut to n_cur tokens; n_cur doubles after a fully accepted draft + // (up to n_max) and halves after a rejected one (down to n_start). A copy ends where the model stops, but the + // context continues, so a fixed n_max of 256 threw away ~200 drafted tokens per short copy - at 48K context + // verifying those costs ~0.3 s each. + int n_cur = 0; }; std::vector sinfos; + // LLAMA_ARG_SPEC_NGRAM_ADAPT=0 disables it (fixed n_max as upstream); other values = starting length (default 64). + // B580, n_max 256, t/s fixed vs 16 / 64 / 128: 48K short copy 38.6 / 60.7 / 57.3 / 45.7, 48K long copy 161.6 / 165 / + // 172.9 / 178.9, short rename 428 / 394 / 411 / 416, edit 305 / 296 / 299 / 316 (same outputs). + int n_start = [] { + const char * e = std::getenv("LLAMA_ARG_SPEC_NGRAM_ADAPT"); + return e ? std::atoi(e) : 64; + }(); + common_speculative_impl_ngram_mod( const common_params_speculative & params, uint32_t n_seq) @@ -3016,6 +3030,7 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl { sinfo.i_last = 0; sinfo.n_draft_last = 0; + sinfo.n_cur = n_start > 0 ? std::min(n_start, params.n_max) : params.n_max; const size_t n = mod.get_n(); if (prompt.size() < n) { @@ -3091,6 +3106,11 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl { } result.resize(result.size() - n); + // adaptive length: the n_min check above still sees the whole chain; only the returned draft is cut + if (sinfo.n_cur > 0 && result.size() > (size_t) sinfo.n_cur) { + result.resize(sinfo.n_cur); + } + // store length of drafted n-gram for later acceptance analysis sinfo.n_draft_last = result.size(); } @@ -3120,6 +3140,11 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl { auto & sinfo = sinfos[seq_id]; + if (n_start > 0 && sinfo.n_draft_last > 0) { + sinfo.n_cur = n_accepted >= sinfo.n_draft_last ? std::min(2 * sinfo.n_cur, (int) params.n_max) + : std::max(std::min(n_start, (int) params.n_max), sinfo.n_cur / 2); + } + // compute acceptance fraction if we have a recorded draft length if (sinfo.n_draft_last > 0) { const double f_acc = (double)n_accepted / (double)sinfo.n_draft_last; From 0b8dd646fb82f44ce9bd8aab6eff432981f0fc68 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Mon, 5 Oct 2026 21:30:58 +1100 Subject: [PATCH 44/65] tools: llama-mtp-export; MTP draft log + lab speculative knobs llama-mtp-export writes, per position, the trunk state the NextN (MTP) head consumes (h_nextn) and the trunk's greedy pick, for training / validating the draft head offline. common/speculative: LLAMA_MTP_DRAFT_LOG (top-3 candidates per draft step), LAB_SPEC_DRAFT_N_MAX / LAB_SPEC_NGRAM_START runtime knobs via GGML_SYCL_KNOBS_FILE. Used for the B580 self-distilled MTP head (+4.8% with MTP, outputs identical). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 7c577265b0e554dee001e7a08db9576fff2b10f4 on Torchit1/llama.cpp arc-b580) --- common/speculative.cpp | 67 +++++++++++++++++- tools/CMakeLists.txt | 1 + tools/mtp-export/CMakeLists.txt | 4 ++ tools/mtp-export/mtp-export.cpp | 119 ++++++++++++++++++++++++++++++++ 4 files changed, 189 insertions(+), 2 deletions(-) create mode 100644 tools/mtp-export/CMakeLists.txt create mode 100644 tools/mtp-export/mtp-export.cpp diff --git a/common/speculative.cpp b/common/speculative.cpp index f8acb3792bfb..626b376a58a2 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -32,6 +32,55 @@ #include #include +// ARC-LAB: lab knobs from GGML_SYCL_KNOBS_FILE (lines NAME=V, re-read when the file changes) so one server can A/B +// speculative settings: LAB_SPEC_DRAFT_N_MAX caps the MTP draft below --spec-draft-n-max, LAB_SPEC_NGRAM_START replaces +// LLAMA_ARG_SPEC_NGRAM_ADAPT's starting length (both from the next request on). Unset = command line / env. +#include +#include +#include +// ARC-LAB: LLAMA_MTP_DRAFT_LOG= appends one JSON line per MTP draft step (position predicted, depth, top 3 candidates +// with probabilities, whether p_min stopped the draft) plus a "begin" line per request, for offline acceptance / tree +// simulation (arctune, 2026-10-05). +static FILE * common_mtp_draft_log() { + static FILE * f = [] { + const char * p = std::getenv("LLAMA_MTP_DRAFT_LOG"); + return p ? fopen(p, "a") : nullptr; + }(); + return f; +} +static int common_lab_knob(const char * name, int def) { + static const char * path = std::getenv("GGML_SYCL_KNOBS_FILE"); + if (!path) { + return def; + } + static std::mutex mtx; + static std::filesystem::file_time_type last{}; + static std::unordered_map m; + std::lock_guard lock(mtx); + std::error_code ec; + const auto t = std::filesystem::last_write_time(path, ec); + if (!ec && t != last) { + last = t; + m.clear(); + if (FILE * fp = fopen(path, "r")) { + char line[256]; + while (fgets(line, sizeof(line), fp)) { + char * eq = strchr(line, '='); + if (!eq || line[0] == '#') { + continue; + } + *eq = 0; + std::string k(line); + k.erase(k.find_last_not_of(" \t") + 1); + m[k] = atoi(eq + 1); + } + fclose(fp); + } + } + const auto it = m.find(name); + return it == m.end() ? def : it->second; +} + #define SPC_DBG(fmt, ...) LOG_DBG("spec %12.*s: " fmt, 12, __func__, __VA_ARGS__) #define SPC_TRC(fmt, ...) LOG_TRC("spec %12.*s: " fmt, 12, __func__, __VA_ARGS__) #define SPC_INF(fmt, ...) LOG_INF("spec %12.*s: " fmt, 12, __func__, __VA_ARGS__) @@ -2385,6 +2434,10 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { } void begin(llama_seq_id seq_id, const llama_tokens & prompt) override { + if (FILE * lf = common_mtp_draft_log()) { + fprintf(lf, "{\"begin\": %zu, \"seq\": %d}\n", prompt.size(), (int) seq_id); + fflush(lf); + } const int32_t N = (int32_t) prompt.size(); if (N <= 0) { return; @@ -2765,6 +2818,14 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { common_token_to_piece(ctx_dft, cur_p->data[k].id).c_str()); } + if (FILE * lf = common_mtp_draft_log()) { + const auto & dpl = dparams.at(seq_id); + fprintf(lf, "{\"pos\": %d, \"d\": %d, \"c\": [", (int) (dpl.n_past + i + 1), i); + for (int k = 0; k < std::min(3, (int) cur_p->size); ++k) { + fprintf(lf, "%s[%d, %.4f]", k ? ", " : "", cur_p->data[k].id, cur_p->data[k].p); + } + fprintf(lf, "], \"stop\": %d}\n", cur_p->data[0].p < params.p_min ? 1 : 0); + } // add drafted token for each sequence const llama_token id = cur_p->data[0].id; @@ -2791,7 +2852,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { result.push_back(id); - if (params.n_max <= (int) result.size()) { + if (std::min((int) params.n_max, common_lab_knob("LAB_SPEC_DRAFT_N_MAX", params.n_max)) <= (int) result.size()) { drafting[seq_id] = false; n_drafting--; continue; @@ -2997,10 +3058,11 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl { // LLAMA_ARG_SPEC_NGRAM_ADAPT=0 disables it (fixed n_max as upstream); other values = starting length (default 64). // B580, n_max 256, t/s fixed vs 16 / 64 / 128: 48K short copy 38.6 / 60.7 / 57.3 / 45.7, 48K long copy 161.6 / 165 / // 172.9 / 178.9, short rename 428 / 394 / 411 / 416, edit 305 / 296 / 299 / 316 (same outputs). - int n_start = [] { + const int n_start_env = [] { const char * e = std::getenv("LLAMA_ARG_SPEC_NGRAM_ADAPT"); return e ? std::atoi(e) : 64; }(); + int n_start = n_start_env; common_speculative_impl_ngram_mod( const common_params_speculative & params, @@ -3030,6 +3092,7 @@ struct common_speculative_impl_ngram_mod : public common_speculative_impl { sinfo.i_last = 0; sinfo.n_draft_last = 0; + n_start = common_lab_knob("LAB_SPEC_NGRAM_START", n_start_env); sinfo.n_cur = n_start > 0 ? std::min(n_start, params.n_max) : params.n_max; const size_t n = mod.get_n(); diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index da1b2bc707d7..986ad620058b 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -28,6 +28,7 @@ else() add_subdirectory(server) endif() add_subdirectory(tokenize) + add_subdirectory(mtp-export) # ARC-LAB: MTP head training data add_subdirectory(tts) add_subdirectory(mtmd) if (GGML_RPC) diff --git a/tools/mtp-export/CMakeLists.txt b/tools/mtp-export/CMakeLists.txt new file mode 100644 index 000000000000..769f6bb07c92 --- /dev/null +++ b/tools/mtp-export/CMakeLists.txt @@ -0,0 +1,4 @@ +set(TARGET llama-mtp-export) +add_executable(${TARGET} mtp-export.cpp) +target_link_libraries(${TARGET} PRIVATE llama ${CMAKE_THREAD_LIBS_INIT}) +target_compile_features(${TARGET} PRIVATE cxx_std_17) diff --git a/tools/mtp-export/mtp-export.cpp b/tools/mtp-export/mtp-export.cpp new file mode 100644 index 000000000000..87e99762c3bf --- /dev/null +++ b/tools/mtp-export/mtp-export.cpp @@ -0,0 +1,119 @@ +// ARC-LAB mtp-export: training / validation data for the NextN (MTP) draft head. +// Runs the trunk over token sequences (teacher forcing) and writes, per position t: +// h[t] the trunk hidden state the MTP head consumes (h_nextn = final-norm output), f16 [n_embd] +// tok[t] the input token at t +// tgt[t] the trunk's own greedy choice for position t + 1 (argmax of the logits at t) - what a draft must match +// The MTP head at position t sees (h[t - 1], tok[t]) and should predict tgt[t]. +// +// llama-mtp-export -m model.gguf -i seqs.i32 -o prefix [-c n_ctx] [-b n_batch] +// seqs.i32: repeated records of int32 n followed by n int32 token ids. Writes prefix.h (f16), prefix.tok (i32), +// prefix.tgt (i32), prefix.idx (int64 start offset of each sequence, plus the total at the end). +#include "llama.h" +#include "ggml.h" +#include "../../src/llama-ext.h" + +#include +#include +#include +#include +#include + +int main(int argc, char ** argv) { + std::string model_path, in_path, out_prefix; + int n_ctx = 4096, n_batch = 512; + for (int i = 1; i < argc; ++i) { + std::string a = argv[i]; + if (a == "-m" && i + 1 < argc) model_path = argv[++i]; + else if (a == "-i" && i + 1 < argc) in_path = argv[++i]; + else if (a == "-o" && i + 1 < argc) out_prefix = argv[++i]; + else if (a == "-c" && i + 1 < argc) n_ctx = atoi(argv[++i]); + else if (a == "-b" && i + 1 < argc) n_batch = atoi(argv[++i]); + } + if (model_path.empty() || in_path.empty() || out_prefix.empty()) { + fprintf(stderr, "usage: %s -m model.gguf -i seqs.i32 -o prefix [-c n_ctx] [-b n_batch]\n", argv[0]); + return 1; + } + + // read the sequences + std::vector> seqs; + { + FILE * f = fopen(in_path.c_str(), "rb"); + if (!f) { fprintf(stderr, "cannot open %s\n", in_path.c_str()); return 1; } + int32_t n; + while (fread(&n, 4, 1, f) == 1) { + std::vector s(n); + if (fread(s.data(), 4, n, f) != (size_t) n) { fprintf(stderr, "short read\n"); return 1; } + if (n > n_ctx) s.resize(n_ctx); + seqs.push_back(std::move(s)); + } + fclose(f); + } + fprintf(stderr, "mtp-export: %zu sequences\n", seqs.size()); + + llama_backend_init(); + llama_model_params mp = llama_model_default_params(); + mp.n_gpu_layers = 999; + llama_model * model = llama_model_load_from_file(model_path.c_str(), mp); + if (!model) { fprintf(stderr, "load failed\n"); return 1; } + const int n_embd = llama_model_n_embd(model); + const int n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model)); + + llama_context_params cp = llama_context_default_params(); + cp.n_ctx = n_ctx; + cp.n_batch = n_batch; + cp.n_ubatch = n_batch; + cp.n_seq_max = 1; + llama_context * ctx = llama_init_from_model(model, cp); + if (!ctx) { fprintf(stderr, "context failed\n"); return 1; } + llama_set_embeddings_nextn(ctx, true, /*masked*/ false); + + FILE * fh = fopen((out_prefix + ".h").c_str(), "wb"); + FILE * ft = fopen((out_prefix + ".tok").c_str(), "wb"); + FILE * fg = fopen((out_prefix + ".tgt").c_str(), "wb"); + FILE * fi = fopen((out_prefix + ".idx").c_str(), "wb"); + if (!fh || !ft || !fg || !fi) { fprintf(stderr, "cannot write outputs\n"); return 1; } + + llama_batch batch = llama_batch_init(n_batch, 0, 1); + std::vector hrow(n_embd); + int64_t total = 0; + for (size_t si = 0; si < seqs.size(); ++si) { + const auto & s = seqs[si]; + fwrite(&total, 8, 1, fi); + llama_memory_clear(llama_get_memory(ctx), true); + for (size_t p0 = 0; p0 < s.size(); p0 += n_batch) { + const int n = (int) std::min(n_batch, s.size() - p0); + batch.n_tokens = n; + for (int i = 0; i < n; ++i) { + batch.token[i] = s[p0 + i]; + batch.pos[i] = (llama_pos) (p0 + i); + batch.n_seq_id[i] = 1; + batch.seq_id[i][0] = 0; + batch.logits[i] = 1; + } + if (llama_decode(ctx, batch) != 0) { fprintf(stderr, "decode failed at seq %zu pos %zu\n", si, p0); return 1; } + for (int i = 0; i < n; ++i) { + const float * h = llama_get_embeddings_nextn_ith(ctx, i); + const float * l = llama_get_logits_ith(ctx, i); + if (!h || !l) { fprintf(stderr, "missing outputs at %d\n", i); return 1; } + int best = 0; + for (int v = 1; v < n_vocab; ++v) if (l[v] > l[best]) best = v; + for (int e = 0; e < n_embd; ++e) hrow[e] = ggml_fp32_to_fp16(h[e]); + fwrite(hrow.data(), sizeof(ggml_fp16_t), n_embd, fh); + const int32_t tok = s[p0 + i], tg = best; + fwrite(&tok, 4, 1, ft); + fwrite(&tg, 4, 1, fg); + } + } + total += (int64_t) s.size(); + if ((si + 1) % 10 == 0 || si + 1 == seqs.size()) { + fprintf(stderr, "mtp-export: %zu / %zu sequences, %lld positions\n", si + 1, seqs.size(), (long long) total); + } + } + fwrite(&total, 8, 1, fi); + fclose(fh); fclose(ft); fclose(fg); fclose(fi); + llama_batch_free(batch); + llama_free(ctx); + llama_model_free(model); + llama_backend_free(); + return 0; +} From 15a9adaa55fbad85897aa7af5661de84f4e2ef23 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:50:59 +1100 Subject: [PATCH 45/65] mtp: the draft head's prompt pass computes only its K/V (LLAMA_MTP_KV_ONLY) Independent implementation of the technique described by GRIMOIRE (written from a functional description and this fork's own code; no GRIMOIRE source read). A draft-context decode without output rows (the NextN head's prompt catch-up) only fills the head's KV cache, so graph_mtp now stops after the cache store: embedding (+ Hadamard inverse), hnorm/enorm, eh_proj, attn_norm, K/V projection, K norm, RoPE and cpy_k/cpy_v. No query, attention, output projection, FFN or head. The K/V ops are the full graph's on the same shapes and in the same expand order, so the cache rows are bit-identical. For batches of <= 8 rows the unused Q projection stays in the graph so the SYCL same-input mat-vec grouping (whose tile depends on the largest member) matches the full graph. Only masked nextn extraction qualifies (unmasked readers copy h for every row). Decodes with outputs build the unchanged graph. LLAMA_MTP_KV_ONLY=0 restores the full graph for every decode. Default on. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 595d554c90d5a15eb22e4c66c4ac2564fd71b4cd on Torchit1/llama.cpp arc-b580) --- src/models/qwen35.cpp | 54 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp index 72fc50a942ce..ecc434f3e31f 100644 --- a/src/models/qwen35.cpp +++ b/src/models/qwen35.cpp @@ -1,5 +1,6 @@ #include "models.h" #include "llama-memory-recurrent.h" +#include "llama-kv-cache.h" void llama_model_qwen35::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); @@ -672,8 +673,19 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr res->add_input(std::move(inp)); + // ARC-LAB: KV-only pass (LLAMA_MTP_KV_ONLY, default on; independent implementation of the technique described by + // GRIMOIRE). A decode with no output rows (the head's prompt catch-up) is only there to fill this layer's K/V, so + // build up to the cache store and stop: no query, attention, output projection, FFN or head. The K/V ops are the + // full graph's, in the same order, so the cache rows are bit-identical. Unmasked nextn readers copy h for every + // token, so they keep the full graph. + static const bool kv_only_on = [] { + const char * e = getenv("LLAMA_MTP_KV_ONLY"); + return e == nullptr || atoi(e) != 0; + }(); + const bool kv_only = kv_only_on && n_outputs == 0 && cparams.embeddings_nextn_masked; + ggml_tensor * inp_pos = build_inp_pos(); - ggml_tensor * inp_out_ids = build_inp_out_ids(); + ggml_tensor * inp_out_ids = kv_only ? nullptr : build_inp_out_ids(); // an unread input would have no buffer auto * inp_attn = build_attn_inp_kv(); @@ -694,6 +706,46 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); cb(cur, "mtp_attn_norm", il); + if (kv_only) { + static bool logged = false; + if (!logged) { + logged = true; + LLAMA_LOG_INFO("%s: MTP KV-only pass for decodes without outputs (LLAMA_MTP_KV_ONLY=0 turns it off)\n", __func__); + } + // the SYCL backend runs same-input mat-vecs of <= 8 rows as one launch whose tile choice depends on the + // largest member; keep the Q projection (result unused) so q + k + v group exactly as in the full graph + if (n_tokens <= 8) { + ggml_build_forward_expand(gf, build_lora_mm(layer.wq, cur, layer.wq_s)); + } + + ggml_tensor * Kcur = build_lora_mm(layer.wk, cur, layer.wk_s); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens); + Kcur = build_norm(Kcur, layer.attn_k_norm, nullptr, LLM_NORM_RMS, il); + cb(Kcur, "mtp_Kcur_normed", il); + + ggml_tensor * Vcur = build_lora_mm(layer.wv, cur, layer.wv_s); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens); + cb(Vcur, "mtp_Vcur", il); + + Kcur = ggml_rope_multi(ctx0, Kcur, inp_pos, nullptr, + n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + + // the store half of build_attn(): same optional cache rotations, same expand order (v, k, then the copies) + if (inp_attn->self_k_rot) { + Kcur = llama_mul_mat_hadamard(ctx0, Kcur, inp_attn->self_k_rot); + } + if (inp_attn->self_v_rot) { + Vcur = llama_mul_mat_hadamard(ctx0, Vcur, inp_attn->self_v_rot); + } + ggml_build_forward_expand(gf, Vcur); + ggml_build_forward_expand(gf, Kcur); + cb(Kcur, "k_cache_in", il); + ggml_build_forward_expand(gf, inp_attn->mctx->cpy_k(ctx0, Kcur, inp_attn->get_k_idxs(), il)); + ggml_build_forward_expand(gf, inp_attn->mctx->cpy_v(ctx0, Vcur, inp_attn->get_v_idxs(), il)); + return; + } + ggml_tensor * Qcur_full = build_lora_mm(layer.wq, cur, layer.wq_s); cb(Qcur_full, "mtp_Qcur_full", il); From c94a7900286d088f88248c1edc9e052bcecf93c1 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 10 Oct 2026 12:57:24 +1100 Subject: [PATCH 46/65] mtp: one-graph draft chain with device draft-vocab embeddings (LLAMA_MTP_CHAIN) All MTP draft steps in one llama_decode: each step takes the previous step's in-graph argmax and head output row; ids read back once. A 36 MiB device copy of the draft-vocabulary token_embd rows (PTQ1_0) plus SYCL GET_ROWS for PTQ1_0 (same arithmetic as the CPU) keep the lookups on the GPU. On by default only with a trimmed draft head; LLAMA_MTP_CHAIN=1 forces it, =0 off; LLAMA_MTP_CHAIN_GPU_EMBD=0 uses host lookups. Independent implementation of the technique described by llamADALL (clean-room: written from a functional spec, without reading llamADALL's source). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit bcbe5a0ebc48d5541f7425af42a0c40c6defe399 on Torchit1/llama.cpp arc-b580) --- common/speculative.cpp | 116 ++++++++++++ ggml/src/ggml-sycl/dequantize.hpp | 13 ++ ggml/src/ggml-sycl/getrows.cpp | 4 + ggml/src/ggml-sycl/ggml-sycl.cpp | 1 + src/llama-context.cpp | 150 +++++++++++++++ src/llama-context.h | 17 ++ src/llama-cparams.h | 3 + src/llama-ext.h | 10 + src/llama-graph.cpp | 2 + src/llama-graph.h | 11 ++ src/llama-kv-cache.cpp | 42 +++++ src/llama-kv-cache.h | 12 ++ src/models/models.h | 3 + src/models/qwen35.cpp | 295 ++++++++++++++++++++++++++++++ tests/test-backend-ops.cpp | 28 ++- 15 files changed, 704 insertions(+), 3 deletions(-) diff --git a/common/speculative.cpp b/common/speculative.cpp index 626b376a58a2..b3ec6ff87aa9 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -2301,6 +2301,12 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { }; std::vector deferred; + // ARC-LAB: chained drafts (LLAMA_MTP_CHAIN; independent implementation of the technique described by llamADALL). + // All draft steps run in one llama_decode (llama_set_mtp_chain): each step's argmax token and output row feed the + // next step on the device, and the n draft tokens are read back once, instead of n decodes with a host sample in + // between. Same greedy drafts and the same draft K/V as the loop below. Set in the ctor. + bool chain_mode = false; + bool can_defer() const { return !is_mem_shared && !chain_heads && !getenv("LLAMA_MTP_EAGER_CATCHUP"); } @@ -2411,6 +2417,108 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { verify_h_rows.assign(n_seq, 0); deferred.assign(n_seq, {}); + + // chained drafts: default on when the head has a trimmed draft vocabulary (its token_embd rows fit on the + // device; measured slower without one), LLAMA_MTP_CHAIN=1 forces, =0 off. Greedy single-sequence drafting only. + { + const char * env = getenv("LLAMA_MTP_CHAIN"); + const int32_t sup = llama_mtp_chain_support(ctx_dft); + const bool want = env ? atoi(env) != 0 : (sup & 2) != 0; + const char * why = nullptr; + if (!want) { + why = env ? "LLAMA_MTP_CHAIN=0" : "no trimmed draft head (LLAMA_MTP_CHAIN=1 forces it)"; + } else if (sup == 0) { + why = "not supported by this context (qwen35 single head, flash attention, plain KV cache)"; + } else if (is_mem_shared || chain_heads) { + why = "shared-memory or multi-head MTP"; + } else if (n_seq != 1) { + why = "more than one sequence"; + } else if (this->params.p_min > 0.0f) { + why = "p_min > 0"; + } + chain_mode = why == nullptr; + if (chain_mode) { + SPC_INF("MTP chained drafts on (%s)\n", (sup & 2) ? "draft head" : "full head, host token lookups"); + } else { + SPC_INF("MTP chained drafts off: %s\n", why); + } + } + } + + // ARC-LAB: one chained decode for the whole draft of sequence 0. Returns false when it does not apply (the loop + // runs), true when it ran (dparams[0].result holds the drafts, empty on failure). + bool draft_chain(common_speculative_draft_params_vec & dparams) { + auto * ctx_dft = params.ctx_dft; + auto & dp = dparams[0]; + auto & d = deferred[0]; + if (!dp.drafting || d.catchup_failed) { + return false; + } + + // the loop's step count: it stops once min(n_max, LAB_SPEC_DRAFT_N_MAX) tokens are drafted (at least one) + const int32_t n_draft = std::max(1, std::min((int) params.n_max, common_lab_knob("LAB_SPEC_DRAFT_N_MAX", params.n_max))); + const int32_t n_catch = d.pending ? d.n_valid : 0; + const int32_t n_rows = n_catch + n_draft; + if (n_draft < 2 || n_rows > (int32_t) std::min(llama_n_batch(ctx_dft), llama_n_ubatch(ctx_dft))) { + return false; + } + + const size_t row_bytes = (size_t) n_embd * sizeof(float); + + common_sampler_reset(smpls[0].get()); + common_batch_clear(batch); + + // catch-up rows and the anchor exactly as the loop's first decode, then one placeholder row per later step + // (its cell receives that step's K/V; token and embd are not read) + for (int32_t k = 0; k < n_catch; ++k) { + common_batch_add(batch, d.tokens[k], d.pos[k], { 0 }, false); + std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, d.h.data() + (size_t) k * n_embd, row_bytes); + } + common_batch_add(batch, dp.id_last, dp.n_past, { 0 }, true); + std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, pending_h[0].data(), row_bytes); + for (int32_t j = 1; j < n_draft; ++j) { + common_batch_add(batch, dp.id_last, dp.n_past + j, { 0 }, false); + std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, pending_h[0].data(), row_bytes); + } + + llama_set_mtp_chain(ctx_dft, n_draft); + const int32_t rc = llama_decode(ctx_dft, batch); + llama_set_mtp_chain(ctx_dft, 0); + if (rc != 0) { + SPC_ERR("llama_decode(ctx_dft) chained draft returned %d\n", (int) rc); + if (n_catch > 0) { + d.catchup_failed = true; + } + return true; + } + if (n_catch > 0) { + d.pending = false; + } + + const llama_token * ids = llama_get_mtp_chain_ids(ctx_dft); + if (ids == nullptr) { + SPC_ERR("%s", "chained draft produced no tokens\n"); + return true; + } + + auto * smpl = smpls[0].get(); + auto & result = *dp.result; + const int32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(llama_get_model(ctx_dft))); + for (int32_t j = 0; j < n_draft; ++j) { + const llama_token id = ids[j]; + if (FILE * lf = common_mtp_draft_log()) { + // the chain reads back token ids only: no candidate list or probabilities (p logged as -1) + fprintf(lf, "{\"pos\": %d, \"d\": %d, \"c\": [[%d, -1.0]], \"stop\": 0, \"chain\": 1}\n", + (int) (dp.n_past + j + 1), j, (int) id); + } + if (id < 0 || id >= n_vocab) { + SPC_ERR("chained draft id %d out of vocab (step %d) - dropping the rest\n", (int) id, j); + break; + } + common_sampler_accept(smpl, id, true); + result.push_back(id); + } + return true; } ~common_speculative_impl_draft_mtp() override { @@ -2716,6 +2824,14 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { } } + if (chain_mode && draft_chain(dparams)) { + auto & dp = dparams[0]; + if (dp.result->size() < (size_t) params.n_min) { + dp.result->clear(); + } + return; + } + common_batch_clear(batch); // keep track of which sequences are still drafting diff --git a/ggml/src/ggml-sycl/dequantize.hpp b/ggml/src/ggml-sycl/dequantize.hpp index 076380eb05a9..8a5c0b58d92e 100644 --- a/ggml/src/ggml-sycl/dequantize.hpp +++ b/ggml/src/ggml-sycl/dequantize.hpp @@ -67,6 +67,19 @@ static __dpct_inline__ int ptq1_0_trit(const uint8_t * qs, const uint8_t * qh, c return (int) ((v * 3) >> 8); } +// ARC-LAB: f32 output whatever GGML_SYCL_F16 says, for GET_ROWS (MTP chain draft-token embeddings). (trit - 1) * d +// with d widened from fp16 is exact in f32 (the product of 0 / +-1 and d), so the rows match the CPU's +// dequantize_row_ptq1_0 bit for bit. +static __dpct_inline__ void dequantize_ptq1_0_f32(const void *vx, const int64_t ib, + const int iqs, float &v0, float &v1) { + const block_ptq1_0 * x = (const block_ptq1_0 *) vx; + + const float d = (float) x[ib].d; + + v0 = (float) (ptq1_0_trit(x[ib].qs, x[ib].qh, iqs + 0) - 1) * d; + v1 = (float) (ptq1_0_trit(x[ib].qs, x[ib].qh, iqs + 1) - 1) * d; +} + static __dpct_inline__ void dequantize_q4_0(const void *vx, const int64_t ib, const int iqs, dfloat2 &v) { const block_q4_0 * x = (const block_q4_0 *) vx; diff --git a/ggml/src/ggml-sycl/getrows.cpp b/ggml/src/ggml-sycl/getrows.cpp index 187640446b48..c6029be7215d 100644 --- a/ggml/src/ggml-sycl/getrows.cpp +++ b/ggml/src/ggml-sycl/getrows.cpp @@ -334,6 +334,10 @@ void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, src1_i32, (float *)dst->data, ctx.stream()); break; + case GGML_TYPE_PTQ1_0: + get_rows_sycl_f32(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, + src1_i32, (float *)dst->data, ctx.stream()); + break; case GGML_TYPE_Q2_K: get_rows_sycl_f32(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, src1_i32, (float *)dst->data, ctx.stream()); diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 963c50806142..354b09854d40 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -7743,6 +7743,7 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons { switch (op->src[0]->type) { case GGML_TYPE_I32: + case GGML_TYPE_PTQ1_0: // ARC-LAB: dequantize_ptq1_0_f32, bit-exact with the CPU case GGML_TYPE_F16: case GGML_TYPE_BF16: case GGML_TYPE_F32: diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 77eb959cae7e..223273004829 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -1338,6 +1338,119 @@ void llama_context::set_nextn_layer_offset(int32_t offset) { cparams.nextn_layer_offset = offset; } +// ARC-LAB: chained MTP drafts (independent implementation of the technique described by llamADALL). The draft loop in +// common/speculative.cpp decodes the NextN head once per drafted token, with a host round trip in between. A chain +// decode runs all steps in one graph (qwen35 graph_mtp::build_chain): step j+1 takes step j's argmax token and output +// state on the device. Bit 0: the chain graph applies to this context; bit 1: the head has a trimmed draft vocabulary, +// so in-graph tokens can be looked up in a small device copy of their token_embd rows. +int32_t llama_context::mtp_chain_support() const { + if (cparams.ctx_type != LLAMA_CONTEXT_TYPE_MTP || model.arch != LLM_ARCH_QWEN35 || model.hparams.n_layer_nextn != 1 || + cparams.nextn_layer_offset != 0 || !cparams.flash_attn || !cparams.embeddings_nextn_masked) { + return 0; + } + // per-step K/V views and index slices assume one plain KV cache with a non-transposed V + const auto * kv = dynamic_cast(memory.get()); + if (kv == nullptr || kv->get_v_trans()) { + return 0; + } + const auto & layer = model.layers[model.hparams.n_layer()]; + // same condition as graph_mtp's draft-head branch + const bool draft_head = layer.nextn.draft_head && layer.nextn.draft_head_ids && !layer.nextn.shared_head_head && + getenv("LLAMA_MTP_DRAFT_HEAD_OFF") == nullptr; + return 1 | (draft_head ? 2 : 0); +} + +void llama_context::set_mtp_chain(uint32_t n_steps) { + cparams.mtp_chain_n = n_steps; + if (n_steps == 0) { + return; // keeps the last chain's ids readable + } + mtp_chain_n_out = 0; + + if (mtp_chain_embd_tried) { + return; + } + mtp_chain_embd_tried = true; + + if ((mtp_chain_support() & 2) == 0) { + LLAMA_LOG_INFO("%s: MTP chain: no draft head, in-graph tokens use the host token_embd lookup\n", __func__); + return; + } + + const auto & layer = model.layers[model.hparams.n_layer()]; + ggml_tensor * ids = layer.nextn.draft_head_ids; + GGML_ASSERT(ids->type == GGML_TYPE_I32 && ggml_n_dims(ids) == 1); + mtp_chain_vocab.resize(ids->ne[0]); + ggml_backend_tensor_get(ids, mtp_chain_vocab.data(), 0, ggml_nbytes(ids)); + + const char * env_gpu = getenv("LLAMA_MTP_CHAIN_GPU_EMBD"); + if (env_gpu && atoi(env_gpu) == 0) { + LLAMA_LOG_INFO("%s: MTP chain: LLAMA_MTP_CHAIN_GPU_EMBD=0, in-graph tokens use the host token_embd lookup\n", __func__); + return; + } + + ggml_tensor * src = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd; + const ggml_backend_dev_t dev = model.dev_layer(model.hparams.n_layer()); + if (dev == nullptr || ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU || + (src->type != GGML_TYPE_PTQ1_0 && src->type != GGML_TYPE_F16 && src->type != GGML_TYPE_F32)) { + LLAMA_LOG_INFO("%s: MTP chain: no device copy of token_embd (%s), host lookups\n", __func__, ggml_type_name(src->type)); + return; + } + + const int64_t n_rows = ids->ne[0]; + const size_t row = ggml_row_size(src->type, src->ne[0]); + GGML_ASSERT(src->nb[1] == row); + + ggml_init_params ip = { + /*.mem_size =*/ ggml_tensor_overhead(), + /*.mem_buffer =*/ nullptr, + /*.no_alloc =*/ true, + }; + mtp_chain_ctx.reset(ggml_init(ip)); + ggml_tensor * t = ggml_new_tensor_2d(mtp_chain_ctx.get(), src->type, src->ne[0], n_rows); + ggml_set_name(t, "mtp_chain_embd"); + mtp_chain_buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(mtp_chain_ctx.get(), ggml_backend_dev_buffer_type(dev))); + if (!mtp_chain_buf) { + LLAMA_LOG_WARN("%s: MTP chain: could not allocate the device token_embd copy, host lookups\n", __func__); + mtp_chain_ctx.reset(); + return; + } + // a weight to the scheduler: ops reading it run on its device (no copy of the table per graph) + ggml_backend_buffer_set_usage(mtp_chain_buf.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + + // row k of the copy = token_embd row of draft-head index k, bytes unchanged (same quantization, same latent basis) + std::vector staging((size_t) n_rows * row); + for (int64_t k = 0; k < n_rows; ++k) { + const int32_t id = mtp_chain_vocab[k]; + GGML_ASSERT(id >= 0 && id < src->ne[1]); + ggml_backend_tensor_get(src, staging.data() + (size_t) k * row, (size_t) id * src->nb[1], row); + } + ggml_backend_tensor_set(t, staging.data(), 0, staging.size()); + mtp_chain_embd = t; + + LLAMA_LOG_INFO("%s: MTP chain: %lld draft-vocab token_embd rows (%s, %.1f MiB) on %s\n", __func__, (long long) n_rows, + ggml_type_name(src->type), staging.size() / 1048576.0, ggml_backend_dev_name(dev)); +} + +const int32_t * llama_context::get_mtp_chain_ids() { + synchronize(); + + if (mtp_chain_n_out == 0) { + return nullptr; + } + auto & tok = mtp_chain_tok; + tok.resize(mtp_chain_n_out); + for (uint32_t s = 0; s < mtp_chain_n_out; ++s) { + const int32_t v = mtp_chain_out[s]; + if (mtp_chain_vocab.empty()) { + tok[s] = v; + } else { + tok[s] = v >= 0 && v < (int32_t) mtp_chain_vocab.size() ? mtp_chain_vocab[v] : -1; + } + } + return tok.data(); +} + void llama_context::set_capture_layers(const std::vector & layer_ids) { const int32_t n_layer = (int32_t) model.hparams.n_layer(); if (layer_ids.size() > LLAMA_MAX_LAYERS) { @@ -1940,6 +2053,13 @@ int llama_context::decode(const llama_batch & batch_inp) { const uint32_t n_tokens_all = balloc->get_n_tokens(); const uint32_t n_outputs_all = balloc->get_n_outputs(); + // ARC-LAB: a chained MTP draft is one graph over the whole batch (catch-up rows, anchor, one row per later step) + if (cparams.mtp_chain_n > 0 && (n_tokens_all > cparams.n_ubatch || n_tokens_all < cparams.mtp_chain_n)) { + LLAMA_LOG_ERROR("%s: MTP chain of %u steps needs %u..%u tokens in one ubatch, got %u\n", __func__, + cparams.mtp_chain_n, cparams.mtp_chain_n, cparams.n_ubatch, n_tokens_all); + return -1; + } + if (output_all) { // require that all tokens are output if (n_outputs_all != n_tokens_all) { @@ -2196,6 +2316,18 @@ int llama_context::decode(const llama_batch & batch_inp) { extract_layer_inputs(res, n_tokens_prev, ubatch.n_tokens); + // ARC-LAB: chained MTP draft: the per-step argmax rows, mapped to tokens in get_mtp_chain_ids() + if (!res->t_mtp_chain.empty()) { + mtp_chain_out.resize(res->t_mtp_chain.size()); + for (size_t s = 0; s < res->t_mtp_chain.size(); ++s) { + ggml_tensor * t = res->t_mtp_chain[s]; + ggml_backend_t b = ggml_backend_sched_get_tensor_backend(sched.get(), t); + GGML_ASSERT(b != nullptr); + ggml_backend_tensor_get_async(b, t, &mtp_chain_out[s], 0, sizeof(int32_t)); + } + mtp_chain_n_out = (uint32_t) res->t_mtp_chain.size(); + } + // extract nextn embeddings before // only meaningful in LLAMA_POOLING_TYPE_NONE (per-token); other pooling modes are ignored. { @@ -2695,6 +2827,10 @@ ggml_cgraph * llama_context::graph_reserve( this->n_outputs = n_outputs; + // ARC-LAB: reserve with the regular graph, never a chained MTP draft (it needs real tokens and hidden rows) + const uint32_t save_mtp_chain_n = cparams.mtp_chain_n; + cparams.mtp_chain_n = 0; + llama_batch_allocr balloc(model.hparams.n_pos_per_embd()); llama_ubatch ubatch = balloc.ubatch_reserve(n_tokens/n_seqs, n_seqs); @@ -2716,6 +2852,7 @@ ggml_cgraph * llama_context::graph_reserve( } this->n_outputs = save_n_outputs; + cparams.mtp_chain_n = save_mtp_chain_n; // initialize scheduler with the specified graph if (split_only) { @@ -2756,6 +2893,7 @@ llm_graph_params llama_context::graph_params( /*.dspark_ctx_width =*/dspark_ctx.n_embd_cap, /*.hadamard_rotations =*/&hadamard_rotations, /*.hadamard_inverses =*/&hadamard_inverses, + /*.mtp_chain_embd =*/mtp_chain_embd, /*.samplers =*/sampling.samplers, /*.n_outputs =*/n_outputs, /*.cb =*/graph_get_cb(), @@ -4081,6 +4219,18 @@ void llama_set_nextn_layer_offset(llama_context * ctx, int32_t offset) { ctx->set_nextn_layer_offset(offset); } +int32_t llama_mtp_chain_support(const llama_context * ctx) { + return ctx->mtp_chain_support(); +} + +void llama_set_mtp_chain(llama_context * ctx, int32_t n_steps) { + ctx->set_mtp_chain(n_steps > 0 ? (uint32_t) n_steps : 0u); +} + +const llama_token * llama_get_mtp_chain_ids(llama_context * ctx) { + return ctx->get_mtp_chain_ids(); +} + llama_memory_t llama_get_memory(const struct llama_context * ctx) { if (!ctx) { return nullptr; diff --git a/src/llama-context.h b/src/llama-context.h index 6057da7f435f..ef04bf85de9c 100644 --- a/src/llama-context.h +++ b/src/llama-context.h @@ -134,6 +134,11 @@ struct llama_context { void set_embeddings_layer_inp(uint32_t lid, bool enable); void set_nextn_layer_offset(int32_t offset); + + // ARC-LAB: chained MTP drafts (llama-ext.h llama_mtp_chain_support / llama_set_mtp_chain / llama_get_mtp_chain_ids) + int32_t mtp_chain_support() const; + void set_mtp_chain(uint32_t n_steps); + const int32_t * get_mtp_chain_ids(); void set_causal_attn(bool value); void set_warmup(bool value); @@ -403,6 +408,18 @@ struct llama_context { // one-time Hadamard transform-coverage check on the first built graph bool hadamard_verified = false; + // ARC-LAB: chained MTP drafts. mtp_chain_embd: device copy of the token_embd rows the draft head can produce (row k + // = token mtp_chain_vocab[k]), built on first use unless LLAMA_MTP_CHAIN_GPU_EMBD=0. mtp_chain_out: the last chain + // decode's per-step argmax rows, read back asynchronously. + ggml_tensor * mtp_chain_embd = nullptr; + ggml_context_ptr mtp_chain_ctx; + ggml_backend_buffer_ptr mtp_chain_buf; + bool mtp_chain_embd_tried = false; + std::vector mtp_chain_vocab; // draft-head index -> token id (empty: no draft head, indices are tokens) + std::vector mtp_chain_out; + std::vector mtp_chain_tok; // mtp_chain_out mapped to token ids + uint32_t mtp_chain_n_out = 0; + // host buffer for the model output (logits and embeddings) ggml_backend_buffer_ptr buf_output; diff --git a/src/llama-cparams.h b/src/llama-cparams.h index 1f013e2f0027..248149614401 100644 --- a/src/llama-cparams.h +++ b/src/llama-cparams.h @@ -23,6 +23,9 @@ struct llama_cparams { int32_t nextn_layer_offset = 0; + // ARC-LAB: > 0: the next DECODER_MTP graph drafts this many tokens in one graph (llama_set_mtp_chain) + uint32_t mtp_chain_n = 0; + float rope_freq_base; float rope_freq_scale; diff --git a/src/llama-ext.h b/src/llama-ext.h index c6cfc35bc3cc..55de2f595e27 100644 --- a/src/llama-ext.h +++ b/src/llama-ext.h @@ -101,6 +101,16 @@ LLAMA_API void llama_set_embeddings_nextn(struct llama_context * ctx, bool value // chain multiple trained NextN heads. Default 0 (first head). LLAMA_API void llama_set_nextn_layer_offset(struct llama_context * ctx, int32_t offset); +// ARC-LAB: chained MTP drafts (single NextN head, qwen35). llama_mtp_chain_support: 0 = not available on this context, +// bit 0 = available, bit 1 = the head has a trimmed draft vocabulary (in-graph token lookups can stay on the device). +// llama_set_mtp_chain(ctx, n): the next llama_decode on this MTP context drafts n tokens in one graph. Its batch must be +// one ubatch: catch-up rows (no output), the anchor row (id_last at n_past, output), then n-1 rows at n_past+1.. whose +// token and embd are placeholders (the graph writes their K/V). Reset with n = 0 after the decode. +// llama_get_mtp_chain_ids: after that decode, the n greedy draft tokens (synchronizes), or nullptr. +LLAMA_API int32_t llama_mtp_chain_support(const struct llama_context * ctx); +LLAMA_API void llama_set_mtp_chain (struct llama_context * ctx, int32_t n_steps); +LLAMA_API const llama_token * llama_get_mtp_chain_ids(struct llama_context * ctx); + // mirrors: // LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); LLAMA_API float * llama_get_embeddings_nextn(struct llama_context * ctx); diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 9ad7fa925e6b..ceabee25866a 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1354,6 +1354,7 @@ void llm_graph_result::reset() { t_sampled.clear(); t_dspark_greedy.clear(); + t_mtp_chain.clear(); t_sampled_probs.clear(); t_sampled_logits.clear(); t_candidates.clear(); @@ -1521,6 +1522,7 @@ llm_graph_context::llm_graph_context(const llm_graph_params & params) : dspark_ctx_width(params.dspark_ctx_width), hadamard_rotations(params.hadamard_rotations), hadamard_inverses(params.hadamard_inverses), + mtp_chain_embd(params.mtp_chain_embd), samplers(params.samplers), cb_func(params.cb), res(params.res), diff --git a/src/llama-graph.h b/src/llama-graph.h index 6c2db6420d64..ea9952557a47 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -863,6 +863,7 @@ struct llm_graph_params { int64_t dspark_ctx_width; const llama_hadamard_rotations * hadamard_rotations; const llama_hadamard_rotations * hadamard_inverses; + ggml_tensor * mtp_chain_embd; // ARC-LAB: device copy of the draft-vocab token_embd rows, or null std::map samplers; @@ -931,6 +932,11 @@ struct llm_graph_params { return false; } + // ARC-LAB: a chained MTP draft graph bakes per-step KV lengths in; never reuse it or reuse into it + if (cparams.mtp_chain_n > 0 || other.cparams.mtp_chain_n > 0) { + return false; + } + if (!samplers_equal(samplers, other.samplers)) { return false; } @@ -1035,6 +1041,10 @@ class llm_graph_result { std::vector t_sampled; std::vector t_dspark_greedy; + + // ARC-LAB: chained MTP draft (cparams.mtp_chain_n): per step, the argmax row of the draft logits (I32 [1]); an + // index into the draft head's vocabulary when it has one, else a token id + std::vector t_mtp_chain; std::vector t_sampled_probs; std::vector t_sampled_logits; std::vector t_candidates; @@ -1127,6 +1137,7 @@ struct llm_graph_context { int64_t dspark_ctx_width; const llama_hadamard_rotations * hadamard_rotations; const llama_hadamard_rotations * hadamard_inverses; + ggml_tensor * mtp_chain_embd; // Transforms shared by folded weights on the same activation. Key is (input, rotation); // both must match. Valid for one graph build only. diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index d406353d564c..cb806c688166 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -1236,6 +1236,32 @@ const llama_kv_cells & llama_kv_cache::get_cells(llama_seq_id seq_id) const { return v_cells[seq_to_stream[seq_id]]; } +uint32_t llama_kv_cache::get_n_kv_prefix(const slot_info & sinfo, uint32_t n_rows) const { + GGML_ASSERT(sinfo.n_stream() == 1); + + const auto & cells = v_cells[sinfo.strm[0]]; + const auto & idxs = sinfo.idxs[0]; + + // highest used cell, not counting the cells of rows >= n_rows (they were empty before this ubatch) + uint32_t p1 = cells.used_max_p1(); + while (p1 > 0) { + const uint32_t c = p1 - 1; + bool later = false; + for (size_t k = n_rows; k < idxs.size() && !later; ++k) { + later = idxs[k] == c; + } + if (!later && !cells.is_empty(c)) { + break; + } + --p1; + } + + // same padding as get_n_kv + const uint32_t n_pad_cur = std::max(n_pad, 256u); + + return std::min(cells.size(), std::max(n_pad_cur, GGML_PAD(p1, n_pad_cur))); +} + uint32_t llama_kv_cache::get_n_kv(const slot_info & sinfo) const { uint32_t result = 0; @@ -2783,6 +2809,22 @@ uint32_t llama_kv_cache_context::get_n_kv() const { return n_kv; } +uint32_t llama_kv_cache_context::get_n_kv_prefix(uint32_t n_rows) const { + return kv->get_n_kv_prefix(sinfos[i_cur], n_rows); +} + +ggml_tensor * llama_kv_cache_context::get_k_n(ggml_context * ctx, int32_t il, uint32_t n_kv_view) const { + return kv->get_k(ctx, il, n_kv_view, sinfos[i_cur]); +} + +ggml_tensor * llama_kv_cache_context::get_v_n(ggml_context * ctx, int32_t il, uint32_t n_kv_view) const { + return kv->get_v(ctx, il, n_kv_view, sinfos[i_cur]); +} + +bool llama_kv_cache_context::is_v_trans() const { + return kv->get_v_trans(); +} + ggml_type llama_kv_cache_context::type_k() const { return kv->type_k(); } diff --git a/src/llama-kv-cache.h b/src/llama-kv-cache.h index 4d2229b90274..05add1b853c1 100644 --- a/src/llama-kv-cache.h +++ b/src/llama-kv-cache.h @@ -172,6 +172,11 @@ class llama_kv_cache : public llama_memory_i { uint32_t get_n_kv(const slot_info & sinfo) const; + // ARC-LAB: get_n_kv as it would be if only the first n_rows of the applied ubatch held their cells (the KV length + // a decode of just those rows would attend over; chained MTP drafts size each step's view with it) + uint32_t get_n_kv_prefix(const slot_info & sinfo, uint32_t n_rows) const; + bool get_v_trans() const { return v_trans; } + // get views of the current state of the cache ggml_tensor * get_k(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const; ggml_tensor * get_v(ggml_context * ctx, int32_t il, uint32_t n_kv, const slot_info & sinfo) const; @@ -392,6 +397,13 @@ class llama_kv_cache_context : public llama_memory_context_i { uint32_t get_n_kv() const; + // ARC-LAB: chained MTP drafts. KV length a decode of only the first n_rows of this ubatch would see, and views + // of the cache with an explicit length (the same views get_k / get_v return for that length) + uint32_t get_n_kv_prefix(uint32_t n_rows) const; + ggml_tensor * get_k_n(ggml_context * ctx, int32_t il, uint32_t n_kv_view) const; + ggml_tensor * get_v_n(ggml_context * ctx, int32_t il, uint32_t n_kv_view) const; + bool is_v_trans() const; + ggml_type type_k() const; ggml_type type_v() const; diff --git a/src/models/models.h b/src/models/models.h index a82133099d5c..81993e40ab24 100644 --- a/src/models/models.h +++ b/src/models/models.h @@ -2302,6 +2302,9 @@ struct llama_model_qwen35 : public llama_model_base { struct graph_mtp : public llm_graph_context { graph_mtp(const llama_model & model, const llm_graph_params & params); + + // ARC-LAB: cparams.mtp_chain_n > 0: all draft steps in one graph (see llama_set_mtp_chain) + void build_chain(const llama_model & model); }; std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp index ecc434f3e31f..c5fedfead481 100644 --- a/src/models/qwen35.cpp +++ b/src/models/qwen35.cpp @@ -609,6 +609,52 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_ffn(ggml_tensor * cur, cons return cur; } +// ARC-LAB: per-step inputs of a chained MTP draft graph (graph_mtp::build_chain): the positions of each step's rows in +// the layout build_inp_pos() gives a decode of just those rows, and the step's output row +class llm_graph_input_mtp_chain : public llm_graph_input_i { +public: + explicit llm_graph_input_mtp_chain(uint32_t n_pos_per_embd) : n_pos_per_embd(n_pos_per_embd) {} + + struct step { + int64_t r0; // first ubatch row of the step + int64_t nr; // rows + ggml_tensor * pos; // I32 [nr * n_pos_per_embd] + ggml_tensor * out_idx; // I32 [1] = nr - 1 (the step's last row produces its draft) + }; + std::vector steps; + + void set_input(const llama_ubatch * ubatch) override { + const int64_t n_tokens = ubatch->n_tokens; + std::vector v; + for (const auto & st : steps) { + v.assign((size_t) st.nr * n_pos_per_embd, 0); + for (int64_t r = 0; r < st.nr; ++r) { + if (ubatch->token && n_pos_per_embd == 4) { + // text tokens under M-RoPE, as llm_graph_input_pos: three equal dims, the fourth 0 + for (int d = 0; d < 3; ++d) { + v[d * st.nr + r] = ubatch->pos[st.r0 + r]; + } + } else { + for (uint32_t d = 0; d < n_pos_per_embd; ++d) { + v[d * st.nr + r] = ubatch->pos[d * n_tokens + st.r0 + r]; + } + } + } + ggml_backend_tensor_set(st.pos, v.data(), 0, v.size() * sizeof(int32_t)); + const int32_t out_row = (int32_t) st.nr - 1; + ggml_backend_tensor_set(st.out_idx, &out_row, 0, sizeof(out_row)); + } + } + + // never reused (llm_graph_params::allow_reuse refuses chain graphs) + bool can_reuse(const llm_graph_params & params) override { + GGML_UNUSED(params); + return false; + } + + const uint32_t n_pos_per_embd; +}; + // LLM_GRAPH_TYPE_DECODER_MTP draft head for Qwen3.5/3.6 dense series llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) { @@ -618,6 +664,11 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr const int64_t n_embd_head = hparams.n_embd_head_v(); GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); + if (cparams.mtp_chain_n > 0) { + build_chain(model); + return; + } + // hparams.n_layer includes both main model layers and MTP layers. The MTP // layer is stored immediately after the main layers in model.layers[]. const int il = hparams.n_layer(); @@ -856,3 +907,247 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr res->t_logits = cur; ggml_build_forward_expand(gf, cur); } + +// ARC-LAB: chained MTP drafts, n = cparams.mtp_chain_n steps in one graph (independent implementation of the technique +// described by llamADALL). The ubatch holds the deferred catch-up rows and the anchor (step 0, as the first decode of the +// draft loop in common/speculative.cpp) followed by one placeholder row per later step, so the KV cache has given every +// draft position its cell and the mask rows exist. Step j > 0 reads step j-1's argmax token and output state (h_nextn +// row) on the device and writes its K/V into its row's cell, as decode j of the loop does. +// +// Each step is built with the loop's shapes, so the kernels and their rounding are the loop's: its own rows only (step 0 +// the catch-up rows + anchor, later steps one row), its own positions, KV index slices and mask rows, and K/V views and +// mask columns of the KV length a decode of the rows up to and including its own would see (get_n_kv_prefix; flash +// attention splits the KV range by that length). The ops mirror the single-step graph above; keep the two in sync. +void llama_model_qwen35::graph_mtp::build_chain(const llama_model & model) { + const int64_t n_embd_head = hparams.n_embd_head_v(); + const int il = hparams.n_layer(); + const auto & layer = model.layers[il]; + + const int64_t n_steps = cparams.mtp_chain_n; + GGML_ASSERT(n_steps >= 1 && n_tokens >= n_steps); + GGML_ASSERT(ubatch.token && ubatch.embd && "MTP chain needs tokens and hidden rows"); + const int64_t n_lead = n_tokens - n_steps + 1; // step 0: catch-up rows + the anchor + + int sections[4]; + std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections); + + // inputs as the single-step graph creates them + auto inp = std::make_unique(hparams.n_embd); + inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens); + ggml_set_input(inp->tokens); + inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens); + ggml_set_input(inp->embd); + inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens); + ggml_set_input(inp->h); + ggml_set_name(inp->h, "mtp_h_input"); + ggml_tensor * tokens_all = inp->tokens; + ggml_tensor * h_all = inp->h; + res->add_input(std::move(inp)); + + auto * inp_attn = build_attn_inp_kv(); + const llama_kv_cache_context * kvc = inp_attn->mctx; + + auto chain_inp = std::make_unique(hparams.n_pos_per_embd()); + for (int64_t s = 0; s < n_steps; ++s) { + llm_graph_input_mtp_chain::step st; + st.r0 = s == 0 ? 0 : n_lead - 1 + s; + st.nr = s == 0 ? n_lead : 1; + st.pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, st.nr * hparams.n_pos_per_embd()); + st.out_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1); + ggml_set_input(st.pos); + ggml_set_input(st.out_idx); + chain_inp->steps.push_back(st); + } + const auto steps = chain_inp->steps; + res->add_input(std::move(chain_inp)); + + // token embeddings: the host table for the ubatch's own tokens (catch-up tokens need not be in the draft + // vocabulary), the draft-vocab device copy (row k = token draft_head_ids[k]) for the argmax tokens + ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd; + const llama_hadamard_transform * inv = nullptr; + if (hadamard_inverses) { + const auto it = hadamard_inverses->find(tok_embd_w); + if (it != hadamard_inverses->end()) { + inv = &it->second; + } + } + auto unrotate = [&](ggml_tensor * e) { + // the single-step graph's inverse transform after the lookup (also applies to the copy: same rows) + if (inv) { + e = llama_mul_mat_hadamard(ctx0, e, inv->rot); + if (inv->signs) { + e = ggml_mul(ctx0, e, inv->signs); + } + } + return e; + }; + + static const bool draft_head_off = getenv("LLAMA_MTP_DRAFT_HEAD_OFF") != nullptr; + const bool use_draft_head = layer.nextn.draft_head && layer.nextn.draft_head_ids && !layer.nextn.shared_head_head && + !draft_head_off; + ggml_tensor * head_w = layer.nextn.shared_head_head ? layer.nextn.shared_head_head : model.output; + ggml_tensor * head_s = layer.nextn.shared_head_head ? layer.nextn.shared_head_head_s : model.output_s; + ggml_tensor * head_norm_w = layer.nextn.shared_head_norm ? layer.nextn.shared_head_norm : model.output_norm; + GGML_ASSERT(head_w && head_norm_w); + + static bool logged = false; + if (!logged) { + logged = true; + LLAMA_LOG_INFO("%s: MTP chain graph (%lld steps, %s, %s token lookups)\n", __func__, (long long) n_steps, + use_draft_head ? "draft head" : "full head", use_draft_head && mtp_chain_embd ? "device" : "host"); + } + + const float kq_scale = hparams.f_attention_scale == 0.0f + ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale; + + ggml_tensor * tok_embd = unrotate(ggml_get_rows(ctx0, tok_embd_w, ggml_view_1d(ctx0, tokens_all, n_lead, 0))); + ggml_tensor * h_embd = ggml_view_2d(ctx0, h_all, h_all->ne[0], n_lead, h_all->nb[1], 0); + + for (int64_t s = 0; s < n_steps; ++s) { + const auto & st = steps[s]; + const int64_t nr = st.nr; + cb(tok_embd, "mtp_tok_embd", il); + + // this step's slice of the ubatch's KV indices and mask, sized as a decode of rows [0, r0 + nr) would be + const uint32_t n_kv_s = kvc->get_n_kv_prefix((uint32_t) (st.r0 + nr)); + ggml_tensor * k_idxs = inp_attn->get_k_idxs(); + ggml_tensor * v_idxs = inp_attn->get_v_idxs(); + k_idxs = ggml_view_1d(ctx0, k_idxs, nr, st.r0 * ggml_element_size(k_idxs)); + v_idxs = ggml_view_1d(ctx0, v_idxs, nr, st.r0 * ggml_element_size(v_idxs)); + ggml_tensor * kq_mask = inp_attn->get_kq_mask(); + kq_mask = ggml_cont(ctx0, ggml_view_2d(ctx0, kq_mask, n_kv_s, nr, kq_mask->nb[1], st.r0 * kq_mask->nb[1])); + + ggml_tensor * h_norm = build_norm(h_embd, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il); + cb(h_norm, "mtp_hnorm", il); + + ggml_tensor * e_norm = build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il); + cb(e_norm, "mtp_enorm", il); + + ggml_tensor * concat = ggml_concat(ctx0, e_norm, h_norm, /*dim=*/ 0); + cb(concat, "mtp_concat", il); + + ggml_tensor * cur = build_lora_mm(layer.nextn.eh_proj, concat, layer.nextn.eh_proj_s); + cb(cur, "mtp_eh_proj", il); + + ggml_tensor * inpSA = cur; + + cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "mtp_attn_norm", il); + + ggml_tensor * Qcur_full = build_lora_mm(layer.wq, cur, layer.wq_s); + cb(Qcur_full, "mtp_Qcur_full", il); + + ggml_tensor * Qcur = ggml_view_3d(ctx0, Qcur_full, + n_embd_head, n_head, nr, + ggml_element_size(Qcur_full) * n_embd_head * 2, + ggml_element_size(Qcur_full) * n_embd_head * 2 * n_head, + 0); + Qcur = build_norm(Qcur, layer.attn_q_norm, nullptr, LLM_NORM_RMS, il); + cb(Qcur, "mtp_Qcur_normed", il); + + ggml_tensor * gate = ggml_view_3d(ctx0, Qcur_full, + n_embd_head, n_head, nr, + ggml_element_size(Qcur_full) * n_embd_head * 2, + ggml_element_size(Qcur_full) * n_embd_head * 2 * n_head, + ggml_element_size(Qcur_full) * n_embd_head); + gate = ggml_cont_2d(ctx0, gate, n_embd_head * n_head, nr); + cb(gate, "mtp_gate", il); + + ggml_tensor * Kcur = build_lora_mm(layer.wk, cur, layer.wk_s); + Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, nr); + Kcur = build_norm(Kcur, layer.attn_k_norm, nullptr, LLM_NORM_RMS, il); + cb(Kcur, "mtp_Kcur_normed", il); + + ggml_tensor * Vcur = build_lora_mm(layer.wv, cur, layer.wv_s); + Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, nr); + cb(Vcur, "mtp_Vcur", il); + + Qcur = ggml_rope_multi(ctx0, Qcur, st.pos, nullptr, + n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + Kcur = ggml_rope_multi(ctx0, Kcur, st.pos, nullptr, + n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, + ext_factor, attn_factor, beta_fast, beta_slow); + + // build_attn() for this step's rows: same rotations, same expand order, store, then attend + if (inp_attn->self_k_rot) { + Qcur = llama_mul_mat_hadamard(ctx0, Qcur, inp_attn->self_k_rot); + Kcur = llama_mul_mat_hadamard(ctx0, Kcur, inp_attn->self_k_rot); + } + if (inp_attn->self_v_rot) { + Vcur = llama_mul_mat_hadamard(ctx0, Vcur, inp_attn->self_v_rot); + } + ggml_build_forward_expand(gf, Qcur); + ggml_build_forward_expand(gf, Vcur); + ggml_build_forward_expand(gf, Kcur); + cb(Kcur, "k_cache_in", il); + ggml_build_forward_expand(gf, kvc->cpy_k(ctx0, Kcur, k_idxs, il)); + ggml_build_forward_expand(gf, kvc->cpy_v(ctx0, Vcur, v_idxs, il)); + + cur = build_attn_mha(Qcur, kvc->get_k_n(ctx0, il, n_kv_s), kvc->get_v_n(ctx0, il, n_kv_s), nullptr, kq_mask, + nullptr, nullptr, kq_scale, il); + cb(cur, "kqv_out", il); + if (inp_attn->self_v_rot) { + cur = llama_mul_mat_hadamard(ctx0, cur, inp_attn->self_v_rot); + } + cb(cur, "mtp_attn_pregate", il); + + cur = ggml_mul(ctx0, cur, ggml_sigmoid(ctx0, gate)); + cur = build_lora_mm(layer.wo, cur, layer.wo_s); + cb(cur, "mtp_attn_out", il); + + cur = ggml_add(ctx0, cur, inpSA); + cb(cur, "mtp_attn_residual", il); + + ggml_tensor * ffn_residual = cur; + cur = build_norm(cur, layer.attn_post_norm, nullptr, LLM_NORM_RMS, il); + cb(cur, "mtp_attn_post_norm", il); + + cur = build_ffn(cur, + layer.ffn_up, nullptr, layer.ffn_up_s, + layer.ffn_gate, nullptr, layer.ffn_gate_s, + layer.ffn_down, nullptr, layer.ffn_down_s, + nullptr, + LLM_FFN_SILU, LLM_FFN_PAR, il); + cb(cur, "mtp_ffn_out", il); + + cur = ggml_add(ctx0, cur, ffn_residual); + cb(cur, "mtp_post_ffn", il); + + cur = build_norm(cur, head_norm_w, nullptr, LLM_NORM_RMS, -1); + cb(cur, "h_nextn", -1); + + // the step's output row (masked h_nextn: the next step's hidden input) + ggml_tensor * h_out = ggml_get_rows(ctx0, cur, st.out_idx); + cb(h_out, "mtp_shared_head_norm", -1); + + // greedy draft: argmax of the head's logits. With a draft head that is an index into its vocabulary (the + // -inf scatter of the single-step graph only maps it to a token id, which the context does on the host) + ggml_tensor * logits = use_draft_head ? build_lora_mm(layer.nextn.draft_head, h_out, nullptr) + : build_lora_mm(head_w, h_out, head_s); + cb(logits, "result_output", -1); + ggml_tensor * best = ggml_argmax(ctx0, logits); + ggml_set_name(best, "mtp_chain_argmax"); + ggml_set_output(best); + ggml_build_forward_expand(gf, best); + res->t_mtp_chain.push_back(best); + + if (s + 1 == n_steps) { + break; + } + + // next step's inputs: this step's draft token and output row + ggml_tensor * e; + if (use_draft_head && mtp_chain_embd) { + e = ggml_get_rows(ctx0, mtp_chain_embd, best); + } else if (use_draft_head) { + ggml_tensor * ids = layer.nextn.draft_head_ids; + ggml_tensor * tok = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, ids, 1, ids->ne[0]), best); // I32 [1, 1] + e = ggml_get_rows(ctx0, tok_embd_w, ggml_reshape_1d(ctx0, tok, 1)); + } else { + e = ggml_get_rows(ctx0, tok_embd_w, best); + } + tok_embd = unrotate(e); + h_embd = h_out; + } +} diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 327a63de1ed2..3c25cc124f14 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -2252,13 +2252,30 @@ struct test_get_rows : public test_case { const int be1; // batch size const int be2; // batch size const bool v; // view (non-contiguous src1) + const bool exact; // ARC-LAB: require bit-identical rows (max NMSE 0), e.g. PTQ1_0 MTP draft embeddings std::string vars() override { - return VARS_TO_STR7(type, n, m, r, be1, be2, v); + return exact ? VARS_TO_STR8(type, n, m, r, be1, be2, v, exact) : VARS_TO_STR7(type, n, m, r, be1, be2, v); } - test_get_rows(ggml_type type = GGML_TYPE_F32, int n = 10, int m = 5, int r = 3, int be1 = 1, int be2 = 1, bool v = false) - : type(type), n(n), m(m), r(r), be1(be1), be2(be2), v(v) {} + double max_nmse_err() override { + return exact ? 0.0 : 1e-7; + } + + double err(const float * a, const float * b, size_t n) override { + if (!exact) { + return test_case::err(a, b, n); + } + size_t n_diff = 0; // bit patterns, so -0.0 vs 0.0 counts too + for (size_t i = 0; i < n; ++i) { + n_diff += memcmp(a + i, b + i, sizeof(float)) != 0; + } + return (double) n_diff; + } + + test_get_rows(ggml_type type = GGML_TYPE_F32, int n = 10, int m = 5, int r = 3, int be1 = 1, int be2 = 1, bool v = false, + bool exact = false) + : type(type), n(n), m(m), r(r), be1(be1), be2(be2), v(v), exact(exact) {} ggml_tensor * build_graph(ggml_context * ctx) override { ggml_tensor * in = ggml_new_tensor_4d(ctx, type, n, m, be1, be2); @@ -8868,6 +8885,11 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_get_rows(GGML_TYPE_I32, 256, 5, 4, b, 1, v)); } } + // ARC-LAB: MTP chain draft-token lookups (PTQ1_0 token_embd rows, Bonsai width 5120, 1..8 rows): bit-exact + for (int r : {1, 4, 8}) { + test_cases.emplace_back(new test_get_rows(GGML_TYPE_PTQ1_0, 5120, 300, r, 1, 1, false, true)); + } + test_cases.emplace_back(new test_get_rows(GGML_TYPE_PTQ1_0, 5120, 300, 8, 3, 1, true, true)); test_cases.emplace_back(new test_get_rows_back(GGML_TYPE_F32, 1, 8, 2, 1, false)); test_cases.emplace_back(new test_get_rows_back(GGML_TYPE_F32, 1, 70000, 4, 1, false)); // row count > CUDA grid-y limit (65535) From 24b56006672cd0b43a49195073b41b72c699b5aa Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:15:45 +1100 Subject: [PATCH 47/65] gdn: rollback by replay instead of state snapshots (LLAMA_GDN_REPLAY) Independent implementation of the technique described by llamADALL (clean-room, from the spec and this fork's own code only). With MTP, each verify pass used to write K = n_rs_seq + 1 full DeltaNet states per layer. The new op GGML_OP_GATED_DELTA_NET_REPLAY keeps one base state (the oldest one a rollback can need) plus a small update record (delta, k, g) per later token. The next pass first re-applies the records of the tokens that were kept, then runs the new tokens. The replay repeats the step's own state update, so the state is bit-identical to the snapshot path. - ggml: new op appended at the end of the enum (GGML_OP_COUNT 102, RPC patch version bump); CPU reference. - SYCL: sequential replay kernel. The blocked kernel can emit records (template flag; the existing instantiations are unchanged). A fused cache write (CPY + SET_ROWS skipped) and the gather-skip are used. - llama: recurrent memory keeps the base in plane 0 and two record banks in plane 1; the tensor layout is unchanged. Partial rollback = fewer records to replay. State save/restore carries the pending records (s_trans = 2). The conv state keeps its snapshot planes. - On by default for a single sequence, qwen35, all recurrent layers on SYCL. LLAMA_GDN_REPLAY=0 restores the snapshot path. - test-backend-ops: GATED_DELTA_NET_REPLAY chain tests (replay == snapshot exactly, and vs CPU). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit e37b71a33b4b55853859ca096c156d313732bfc9 on Torchit1/llama.cpp arc-b580) --- ggml/include/ggml-rpc.h | 4 +- ggml/include/ggml.h | 30 +++ ggml/src/ggml-cpu/ggml-cpu.c | 11 + ggml/src/ggml-cpu/ops.cpp | 145 ++++++++++ ggml/src/ggml-cpu/ops.h | 1 + ggml/src/ggml-sycl/gated_delta_net.cpp | 353 ++++++++++++++++++++++++- ggml/src/ggml-sycl/gated_delta_net.hpp | 13 + ggml/src/ggml-sycl/ggml-sycl.cpp | 94 ++++++- ggml/src/ggml.c | 67 ++++- src/llama-graph.cpp | 39 +++ src/llama-graph.h | 7 + src/llama-memory-recurrent.cpp | 165 +++++++++++- src/llama-memory-recurrent.h | 22 +- src/models/delta-net-base.cpp | 40 +++ src/models/qwen35.cpp | 13 +- tests/test-backend-ops.cpp | 213 ++++++++++++++- 16 files changed, 1192 insertions(+), 25 deletions(-) diff --git a/ggml/include/ggml-rpc.h b/ggml/include/ggml-rpc.h index 059e4496269a..545b892c0eed 100644 --- a/ggml/include/ggml-rpc.h +++ b/ggml/include/ggml-rpc.h @@ -8,10 +8,10 @@ extern "C" { #define RPC_PROTO_MAJOR_VERSION 5 #define RPC_PROTO_MINOR_VERSION 1 -#define RPC_PROTO_PATCH_VERSION 0 +#define RPC_PROTO_PATCH_VERSION 1 #ifdef __cplusplus -static_assert(GGML_OP_COUNT == 101, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION"); +static_assert(GGML_OP_COUNT == 102, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION"); #endif #define GGML_RPC_MAX_SERVERS 16 diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index d875a15ed066..c9339461cf8c 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -596,6 +596,8 @@ extern "C" { GGML_OP_GLU, + GGML_OP_GATED_DELTA_NET_REPLAY, // ARC-LAB: appended last so the existing op numbers do not move + GGML_OP_COUNT, }; @@ -2628,6 +2630,34 @@ extern "C" { struct ggml_tensor * dt_bias, struct ggml_tensor * a); + // ARC-LAB: gated delta net with rollback by replay (independent implementation of the technique described by + // llamADALL). Same recurrence as ggml_gated_delta_net (scalar gate only), but instead of K state snapshots the op + // keeps ONE base state plus, for each later token, its update record: the per-token update is + // S <- g * S + k (x) delta, so a record is (delta [S_v], k [S_v], g [1]) per head, R = H*(2*S_v + 1) floats: + // record = [ delta: S_v*H | k: S_v*H (head h holds k of head h % H_k) | g: H ] (g = the decay factor exp(gate)) + // + // q, k, v, g, beta : as ggml_gated_delta_net (g: [1, H, n_tokens, n_seqs]) + // state : [S_v, S_v, H, n_seqs] stored base state + // rec : F32 [R*(K-1), 2, n_seqs] stored records, two banks per seq (a pass reads one bank, the caller + // stores the new records in the other one) + // ctrl : I32 [2, n_seqs] = {n_rec, bank}: records 0..n_rec-1 of `bank` are first re-applied to state + // + // output, packed: [ attn scores S_v*H*n_tokens*n_seqs | base D*n_seqs | records (K-1)*R*n_seqs ], D = S_v*S_v*H + // with m = min(n_tokens - 1, K - 1): base = state after token n_tokens-1-m (the oldest state a rollback of up + // to m tokens can need), record j = token n_tokens-m+j for j < m; records m..K-2 are left unwritten. + // Rolling back r <= m tokens = re-applying m - r records to base on the next call. + GGML_API struct ggml_tensor * ggml_gated_delta_net_replay( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * g, + struct ggml_tensor * beta, + struct ggml_tensor * state, + struct ggml_tensor * rec, + struct ggml_tensor * ctrl, + int K); + // DSA lightning indexer // // q: [n_embd_idx, n_head_idx, n_batch, ne3 ] diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 10d4d690c862..666c1576011a 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -2112,6 +2112,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm { ggml_compute_forward_gated_delta_net(params, tensor); } break; + case GGML_OP_GATED_DELTA_NET_REPLAY: + { + ggml_compute_forward_gated_delta_net_replay(params, tensor); + } break; case GGML_OP_LIGHTNING_INDEXER: { ggml_compute_forward_lightning_indexer(params, tensor); @@ -2308,6 +2312,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) { case GGML_OP_COUNT_EQUAL: case GGML_OP_SOLVE_TRI: case GGML_OP_GATED_DELTA_NET: + case GGML_OP_GATED_DELTA_NET_REPLAY: case GGML_OP_DSV4_HC_COMB: case GGML_OP_DSV4_HC_PRE: case GGML_OP_DSV4_HC_POST: @@ -3045,6 +3050,12 @@ struct ggml_cplan ggml_graph_plan( const int64_t per_thread = S_v + (K > 1 ? S_v * S_v : 0); cur = per_thread * sizeof(float) * n_tasks; } break; + case GGML_OP_GATED_DELTA_NET_REPLAY: + { + // per thread: working state [S_v, S_v] + delta [S_v] + const int64_t S_v = node->src[2]->ne[0]; + cur = (S_v * S_v + S_v) * sizeof(float) * n_tasks; + } break; case GGML_OP_COUNT: { GGML_ABORT("fatal error"); diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index c7992c78bd14..ccb5ec86b4bb 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -11171,6 +11171,151 @@ void ggml_compute_forward_gated_delta_net( } } +// ARC-LAB: ggml_compute_forward_gated_delta_net_replay (rollback by replay, see ggml_gated_delta_net_replay). +// The per-token step is the scalar-gate path of ggml_compute_forward_gated_delta_net_one_chunk, operation for +// operation, and a replayed record repeats exactly its state update (scale by g, then mad k * delta), so the replayed +// state is bit-identical to the state that path computes for the same token. +static void ggml_compute_forward_gated_delta_net_replay_f32( + const ggml_compute_params * params, + ggml_tensor * dst) { + const ggml_tensor * src_q = dst->src[0]; + const ggml_tensor * src_k = dst->src[1]; + const ggml_tensor * src_v = dst->src[2]; + const ggml_tensor * src_g = dst->src[3]; + const ggml_tensor * src_beta = dst->src[4]; + const ggml_tensor * src_state = dst->src[5]; + const ggml_tensor * src_rec = dst->src[6]; + const ggml_tensor * src_ctrl = dst->src[7]; + + GGML_TENSOR_LOCALS(int64_t, neq, src_q, ne); + GGML_TENSOR_LOCALS(size_t, nbq, src_q, nb); + GGML_TENSOR_LOCALS(int64_t, nek, src_k, ne); + GGML_TENSOR_LOCALS(size_t, nbk, src_k, nb); + GGML_TENSOR_LOCALS(int64_t, nev, src_v, ne); + GGML_TENSOR_LOCALS(size_t, nbv, src_v, nb); + GGML_TENSOR_LOCALS(size_t, nbg, src_g, nb); + GGML_TENSOR_LOCALS(size_t, nbb, src_beta, nb); + + const int64_t S_v = nev0; + const int64_t H = nev1; + const int64_t n_tokens = nev2; + const int64_t n_seqs = nev3; + const int64_t K = ggml_get_op_params_i32(dst, 0); + const int64_t R = H * (2 * S_v + 1); + const int64_t D = S_v * S_v * H; + + // base = state after token t_base; tokens after it leave a record + const int64_t m = std::min(n_tokens - 1, K - 1); + const int64_t t_base = n_tokens - 1 - m; + + float * attn_out_base = (float *) dst->data; + float * base_out = attn_out_base + S_v * H * n_tokens * n_seqs; + float * rec_out = base_out + D * n_seqs; + + const int32_t * ctrl = (const int32_t *) src_ctrl->data; + + const int64_t rq3 = nev3 / neq3; + const int64_t rk3 = nev3 / nek3; + + const float scale = 1.0f / sqrtf((float) S_v); + + const int ith = params->ith; + const int nth = params->nth; + + float * s = (float *) params->wdata + ith * (S_v * S_v + S_v + CACHE_LINE_SIZE_F32); + float * delta = s + S_v * S_v; + + const int64_t nr = H * n_seqs; + const int64_t dr = (nr + nth - 1) / nth; + const int64_t ir0 = dr * ith; + const int64_t ir1 = std::min(ir0 + dr, nr); + + for (int64_t ir = ir0; ir < ir1; ++ir) { + const int64_t iv1 = ir % H; // head + const int64_t iv3 = ir / H; // sequence + + const int64_t iq1 = iv1 % neq1; + const int64_t ik1 = iv1 % nek1; + const int64_t iq3 = iv3 / rq3; + const int64_t ik3 = iv3 / rk3; + + // state layout: s[j*S_v + i] = S[i][j] (row j = value column j) + memcpy(s, (const char *) src_state->data + iv3 * src_state->nb[3] + iv1 * S_v * S_v * sizeof(float), + S_v * S_v * sizeof(float)); + + // replay: re-apply the stored updates of the tokens kept since the base state + const int32_t n_rec = ctrl[2 * iv3 + 0]; + const int32_t bank = ctrl[2 * iv3 + 1]; + GGML_ASSERT(n_rec >= 0 && n_rec <= K - 1 && (bank == 0 || bank == 1)); + const float * rec_in = (const float *) ((const char *) src_rec->data + iv3 * src_rec->nb[2] + bank * src_rec->nb[1]); + for (int32_t j = 0; j < n_rec; ++j) { + const float * r = rec_in + j * R; + const float * rd = r + iv1 * S_v; + const float * rk = r + H * S_v + iv1 * S_v; + ggml_vec_scale_f32(S_v * S_v, s, r[2 * H * S_v + iv1]); + for (int64_t jj = 0; jj < S_v; ++jj) { + ggml_vec_mad_f32(S_v, &s[jj * S_v], rk, rd[jj]); + } + } + + float * attn_data = attn_out_base + (iv3 * n_tokens * H + iv1) * S_v; + + for (int64_t t = 0; t < n_tokens; t++) { + const float * q_d = (const float *)((const char *)src_q->data + iq3 * nbq3 + t * nbq2 + iq1 * nbq1); + const float * k_d = (const float *)((const char *)src_k->data + ik3 * nbk3 + t * nbk2 + ik1 * nbk1); + const float * v_d = (const float *)((const char *)src_v->data + iv3 * nbv3 + t * nbv2 + iv1 * nbv1); + + const float beta_val = *(const float *)((const char *)src_beta->data + iv3 * nbb3 + t * nbb2 + iv1 * nbb1); + const float g0 = *(const float *)((const char *)src_g->data + iv3 * nbg3 + t * nbg2 + iv1 * nbg1); + const float g_val = expf(g0); + + ggml_vec_scale_f32(S_v * S_v, s, g_val); + + for (int64_t j = 0; j < S_v; ++j) { + float sum = 0.0f; + ggml_vec_dot_f32(S_v, &sum, 0, &s[j * S_v], 0, k_d, 0, 1); + delta[j] = (v_d[j] - sum) * beta_val; + } + + for (int64_t j = 0; j < S_v; ++j) { + ggml_vec_mad_f32(S_v, &s[j * S_v], k_d, delta[j]); + } + + for (int64_t j = 0; j < S_v; ++j) { + float sum = 0.0f; + ggml_vec_dot_f32(S_v, &sum, 0, &s[j * S_v], 0, q_d, 0, 1); + attn_data[j] = sum * scale; + } + + attn_data += S_v * H; + + if (t == t_base) { + memcpy(base_out + (iv3 * H + iv1) * S_v * S_v, s, S_v * S_v * sizeof(float)); + } else if (t > t_base) { + float * r = rec_out + iv3 * (K - 1) * R + (t - t_base - 1) * R; + memcpy(r + iv1 * S_v, delta, S_v * sizeof(float)); + memcpy(r + H * S_v + iv1 * S_v, k_d, S_v * sizeof(float)); + r[2 * H * S_v + iv1] = g_val; + } + } + } +} + +void ggml_compute_forward_gated_delta_net_replay( + const ggml_compute_params * params, + ggml_tensor * dst) { + switch (dst->src[0]->type) { + case GGML_TYPE_F32: + { + ggml_compute_forward_gated_delta_net_replay_f32(params, dst); + } break; + default: + { + GGML_ABORT("fatal error"); + } + } +} + // ggml_compute_forward_dsv4_hc_comb diff --git a/ggml/src/ggml-cpu/ops.h b/ggml/src/ggml-cpu/ops.h index 4c1642a67603..ed645ad7a852 100644 --- a/ggml/src/ggml-cpu/ops.h +++ b/ggml/src/ggml-cpu/ops.h @@ -105,6 +105,7 @@ void ggml_compute_forward_rwkv_wkv7(const struct ggml_compute_params * params, s void ggml_compute_forward_solve_tri(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_gla(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_gated_delta_net(const struct ggml_compute_params * params, struct ggml_tensor * dst); +void ggml_compute_forward_gated_delta_net_replay(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_lightning_indexer(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_dsv4_hc_comb(const struct ggml_compute_params * params, struct ggml_tensor * dst); void ggml_compute_forward_dsv4_hc_pre(const struct ggml_compute_params * params, struct ggml_tensor * dst); diff --git a/ggml/src/ggml-sycl/gated_delta_net.cpp b/ggml/src/ggml-sycl/gated_delta_net.cpp index 740a3a1efded..c5e0c7a7eefd 100644 --- a/ggml/src/ggml-sycl/gated_delta_net.cpp +++ b/ggml/src/ggml-sycl/gated_delta_net.cpp @@ -167,19 +167,33 @@ void gated_delta_net_sycl(const float * q, } } +// ARC-LAB replay (Feature A; independent implementation of the technique described by llamADALL): where a +// GATED_DELTA_NET_REPLAY pass puts the state after token t_base (the base) and the update records of the tokens after +// it. Record layout (R = H*(2*S + 1) floats): [delta S*H | k S*H | g H], see ggml_gated_delta_net_replay. +struct gdn_emit { + float * base = nullptr; // [S, S, H] per seq; nullptr = not written + float * rec = nullptr; // record 0 of seq s at rec + (rec_rows ? rec_rows[s] : s) * rec_row_stride + const int64_t * rec_rows = nullptr; // fused cache write: the SET_ROWS row (bank) per seq + int64_t rec_row_stride = 0; // floats + int64_t t_base = 0; // -1: base = the replayed input state (replay-only launch) + int64_t R = 0; +}; + // ARC-LAB token-blocked sequential kernel (non-KDA, S_v = 128). Same per-token recurrence and per-column // arithmetic/reduction order as gated_delta_net_sycl, but the sequential kernel stalls on global-memory latency // every token (~0.4 us/step on the B580 even with the GPU mostly idle) and needs ~5 rounds of threads at 48 heads. // Here each subgroup owns NC columns and shares its k/q registers across them (CUDA's cols_per_warp), and the // work-group stages T tokens of k, q, v, exp(g), beta in SLM; the next block's global loads are issued into // registers before the current block is computed, so their latency overlaps T steps of compute. -template +// emit_t (ARC-LAB replay): also write the base state and the update records described by `em`; the per-token +// arithmetic is untouched, the emission only stores values the step already computed. +template static void gdn_blocked_sycl(const float * q, const float * k, const float * v, const float * g, const float * beta, const float * curr_state, float * dst, float * state, int64_t H, int64_t n_tokens, int64_t n_seqs, int64_t sq1, int64_t sq2, int64_t sq3, int64_t sv1, int64_t sv2, int64_t sv3, int64_t sb1, int64_t sb2, int64_t sb3, int64_t neqk1, int64_t rq3, float scale, int64_t state_slot_stride, int K, const int32_t * s_ids, - int64_t s_row_stride, dpct::queue_ptr stream) { + int64_t s_row_stride, dpct::queue_ptr stream, const gdn_emit em = gdn_emit()) { constexpr int S = 128, L = 16, RPL = S / L; constexpr int NCW = NC * NW, WGS = NW * L, ncb = S / NCW; constexpr int PK = T * S / WGS, PV = T * NCW / WGS; // per-thread prefetch registers @@ -272,6 +286,7 @@ static void gdn_blocked_sycl(const float * q, const float * k, const float * v, const float g_val = sgb[t]; const float beta_val = sgb[T + t]; float out = 0.0f; + [[maybe_unused]] float dl[NC]; // emit_t: this token's delta per column #pragma unroll for (int c = 0; c < NC; ++c) { float kv_shard = 0.0f; @@ -281,6 +296,9 @@ static void gdn_blocked_sycl(const float * q, const float * k, const float * v, } const float kv_col = warp_reduce_sum(kv_shard); const float delta_col = (sv[t * NCW + w * NC + c] - g_val * kv_col) * beta_val; + if constexpr (emit_t) { + dl[c] = delta_col; + } float attn_partial = 0.0f; #pragma unroll for (int r = 0; r < RPL; ++r) { @@ -295,6 +313,37 @@ static void gdn_blocked_sycl(const float * q, const float * k, const float * v, if (lane < NC) { attn[(t0 + t) * S * H + col0 + lane] = out * scale; } + if constexpr (emit_t) { + const int64_t tt = t0 + t; + if (tt == em.t_base) { + float * bs = em.base + (seq * H + h) * S * S; +#pragma unroll + for (int c = 0; c < NC; ++c) { +#pragma unroll + for (int r = 0; r < RPL; ++r) { + bs[(col0 + c) * S + r * L + lane] = s[c][r]; + } + } + } else if (tt > em.t_base) { + float * rr = em.rec + (em.rec_rows ? em.rec_rows[seq] : seq) * em.rec_row_stride + + (tt - em.t_base - 1) * em.R; +#pragma unroll + for (int c = 0; c < NC; ++c) { + if (lane == c) { + rr[h * S + col0 + c] = dl[c]; + } + } + if (colb == 0 && w == 0) { +#pragma unroll + for (int r = 0; r < RPL; ++r) { + rr[H * S + h * S + r * L + lane] = kr[r]; + } + if (lane == 0) { + rr[2 * H * S + h] = g_val; + } + } + } + } if constexpr (keep_rs_t) { const int target_slot = (int) (n_tokens - 1 - (t0 + t)); if (target_slot >= 0 && target_slot < K) { @@ -324,15 +373,16 @@ static void gdn_blocked_sycl(const float * q, const float * k, const float * v, }); } -template +template static void launch_gdn_blocked(int variant, const float * q, const float * k, const float * v, const float * g, const float * beta, const float * s0, float * dst, float * state, int64_t H, int64_t n_tokens, int64_t n_seqs, int64_t sq1, int64_t sq2, int64_t sq3, int64_t sv1, int64_t sv2, int64_t sv3, int64_t sb1, int64_t sb2, int64_t sb3, int64_t neqk1, int64_t rq3, float scale, int64_t state_slot_stride, int K, const int32_t * s_ids, - int64_t s_row_stride, dpct::queue_ptr stream) { -#define GDN_BLK(NC, NW, T) gdn_blocked_sycl(q, k, v, g, beta, s0, dst, state, H, n_tokens, n_seqs, \ - sq1, sq2, sq3, sv1, sv2, sv3, sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, s_ids, s_row_stride, stream) + int64_t s_row_stride, dpct::queue_ptr stream, const gdn_emit & em = gdn_emit()) { +#define GDN_BLK(NC, NW, T) gdn_blocked_sycl(q, k, v, g, beta, s0, dst, state, H, n_tokens, \ + n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, sb1, sb2, sb3, neqk1, rq3, scale, state_slot_stride, K, s_ids, s_row_stride, \ + stream, em) if (variant < 0) { // auto: widest NC that still leaves ~768 subgroups (~60% of the B580's 1280 thread slots) to hide latency. // B580, 1024 tokens (old kernel -> this): 48 heads 3383 -> 1688 us (NC 8, T 8); 32 heads 2762 -> 1024 (NC 4); @@ -803,3 +853,294 @@ void ggml_sycl_op_gated_delta_net_fused_cache(ggml_backend_sycl_context & ctx, g scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/6); ggml_sycl_op_gated_delta_net_impl(ctx, dst, &cache); } + +// ARC-LAB: GATED_DELTA_NET_REPLAY (Feature A: DeltaNet rollback by replay instead of snapshots; independent +// implementation of the technique described by llamADALL). See ggml_gated_delta_net_replay for the data layout. +// +// Sequential kernel: gated_delta_net_sycl's scalar-gate path (same grid, same per-column arithmetic and reduction +// order, lines kept verbatim) with +// - a replay prologue that re-applies n_rec stored records to the loaded state, using the step's own update +// expression (g * S + k * delta) on the stored g, k and delta, so the state matches what the step produced; +// - an epilogue that stores the base state (after token t_base) and the records of the tokens after it. +// With n_tokens == 0 and t_base == -1 it only replays and writes the result to em.base (used before the blocked +// kernel, which has no replay prologue). +template +static void gdn_replay_seq_sycl(const float * q, const float * k, const float * v, const float * g, const float * beta, + const float * curr_state, float * dst, int64_t H, int64_t n_tokens, int64_t sq1, + int64_t sq2, int64_t sq3, int64_t sv1, int64_t sv2, int64_t sv3, int64_t sb1, + int64_t sb2, int64_t sb3, const sycl::uint3 neqk1_magic, const sycl::uint3 rq3_magic, + float scale, const int32_t * s_ids, int64_t s_row_stride, const float * rec_in, + int64_t rec_in_seq_stride, int64_t rec_in_bank_stride, const int32_t * ctrl, + const gdn_emit em) { + auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>(); + const uint32_t h_idx = item_ct1.get_group(2); + const uint32_t sequence = item_ct1.get_group(1); + const int lane = item_ct1.get_local_id(2); + const int col = item_ct1.get_group(0) * item_ct1.get_local_range(1) + item_ct1.get_local_id(1); + + const uint32_t iq1 = fastmodulo(h_idx, neqk1_magic); + const uint32_t iq3 = fastdiv(sequence, rq3_magic); + + float * attn_data = dst; + + const int64_t state_in_offset = (s_ids ? (int64_t) s_ids[sequence] * s_row_stride : (int64_t) sequence * H * S_v * S_v) + + h_idx * S_v * S_v; + curr_state += state_in_offset + col * S_v; + attn_data += (sequence * n_tokens * H + h_idx) * S_v; + + constexpr int warp_size = ggml_sycl_get_physical_warp_size() < S_v ? ggml_sycl_get_physical_warp_size() : S_v; + static_assert(S_v % warp_size == 0, "S_v must be a multiple of warp_size"); + constexpr int rows_per_lane = (S_v + warp_size - 1) / warp_size; + float s_shard[rows_per_lane]; +#pragma unroll + for (int r = 0; r < rows_per_lane; r++) { + const int i = r * warp_size + lane; + s_shard[r] = curr_state[i]; + } + + // replay prologue + if (rec_in != nullptr) { + const int32_t n_rec = ctrl[2 * sequence + 0]; + const int32_t bank = ctrl[2 * sequence + 1]; + const float * rb = rec_in + sequence * rec_in_seq_stride + bank * rec_in_bank_stride; + for (int32_t j = 0; j < n_rec; j++) { + const float * rj = rb + j * em.R; + const float g_val = rj[2 * H * S_v + h_idx]; + const float delta_col = rj[h_idx * S_v + col]; + const float * k_t = rj + H * S_v + h_idx * S_v; +#pragma unroll + for (int r = 0; r < rows_per_lane; r++) { + const int i = r * warp_size + lane; + s_shard[r] = g_val * s_shard[r] + k_t[i] * delta_col; + } + } + } + + float * base_out = em.base ? em.base + (sequence * H + h_idx) * S_v * S_v + col * S_v : nullptr; + float * rec_seq = em.rec ? em.rec + (em.rec_rows ? em.rec_rows[sequence] : (int64_t) sequence) * em.rec_row_stride + : nullptr; + + if (em.t_base < 0 && base_out) { +#pragma unroll + for (int r = 0; r < rows_per_lane; r++) { + base_out[r * warp_size + lane] = s_shard[r]; + } + } + + for (int t = 0; t < n_tokens; t++) { + const float * q_t = q + iq3 * sq3 + t * sq2 + iq1 * sq1; + const float * k_t = k + iq3 * sq3 + t * sq2 + iq1 * sq1; + const float * v_t = v + sequence * sv3 + t * sv2 + h_idx * sv1; + + const int64_t gb_offset = sequence * sb3 + t * sb2 + h_idx * sb1; + const float * beta_t = beta + gb_offset; + const float * g_t = g + gb_offset; + + const float beta_val = *beta_t; + + const float g_val = sycl::native::exp(*g_t); + + // kv[col] = (S^T @ k)[col] = sum_i S[i][col] * k[i] + float kv_shard = 0.0f; +#pragma unroll + for (int r = 0; r < rows_per_lane; r++) { + const int i = r * warp_size + lane; + kv_shard += s_shard[r] * k_t[i]; + } + float kv_col = warp_reduce_sum(kv_shard); + + // delta[col] = (v[col] - g * kv[col]) * beta + float delta_col = (v_t[col] - g_val * kv_col) * beta_val; + + // fused: S[i][col] = g * S[i][col] + k[i] * delta[col] + // attn[col] = (S^T @ q)[col] = sum_i S[i][col] * q[i] + float attn_partial = 0.0f; +#pragma unroll + for (int r = 0; r < rows_per_lane; r++) { + const int i = r * warp_size + lane; + s_shard[r] = g_val * s_shard[r] + k_t[i] * delta_col; + attn_partial += s_shard[r] * q_t[i]; + } + + float attn_col = warp_reduce_sum(attn_partial); + + if (lane == 0) { + attn_data[col] = attn_col * scale; + } + + attn_data += S_v * H; + + // epilogue: base state, then one record per later token + if (t == em.t_base) { + if (base_out) { +#pragma unroll + for (int r = 0; r < rows_per_lane; r++) { + base_out[r * warp_size + lane] = s_shard[r]; + } + } + } else if (t > em.t_base) { + float * rt = rec_seq + (t - em.t_base - 1) * em.R; + if (lane == 0) { + rt[h_idx * S_v + col] = delta_col; + } + if (col == 0) { +#pragma unroll + for (int r = 0; r < rows_per_lane; r++) { + const int i = r * warp_size + lane; + rt[H * S_v + h_idx * S_v + i] = k_t[i]; + } + if (lane == 0) { + rt[2 * H * S_v + h_idx] = g_val; + } + } + } + } +} + +static void launch_gdn_replay_seq(const float * q_d, const float * k_d, const float * v_d, const float * g_d, + const float * b_d, const float * s_d, float * dst_d, int64_t S_v, int64_t H, + int64_t n_tokens, int64_t n_seqs, int64_t sq1, int64_t sq2, int64_t sq3, int64_t sv1, + int64_t sv2, int64_t sv3, int64_t sb1, int64_t sb2, int64_t sb3, int64_t neqk1, + int64_t rq3, float scale, const int32_t * s_ids, int64_t s_row_stride, + const float * rec_in, int64_t rec_in_seq_stride, int64_t rec_in_bank_stride, + const int32_t * ctrl, const gdn_emit & em, dpct::queue_ptr stream) { + GGML_ASSERT(S_v == 128); // supports_op + const int warp_size = ggml_sycl_info().devices[ggml_sycl_get_device()].warp_size; + + const int num_warps = 4; + dpct::dim3 grid_dims(H, n_seqs, (S_v + num_warps - 1) / num_warps); + dpct::dim3 block_dims(warp_size <= S_v ? warp_size : S_v, num_warps, 1); + + const sycl::uint3 neqk1_magic = init_fastdiv_values(neqk1); + const sycl::uint3 rq3_magic = init_fastdiv_values(rq3); + + constexpr int sv = 128; + const gdn_emit e = em; + stream->parallel_for(sycl::nd_range<3>(grid_dims * block_dims, block_dims), + [=](sycl::nd_item<3> /*item_ct1*/) [[sycl::reqd_sub_group_size(WARP_SIZE)]] { + gdn_replay_seq_sycl(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, H, n_tokens, sq1, sq2, sq3, + sv1, sv2, sv3, sb1, sb2, sb3, neqk1_magic, rq3_magic, scale, s_ids, + s_row_stride, rec_in, rec_in_seq_stride, rec_in_bank_stride, ctrl, + e); + }); +} + +void ggml_sycl_op_gated_delta_net_replay(ggml_backend_sycl_context & ctx, ggml_tensor * dst, + const ggml_sycl_gdn_replay_fused * fused) { + scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/8); + + const ggml_tensor * src_q = dst->src[0]; + const ggml_tensor * src_k = dst->src[1]; + const ggml_tensor * src_v = dst->src[2]; + const ggml_tensor * src_g = dst->src[3]; + const ggml_tensor * src_beta = dst->src[4]; + const ggml_tensor * src_s = dst->src[5]; + const ggml_tensor * src_rec = dst->src[6]; + const ggml_tensor * src_ctrl = dst->src[7]; + + GGML_TENSOR_LOCALS(int64_t, neq, src_q, ne); + GGML_TENSOR_LOCALS(size_t , nbq, src_q, nb); + GGML_TENSOR_LOCALS(int64_t, nev, src_v, ne); + GGML_TENSOR_LOCALS(size_t, nbv, src_v, nb); + GGML_TENSOR_LOCALS(size_t, nbb, src_beta, nb); + + const int64_t S_v = nev0; + const int64_t H = nev1; + const int64_t n_tokens = nev2; + const int64_t n_seqs = nev3; + const int64_t K = ggml_get_op_params_i32(dst, 0); + const int64_t R = H * (2 * S_v + 1); + const int64_t D = S_v * S_v * H; + const int64_t m = std::min(n_tokens - 1, K - 1); + + GGML_ASSERT(ggml_are_same_stride(src_q, src_k)); + GGML_ASSERT(ggml_is_contiguous(src_g) && ggml_is_contiguous(src_beta) && ggml_is_contiguous(src_s)); + + const int64_t neqk1 = neq1; + const int64_t rq3 = nev3 / neq3; + + const float * q_d = (const float *) src_q->data; + const float * k_d = (const float *) src_k->data; + const float * v_d = (const float *) src_v->data; + const float * g_d = (const float *) src_g->data; + const float * b_d = (const float *) src_beta->data; + float * dst_d = (float *) dst->data; + + // fused recurrent-state gather (GET_ROWS skipped, see ggml_sycl_try_gdn_gather_skip) + const float * s_d = (const float *) src_s->data; + const int32_t * s_ids = nullptr; + int64_t s_row_stride = 0; + { + auto & mg = g_gdn_gathers[&ctx]; + auto it = mg.find(dst); + if (it != mg.end()) { + s_d = it->second.base; + s_ids = it->second.ids; + s_row_stride = it->second.row_stride; + } + } + + const int64_t sq1 = nbq1 / sizeof(float); + const int64_t sq2 = nbq2 / sizeof(float); + const int64_t sq3 = nbq3 / sizeof(float); + const int64_t sv1 = nbv1 / sizeof(float); + const int64_t sv2 = nbv2 / sizeof(float); + const int64_t sv3 = nbv3 / sizeof(float); + const int64_t sb1 = nbb1 / sizeof(float); + const int64_t sb2 = nbb2 / sizeof(float); + const int64_t sb3 = nbb3 / sizeof(float); + + const float scale = 1.0f / sqrtf((float) S_v); + + const float * rec_in = (const float *) src_rec->data; + const int64_t rec_in_bank_stride = src_rec->nb[1] / sizeof(float); + const int64_t rec_in_seq_stride = src_rec->nb[2] / sizeof(float); + const int32_t * ctrl = (const int32_t *) src_ctrl->data; + + gdn_emit em; + em.R = R; + em.t_base = n_tokens - 1 - m; + if (fused) { + em.base = fused->base; + em.rec = fused->rec; + em.rec_rows = fused->rec_rows; + em.rec_row_stride = fused->rec_row_stride; + } else { + em.base = dst_d + S_v * H * n_tokens * n_seqs; + em.rec = em.base + D * n_seqs; + em.rec_row_stride = (K - 1) * R; + } + + static bool logged = false; + if (!logged) { + logged = true; + GGML_LOG_INFO("%s: ARC-LAB GDN rollback by replay active (cache write %s)\n", __func__, + fused ? "fused" : "via copy"); + } + + dpct::queue_ptr stream = ctx.stream(); + + // same kernel choice as the snapshot path (launch_gated_delta_net, scalar gate, S_v == 128) + static const bool blk_off = getenv("GGML_SYCL_GDN_BLOCKED_OFF") != nullptr; + static const int blk_variant = getenv("GGML_SYCL_GDN_BLK_VARIANT") ? atoi(getenv("GGML_SYCL_GDN_BLK_VARIANT")) : -1; + static const int blk_min = getenv("GGML_SYCL_GDN_BLK_MIN") ? atoi(getenv("GGML_SYCL_GDN_BLK_MIN")) : 16; + if (!blk_off && n_tokens >= blk_min) { + // the blocked kernel has no replay prologue: replay into a scratch state first, then run it from there + ggml_sycl_pool_alloc s_rpl(ctx.pool(), (size_t) D * n_seqs); + gdn_emit em0; + em0.base = s_rpl.get(); + em0.t_base = -1; + em0.R = R; + launch_gdn_replay_seq(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, S_v, H, 0, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, + sb1, sb2, sb3, neqk1, rq3, scale, s_ids, s_row_stride, rec_in, rec_in_seq_stride, + rec_in_bank_stride, ctrl, em0, stream); + // K = 0 with keep_rs_t: no snapshot slots and no final-state store, only the emission + launch_gdn_blocked(blk_variant, q_d, k_d, v_d, g_d, b_d, s_rpl.get(), dst_d, em.base, H, n_tokens, + n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, sb1, sb2, sb3, neqk1, rq3, scale, 0, 0, + nullptr, 0, stream, em); + return; + } + launch_gdn_replay_seq(q_d, k_d, v_d, g_d, b_d, s_d, dst_d, S_v, H, n_tokens, n_seqs, sq1, sq2, sq3, sv1, sv2, sv3, + sb1, sb2, sb3, neqk1, rq3, scale, s_ids, s_row_stride, rec_in, rec_in_seq_stride, + rec_in_bank_stride, ctrl, em, stream); +} diff --git a/ggml/src/ggml-sycl/gated_delta_net.hpp b/ggml/src/ggml-sycl/gated_delta_net.hpp index da5dda3f4969..c086986195da 100644 --- a/ggml/src/ggml-sycl/gated_delta_net.hpp +++ b/ggml/src/ggml-sycl/gated_delta_net.hpp @@ -22,3 +22,16 @@ void ggml_sycl_op_gated_delta_net_fused_cache(ggml_backend_sycl_context & ctx, g void ggml_sycl_gdn_gather_reset(const ggml_backend_sycl_context & ctx); void ggml_sycl_gdn_gather_set(const ggml_backend_sycl_context & ctx, const ggml_tensor * gdn, const float * base, const int32_t * ids, int64_t row_stride); + +// ARC-LAB: GATED_DELTA_NET_REPLAY (rollback by replay). fused = nullptr: base state and records go to dst (the graph +// copies them into the cache); otherwise the kernel writes them straight into the cache (see +// ggml_sycl_try_gdn_replay_fusion): base -> the CPY destination, records -> row rec_rows[seq] of the SET_ROWS target. +struct ggml_sycl_gdn_replay_fused { + float * base; + float * rec; + const int64_t * rec_rows; // device I64, one per seq + int64_t rec_row_stride; // floats +}; + +void ggml_sycl_op_gated_delta_net_replay(ggml_backend_sycl_context & ctx, ggml_tensor * dst, + const ggml_sycl_gdn_replay_fused * fused); diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 354b09854d40..f7037bf36087 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -6024,6 +6024,9 @@ static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct gg case GGML_OP_GATED_DELTA_NET: ggml_sycl_gated_delta_net(ctx, dst); break; + case GGML_OP_GATED_DELTA_NET_REPLAY: + ggml_sycl_op_gated_delta_net_replay(ctx, dst, nullptr); + break; case GGML_OP_OPT_STEP_ADAMW: ggml_sycl_opt_step_adamw(ctx, dst); break; @@ -6359,6 +6362,83 @@ static int ggml_sycl_try_gdn_cache_fusion(const ggml_cgraph * cgraph, int node_i return skip; } +// ARC-LAB: match GATED_DELTA_NET_REPLAY + the two nodes that store its output in the recurrent cache - a CPY of the +// base state and a SET_ROWS of the update records into the bank picked by its row index - so the kernel writes both +// straight into the cache and the two copies are skipped. Single sequence only. Returns the nodes to skip (0 = none). +static int ggml_sycl_try_gdn_replay_fusion(const ggml_cgraph * cgraph, int node_idx, ggml_sycl_gdn_replay_fused & fused) { + if (!g_ggml_sycl_enable_fusion) { + return 0; + } + const ggml_tensor * gdn = cgraph->nodes[node_idx]; + if (gdn->op != GGML_OP_GATED_DELTA_NET_REPLAY || gdn->type != GGML_TYPE_F32 || (gdn->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return 0; + } + const ggml_tensor * src_v = gdn->src[2]; + const int64_t S_v = src_v->ne[0]; + const int64_t H = src_v->ne[1]; + const int64_t n_tokens = src_v->ne[2]; + const int64_t n_seqs = src_v->ne[3]; + const int64_t K = ggml_get_op_params_i32(gdn, 0); + const int64_t D = S_v * S_v * H; + const int64_t R = H * (2 * S_v + 1); + if (n_seqs != 1) { + return 0; + } + const size_t base_off = ggml_row_size(GGML_TYPE_F32, S_v * H * n_tokens * n_seqs); + const size_t rec_off = base_off + ggml_row_size(GGML_TYPE_F32, D * n_seqs); + + // the next two computed nodes must be the CPY and the SET_ROWS (either order), nothing in between + const ggml_tensor * cpy = nullptr; + const ggml_tensor * sr = nullptr; + int skip = 0; + for (int j = node_idx + 1; j < cgraph->n_nodes && (cpy == nullptr || sr == nullptr); ++j) { + const ggml_tensor * n = cgraph->nodes[j]; + if (ggml_sycl_is_view_or_noop(n) || (n->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { + continue; + } + if ((n->flags & GGML_TENSOR_FLAG_OUTPUT)) { + return 0; + } + if (n->op == GGML_OP_CPY && cpy == nullptr) { + cpy = n; + } else if (n->op == GGML_OP_SET_ROWS && sr == nullptr) { + sr = n; + } else { + return 0; + } + skip = j - node_idx; + } + if (cpy == nullptr || sr == nullptr) { + return 0; + } + + // CPY: base region of this op's output -> a contiguous F32 cache view of D * n_seqs floats + const ggml_tensor * cs = cpy->src[0]; + const ggml_tensor * cd = cpy->src[1]; + if (cs->op != GGML_OP_VIEW || cs->view_src != gdn || cs->view_offs != base_off || !ggml_is_contiguous(cs) || + ggml_nelements(cs) != D * n_seqs || cd->type != GGML_TYPE_F32 || cd->data == nullptr || + !ggml_is_contiguous(cd)) { + return 0; + } + + // SET_ROWS: one row of R * (K - 1) records from the record region -> F32 rows (banks), I64 index on the device + const ggml_tensor * rs = sr->src[0]; + const ggml_tensor * ri = sr->src[1]; + if (rs->op != GGML_OP_VIEW || rs->view_src != gdn || rs->view_offs != rec_off || !ggml_is_contiguous(rs) || + rs->ne[0] != R * (K - 1) || ggml_nrows(rs) != 1 || sr->type != GGML_TYPE_F32 || sr->data == nullptr || + sr->ne[0] != R * (K - 1) || sr->nb[0] != sizeof(float) || sr->nb[1] % sizeof(float) != 0 || + ri->type != GGML_TYPE_I64 || ggml_nelements(ri) != 1 || ri->data == nullptr || + (ri->buffer && ggml_backend_buffer_is_host(ri->buffer))) { + return 0; + } + + fused.base = (float *) cd->data; + fused.rec = (float *) sr->data; + fused.rec_rows = (const int64_t *) ri->data; + fused.rec_row_stride = (int64_t) (sr->nb[1] / sizeof(float)); + return skip; +} + // GET_ROWS that gathers a recurrent state row feeding only GATED_DELTA_NET src[5] (possibly through a RESHAPE): // skip it and let the kernel read the cache row directly. Single sequence only. GGML_SYCL_GDN_GATHER=0 disables. static bool ggml_sycl_try_gdn_gather_skip(ggml_backend_sycl_context & ctx, const ggml_cgraph * cgraph, int node_idx) { @@ -6385,7 +6465,7 @@ static bool ggml_sycl_try_gdn_gather_skip(ggml_backend_sycl_context & ctx, const const ggml_tensor * cur = gr; for (int j = node_idx + 1; j < cgraph->n_nodes; ++j) { const ggml_tensor * n = cgraph->nodes[j]; - if (n->op == GGML_OP_GATED_DELTA_NET && n->src[5] == cur) { + if ((n->op == GGML_OP_GATED_DELTA_NET || n->op == GGML_OP_GATED_DELTA_NET_REPLAY) && n->src[5] == cur) { const ggml_tensor * v = n->src[2]; const int64_t D = v->ne[0] * v->ne[0] * v->ne[1]; if (gr->ne[0] != D || v->ne[3] != 1 || ggml_nelements(cur) != D) { @@ -7220,6 +7300,15 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc continue; } } + if (node->op == GGML_OP_GATED_DELTA_NET_REPLAY) { + ggml_sycl_gdn_replay_fused fused; + const int rpl_nodes_to_skip = ggml_sycl_try_gdn_replay_fusion(cgraph, i, fused); + if (rpl_nodes_to_skip > 0) { + ggml_sycl_op_gated_delta_net_replay(*sycl_ctx, node, &fused); + i += rpl_nodes_to_skip; + continue; + } + } if (node->op == GGML_OP_MUL_MAT && ggml_sycl_try_dn_gates(*sycl_ctx, cgraph, i)) { continue; } @@ -8033,6 +8122,9 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons case GGML_OP_GATED_DELTA_NET: // rows-indexed state read (src[6]) and raw gates (op_params[1]) not implemented here return op->src[6] == NULL && ggml_get_op_params_i32(op, 1) == 0; + case GGML_OP_GATED_DELTA_NET_REPLAY: + // ARC-LAB: kernels exist for the scalar gate at head size 128 (Bonsai / Qwen3.5) only + return op->src[2]->ne[0] == 128 && op->src[3]->ne[0] == 1 && ggml_are_same_stride(op->src[0], op->src[1]); case GGML_OP_SSM_CONV: return op->type == GGML_TYPE_F32 && op->src[0]->type == GGML_TYPE_F32 && diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index fa2a9c44f5cf..762655731dbb 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -1114,9 +1114,11 @@ static const char * GGML_OP_NAME[GGML_OP_COUNT] = { "OPT_STEP_SGD", "GLU", + + "GATED_DELTA_NET_REPLAY", }; -static_assert(GGML_OP_COUNT == 101, "GGML_OP_COUNT != 101"); +static_assert(GGML_OP_COUNT == 102, "GGML_OP_COUNT != 102"); static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "none", @@ -1229,9 +1231,11 @@ static const char * GGML_OP_SYMBOL[GGML_OP_COUNT] = { "sgd(x)", "glu(x)", + + "gated_delta_net_replay(q, k, v, g, beta, s, rec, ctrl)", }; -static_assert(GGML_OP_COUNT == 101, "GGML_OP_COUNT != 101"); +static_assert(GGML_OP_COUNT == 102, "GGML_OP_COUNT != 102"); static_assert(GGML_OP_POOL_COUNT == 2, "GGML_OP_POOL_COUNT != 2"); @@ -6441,6 +6445,65 @@ void ggml_gated_delta_net_set_raw_gates( gdn->src[8] = a; } +// ARC-LAB: ggml_gated_delta_net_replay (see ggml.h) +struct ggml_tensor * ggml_gated_delta_net_replay( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * g, + struct ggml_tensor * beta, + struct ggml_tensor * state, + struct ggml_tensor * rec, + struct ggml_tensor * ctrl, + int K) { + GGML_ASSERT(ggml_is_contiguous_rows(q)); + GGML_ASSERT(ggml_is_contiguous_rows(k)); + GGML_ASSERT(ggml_is_contiguous_rows(v)); + GGML_ASSERT(ggml_is_contiguous(g)); + GGML_ASSERT(ggml_is_contiguous(beta)); + GGML_ASSERT(ggml_is_contiguous(state)); + GGML_ASSERT(ggml_is_contiguous(ctrl)); + + GGML_ASSERT(q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F32 && v->type == GGML_TYPE_F32); + GGML_ASSERT(g->type == GGML_TYPE_F32 && beta->type == GGML_TYPE_F32 && state->type == GGML_TYPE_F32); + GGML_ASSERT(rec->type == GGML_TYPE_F32); + GGML_ASSERT(ctrl->type == GGML_TYPE_I32); + + const int64_t S_v = v->ne[0]; + const int64_t H = v->ne[1]; + const int64_t n_tokens = v->ne[2]; + const int64_t n_seqs = v->ne[3]; + + GGML_ASSERT(K >= 2 && n_tokens >= 1); + GGML_ASSERT(q->ne[0] == S_v && k->ne[0] == S_v && q->ne[1] == k->ne[1] && H % q->ne[1] == 0); + GGML_ASSERT(g->ne[0] == 1 && g->ne[1] == H && g->ne[2] == n_tokens && g->ne[3] == n_seqs); // scalar gate only + GGML_ASSERT(beta->ne[0] == 1 && beta->ne[1] == H && beta->ne[2] == n_tokens && beta->ne[3] == n_seqs); + GGML_ASSERT(state->ne[0] == S_v && state->ne[1] == S_v && state->ne[2] == H && state->ne[3] == n_seqs); + + const int64_t R = H * (2 * S_v + 1); + GGML_ASSERT(rec->ne[0] == R * (K - 1) && rec->ne[1] == 2 && rec->ne[2] == n_seqs); + GGML_ASSERT(rec->nb[0] == sizeof(float) && rec->nb[1] == (size_t) rec->ne[0] * sizeof(float)); + GGML_ASSERT(ggml_nelements(ctrl) == 2 * n_seqs); + + const int64_t ne[4] = { S_v * H * n_tokens * n_seqs + S_v * S_v * H * n_seqs + (K - 1) * R * n_seqs, 1, 1, 1 }; + struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne); + + ggml_set_op_params_i32(result, 0, (int32_t) K); + + result->op = GGML_OP_GATED_DELTA_NET_REPLAY; + result->src[0] = q; + result->src[1] = k; + result->src[2] = v; + result->src[3] = g; + result->src[4] = beta; + result->src[5] = state; + result->src[6] = rec; + result->src[7] = ctrl; + + return result; +} + // ggml_lightning_indexer struct ggml_tensor * ggml_lightning_indexer( diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index ceabee25866a..44a31198b323 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -385,6 +385,27 @@ void llm_graph_input_rs::set_input_rs(const llama_memory_recurrent_context * mct } } + if (s_rpl_ctrl) { + GGML_ASSERT(ggml_backend_buffer_is_host(s_rpl_rows->buffer)); + GGML_ASSERT(ggml_backend_buffer_is_host(s_rpl_ctrl->buffer)); + GGML_ASSERT(ggml_backend_buffer_is_host(s_rpl_wrow->buffer)); + int32_t * rows = (int32_t *) s_rpl_rows->data; + int32_t * ctrl = (int32_t *) s_rpl_ctrl->data; + int64_t * wrow = (int64_t *) s_rpl_wrow->data; + + for (uint32_t i = 0; i < n_rs; ++i) { + rows[i] = mctx_cur->rpl_row(i); + } + for (uint32_t i = 0; i < ubatch->n_seqs; ++i) { + int32_t n_rec = 0; + int32_t bank = 0; + mctx_cur->rpl_take(i, ubatch->n_seq_tokens, n_rec, bank); + ctrl[2*i + 0] = n_rec; + ctrl[2*i + 1] = bank; + wrow[i] = bank ^ 1; + } + } + if (s_write_rows) { GGML_ASSERT(ggml_backend_buffer_is_host(s_write_rows->buffer)); int64_t * data = (int64_t *) s_write_rows->data; @@ -417,6 +438,12 @@ bool llm_graph_input_rs::can_reuse_rs(const llama_memory_recurrent_context * mct res &= head == mctx_cur->get_head(); res &= rs_z == mctx_cur->get_rs_z(); + res &= (s_rpl_ctrl != nullptr) == mctx_cur->get_rpl(); + if (s_rpl_ctrl) { + res &= s_rpl_rows->ne[0] == mctx_cur->get_n_rs(); + res &= s_rpl_ctrl->ne[0] == 2 * params.ubatch.n_seqs; + } + return res; } @@ -3578,6 +3605,18 @@ static std::unique_ptr build_rs_inp_impl( inp->s_copy_main = ggml_view_1d(ctx0, inp->s_copy, n_seqs, 0); inp->s_copy_extra = ggml_view_1d(ctx0, inp->s_copy, n_rs - n_seqs, n_seqs * inp->s_copy->nb[0]); + if (mctx_cur->get_rpl()) { + inp->s_rpl_rows = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_rs); + ggml_set_input(inp->s_rpl_rows); + inp->s_rpl_main = ggml_view_1d(ctx0, inp->s_rpl_rows, n_seqs, 0); + inp->s_rpl_extra = ggml_view_1d(ctx0, inp->s_rpl_rows, n_rs - n_seqs, n_seqs * inp->s_rpl_rows->nb[0]); + + inp->s_rpl_ctrl = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 2 * n_seqs); + ggml_set_input(inp->s_rpl_ctrl); + inp->s_rpl_wrow = ggml_new_tensor_1d(ctx0, GGML_TYPE_I64, n_seqs); + ggml_set_input(inp->s_rpl_wrow); + } + inp->head = mctx_cur->get_head(); inp->rs_z = mctx_cur->get_rs_z(); diff --git a/src/llama-graph.h b/src/llama-graph.h index ea9952557a47..0a5b378ddc58 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -346,6 +346,13 @@ class llm_graph_input_rs : public llm_graph_input_i { ggml_tensor * s_write_rows = nullptr; int64_t s_write_K = 0; + // ARC-LAB GDN rollback by replay (llama_memory_recurrent::rpl), null when off: + ggml_tensor * s_rpl_rows = nullptr; // I32 [n_rs]: plane-0 S row each state is read from (no rollback plane) + ggml_tensor * s_rpl_main = nullptr; // view [n_seqs] + ggml_tensor * s_rpl_extra = nullptr; // view [n_rs - n_seqs] + ggml_tensor * s_rpl_ctrl = nullptr; // I32 [2, n_seqs]: {records to re-apply, their bank} + ggml_tensor * s_rpl_wrow = nullptr; // I64 [n_seqs]: bank the new records go to + const llama_memory_recurrent_context * mctx; // used in view offsets, need to match for valid graph reuse diff --git a/src/llama-memory-recurrent.cpp b/src/llama-memory-recurrent.cpp index e2990972ef7e..f5d33f30397b 100644 --- a/src/llama-memory-recurrent.cpp +++ b/src/llama-memory-recurrent.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -34,6 +35,8 @@ llama_memory_recurrent::llama_memory_recurrent( this->n_rs_seq = n_rs_seq; rs_idx.assign(n_seq_max, 0); + rpl_n.assign(n_seq_max, 0); + rpl_bank.assign(n_seq_max, 0); cells.clear(); cells.resize(mem_size); @@ -105,6 +108,39 @@ llama_memory_recurrent::llama_memory_recurrent( s_l[i] = s; } + // ARC-LAB: GDN rollback by replay (LLAMA_GDN_REPLAY=0 keeps the snapshot planes). Applies to a single sequence + // of a qwen35 model whose recurrent layers all run on SYCL; anything else silently keeps the snapshot path. + // The tensor layout is the same either way (planes 0 and 1 of S are reused), only their meaning changes. + { + const char * env = getenv("LLAMA_GDN_REPLAY"); + bool on = !(env && atoi(env) == 0) && n_rs_seq > 0 && n_seq_max == 1 && mem_size == 1 && offload && + model.arch == LLM_ARCH_QWEN35 && type_s == GGML_TYPE_F32 && + getenv("GGML_SYCL_GDN_CHUNKED") == nullptr; // the opt-in chunked prefill has no replay variant + const uint32_t S = hparams.ssm_d_state; + const uint32_t H = hparams.ssm_dt_rank; + on = on && S == 128 && H > 0 && hparams.ssm_d_inner == S * H && hparams.n_embd_s() == S * S * H; + const uint32_t R = H * (2 * S + 1); + // both banks of n_rs_seq records must fit in one S row + on = on && (uint64_t) 2 * n_rs_seq * R <= (uint64_t) hparams.n_embd_s(); + bool any = false; + for (int i = 0; on && i < n_layer; i++) { + if (s_l[i] == nullptr) { + continue; + } + any = true; + ggml_backend_dev_t dev = model.dev_layer(i); + ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr; + on = reg && strcmp(ggml_backend_reg_name(reg), "SYCL") == 0; + } + rpl = on && any; + rpl_R = rpl ? R : 0; + if (rpl) { + LLAMA_LOG_INFO("%s: GDN rollback by replay: on (%u records of %u floats per bank)\n", __func__, n_rs_seq, R); + } else if (n_rs_seq > 0) { + LLAMA_LOG_INFO("%s: GDN rollback by replay: off (snapshot planes)\n", __func__); + } + } + // allocate tensors and initialize the buffers to avoid NaNs in the padding for (auto & [buft, ctx] : ctx_map) { ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); @@ -145,6 +181,7 @@ void llama_memory_recurrent::clear(bool data) { } std::fill(rs_idx.begin(), rs_idx.end(), 0); + rpl_reset(-1); } bool llama_memory_recurrent::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos p1) { @@ -166,6 +203,7 @@ bool llama_memory_recurrent::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos const bool rm_all = p0 == 0 && p1 == std::numeric_limits::max(); if (rm_all) { set_rs_idx(seq_id, 0); + rpl_reset(seq_id); } // models like Mamba or RWKV can't have a state partially erased at the end @@ -185,6 +223,14 @@ bool llama_memory_recurrent::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos // pending rollback is single-use const bool pending = rs_idx[seq_id] != 0; if (!pending && rollback >= 1 && rollback <= (llama_pos) n_rs_seq) { + if (rpl) { + // ARC-LAB replay: only the tokens that left a record can be rolled back + if ((uint32_t) rollback > rpl_n[seq_id]) { + LLAMA_LOG_WARN("%s: cannot roll back %d tokens, only %u kept\n", __func__, rollback, rpl_n[seq_id]); + return false; + } + rpl_n[seq_id] -= (uint32_t) rollback; + } set_rs_idx(seq_id, (uint32_t) rollback); cell.pos = p0 - 1; return true; @@ -194,6 +240,7 @@ bool llama_memory_recurrent::seq_rm(llama_seq_id seq_id, llama_pos p0, llama_pos // invalidate tails which will be cleared if (p0 <= cell.pos && cell.pos < p1) { tail_id = -1; + rpl_reset(seq_id); } } } else { @@ -406,6 +453,18 @@ void llama_memory_recurrent::set_rs_idx(llama_seq_id seq_id, uint32_t idx) { rs_idx[seq_id] = idx; } +void llama_memory_recurrent::rpl_reset(llama_seq_id seq_id) { + if (seq_id < 0) { + std::fill(rpl_n.begin(), rpl_n.end(), 0); + std::fill(rpl_bank.begin(), rpl_bank.end(), 0); + return; + } + if ((size_t) seq_id < rpl_n.size()) { + rpl_n[seq_id] = 0; + rpl_bank[seq_id] = 0; + } +} + std::map llama_memory_recurrent::memory_breakdown() const { std::map ret; for (const auto & [_, buf] : ctxs_bufs) { @@ -745,7 +804,10 @@ void llama_memory_recurrent::state_write(llama_io_write_i & io, llama_seq_id seq std::vector> cell_ranges; // ranges, from inclusive, to exclusive std::vector> cell_ranges_data; // logical source row ranges + std::vector> cell_ranges_data_s; // ARC-LAB replay: S rows (base, plane 0) uint32_t cell_count = 0; + uint32_t rpl_pending = 0; // ARC-LAB replay: records still to apply to the written base state + uint32_t rpl_bank_cur = 0; // Count the number of cells with the specified seq_id // Find all the ranges of cells with this seq id (or all, when -1) @@ -784,6 +846,23 @@ void llama_memory_recurrent::state_write(llama_io_write_i & io, llama_seq_id seq cell_ranges_data.back().second++; } + if (rpl) { + // the base state lives in plane 0 whatever rs_idx says (rs_idx only picks the conv snapshot) + const uint32_t cell_id_s = (cell.src >= 0 ? cell.src : (int32_t) i); + if (cell_ranges_data_s.empty() || cell_ranges_data_s.back().second != cell_id_s) { + cell_ranges_data_s.emplace_back(cell_id_s, cell_id_s + 1); + } else { + cell_ranges_data_s.back().second++; + } + // single sequence (rpl requires n_seq_max == 1): the pending records of the cell's sequence + for (const llama_seq_id cell_seq_id : cell.seq_id) { + if (cell_seq_id >= 0 && (size_t) cell_seq_id < rpl_n.size()) { + rpl_pending = rpl_n[cell_seq_id]; + rpl_bank_cur = rpl_bank[cell_seq_id]; + } + } + } + if (cell_range_begin == size) { cell_range_begin = i; } @@ -818,7 +897,12 @@ void llama_memory_recurrent::state_write(llama_io_write_i & io, llama_seq_id seq io.write(&cell_count, sizeof(cell_count)); state_write_meta(io, cell_ranges, seq_id); - state_write_data(io, cell_ranges_data); + if (!rpl) { + state_write_data(io, cell_ranges_data, cell_ranges_data, 0, 0); + } else { + GGML_ASSERT(cell_count <= 1); // single sequence + state_write_data(io, cell_ranges_data, cell_ranges_data_s, rpl_pending, rpl_bank_cur); + } } void llama_memory_recurrent::state_read(llama_io_read_i & io, llama_seq_id seq_id, llama_state_seq_flags flags) { @@ -871,8 +955,11 @@ void llama_memory_recurrent::state_write_meta(llama_io_write_i & io, const std:: } } -void llama_memory_recurrent::state_write_data(llama_io_write_i & io, const std::vector> & cell_ranges) const { - const uint32_t s_trans = 0; +void llama_memory_recurrent::state_write_data(llama_io_write_i & io, const std::vector> & cell_ranges, + const std::vector> & cell_ranges_s, uint32_t rpl_pending, uint32_t rpl_bank_cur) const { + // ARC-LAB replay: with records pending the state is base + records; the header then says 2 (a reader without + // replay rejects it as an incompatible S layout instead of loading a stale state), and the records follow S + const uint32_t s_trans = rpl_pending > 0 ? 2 : 0; const uint32_t n_layer = hparams.n_layer(); io.write(&s_trans, sizeof(s_trans)); @@ -901,7 +988,7 @@ void llama_memory_recurrent::state_write_data(llama_io_write_i & io, const std:: } } - if (!s_trans) { + if (s_trans != 1) { for (uint32_t il = 0; il < n_layer; ++il) { // skip null layers (read_data will handle this by checking "r_l" and "s_l" for null) if (s_l[il] == nullptr) continue; @@ -916,7 +1003,7 @@ void llama_memory_recurrent::state_write_data(llama_io_write_i & io, const std:: // Write each logical cell row range. With pending recurrent rollback, // the logical current state may live in a rollback snapshot plane. - for (const auto & range : cell_ranges) { + for (const auto & range : cell_ranges_s) { const size_t range_size = range.second - range.first; const size_t buf_size = range_size * s_size_row; io.write_tensor(s_l[il], range.first * s_size_row, buf_size); @@ -953,6 +1040,19 @@ void llama_memory_recurrent::state_write_data(llama_io_write_i & io, const std:: } } } + + if (rpl_pending > 0) { + // ARC-LAB replay: the pending update records of the (single) written cell, per layer + GGML_ASSERT(cell_ranges_s.size() == 1 && cell_ranges_s[0].second - cell_ranges_s[0].first == 1); + const uint32_t cell = cell_ranges_s[0].first; + io.write(&rpl_pending, sizeof(rpl_pending)); + io.write(&rpl_R, sizeof(rpl_R)); + for (uint32_t il = 0; il < n_layer; ++il) { + if (s_l[il] == nullptr) continue; + const size_t off = ((size_t) (size + cell) * hparams.n_embd_s() + (size_t) rpl_bank_cur * n_rs_seq * rpl_R) * sizeof(float); + io.write_tensor(s_l[il], off, (size_t) rpl_pending * rpl_R * sizeof(float)); + } + } } bool llama_memory_recurrent::state_read_meta(llama_io_read_i & io, uint32_t cell_count, llama_seq_id dest_seq_id) { @@ -1065,7 +1165,8 @@ bool llama_memory_recurrent::state_read_data(llama_io_read_i & io, uint32_t cell LLAMA_LOG_ERROR("%s: not enough cells in kv cache to restore state (%u > %u)\n", __func__, cell_count, size); return false; } - if (false != (bool) s_trans) { + // ARC-LAB replay: 2 = plain S layout followed by pending update records (written only when replay is on) + if (s_trans != 0 && !(s_trans == 2 && rpl)) { LLAMA_LOG_ERROR("%s: incompatible s transposition\n", __func__); return false; } @@ -1099,7 +1200,7 @@ bool llama_memory_recurrent::state_read_data(llama_io_read_i & io, uint32_t cell } } - if (!s_trans) { + if (s_trans != 1) { for (uint32_t il = 0; il < n_layer; ++il) { // skip null layers if (s_l[il] == nullptr) continue; @@ -1172,6 +1273,28 @@ bool llama_memory_recurrent::state_read_data(llama_io_read_i & io, uint32_t cell } } + if (s_trans == 2) { + // ARC-LAB replay: pending update records -> bank 0 of the restored cell + uint32_t n_rec = 0; + uint32_t R_ref = 0; + io.read(&n_rec, sizeof(n_rec)); + io.read(&R_ref, sizeof(R_ref)); + if (cell_count != 1 || R_ref != rpl_R || n_rec > n_rs_seq) { + LLAMA_LOG_ERROR("%s: mismatched replay records (%u cells, %u records of %u floats)\n", __func__, cell_count, n_rec, R_ref); + return false; + } + for (uint32_t il = 0; il < n_layer; ++il) { + if (s_l[il] == nullptr) continue; + io.read_tensor(s_l[il], (size_t) (size + head) * hparams.n_embd_s() * sizeof(float), (size_t) n_rec * rpl_R * sizeof(float)); + } + for (const llama_seq_id seq : cells[head].seq_id) { + if (seq >= 0 && (size_t) seq < rpl_n.size()) { + rpl_n[seq] = n_rec; + rpl_bank[seq] = 0; + } + } + } + return true; } @@ -1270,3 +1393,31 @@ int32_t llama_memory_recurrent_context::s_copy(int i) const { } return (int32_t)(idx * mem->size) + src0; } + +bool llama_memory_recurrent_context::get_rpl() const { + return mem->rpl; +} + +int32_t llama_memory_recurrent_context::rpl_row(int i) const { + return mem->cells[i + mem->head].src0; +} + +void llama_memory_recurrent_context::rpl_take(int i, uint32_t n_seq_tokens, int32_t & n_rec, int32_t & bank) const { + const uint32_t cell_idx = i + mem->head; + + n_rec = 0; + bank = 0; + if (mem->cells[cell_idx].seq_id.empty()) { + return; + } + const llama_seq_id seq = *mem->cells[cell_idx].seq_id.begin(); + if (seq < 0 || (size_t) seq >= mem->rpl_n.size()) { + return; + } + n_rec = (int32_t) mem->rpl_n[seq]; + bank = (int32_t) mem->rpl_bank[seq]; + + // this ubatch leaves a base state after token n-1-m and records for its last m tokens, in the other bank + mem->rpl_n[seq] = std::min(n_seq_tokens - 1, mem->n_rs_seq); + mem->rpl_bank[seq] = (uint8_t) (bank ^ 1); +} diff --git a/src/llama-memory-recurrent.h b/src/llama-memory-recurrent.h index b13b7b748f5e..b44915e1a690 100644 --- a/src/llama-memory-recurrent.h +++ b/src/llama-memory-recurrent.h @@ -78,6 +78,18 @@ class llama_memory_recurrent : public llama_memory_i { void set_rs_idx(llama_seq_id seq_id, uint32_t idx); + // ARC-LAB: rollback by replay for the gated delta net state (LLAMA_GDN_REPLAY, default on where it applies; + // independent implementation of the technique described by llamADALL). The S rows of plane 0 hold the base + // state; plane 1 of each cell holds two banks of per-token update records (ggml_gated_delta_net_replay). + // Rolling back r tokens drops the last r pending records instead of selecting a snapshot plane. The conv + // state (r_l) keeps its snapshot planes and rs_idx. + bool rpl = false; + uint32_t rpl_R = 0; // floats per record: n_v_heads * (2 * head_dim + 1) + std::vector rpl_n; // per seq: records still to be re-applied onto the base state + std::vector rpl_bank; // per seq: bank holding them + + void rpl_reset(llama_seq_id seq_id); // seq_id < 0: all + // computed before each graph build uint32_t n = 0; @@ -127,7 +139,8 @@ class llama_memory_recurrent : public llama_memory_i { size_t size_s_bytes() const; void state_write_meta(llama_io_write_i & io, const std::vector> & cell_ranges, llama_seq_id seq_id = -1) const; - void state_write_data(llama_io_write_i & io, const std::vector> & cell_ranges) const; + void state_write_data(llama_io_write_i & io, const std::vector> & cell_ranges, + const std::vector> & cell_ranges_s, uint32_t rpl_pending, uint32_t rpl_bank_cur) const; bool state_read_meta(llama_io_read_i & io, uint32_t cell_count, llama_seq_id dest_seq_id = -1); bool state_read_data(llama_io_read_i & io, uint32_t cell_count); @@ -173,6 +186,13 @@ class llama_memory_recurrent_context : public llama_memory_context_i { int32_t s_copy(int i) const; + // ARC-LAB GDN replay (see llama_memory_recurrent::rpl) + bool get_rpl() const; + int32_t rpl_row(int i) const; // plane-0 S row the state of rs i is read from + // records to re-apply for seq set i of the ubatch and their bank; also books this ubatch's records (call once + // per ubatch, when setting the inputs - like s_copy() consumes rs_idx) + void rpl_take(int i, uint32_t n_seq_tokens, int32_t & n_rec, int32_t & bank) const; + private: const llama_memory_status status; diff --git a/src/models/delta-net-base.cpp b/src/models/delta-net-base.cpp index ae67400c6131..ca8e69085183 100644 --- a/src/models/delta-net-base.cpp +++ b/src/models/delta-net-base.cpp @@ -573,6 +573,46 @@ ggml_tensor * llm_build_delta_net_base::build_recurrent_attn( const int64_t D = S_v * S_v * H_v; const int64_t K = cparams.n_rs_seq + 1; + if (inp->s_rpl_ctrl) { + // ARC-LAB: rollback by replay (independent implementation of the technique described by llamADALL). One + // base state in plane 0 plus the update records of the last K-1 tokens in plane 1 (two banks: this pass + // reads one, its records go to the other) instead of K snapshot planes; see ggml_gated_delta_net_replay. + GGML_ASSERT(state_rows == nullptr && n_seqs == 1 && g->ne[0] == 1); + const int64_t R = H_v * (2 * S_v + 1); + const size_t row_size = ssm_states_all->nb[1]; + const size_t rec_size = ggml_row_size(ssm_states_all->type, R * (K - 1)); + + // both banks of this cell's records: [R*(K-1), 2, n_seqs] + ggml_tensor * rec = ggml_view_3d(ctx0, ssm_states_all, R * (K - 1), 2, n_seqs, rec_size, row_size, + (size_t) (mem_size + kv_head) * row_size); + + ggml_tensor * gdn_out = ggml_gated_delta_net_replay(ctx0, q, k, v, g, b, s, rec, inp->s_rpl_ctrl, (int) K); + res->add_fused_node({n_seq_tokens > 1 ? LLM_FUSED_OP_GDN_CH : LLM_FUSED_OP_GDN_AR, gdn_out, il}); + + ggml_tensor * output = ggml_view_4d(ctx0, gdn_out, + S_v, H_v, n_seq_tokens, n_seqs, + ggml_row_size(gdn_out->type, S_v), + ggml_row_size(gdn_out->type, S_v * H_v), + ggml_row_size(gdn_out->type, S_v * H_v * n_seq_tokens), + 0); + cb(output, "attn_output", il); + + const size_t base_off = ggml_row_size(gdn_out->type, S_v * H_v * n_seq_tokens * n_seqs); + const size_t rec_off = base_off + ggml_row_size(gdn_out->type, D * n_seqs); + + // base state -> plane 0; records -> the bank s_rpl_wrow picks (the SYCL backend fuses both into the op) + ggml_build_forward_expand(gf, ggml_cpy(ctx0, + ggml_view_2d(ctx0, gdn_out, D, n_seqs, ggml_row_size(gdn_out->type, D), base_off), + ggml_view_2d(ctx0, ssm_states_all, D, n_seqs, row_size, (size_t) kv_head * row_size))); + + ggml_build_forward_expand(gf, ggml_set_rows(ctx0, + ggml_view_2d(ctx0, ssm_states_all, R * (K - 1), 2, rec_size, (size_t) (mem_size + kv_head) * row_size), + ggml_view_2d(ctx0, gdn_out, R * (K - 1), n_seqs, rec_size, rec_off), + inp->s_rpl_wrow)); + + return output; + } + const bool raw = gdn_raw_beta && gdn_raw_alpha && gdn_raw_dt_bias && gdn_raw_a; ggml_tensor * gg = raw ? gdn_raw_alpha : g; ggml_tensor * bb = raw ? gdn_raw_beta : b; diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp index c5fedfead481..fc7841e004b2 100644 --- a/src/models/qwen35.cpp +++ b/src/models/qwen35.cpp @@ -491,10 +491,19 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_attn_linear( // GPU device in the model is Metal. static const bool gdn_state_rows_env = getenv("GGML_GDN_STATE_GATHER") == nullptr; - const bool gdn_state_rows = gdn_state_rows_env && gdn_state_rows_dev_ok && cparams.n_rs_seq > 0; + // ARC-LAB: GDN rollback by replay (the memory decides, see llama_memory_recurrent::rpl): the state is the base + // row in plane 0, gathered with its own row indices (s_copy carries the conv rollback plane) + const bool gdn_rpl = inp->s_rpl_ctrl != nullptr; + + const bool gdn_state_rows = gdn_state_rows_env && gdn_state_rows_dev_ok && cparams.n_rs_seq > 0 && !gdn_rpl; ggml_tensor * state; - if (gdn_state_rows) { + if (gdn_rpl) { + state = build_rs(ssm_states_all, inp->s_rpl_main, inp->s_rpl_extra, hparams.n_embd_s(), n_seqs, + mctx_cur->get_n_rs(), mctx_cur->get_head(), mctx_cur->get_size(), mctx_cur->get_rs_z()); + state = ggml_reshape_4d(ctx0, state, head_v_dim, head_v_dim, num_v_heads, n_seqs); + cb(state, "state_predelta", il); + } else if (gdn_state_rows) { state = build_rs_cache_view(inp, ssm_states_all, hparams.n_embd_s(), n_seqs); cb(state, "state_cache_view", il); } else { diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 3c25cc124f14..bb39c3de59e5 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -1218,6 +1218,8 @@ struct test_case { virtual bool run_whole_graph() { return false; } virtual std::vector fusion_test_nodes() { return {}; } virtual bool use_weight_context() { return false; } + // tensor capacity of the test contexts (ARC-LAB: multi-launch chain tests need more than the default) + virtual size_t ctx_tensors() { return 128; } ggml_cgraph * gf = nullptr; ggml_cgraph * gb = nullptr; @@ -1320,7 +1322,7 @@ struct test_case { mode = MODE_TEST; ggml_init_params params = { - /* .mem_size = */ ggml_tensor_overhead()*128 + ggml_graph_overhead(), + /* .mem_size = */ ggml_tensor_overhead()*ctx_tensors() + ggml_graph_overhead(), /* .mem_base = */ NULL, /* .no_alloc = */ true, }; @@ -1511,7 +1513,7 @@ struct test_case { static const size_t graph_nodes = 8192; ggml_init_params params = { - /* .mem_size = */ ggml_tensor_overhead()*128 + ggml_graph_overhead_custom(graph_nodes, false), + /* .mem_size = */ ggml_tensor_overhead()*ctx_tensors() + ggml_graph_overhead_custom(graph_nodes, false), /* .mem_base = */ NULL, /* .no_alloc = */ true, }; @@ -1658,7 +1660,7 @@ struct test_case { static const size_t graph_nodes = 8192; ggml_init_params params = { - /* .mem_size = */ ggml_tensor_overhead()*128 + ggml_graph_overhead_custom(graph_nodes, false), + /* .mem_size = */ ggml_tensor_overhead()*ctx_tensors() + ggml_graph_overhead_custom(graph_nodes, false), /* .mem_base = */ NULL, /* .no_alloc = */ true, }; @@ -1692,7 +1694,7 @@ struct test_case { const std::vector expect = grad_expect(); ggml_init_params params = { - /* .mem_size = */ ggml_tensor_overhead()*128 + 2*ggml_graph_overhead_custom(GGML_DEFAULT_GRAPH_SIZE, true), + /* .mem_size = */ ggml_tensor_overhead()*ctx_tensors() + 2*ggml_graph_overhead_custom(GGML_DEFAULT_GRAPH_SIZE, true), /* .mem_base = */ NULL, /* .no_alloc = */ true, }; @@ -4350,6 +4352,197 @@ struct test_rwkv_wkv6 : public test_case { } }; +// ARC-LAB: GGML_OP_GATED_DELTA_NET_REPLAY (rollback by replay; independent implementation of the technique described +// by llamADALL). One graph runs a chain of launches twice, the way the recurrent memory does across MTP steps: +// snapshot path: GATED_DELTA_NET(K) -> CPY of the snapshots into a K-plane cache; the next launch reads plane r +// replay path: GATED_DELTA_NET_REPLAY -> CPY of the base into row 0 + SET_ROWS of the records into one bank of +// row 1; the next launch re-applies m - r records ({n_rec, bank} in its ctrl input) +// where r is the rollback (rejected drafts) chosen per step. Each launch reads the cache through the previous CPY / +// SET_ROWS result, which orders the launches and keeps the cache-write fusion of the real graph. +// exact = true : output = snapshot results - replay results (attention outputs of every step + final state); +// must be exactly 0 on each backend (max |x|, threshold 0) +// exact = false: output = replay results, compared with the CPU backend (nmse) +// The last step must have 1 token, so both paths hold the final state in their first row. +struct test_gdn_replay_chain : public test_case { + const int64_t head_count; // q/k heads + const int64_t v_repeat; // v heads = head_count * v_repeat + const int64_t K; // snapshot slots = 1 + records per bank + const std::vector> steps; // (n_tokens, rollback after the step) + const bool exact; + + static constexpr int64_t S = 128; + + test_gdn_replay_chain(int64_t head_count, int64_t v_repeat, int64_t K, + std::vector> steps, bool exact) + : head_count(head_count), v_repeat(v_repeat), K(K), steps(std::move(steps)), exact(exact) {} + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "GATED_DELTA_NET_REPLAY"; + } + + std::string vars() override { + std::string st; + for (const auto & p : steps) { + st += (st.empty() ? "" : " ") + std::to_string(p.first) + "-" + std::to_string(p.second); + } + return VARS_TO_STR4(head_count, v_repeat, K, exact) + ",steps=[" + st + "]"; + } + + bool run_whole_graph() override { return true; } + size_t ctx_tensors() override { return 64 * steps.size() + 128; } + + double max_nmse_err() override { return exact ? 0.0 : 1e-7; } + + double err(const float * a, const float * b, size_t n) override { + if (!exact) { + return nmse(a, b, n); + } + double mx = 0.0; + for (size_t i = 0; i < n; i++) { + mx = std::max(mx, (double) std::fabs(a[i])); + mx = std::max(mx, (double) std::fabs(b[i])); + } + if (mx > 0.0) { + printf("[replay != snapshot: max |diff| %g] ", mx); + } + return mx; + } + + ggml_tensor * build_graph(ggml_context * ctx) override { + const int64_t H = head_count * v_repeat; + const int64_t D = S * S * H; + const int64_t R = H * (2 * S + 1); + const int64_t RK = R * (K - 1); + GGML_ASSERT(!steps.empty() && steps.back().first == 1 && steps.back().second == 0); + GGML_ASSERT(2 * RK <= D); + + ggml_tensor * snap_cache = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, D, K); + ggml_tensor * rpl_cache = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, D, 2); + ggml_set_name(snap_cache, "snap_cache"); + ggml_set_name(rpl_cache, "rpl_cache"); + + ggml_tensor * snap_src = snap_cache; // [D, rows]: the rows the next snapshot launch can read + ggml_tensor * base_src = rpl_cache; // [D, rows]: row 0 = base state + ggml_tensor * rec_src = ggml_view_3d(ctx, rpl_cache, RK, 2, 1, RK * sizeof(float), rpl_cache->nb[1], rpl_cache->nb[1]); + + ggml_tensor * zero_idx = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1); + ggml_set_name(zero_idx, "idx_0"); + + ggml_tensor * out_snap = nullptr; + ggml_tensor * out_rpl = nullptr; + auto append = [&](ggml_tensor * acc, ggml_tensor * t) { + t = ggml_view_1d(ctx, t, ggml_nelements(t), 0); + return acc ? ggml_concat(ctx, acc, t, 0) : t; + }; + + int64_t snap_plane = 0; // rollback plane the next snapshot launch reads + int64_t n_rec = 0, bank = 0; + for (size_t i = 0; i < steps.size(); ++i) { + const int64_t n = steps[i].first; + const int64_t r = steps[i].second; + const int64_t m = std::min(n - 1, K - 1); + GGML_ASSERT(r <= m); + + ggml_tensor * q = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, S, head_count, n, 1); + ggml_tensor * k = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, S, head_count, n, 1); + ggml_tensor * v = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, S, H, n, 1); + ggml_tensor * g = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 1, H, n, 1); + ggml_tensor * b = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, 1, H, n, 1); + ggml_set_name(v, "v"); + ggml_set_name(g, "g"); + ggml_set_name(b, "beta"); + q = ggml_l2_norm(ctx, q, 1e-6f); + k = ggml_l2_norm(ctx, k, 1e-6f); + + // snapshot path + ggml_tensor * sidx = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1); + ggml_set_name(sidx, ("idx_" + std::to_string(snap_plane)).c_str()); + ggml_tensor * st = ggml_reshape_4d(ctx, ggml_get_rows(ctx, snap_src, sidx), S, S, H, 1); + ggml_tensor * o = ggml_gated_delta_net(ctx, q, k, v, g, b, st, K); + const int64_t n_written = std::min(n, K); + ggml_tensor * cs = ggml_cpy(ctx, + ggml_view_3d(ctx, o, D, 1, n_written, D * sizeof(float), D * sizeof(float), S * H * n * sizeof(float)), + ggml_view_3d(ctx, snap_cache, D, 1, n_written, snap_cache->nb[1], snap_cache->nb[1], 0)); + if (gf) { + ggml_build_forward_expand(gf, cs); + } + snap_src = ggml_reshape_2d(ctx, cs, D, n_written); + snap_plane = r; + out_snap = append(out_snap, ggml_view_1d(ctx, o, S * H * n, 0)); + + // replay path + ggml_tensor * ctrl = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 2); + ggml_set_name(ctrl, ("ctrl_" + std::to_string(n_rec) + "_" + std::to_string(bank)).c_str()); + ggml_tensor * wrow = ggml_new_tensor_1d(ctx, GGML_TYPE_I64, 1); + ggml_set_name(wrow, ("wrow_" + std::to_string(bank ^ 1)).c_str()); + ggml_tensor * bs = ggml_reshape_4d(ctx, ggml_get_rows(ctx, base_src, zero_idx), S, S, H, 1); + ggml_tensor * o2 = ggml_gated_delta_net_replay(ctx, q, k, v, g, b, bs, rec_src, ctrl, (int) K); + const size_t base_off = S * H * n * sizeof(float); + ggml_tensor * cb = ggml_cpy(ctx, ggml_view_2d(ctx, o2, D, 1, D * sizeof(float), base_off), + ggml_view_2d(ctx, rpl_cache, D, 1, rpl_cache->nb[1], 0)); + ggml_tensor * sr = ggml_set_rows(ctx, + ggml_view_2d(ctx, rpl_cache, RK, 2, RK * sizeof(float), rpl_cache->nb[1]), + ggml_view_2d(ctx, o2, RK, 1, RK * sizeof(float), base_off + D * sizeof(float)), wrow); + if (gf) { + ggml_build_forward_expand(gf, cb); + ggml_build_forward_expand(gf, sr); + } + base_src = ggml_reshape_2d(ctx, cb, D, 1); + rec_src = ggml_reshape_3d(ctx, sr, RK, 2, 1); + n_rec = m - r; + bank ^= 1; + out_rpl = append(out_rpl, ggml_view_1d(ctx, o2, S * H * n, 0)); + + if (i + 1 == steps.size()) { + out_snap = append(out_snap, ggml_view_1d(ctx, cs, D, 0)); + out_rpl = append(out_rpl, ggml_view_1d(ctx, cb, D, 0)); + } + } + + ggml_tensor * out = exact ? ggml_sub(ctx, out_snap, out_rpl) : out_rpl; + ggml_set_name(out, "out"); + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + std::mt19937 rng(1234); + std::uniform_real_distribution u(-1.0f, 1.0f); + std::vector s0(S * S * head_count * v_repeat); + for (auto & x : s0) { + x = u(rng); + } + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + if (ggml_is_view_op(t->op) || t->op != GGML_OP_NONE) { continue; } + const std::string name = t->name; + if (name == "snap_cache" || name == "rpl_cache") { + // the same initial state in row 0 of both caches, zeros elsewhere (no records to replay yet) + std::vector z(ggml_nelements(t), 0.0f); + std::copy(s0.begin(), s0.end(), z.begin()); + ggml_backend_tensor_set(t, z.data(), 0, ggml_nbytes(t)); + } else if (name.rfind("idx_", 0) == 0) { + const int32_t x = std::stoi(name.substr(4)); + ggml_backend_tensor_set(t, &x, 0, sizeof(x)); + } else if (name.rfind("ctrl_", 0) == 0) { + const size_t us = name.find('_', 5); + const int32_t x[2] = { std::stoi(name.substr(5, us - 5)), std::stoi(name.substr(us + 1)) }; + ggml_backend_tensor_set(t, x, 0, sizeof(x)); + } else if (name.rfind("wrow_", 0) == 0) { + const int64_t x = std::stoll(name.substr(5)); + ggml_backend_tensor_set(t, &x, 0, sizeof(x)); + } else if (name == "g") { + init_tensor_uniform(t, -3.0f, -1e-4f); // keep some of the state alive across the chain + } else if (name == "beta") { + init_tensor_uniform(t, 0.0f, 1.0f); + } else if (name == "v") { + init_tensor_uniform(t, -0.3f, 5.0f); + } else if (t->type == GGML_TYPE_F32) { + init_tensor_uniform(t); + } + } + } +}; + // GGML_OP_GATED_DELTA_NET struct test_gated_delta_net : public test_case { const ggml_type type; @@ -10795,6 +10988,18 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 1, 2, 2, false, false, 1, false, -1, true)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 64, 1, 1, false, false, 1, false, -1, true)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 128, 1, 2, 1, false, false, 2, true, -1, true)); + + // ARC-LAB: GDN rollback by replay - replay chain == snapshot chain (exact), and replay chain vs CPU. + // steps (n_tokens, rollback): prompt-sized launches (>= 16 tokens: blocked kernel on SYCL), MTP verify launches + // with partial acceptance, full rejection (rollback = n - 1) and a prompt after pending records. + for (bool exact : { true, false }) { + const std::vector> chain_a = { {20, 0}, {5, 2}, {5, 0}, {3, 1}, {5, 4}, {2, 0}, {1, 0} }; + const std::vector> chain_b = { {5, 1}, {17, 0}, {4, 3}, {18, 2}, {5, 1}, {1, 0} }; + test_cases.emplace_back(new test_gdn_replay_chain(4, 1, 5, chain_a, exact)); + test_cases.emplace_back(new test_gdn_replay_chain(2, 3, 5, chain_b, exact)); + test_cases.emplace_back(new test_gdn_replay_chain(16, 3, 5, chain_a, exact)); // Bonsai: 16 q/k, 48 v heads + test_cases.emplace_back(new test_gdn_replay_chain(2, 2, 3, { {7, 1}, {3, 2}, {3, 0}, {1, 0} }, exact)); + } test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 32, 16, 1, 1, 1, false, true)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 16, 64, 1, 2)); test_cases.emplace_back(new test_gated_delta_net(GGML_TYPE_F32, 4, 64, 4, 1)); From 602cd2f0621f03f494b6657650d00c3c529dd358 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:50:04 +1100 Subject: [PATCH 48/65] mtp: draft chain off by default until its graph is reused (measured -0.2..-0.8% on the B580) Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 4798af11c85414539b2c8f8b7c3b5b9d049ba4db on Torchit1/llama.cpp arc-b580) --- common/speculative.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/common/speculative.cpp b/common/speculative.cpp index b3ec6ff87aa9..e9ece1006bb1 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -2418,15 +2418,15 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { deferred.assign(n_seq, {}); - // chained drafts: default on when the head has a trimmed draft vocabulary (its token_embd rows fit on the - // device; measured slower without one), LLAMA_MTP_CHAIN=1 forces, =0 off. Greedy single-sequence drafting only. + // chained drafts: default OFF for now (B580, 2026-10-10: identical drafts but -0.2..-0.8%, the chain graph is rebuilt + // on every draft; revisit with graph reuse). LLAMA_MTP_CHAIN=1 turns it on. Greedy single-sequence drafting only. { const char * env = getenv("LLAMA_MTP_CHAIN"); const int32_t sup = llama_mtp_chain_support(ctx_dft); - const bool want = env ? atoi(env) != 0 : (sup & 2) != 0; + const bool want = env ? atoi(env) != 0 : false; const char * why = nullptr; if (!want) { - why = env ? "LLAMA_MTP_CHAIN=0" : "no trimmed draft head (LLAMA_MTP_CHAIN=1 forces it)"; + why = env ? "LLAMA_MTP_CHAIN=0" : "off by default (LLAMA_MTP_CHAIN=1 turns it on)"; } else if (sup == 0) { why = "not supported by this context (qwen35 single head, flash attention, plain KV cache)"; } else if (is_mem_shared || chain_heads) { From fdb28c12ec3c49ed0054e33870bc7c8c697395cb Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:04:20 +1100 Subject: [PATCH 49/65] mtp: reuse the draft-chain graph across drafts (cached per batch shape) Independent implementation of the technique described by llamADALL (follow-up). The chain graph was built, split and allocated on every draft because allow_reuse refused chain graphs, which cost more host time than the chain saved (-0.2..-0.8% on the B580, drafts identical). Now: - llama_context keeps a scheduler + graph result per chain batch shape (steps, rows) and swaps them in around process_ubatch, so a draft of a known shape only sets inputs and computes; regular decodes keep their own gf_res_prev. - allow_reuse accepts chain graphs with the same step count; the chain input's can_reuse also requires the same rows and the same per-step KV lengths (get_n_kv_prefix), which are baked into the step views. - per-step position and output-row inputs are packed into two tensors (two uploads per draft instead of 2 x steps). Same graph, same values: outputs unchanged. Still default off. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit d6435c6f00c28bf31a463f6b557f9a0ca4fa0e7d on Torchit1/llama.cpp arc-b580) --- src/llama-context.cpp | 55 +++++++++++++++++++++------- src/llama-context.h | 10 ++++++ src/llama-graph.h | 4 +-- src/models/qwen35.cpp | 83 +++++++++++++++++++++++++++---------------- 4 files changed, 108 insertions(+), 44 deletions(-) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 223273004829..eb4f6ea76edc 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -711,6 +711,8 @@ void llama_context::sched_reserve() { synchronize(); + mtp_chain_cache.clear(); // ARC-LAB: built under the old settings + const int64_t t_start_us = ggml_time_us(); const uint32_t n_seqs = cparams.n_seq_max; @@ -1432,6 +1434,24 @@ void llama_context::set_mtp_chain(uint32_t n_steps) { ggml_type_name(src->type), staging.size() / 1048576.0, ggml_backend_dev_name(dev)); } +llama_context::mtp_chain_entry * llama_context::mtp_chain_entry_for(uint32_t n_tokens) { + const uint32_t key = (cparams.mtp_chain_n << 16) | n_tokens; + auto it = mtp_chain_cache.find(key); + if (it != mtp_chain_cache.end()) { + return &it->second; + } + if (mtp_chain_cache.size() >= 16) { // shapes are catch-up rows (1..n_max+1) + steps: a handful in practice + synchronize(); + mtp_chain_cache.clear(); + } + const size_t max_nodes = graph_max_nodes(std::min(cparams.n_ctx, cparams.n_ubatch)); + mtp_chain_entry e; + e.sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, false, + cparams.op_offload)); + e.res.reset(new llm_graph_result(max_nodes)); + return &(mtp_chain_cache[key] = std::move(e)); +} + const int32_t * llama_context::get_mtp_chain_ids() { synchronize(); @@ -2174,8 +2194,31 @@ int llama_context::decode(const llama_batch & batch_inp) { ggml_status status; + // ARC-LAB: a chained MTP draft runs on the cached scheduler + graph of its batch shape (mtp_chain_cache) + mtp_chain_entry * chain_e = cparams.mtp_chain_n > 0 ? mtp_chain_entry_for(ubatch.n_tokens) : nullptr; + if (chain_e) { + std::swap(sched, chain_e->sched); + std::swap(gf_res_prev, chain_e->res); + } + const auto * res = process_ubatch(ubatch, ctx_type_to_graph_type(cparams.ctx_type), mctx.get(), status); + if (chain_e) { + // the per-step argmax rows, mapped to tokens in get_mtp_chain_ids() + if (res && !res->t_mtp_chain.empty()) { + mtp_chain_out.resize(res->t_mtp_chain.size()); + for (size_t s = 0; s < res->t_mtp_chain.size(); ++s) { + ggml_tensor * t = res->t_mtp_chain[s]; + ggml_backend_t b = ggml_backend_sched_get_tensor_backend(sched.get(), t); + GGML_ASSERT(b != nullptr); + ggml_backend_tensor_get_async(b, t, &mtp_chain_out[s], 0, sizeof(int32_t)); + } + mtp_chain_n_out = (uint32_t) res->t_mtp_chain.size(); + } + std::swap(sched, chain_e->sched); + std::swap(gf_res_prev, chain_e->res); + } + if (!res) { // the last ubatch failed or was aborted -> remove all positions of that ubatch from the memory module llama_pos pos_min[LLAMA_MAX_SEQ]; @@ -2316,18 +2359,6 @@ int llama_context::decode(const llama_batch & batch_inp) { extract_layer_inputs(res, n_tokens_prev, ubatch.n_tokens); - // ARC-LAB: chained MTP draft: the per-step argmax rows, mapped to tokens in get_mtp_chain_ids() - if (!res->t_mtp_chain.empty()) { - mtp_chain_out.resize(res->t_mtp_chain.size()); - for (size_t s = 0; s < res->t_mtp_chain.size(); ++s) { - ggml_tensor * t = res->t_mtp_chain[s]; - ggml_backend_t b = ggml_backend_sched_get_tensor_backend(sched.get(), t); - GGML_ASSERT(b != nullptr); - ggml_backend_tensor_get_async(b, t, &mtp_chain_out[s], 0, sizeof(int32_t)); - } - mtp_chain_n_out = (uint32_t) res->t_mtp_chain.size(); - } - // extract nextn embeddings before // only meaningful in LLAMA_POOLING_TYPE_NONE (per-token); other pooling modes are ignored. { diff --git a/src/llama-context.h b/src/llama-context.h index ef04bf85de9c..16f308b140b2 100644 --- a/src/llama-context.h +++ b/src/llama-context.h @@ -418,6 +418,16 @@ struct llama_context { std::vector mtp_chain_vocab; // draft-head index -> token id (empty: no draft head, indices are tokens) std::vector mtp_chain_out; std::vector mtp_chain_tok; // mtp_chain_out mapped to token ids + + // ARC-LAB: chain graphs are kept per batch shape, each with its own scheduler, so a draft whose shape (rows, steps) + // and per-step KV lengths match an earlier one reuses its built and allocated graph: no build, split or allocation + // per draft, and the regular decodes keep their own reusable graph in gf_res_prev + struct mtp_chain_entry { + ggml_backend_sched_ptr sched; + llm_graph_result_ptr res; + }; + std::map mtp_chain_cache; + mtp_chain_entry * mtp_chain_entry_for(uint32_t n_tokens); uint32_t mtp_chain_n_out = 0; // host buffer for the model output (logits and embeddings) diff --git a/src/llama-graph.h b/src/llama-graph.h index 0a5b378ddc58..7b1fd48989f8 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -939,8 +939,8 @@ struct llm_graph_params { return false; } - // ARC-LAB: a chained MTP draft graph bakes per-step KV lengths in; never reuse it or reuse into it - if (cparams.mtp_chain_n > 0 || other.cparams.mtp_chain_n > 0) { + // ARC-LAB: a chained MTP draft graph only for the same step count (its input checks the per-step KV lengths) + if (cparams.mtp_chain_n != other.cparams.mtp_chain_n) { return false; } diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp index fc7841e004b2..0e81f3dba9a9 100644 --- a/src/models/qwen35.cpp +++ b/src/models/qwen35.cpp @@ -618,47 +618,63 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_ffn(ggml_tensor * cur, cons return cur; } -// ARC-LAB: per-step inputs of a chained MTP draft graph (graph_mtp::build_chain): the positions of each step's rows in -// the layout build_inp_pos() gives a decode of just those rows, and the step's output row +// ARC-LAB: per-step inputs of a chained MTP draft graph (graph_mtp::build_chain), packed into two tensors so a draft +// uploads them in two writes: pos holds each step's positions in the layout build_inp_pos() gives a decode of just +// that step's rows (step after step), out_idx each step's output row. The graph reads them through per-step views. class llm_graph_input_mtp_chain : public llm_graph_input_i { public: explicit llm_graph_input_mtp_chain(uint32_t n_pos_per_embd) : n_pos_per_embd(n_pos_per_embd) {} struct step { - int64_t r0; // first ubatch row of the step - int64_t nr; // rows - ggml_tensor * pos; // I32 [nr * n_pos_per_embd] - ggml_tensor * out_idx; // I32 [1] = nr - 1 (the step's last row produces its draft) + int64_t r0; // first ubatch row of the step + int64_t nr; // rows + uint32_t n_kv; // KV length the step attends over (baked into the graph's views) }; std::vector steps; + int64_t n_tokens = 0; + ggml_tensor * pos = nullptr; // I32 [n_tokens * n_pos_per_embd] + ggml_tensor * out_idx = nullptr; // I32 [n_steps], step s: nr - 1 (the step's last row produces its draft) void set_input(const llama_ubatch * ubatch) override { - const int64_t n_tokens = ubatch->n_tokens; - std::vector v; - for (const auto & st : steps) { - v.assign((size_t) st.nr * n_pos_per_embd, 0); + GGML_ASSERT(ubatch->n_tokens == n_tokens); + std::vector v((size_t) n_tokens * n_pos_per_embd, 0); + std::vector o(steps.size()); + size_t off = 0; + for (size_t s = 0; s < steps.size(); ++s) { + const auto & st = steps[s]; + int32_t * vs = v.data() + off; for (int64_t r = 0; r < st.nr; ++r) { if (ubatch->token && n_pos_per_embd == 4) { // text tokens under M-RoPE, as llm_graph_input_pos: three equal dims, the fourth 0 for (int d = 0; d < 3; ++d) { - v[d * st.nr + r] = ubatch->pos[st.r0 + r]; + vs[d * st.nr + r] = ubatch->pos[st.r0 + r]; } } else { for (uint32_t d = 0; d < n_pos_per_embd; ++d) { - v[d * st.nr + r] = ubatch->pos[d * n_tokens + st.r0 + r]; + vs[d * st.nr + r] = ubatch->pos[d * n_tokens + st.r0 + r]; } } } - ggml_backend_tensor_set(st.pos, v.data(), 0, v.size() * sizeof(int32_t)); - const int32_t out_row = (int32_t) st.nr - 1; - ggml_backend_tensor_set(st.out_idx, &out_row, 0, sizeof(out_row)); + off += (size_t) st.nr * n_pos_per_embd; + o[s] = (int32_t) st.nr - 1; } + ggml_backend_tensor_set(pos, v.data(), 0, v.size() * sizeof(int32_t)); + ggml_backend_tensor_set(out_idx, o.data(), 0, o.size() * sizeof(int32_t)); } - // never reused (llm_graph_params::allow_reuse refuses chain graphs) + // reusable when the batch has the same rows and every step would attend over the same KV length (the rest of the + // topology is checked by allow_reuse and the other inputs) bool can_reuse(const llm_graph_params & params) override { - GGML_UNUSED(params); - return false; + if (params.ubatch.n_tokens != n_tokens || params.cparams.mtp_chain_n != steps.size()) { + return false; + } + const auto * kvc = static_cast(params.mctx); + for (const auto & st : steps) { + if (kvc->get_n_kv_prefix((uint32_t) (st.r0 + st.nr)) != st.n_kv) { + return false; + } + } + return true; } const uint32_t n_pos_per_embd; @@ -956,18 +972,23 @@ void llama_model_qwen35::graph_mtp::build_chain(const llama_model & model) { auto * inp_attn = build_attn_inp_kv(); const llama_kv_cache_context * kvc = inp_attn->mctx; - auto chain_inp = std::make_unique(hparams.n_pos_per_embd()); + const uint32_t n_pos = hparams.n_pos_per_embd(); + auto chain_inp = std::make_unique(n_pos); + chain_inp->n_tokens = n_tokens; + chain_inp->pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens * n_pos); + chain_inp->out_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_steps); + ggml_set_input(chain_inp->pos); + ggml_set_input(chain_inp->out_idx); for (int64_t s = 0; s < n_steps; ++s) { llm_graph_input_mtp_chain::step st; - st.r0 = s == 0 ? 0 : n_lead - 1 + s; - st.nr = s == 0 ? n_lead : 1; - st.pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, st.nr * hparams.n_pos_per_embd()); - st.out_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, 1); - ggml_set_input(st.pos); - ggml_set_input(st.out_idx); + st.r0 = s == 0 ? 0 : n_lead - 1 + s; + st.nr = s == 0 ? n_lead : 1; + st.n_kv = kvc->get_n_kv_prefix((uint32_t) (st.r0 + st.nr)); chain_inp->steps.push_back(st); } - const auto steps = chain_inp->steps; + const auto steps = chain_inp->steps; + ggml_tensor * pos_all = chain_inp->pos; + ggml_tensor * out_all = chain_inp->out_idx; res->add_input(std::move(chain_inp)); // token embeddings: the host table for the ubatch's own tokens (catch-up tokens need not be in the draft @@ -1018,7 +1039,9 @@ void llama_model_qwen35::graph_mtp::build_chain(const llama_model & model) { cb(tok_embd, "mtp_tok_embd", il); // this step's slice of the ubatch's KV indices and mask, sized as a decode of rows [0, r0 + nr) would be - const uint32_t n_kv_s = kvc->get_n_kv_prefix((uint32_t) (st.r0 + nr)); + const uint32_t n_kv_s = st.n_kv; + ggml_tensor * st_pos = ggml_view_1d(ctx0, pos_all, nr * n_pos, st.r0 * n_pos * ggml_element_size(pos_all)); + ggml_tensor * st_out = ggml_view_1d(ctx0, out_all, 1, s * ggml_element_size(out_all)); ggml_tensor * k_idxs = inp_attn->get_k_idxs(); ggml_tensor * v_idxs = inp_attn->get_v_idxs(); k_idxs = ggml_view_1d(ctx0, k_idxs, nr, st.r0 * ggml_element_size(k_idxs)); @@ -1071,10 +1094,10 @@ void llama_model_qwen35::graph_mtp::build_chain(const llama_model & model) { Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, nr); cb(Vcur, "mtp_Vcur", il); - Qcur = ggml_rope_multi(ctx0, Qcur, st.pos, nullptr, + Qcur = ggml_rope_multi(ctx0, Qcur, st_pos, nullptr, n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); - Kcur = ggml_rope_multi(ctx0, Kcur, st.pos, nullptr, + Kcur = ggml_rope_multi(ctx0, Kcur, st_pos, nullptr, n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale, ext_factor, attn_factor, beta_fast, beta_slow); @@ -1127,7 +1150,7 @@ void llama_model_qwen35::graph_mtp::build_chain(const llama_model & model) { cb(cur, "h_nextn", -1); // the step's output row (masked h_nextn: the next step's hidden input) - ggml_tensor * h_out = ggml_get_rows(ctx0, cur, st.out_idx); + ggml_tensor * h_out = ggml_get_rows(ctx0, cur, st_out); cb(h_out, "mtp_shared_head_norm", -1); // greedy draft: argmax of the head's logits. With a draft head that is an index into its vocabulary (the From c18c801257de709f6e055f5897bd8dd0c59a157d Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:20:33 +1100 Subject: [PATCH 50/65] mtp: draft-time instrumentation (LLAMA_MTP_DRAFT_TIMING, GGML_SYCL_OP_TIMING) Both measured runs of the draft chain spend ~6.5 ms per draft call, chain or loop, against a ~3.4 ms weight-read floor. To find where it goes: - LLAMA_MTP_DRAFT_TIMING=1: host phases of every MTP draft() call (pre-draft flushes, batch building, llama_decode submission, GPU wait + readback), averaged and printed every 64 calls, for both the chain and the loop. - GGML_SYCL_OP_TIMING=1: GPU time per op type for draft-sized graphs (< 2000 nodes, every mat-mul <= 8 rows), draining the queue after every kernel group (serialised upper bound incl. one launch latency each), every 64 graphs. Both off by default; no change to what is computed. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 60edb38e7d8833629684c07ac6925f4ce88bc0e3 on Torchit1/llama.cpp arc-b580) --- common/speculative.cpp | 55 +++++++++++++++++++++++++ ggml/src/ggml-sycl/ggml-sycl.cpp | 69 ++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) diff --git a/common/speculative.cpp b/common/speculative.cpp index e9ece1006bb1..ed482143e115 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -6,6 +6,7 @@ #include "ggml.h" #include "llama.h" +#include #include #include #ifdef LLAMA_DSPARK_MARKOV_CUDA @@ -41,6 +42,40 @@ // ARC-LAB: LLAMA_MTP_DRAFT_LOG= appends one JSON line per MTP draft step (position predicted, depth, top 3 candidates // with probabilities, whether p_min stopped the draft) plus a "begin" line per request, for offline acceptance / tree // simulation (arctune, 2026-10-05). +// ARC-LAB: LLAMA_MTP_DRAFT_TIMING=1: host-side phase timing of the MTP draft() calls (chain and loop), averaged and +// printed to stderr every 64 calls. pre = deferred flushes before the draft, batch = building batches, decode = the +// llama_decode calls (batch split, slot search, input uploads, kernel submission; no wait), wait = waiting for the GPU +// and reading the draft back (chain: llama_get_mtp_chain_ids; loop: the per-step sampling, which syncs). +struct common_mtp_draft_timing { + using clk = std::chrono::steady_clock; + const bool on = [] { const char * e = std::getenv("LLAMA_MTP_DRAFT_TIMING"); return e && std::atoi(e) != 0; }(); + clk::time_point t0, t; + int calls = 0, chain_calls = 0, decodes = 0; + double total = 0, pre = 0, batch = 0, decode = 0, wait = 0, rest = 0; + void begin() { if (on) { t0 = t = clk::now(); } } + void lap(double & acc) { + if (!on) { return; } + const auto n = clk::now(); + acc += std::chrono::duration(n - t).count(); + t = n; + } + void end(bool chain) { + if (!on) { return; } + lap(rest); + total += std::chrono::duration(t - t0).count(); + calls++; + chain_calls += chain; + if (calls == 64) { + fprintf(stderr, "mtp-draft-timing: %d calls (%d chained), per call: total %.3f ms = pre %.3f + batch %.3f + " + "decode %.3f (%.2f llama_decode) + wait/readback %.3f + rest %.3f\n", calls, chain_calls, total / calls, + pre / calls, batch / calls, decode / calls, (double) decodes / calls, wait / calls, rest / calls); + calls = chain_calls = decodes = 0; + total = pre = batch = decode = wait = rest = 0; + } + } +}; +static common_mtp_draft_timing g_mtp_dt; + static FILE * common_mtp_draft_log() { static FILE * f = [] { const char * p = std::getenv("LLAMA_MTP_DRAFT_LOG"); @@ -2465,6 +2500,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { const size_t row_bytes = (size_t) n_embd * sizeof(float); + g_mtp_dt.lap(g_mtp_dt.pre); common_sampler_reset(smpls[0].get()); common_batch_clear(batch); @@ -2481,9 +2517,12 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { std::memcpy(batch.embd + (size_t) (batch.n_tokens - 1) * n_embd, pending_h[0].data(), row_bytes); } + g_mtp_dt.lap(g_mtp_dt.batch); llama_set_mtp_chain(ctx_dft, n_draft); const int32_t rc = llama_decode(ctx_dft, batch); llama_set_mtp_chain(ctx_dft, 0); + g_mtp_dt.lap(g_mtp_dt.decode); + g_mtp_dt.decodes++; if (rc != 0) { SPC_ERR("llama_decode(ctx_dft) chained draft returned %d\n", (int) rc); if (n_catch > 0) { @@ -2496,6 +2535,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { } const llama_token * ids = llama_get_mtp_chain_ids(ctx_dft); + g_mtp_dt.lap(g_mtp_dt.wait); if (ids == nullptr) { SPC_ERR("%s", "chained draft produced no tokens\n"); return true; @@ -2768,9 +2808,18 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { } void draft(common_speculative_draft_params_vec & dparams) override { + g_mtp_dt.begin(); + draft_impl(dparams); + g_mtp_dt.end(last_draft_chained); + } + + bool last_draft_chained = false; + + void draft_impl(common_speculative_draft_params_vec & dparams) { auto & ctx_dft = params.ctx_dft; const size_t row_bytes = (size_t) n_embd * sizeof(float); + last_draft_chained = false; // deferred catch-up rows that do not lead straight into this draft position are decoded // on their own first (uses `batch`, so before it is built) @@ -2825,6 +2874,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { } if (chain_mode && draft_chain(dparams)) { + last_draft_chained = true; auto & dp = dparams[0]; if (dp.result->size() < (size_t) params.n_min) { dp.result->clear(); @@ -2832,6 +2882,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { return; } + g_mtp_dt.lap(g_mtp_dt.pre); common_batch_clear(batch); // keep track of which sequences are still drafting @@ -2891,7 +2942,10 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { llama_set_nextn_layer_offset(ctx_dft, i); } + g_mtp_dt.lap(g_mtp_dt.batch); int ret = llama_decode(ctx_dft, batch); + g_mtp_dt.lap(g_mtp_dt.decode); + g_mtp_dt.decodes++; if (ret != 0) { SPC_ERR("llama_decode[%d] returned %d\n", i, ret); if (i == 0) { @@ -2924,6 +2978,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { auto * smpl = smpls[seq_id].get(); common_sampler_sample(smpl, ctx_dft, i_last[seq_id], true); + g_mtp_dt.lap(g_mtp_dt.wait); const float * h_row = llama_get_embeddings_nextn_ith(ctx_dft, i_last[seq_id]); const auto * cur_p = common_sampler_get_candidates(smpl, true); diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index f7037bf36087..dfd77750deb8 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -6255,6 +6255,45 @@ struct sycl_host_timing { }; static sycl_host_timing g_ht; +// ARC-LAB: GGML_SYCL_OP_TIMING=1 -> GPU time per op type for draft-sized graphs (fewer than 2000 nodes, every mat-mul at +// most 8 rows: the MTP draft decodes and chains, not the target's verify graph). The queue is drained after every kernel +// group, so each entry is its kernel time plus one launch latency (a serialised upper bound); the sum is reported next to +// the kernel-group count. Printed every 64 such graphs. Timing-only: it slows the draft, never use it for speed runs. +struct sycl_op_timing { + bool on = getenv("GGML_SYCL_OP_TIMING") && atoi(getenv("GGML_SYCL_OP_TIMING")); + using clk = std::chrono::steady_clock; + int graphs = 0; + long groups = 0, nodes = 0; + double total = 0; + std::map> per_op; + static bool draft_sized(const ggml_cgraph * g) { + if (g->n_nodes >= 2000) return false; + bool mm = false; + for (int i = 0; i < g->n_nodes; ++i) { + const ggml_tensor * n = g->nodes[i]; + if (n->op == GGML_OP_MUL_MAT) { + mm = true; + if (n->src[1]->ne[1] * n->src[1]->ne[2] > 8) return false; + } + } + return mm; + } + void report() { + fprintf(stderr, "op-timing: %d draft-sized graphs, per graph: %.3f ms serialised over %.1f kernel groups (%.0f nodes)\n", + graphs, total / graphs, (double) groups / graphs, (double) nodes / graphs); + std::vector> v; + for (auto & kv : per_op) v.push_back({ kv.second.first, kv.first }); + std::sort(v.rbegin(), v.rend()); + for (size_t i = 0; i < v.size() && i < 16; ++i) { + const auto & a = per_op[v[i].second]; + fprintf(stderr, "op-timing: %-22s %.3f ms/graph (%.1f groups/graph, %.1f us each)\n", v[i].second.c_str(), + a.first / graphs, (double) a.second / graphs, 1000.0 * a.first / std::max(1L, a.second)); + } + graphs = 0; groups = 0; nodes = 0; total = 0; per_op.clear(); + } +}; +static sycl_op_timing g_ot; + static void ggml_backend_sycl_synchronize(ggml_backend_t backend) try { GGML_SYCL_DEBUG("[SYCL] call %s\n", __func__); ggml_backend_sycl_context * sycl_ctx = (ggml_backend_sycl_context *)backend->context; @@ -7249,6 +7288,27 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc const bool ht_on = g_ht.on && cgraph->n_nodes > 500; sycl_host_timing::clk::time_point ht_prev = sycl_host_timing::clk::now(); const char * ht_op = nullptr; + const bool ot_on = g_ot.on && sycl_op_timing::draft_sized(cgraph); + sycl_op_timing::clk::time_point ot_prev{}; + const char * ot_op = nullptr; + auto ot_mark = [&](const char * next) { // drain the queue, charge the time since the last mark to the op that ran + SYCL_CHECK(CHECK_TRY_ERROR(sycl_ctx->stream()->wait())); + const auto now = sycl_op_timing::clk::now(); + if (ot_op) { + const double ms = std::chrono::duration(now - ot_prev).count(); + auto & a = g_ot.per_op[ot_op]; + a.first += ms; + a.second++; + g_ot.total += ms; + g_ot.groups++; + } + ot_prev = now; + ot_op = next; + }; + if (ot_on) { + ot_mark(nullptr); // inputs and earlier work are not charged + g_ot.nodes += cgraph->n_nodes; + } for (int i = 0; i < cgraph->n_nodes; i++) { ggml_tensor * node = cgraph->nodes[i]; if (ht_on) { @@ -7273,6 +7333,9 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { continue; } + if (ot_on) { + ot_mark(ggml_op_desc(node)); + } const int nodes_to_skip = ggml_sycl_fuse(*sycl_ctx, cgraph, i); if (nodes_to_skip != 0) { @@ -7412,6 +7475,12 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc } GGML_ASSERT(ok); } + if (ot_on) { + ot_mark(nullptr); + if (++g_ot.graphs == 64) { + g_ot.report(); + } + } } #ifdef GGML_SYCL_GRAPH From 7ac3c1b9c57f703cf6b5a2bc86ae6e42d6fbd507 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:20:56 +1100 Subject: [PATCH 51/65] gdn replay: allocate only S planes 0 and 1 when rollback by replay is on Replay mode keeps the base state in S plane 0 and the two record banks in plane 1; planes 2..n_rs_seq were allocated but never used (432 MiB on Bonsai at n_rs_seq = 4, 48 GDN layers x 3 planes x 3 MiB, independent of -c). - llama_memory_recurrent: the replay decision now runs before the tensors are created (layer selection via the same filter as the allocation), and S gets 2 * mem_size rows in replay mode. The conv state r_l keeps its 1 + n_rs_seq planes and rs_idx. - LLAMA_GDN_REPLAY=0 and every case where replay does not apply keep the full (1 + n_rs_seq)-plane S allocation, unchanged. - No offset changes: the graph, the SYCL fused cache write and the state save/restore only ever address plane 0 (row head) and plane 1 (row size + head), so the s_trans = 0/2 file layout is the same and files from cr-replay still load. - build_recurrent_attn asserts the 2-plane S shape in replay mode. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit bda090edcc7ebc15416b0a368e99d3f276ce4f48 on Torchit1/llama.cpp arc-b580) --- src/llama-memory-recurrent.cpp | 74 ++++++++++++++++++---------------- src/llama-memory-recurrent.h | 4 +- src/models/delta-net-base.cpp | 2 + 3 files changed, 44 insertions(+), 36 deletions(-) diff --git a/src/llama-memory-recurrent.cpp b/src/llama-memory-recurrent.cpp index f5d33f30397b..808524545b1f 100644 --- a/src/llama-memory-recurrent.cpp +++ b/src/llama-memory-recurrent.cpp @@ -72,6 +72,41 @@ llama_memory_recurrent::llama_memory_recurrent( return it->second.get(); }; + // ARC-LAB: GDN rollback by replay (LLAMA_GDN_REPLAY=0 keeps the snapshot planes). Applies to a single sequence + // of a qwen35 model whose recurrent layers all run on SYCL; anything else silently keeps the snapshot path. + // Decided before the tensors are created: in replay mode S gets only planes 0 (base) and 1 (record banks), so + // planes 2..n_rs_seq are not allocated. The snapshot path keeps the full (1 + n_rs_seq)-plane layout. + { + const char * env = getenv("LLAMA_GDN_REPLAY"); + bool on = !(env && atoi(env) == 0) && n_rs_seq > 0 && n_seq_max == 1 && mem_size == 1 && offload && + model.arch == LLM_ARCH_QWEN35 && type_s == GGML_TYPE_F32 && + getenv("GGML_SYCL_GDN_CHUNKED") == nullptr; // the opt-in chunked prefill has no replay variant + const uint32_t S = hparams.ssm_d_state; + const uint32_t H = hparams.ssm_dt_rank; + on = on && S == 128 && H > 0 && hparams.ssm_d_inner == S * H && hparams.n_embd_s() == S * S * H; + const uint32_t R = H * (2 * S + 1); + // both banks of n_rs_seq records must fit in one S row + on = on && (uint64_t) 2 * n_rs_seq * R <= (uint64_t) hparams.n_embd_s(); + bool any = false; + for (int i = 0; on && i < n_layer; i++) { + if (filter && !filter(i)) { // same layer selection as the allocation below + continue; + } + any = true; + ggml_backend_dev_t dev = model.dev_layer(i); + ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr; + on = reg && strcmp(ggml_backend_reg_name(reg), "SYCL") == 0; + } + rpl = on && any; + rpl_R = rpl ? R : 0; + if (rpl) { + LLAMA_LOG_INFO("%s: GDN rollback by replay: on (%u records of %u floats per bank, S planes 2 of %u)\n", __func__, + n_rs_seq, R, 1 + n_rs_seq); + } else if (n_rs_seq > 0) { + LLAMA_LOG_INFO("%s: GDN rollback by replay: off (snapshot planes)\n", __func__); + } + } + r_l.resize(n_layer); s_l.resize(n_layer); @@ -99,48 +134,17 @@ llama_memory_recurrent::llama_memory_recurrent( throw std::runtime_error("failed to create ggml context for rs cache"); } - const uint32_t n_rows = mem_size * (1 + n_rs_seq); + const uint32_t n_rows = mem_size * (1 + n_rs_seq); + // ARC-LAB replay: S needs plane 0 (base) and plane 1 (record banks) only; the conv state keeps all its planes + const uint32_t n_rows_s = rpl ? mem_size * 2 : n_rows; ggml_tensor * r = ggml_new_tensor_2d(ctx, type_r, hparams.n_embd_r(), n_rows); - ggml_tensor * s = ggml_new_tensor_2d(ctx, type_s, hparams.n_embd_s(), n_rows); + ggml_tensor * s = ggml_new_tensor_2d(ctx, type_s, hparams.n_embd_s(), n_rows_s); ggml_format_name(r, "cache_r_l%d", i); ggml_format_name(s, "cache_s_l%d", i); r_l[i] = r; s_l[i] = s; } - // ARC-LAB: GDN rollback by replay (LLAMA_GDN_REPLAY=0 keeps the snapshot planes). Applies to a single sequence - // of a qwen35 model whose recurrent layers all run on SYCL; anything else silently keeps the snapshot path. - // The tensor layout is the same either way (planes 0 and 1 of S are reused), only their meaning changes. - { - const char * env = getenv("LLAMA_GDN_REPLAY"); - bool on = !(env && atoi(env) == 0) && n_rs_seq > 0 && n_seq_max == 1 && mem_size == 1 && offload && - model.arch == LLM_ARCH_QWEN35 && type_s == GGML_TYPE_F32 && - getenv("GGML_SYCL_GDN_CHUNKED") == nullptr; // the opt-in chunked prefill has no replay variant - const uint32_t S = hparams.ssm_d_state; - const uint32_t H = hparams.ssm_dt_rank; - on = on && S == 128 && H > 0 && hparams.ssm_d_inner == S * H && hparams.n_embd_s() == S * S * H; - const uint32_t R = H * (2 * S + 1); - // both banks of n_rs_seq records must fit in one S row - on = on && (uint64_t) 2 * n_rs_seq * R <= (uint64_t) hparams.n_embd_s(); - bool any = false; - for (int i = 0; on && i < n_layer; i++) { - if (s_l[i] == nullptr) { - continue; - } - any = true; - ggml_backend_dev_t dev = model.dev_layer(i); - ggml_backend_reg_t reg = dev ? ggml_backend_dev_backend_reg(dev) : nullptr; - on = reg && strcmp(ggml_backend_reg_name(reg), "SYCL") == 0; - } - rpl = on && any; - rpl_R = rpl ? R : 0; - if (rpl) { - LLAMA_LOG_INFO("%s: GDN rollback by replay: on (%u records of %u floats per bank)\n", __func__, n_rs_seq, R); - } else if (n_rs_seq > 0) { - LLAMA_LOG_INFO("%s: GDN rollback by replay: off (snapshot planes)\n", __func__); - } - } - // allocate tensors and initialize the buffers to avoid NaNs in the padding for (auto & [buft, ctx] : ctx_map) { ggml_backend_buffer_t buf = ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft); diff --git a/src/llama-memory-recurrent.h b/src/llama-memory-recurrent.h index b44915e1a690..f83671ad8dc6 100644 --- a/src/llama-memory-recurrent.h +++ b/src/llama-memory-recurrent.h @@ -71,6 +71,7 @@ class llama_memory_recurrent : public llama_memory_i { uint32_t used = 0; // used cells (i.e. at least one seq_id) // number of recurrent-state snapshots per seq for rollback; tensors are widened to (1 + n_rs_seq) groups + // (ARC-LAB: with rpl on, S has 2 groups only - base and record banks; r keeps 1 + n_rs_seq) uint32_t n_rs_seq = 0; // per-seq rollback index @@ -80,7 +81,8 @@ class llama_memory_recurrent : public llama_memory_i { // ARC-LAB: rollback by replay for the gated delta net state (LLAMA_GDN_REPLAY, default on where it applies; // independent implementation of the technique described by llamADALL). The S rows of plane 0 hold the base - // state; plane 1 of each cell holds two banks of per-token update records (ggml_gated_delta_net_replay). + // state; plane 1 of each cell holds two banks of per-token update records (ggml_gated_delta_net_replay). S is + // allocated with these 2 planes only (the snapshot path allocates 1 + n_rs_seq). // Rolling back r tokens drops the last r pending records instead of selecting a snapshot plane. The conv // state (r_l) keeps its snapshot planes and rs_idx. bool rpl = false; diff --git a/src/models/delta-net-base.cpp b/src/models/delta-net-base.cpp index ca8e69085183..93213085790a 100644 --- a/src/models/delta-net-base.cpp +++ b/src/models/delta-net-base.cpp @@ -578,6 +578,8 @@ ggml_tensor * llm_build_delta_net_base::build_recurrent_attn( // base state in plane 0 plus the update records of the last K-1 tokens in plane 1 (two banks: this pass // reads one, its records go to the other) instead of K snapshot planes; see ggml_gated_delta_net_replay. GGML_ASSERT(state_rows == nullptr && n_seqs == 1 && g->ne[0] == 1); + // the memory allocates S with planes 0 and 1 only in replay mode (llama_memory_recurrent::rpl) + GGML_ASSERT(ssm_states_all->ne[1] == 2 * (int64_t) mem_size); const int64_t R = H_v * (2 * S_v + 1); const size_t row_size = ssm_states_all->nb[1]; const size_t rec_size = ggml_row_size(ssm_states_all->type, R * (K - 1)); From 9eec75a16ab3ab9ac054aa5650ddbf09d95da365 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:28:02 +1100 Subject: [PATCH 52/65] sycl: GGML_SYCL_OP_TIMING matches the MTP draft graphs; mat-muls by weight type The draft-size test counted the Hadamard transform mat-muls, whose src1 holds each row as 1024-wide blocks (5 per 5120 row, 17 per FFN row), so every draft graph failed the <= 8 rows check and nothing was timed. Only weight mat-muls are checked now. A one-time "op-timing: active" line confirms the hook engaged, and mat-muls are reported per weight type (Q6_K / Q8_0 / PTQ1_0 head weights, the Hadamard transforms apart). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 24424cd538a23f99f3758f9dfd56b5fda6fb5ea6 on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/ggml-sycl.cpp | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index dfd77750deb8..5ea6f2e7aaea 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -6271,13 +6271,31 @@ struct sycl_op_timing { bool mm = false; for (int i = 0; i < g->n_nodes; ++i) { const ggml_tensor * n = g->nodes[i]; - if (n->op == GGML_OP_MUL_MAT) { + // weight mat-muls only: a Hadamard transform reshapes each row into 1024-wide blocks (5..17 per row) + if (n->op == GGML_OP_MUL_MAT && ggml_get_op_params_i32(n, 1) != GGML_HINT_SRC0_IS_HADAMARD) { mm = true; if (n->src[1]->ne[1] * n->src[1]->ne[2] > 8) return false; } } + if (mm) { + static bool logged = false; + if (!logged) { + logged = true; + fprintf(stderr, "op-timing: active (first draft-sized graph: %d nodes)\n", g->n_nodes); + } + } return mm; } + // op label: mat-muls by weight type (and the Hadamard transforms apart) + static std::string label(const ggml_tensor * n) { + if (n->op == GGML_OP_MUL_MAT) { + if (ggml_get_op_params_i32(n, 1) == GGML_HINT_SRC0_IS_HADAMARD) { + return "MUL_MAT hadamard"; + } + return std::string("MUL_MAT ") + ggml_type_name(n->src[0]->type); + } + return ggml_op_desc(n); + } void report() { fprintf(stderr, "op-timing: %d draft-sized graphs, per graph: %.3f ms serialised over %.1f kernel groups (%.0f nodes)\n", graphs, total / graphs, (double) groups / graphs, (double) nodes / graphs); @@ -7290,11 +7308,11 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc const char * ht_op = nullptr; const bool ot_on = g_ot.on && sycl_op_timing::draft_sized(cgraph); sycl_op_timing::clk::time_point ot_prev{}; - const char * ot_op = nullptr; - auto ot_mark = [&](const char * next) { // drain the queue, charge the time since the last mark to the op that ran + std::string ot_op; + auto ot_mark = [&](const std::string & next) { // drain the queue, charge the time since the last mark to the op that ran SYCL_CHECK(CHECK_TRY_ERROR(sycl_ctx->stream()->wait())); const auto now = sycl_op_timing::clk::now(); - if (ot_op) { + if (!ot_op.empty()) { const double ms = std::chrono::duration(now - ot_prev).count(); auto & a = g_ot.per_op[ot_op]; a.first += ms; @@ -7306,7 +7324,7 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc ot_op = next; }; if (ot_on) { - ot_mark(nullptr); // inputs and earlier work are not charged + ot_mark(std::string()); // inputs and earlier work are not charged g_ot.nodes += cgraph->n_nodes; } for (int i = 0; i < cgraph->n_nodes; i++) { @@ -7334,7 +7352,7 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc continue; } if (ot_on) { - ot_mark(ggml_op_desc(node)); + ot_mark(sycl_op_timing::label(node)); } const int nodes_to_skip = ggml_sycl_fuse(*sycl_ctx, cgraph, i); @@ -7476,7 +7494,7 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc GGML_ASSERT(ok); } if (ot_on) { - ot_mark(nullptr); + ot_mark(std::string()); if (++g_ot.graphs == 64) { g_ot.report(); } From f142204f155db65b906b101a1407c87ebeba79c8 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:44:47 +1100 Subject: [PATCH 53/65] mtp chain: per-step input slices made once, up front, from device copies (LLAMA_MTP_CHAIN_TIGHT bit 1) Feature B round 2, idea 1 (independent implementation of the technique described by llamADALL). The chain graph sliced its packed host inputs (positions, output rows, K/V indices, mask) with per-step views built lazily inside each step. The scheduler makes every view of a host input its own split input: ~20 blocking host->device copies per draft (5 per step), each behind a backend sync, before the device split starts. And the views landed between the step's compute nodes. Now each packed input gets one whole-tensor copy (a device memcpy) pinned to the head's device, all per-step slices are views of those copies, and the host token lookup, the copies and every slice are added to the graph before the first compute node. Each step's compute nodes then run uninterrupted, as in the single-step draft graph; the device split keeps ~9 inputs instead of ~24. Same values in the same kernels: drafts unchanged. LLAMA_MTP_CHAIN_TIGHT is a bit mask (default 1 here); 0 restores the previous layout exactly. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 0e1ba9c6ce7b46224c7135beb596b98f06b6147c on Torchit1/llama.cpp arc-b580) --- src/models/qwen35.cpp | 88 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 77 insertions(+), 11 deletions(-) diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp index 0e81f3dba9a9..55dba5f19451 100644 --- a/src/models/qwen35.cpp +++ b/src/models/qwen35.cpp @@ -1030,24 +1030,90 @@ void llama_model_qwen35::graph_mtp::build_chain(const llama_model & model) { const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale; - ggml_tensor * tok_embd = unrotate(ggml_get_rows(ctx0, tok_embd_w, ggml_view_1d(ctx0, tokens_all, n_lead, 0))); + // ARC-LAB: LLAMA_MTP_CHAIN_TIGHT (bit mask, default all on; 0 = the first chain layout): + // 1: every per-step input slice is made once, up front, from ONE device copy of each packed input. A view of a + // host input is its own scheduler split input (one blocking host->device copy per view and step), and a view + // built lazily lands between the step's compute nodes; here the device split starts with 4-5 copy kernels and + // all the slices, then each step's compute nodes run uninterrupted, as in the single-step draft graph. + static const int tight = [] { + const char * e = getenv("LLAMA_MTP_CHAIN_TIGHT"); + return e ? atoi(e) : 1; + }(); + + ggml_tensor * tok_rows = ggml_get_rows(ctx0, tok_embd_w, ggml_view_1d(ctx0, tokens_all, n_lead, 0)); ggml_tensor * h_embd = ggml_view_2d(ctx0, h_all, h_all->ne[0], n_lead, h_all->nb[1], 0); + ggml_tensor * k_all = inp_attn->get_k_idxs(); + ggml_tensor * v_all = inp_attn->get_v_idxs(); + ggml_tensor * mask_all = inp_attn->get_kq_mask(); + + // the device holding the head's weights (none: CPU-only, the slices stay views of the host inputs) + ggml_backend_t dev = nullptr; + if ((tight & 1) && sched && layer.wq->buffer) { + ggml_backend_buffer_type_t wbuft = ggml_backend_buffer_get_type(layer.wq->buffer); + for (int b = 0; b < ggml_backend_sched_get_n_backends(sched) && !dev; ++b) { + ggml_backend_t be = ggml_backend_sched_get_backend(sched, b); + if (be != backend_cpu && ggml_backend_supports_buft(be, wbuft)) { + dev = be; + } + } + } + if (tight & 1) { + ggml_build_forward_expand(gf, tok_rows); // the host-table lookup first: one host split, then one device split + } + if (dev) { + // whole-tensor copies (a device memcpy each), pinned to the device so their views need no upload + auto on_dev = [&](ggml_tensor * t, const char * name) { + ggml_tensor * c = ggml_cont(ctx0, t); + ggml_set_name(c, name); + if (!ggml_backend_supports_op(dev, c)) { + return t; + } + ggml_backend_sched_set_tensor_backend(sched, c, dev); + ggml_build_forward_expand(gf, c); + return c; + }; + pos_all = on_dev(pos_all, "mtp_chain_pos_dev"); + out_all = on_dev(out_all, "mtp_chain_out_dev"); + k_all = on_dev(k_all, "mtp_chain_k_idxs_dev"); + v_all = on_dev(v_all, "mtp_chain_v_idxs_dev"); + mask_all = on_dev(mask_all, "mtp_chain_kq_mask_dev"); + } + + // each step's slices: positions, output row, KV indices and mask rows, sized as a decode of rows [0, r0 + nr) + // would be (created here; added to the graph here with bit 1, else where the step first reads them) + struct step_inputs { + ggml_tensor * pos, * out, * k_idxs, * v_idxs, * kq_mask; + }; + std::vector sin(n_steps); + for (int64_t s = 0; s < n_steps; ++s) { + const auto & st = steps[s]; + const int64_t nr = st.nr; + auto & si = sin[s]; + si.pos = ggml_view_1d(ctx0, pos_all, nr * n_pos, st.r0 * n_pos * ggml_element_size(pos_all)); + si.out = ggml_view_1d(ctx0, out_all, 1, s * ggml_element_size(out_all)); + si.k_idxs = ggml_view_1d(ctx0, k_all, nr, st.r0 * ggml_element_size(k_all)); + si.v_idxs = ggml_view_1d(ctx0, v_all, nr, st.r0 * ggml_element_size(v_all)); + si.kq_mask = ggml_cont(ctx0, ggml_view_2d(ctx0, mask_all, st.n_kv, nr, mask_all->nb[1], st.r0 * mask_all->nb[1])); + if (tight & 1) { + for (ggml_tensor * t : { si.pos, si.out, si.k_idxs, si.v_idxs, si.kq_mask }) { + ggml_build_forward_expand(gf, t); + } + } + } + + ggml_tensor * tok_embd = unrotate(tok_rows); for (int64_t s = 0; s < n_steps; ++s) { const auto & st = steps[s]; const int64_t nr = st.nr; cb(tok_embd, "mtp_tok_embd", il); - // this step's slice of the ubatch's KV indices and mask, sized as a decode of rows [0, r0 + nr) would be - const uint32_t n_kv_s = st.n_kv; - ggml_tensor * st_pos = ggml_view_1d(ctx0, pos_all, nr * n_pos, st.r0 * n_pos * ggml_element_size(pos_all)); - ggml_tensor * st_out = ggml_view_1d(ctx0, out_all, 1, s * ggml_element_size(out_all)); - ggml_tensor * k_idxs = inp_attn->get_k_idxs(); - ggml_tensor * v_idxs = inp_attn->get_v_idxs(); - k_idxs = ggml_view_1d(ctx0, k_idxs, nr, st.r0 * ggml_element_size(k_idxs)); - v_idxs = ggml_view_1d(ctx0, v_idxs, nr, st.r0 * ggml_element_size(v_idxs)); - ggml_tensor * kq_mask = inp_attn->get_kq_mask(); - kq_mask = ggml_cont(ctx0, ggml_view_2d(ctx0, kq_mask, n_kv_s, nr, kq_mask->nb[1], st.r0 * kq_mask->nb[1])); + const uint32_t n_kv_s = st.n_kv; + ggml_tensor * st_pos = sin[s].pos; + ggml_tensor * st_out = sin[s].out; + ggml_tensor * k_idxs = sin[s].k_idxs; + ggml_tensor * v_idxs = sin[s].v_idxs; + ggml_tensor * kq_mask = sin[s].kq_mask; ggml_tensor * h_norm = build_norm(h_embd, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il); cb(h_norm, "mtp_hnorm", il); From 075c4b212bc2be14ee0b773f0702f34f473af057 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:45:03 +1100 Subject: [PATCH 54/65] mtp chain: copy a step's mask rows only when the slice is not contiguous (LLAMA_MTP_CHAIN_TIGHT bit 2) Feature B round 2, idea 2 (independent implementation of the technique described by llamADALL). Every chain step ran a CONT on its mask slice. A 1-row step's slice is one row of the mask and already contiguous, so flash attention reads it in place (the decode kernels take the mask row stride from nb[1]; with one row it is never used). Only step 0 with catch-up rows, narrower than the full mask (its KV length is a prefix), still needs the copy. Saves n-1 copy kernels per draft (n = 4: 3). Same mask values: drafts unchanged. Default LLAMA_MTP_CHAIN_TIGHT is now 3. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit c9c5053ac86184909e77787e396a923f090fc640 on Torchit1/llama.cpp arc-b580) --- src/models/qwen35.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp index 55dba5f19451..fb883e8ade53 100644 --- a/src/models/qwen35.cpp +++ b/src/models/qwen35.cpp @@ -1035,9 +1035,11 @@ void llama_model_qwen35::graph_mtp::build_chain(const llama_model & model) { // host input is its own scheduler split input (one blocking host->device copy per view and step), and a view // built lazily lands between the step's compute nodes; here the device split starts with 4-5 copy kernels and // all the slices, then each step's compute nodes run uninterrupted, as in the single-step draft graph. + // 2: a step's mask rows are copied only when the slice is not contiguous: a 1-row step's row is already + // contiguous, only a multi-row step 0 narrower than the full mask needs the copy (one copy kernel, not n). static const int tight = [] { const char * e = getenv("LLAMA_MTP_CHAIN_TIGHT"); - return e ? atoi(e) : 1; + return e ? atoi(e) : 3; }(); ggml_tensor * tok_rows = ggml_get_rows(ctx0, tok_embd_w, ggml_view_1d(ctx0, tokens_all, n_lead, 0)); @@ -1093,7 +1095,10 @@ void llama_model_qwen35::graph_mtp::build_chain(const llama_model & model) { si.out = ggml_view_1d(ctx0, out_all, 1, s * ggml_element_size(out_all)); si.k_idxs = ggml_view_1d(ctx0, k_all, nr, st.r0 * ggml_element_size(k_all)); si.v_idxs = ggml_view_1d(ctx0, v_all, nr, st.r0 * ggml_element_size(v_all)); - si.kq_mask = ggml_cont(ctx0, ggml_view_2d(ctx0, mask_all, st.n_kv, nr, mask_all->nb[1], st.r0 * mask_all->nb[1])); + si.kq_mask = ggml_view_2d(ctx0, mask_all, st.n_kv, nr, mask_all->nb[1], st.r0 * mask_all->nb[1]); + if (!(tight & 2) || !ggml_is_contiguous(si.kq_mask)) { + si.kq_mask = ggml_cont(ctx0, si.kq_mask); + } if (tight & 1) { for (ggml_tensor * t : { si.pos, si.out, si.k_idxs, si.v_idxs, si.kq_mask }) { ggml_build_forward_expand(gf, t); From 567ba0b893e1e73ced20f01632326ed3e2e14a5b Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:45:55 +1100 Subject: [PATCH 55/65] mtp chain: gather the output row only for a multi-row step (LLAMA_MTP_CHAIN_TIGHT bit 4) Feature B round 2, idea 3 (independent implementation of the technique described by llamADALL). Every chain step ran GET_ROWS(h_nextn, out_idx) to pick its output row. Only step 0 (catch-up rows + the anchor) has a choice; a 1-row step's row is already the output. Now only steps with more than one row gather, and the output-index input holds only their entries (none when step 0 has no catch-up rows: no out_idx input at all). Side effect, on purpose: without the gather, the draft head's input chain of steps 1.. is ADD -> RMS_NORM -> MUL(norm w) -> MUL(signs) -> RESHAPE -> MUL_MAT(Hadamard), which ggml_sycl_try_add_norm_rot folds into one kernel (residual add + head norm + rotation). The loop never takes that path for the head (its out_ids gather sits in between). The fold is documented as the same arithmetic in the same order (byte-identical in the trunk), but it is new for drafts: first suspect if drafts ever differ (bit 4 off isolates it). Default LLAMA_MTP_CHAIN_TIGHT is now 7. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 9d3313b24d9f98fc715486292d7b1e09a14aee17 on Torchit1/llama.cpp arc-b580) --- src/models/qwen35.cpp | 75 +++++++++++++++++++++++++++---------------- 1 file changed, 47 insertions(+), 28 deletions(-) diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp index fb883e8ade53..affb021fcd3c 100644 --- a/src/models/qwen35.cpp +++ b/src/models/qwen35.cpp @@ -620,7 +620,7 @@ ggml_tensor * llama_model_qwen35::graph::build_layer_ffn(ggml_tensor * cur, cons // ARC-LAB: per-step inputs of a chained MTP draft graph (graph_mtp::build_chain), packed into two tensors so a draft // uploads them in two writes: pos holds each step's positions in the layout build_inp_pos() gives a decode of just -// that step's rows (step after step), out_idx each step's output row. The graph reads them through per-step views. +// that step's rows (step after step), out_idx the output row of each step that gathers one. The graph reads them through per-step views. class llm_graph_input_mtp_chain : public llm_graph_input_i { public: explicit llm_graph_input_mtp_chain(uint32_t n_pos_per_embd) : n_pos_per_embd(n_pos_per_embd) {} @@ -629,16 +629,17 @@ class llm_graph_input_mtp_chain : public llm_graph_input_i { int64_t r0; // first ubatch row of the step int64_t nr; // rows uint32_t n_kv; // KV length the step attends over (baked into the graph's views) + bool gather; // picks its output row with a GET_ROWS (else its single row is the output) }; std::vector steps; int64_t n_tokens = 0; ggml_tensor * pos = nullptr; // I32 [n_tokens * n_pos_per_embd] - ggml_tensor * out_idx = nullptr; // I32 [n_steps], step s: nr - 1 (the step's last row produces its draft) + ggml_tensor * out_idx = nullptr; // I32 [gathering steps]: nr - 1 (the step's last row produces its draft); null: none void set_input(const llama_ubatch * ubatch) override { GGML_ASSERT(ubatch->n_tokens == n_tokens); std::vector v((size_t) n_tokens * n_pos_per_embd, 0); - std::vector o(steps.size()); + std::vector o; size_t off = 0; for (size_t s = 0; s < steps.size(); ++s) { const auto & st = steps[s]; @@ -656,10 +657,14 @@ class llm_graph_input_mtp_chain : public llm_graph_input_i { } } off += (size_t) st.nr * n_pos_per_embd; - o[s] = (int32_t) st.nr - 1; + if (st.gather) { + o.push_back((int32_t) st.nr - 1); + } } ggml_backend_tensor_set(pos, v.data(), 0, v.size() * sizeof(int32_t)); - ggml_backend_tensor_set(out_idx, o.data(), 0, o.size() * sizeof(int32_t)); + if (out_idx) { + ggml_backend_tensor_set(out_idx, o.data(), 0, o.size() * sizeof(int32_t)); + } } // reusable when the batch has the same rows and every step would attend over the same KV length (the rest of the @@ -972,20 +977,39 @@ void llama_model_qwen35::graph_mtp::build_chain(const llama_model & model) { auto * inp_attn = build_attn_inp_kv(); const llama_kv_cache_context * kvc = inp_attn->mctx; + // ARC-LAB: LLAMA_MTP_CHAIN_TIGHT (bit mask, default all on; 0 = the first chain layout): + // 1: every per-step input slice is made once, up front, from ONE device copy of each packed input. A view of a + // host input is its own scheduler split input (one blocking host->device copy per view and step), and a view + // built lazily lands between the step's compute nodes; here the device split starts with 4-5 copy kernels and + // all the slices, then each step's compute nodes run uninterrupted, as in the single-step draft graph. + // 2: a step's mask rows are copied only when the slice is not contiguous: a 1-row step's row is already + // contiguous, only a multi-row step 0 narrower than the full mask needs the copy (one copy kernel, not n). + // 4: a step picks its output row with a GET_ROWS only if it has several rows (step 0 with catch-up rows); a + // 1-row step's row is already the output (no gather, no output-index slice). + static const int tight = [] { + const char * e = getenv("LLAMA_MTP_CHAIN_TIGHT"); + return e ? atoi(e) : 7; + }(); + const uint32_t n_pos = hparams.n_pos_per_embd(); auto chain_inp = std::make_unique(n_pos); chain_inp->n_tokens = n_tokens; chain_inp->pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens * n_pos); - chain_inp->out_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_steps); ggml_set_input(chain_inp->pos); - ggml_set_input(chain_inp->out_idx); + int64_t n_gather = 0; for (int64_t s = 0; s < n_steps; ++s) { llm_graph_input_mtp_chain::step st; - st.r0 = s == 0 ? 0 : n_lead - 1 + s; - st.nr = s == 0 ? n_lead : 1; - st.n_kv = kvc->get_n_kv_prefix((uint32_t) (st.r0 + st.nr)); + st.r0 = s == 0 ? 0 : n_lead - 1 + s; + st.nr = s == 0 ? n_lead : 1; + st.n_kv = kvc->get_n_kv_prefix((uint32_t) (st.r0 + st.nr)); + st.gather = !(tight & 4) || st.nr > 1; + n_gather += st.gather; chain_inp->steps.push_back(st); } + if (n_gather > 0) { + chain_inp->out_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_gather); + ggml_set_input(chain_inp->out_idx); + } const auto steps = chain_inp->steps; ggml_tensor * pos_all = chain_inp->pos; ggml_tensor * out_all = chain_inp->out_idx; @@ -1030,18 +1054,6 @@ void llama_model_qwen35::graph_mtp::build_chain(const llama_model & model) { const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale; - // ARC-LAB: LLAMA_MTP_CHAIN_TIGHT (bit mask, default all on; 0 = the first chain layout): - // 1: every per-step input slice is made once, up front, from ONE device copy of each packed input. A view of a - // host input is its own scheduler split input (one blocking host->device copy per view and step), and a view - // built lazily lands between the step's compute nodes; here the device split starts with 4-5 copy kernels and - // all the slices, then each step's compute nodes run uninterrupted, as in the single-step draft graph. - // 2: a step's mask rows are copied only when the slice is not contiguous: a 1-row step's row is already - // contiguous, only a multi-row step 0 narrower than the full mask needs the copy (one copy kernel, not n). - static const int tight = [] { - const char * e = getenv("LLAMA_MTP_CHAIN_TIGHT"); - return e ? atoi(e) : 3; - }(); - ggml_tensor * tok_rows = ggml_get_rows(ctx0, tok_embd_w, ggml_view_1d(ctx0, tokens_all, n_lead, 0)); ggml_tensor * h_embd = ggml_view_2d(ctx0, h_all, h_all->ne[0], n_lead, h_all->nb[1], 0); ggml_tensor * k_all = inp_attn->get_k_idxs(); @@ -1075,7 +1087,9 @@ void llama_model_qwen35::graph_mtp::build_chain(const llama_model & model) { return c; }; pos_all = on_dev(pos_all, "mtp_chain_pos_dev"); - out_all = on_dev(out_all, "mtp_chain_out_dev"); + if (n_gather > 1) { + out_all = on_dev(out_all, "mtp_chain_out_dev"); + } k_all = on_dev(k_all, "mtp_chain_k_idxs_dev"); v_all = on_dev(v_all, "mtp_chain_v_idxs_dev"); mask_all = on_dev(mask_all, "mtp_chain_kq_mask_dev"); @@ -1087,12 +1101,12 @@ void llama_model_qwen35::graph_mtp::build_chain(const llama_model & model) { ggml_tensor * pos, * out, * k_idxs, * v_idxs, * kq_mask; }; std::vector sin(n_steps); - for (int64_t s = 0; s < n_steps; ++s) { + for (int64_t s = 0, g = 0; s < n_steps; ++s) { const auto & st = steps[s]; const int64_t nr = st.nr; auto & si = sin[s]; si.pos = ggml_view_1d(ctx0, pos_all, nr * n_pos, st.r0 * n_pos * ggml_element_size(pos_all)); - si.out = ggml_view_1d(ctx0, out_all, 1, s * ggml_element_size(out_all)); + si.out = !st.gather ? nullptr : n_gather == 1 ? out_all : ggml_view_1d(ctx0, out_all, 1, g++ * ggml_element_size(out_all)); si.k_idxs = ggml_view_1d(ctx0, k_all, nr, st.r0 * ggml_element_size(k_all)); si.v_idxs = ggml_view_1d(ctx0, v_all, nr, st.r0 * ggml_element_size(v_all)); si.kq_mask = ggml_view_2d(ctx0, mask_all, st.n_kv, nr, mask_all->nb[1], st.r0 * mask_all->nb[1]); @@ -1101,7 +1115,9 @@ void llama_model_qwen35::graph_mtp::build_chain(const llama_model & model) { } if (tight & 1) { for (ggml_tensor * t : { si.pos, si.out, si.k_idxs, si.v_idxs, si.kq_mask }) { - ggml_build_forward_expand(gf, t); + if (t) { + ggml_build_forward_expand(gf, t); + } } } } @@ -1221,8 +1237,11 @@ void llama_model_qwen35::graph_mtp::build_chain(const llama_model & model) { cb(cur, "h_nextn", -1); // the step's output row (masked h_nextn: the next step's hidden input) - ggml_tensor * h_out = ggml_get_rows(ctx0, cur, st_out); - cb(h_out, "mtp_shared_head_norm", -1); + ggml_tensor * h_out = cur; + if (st_out) { + h_out = ggml_get_rows(ctx0, cur, st_out); + cb(h_out, "mtp_shared_head_norm", -1); + } // greedy draft: argmax of the head's logits. With a draft head that is an index into its vocabulary (the // -inf scatter of the single-step graph only maps it to a token id, which the context does on the host) From 31ffa174c0f432df7baf2f41ba3f2b5d4684d1cb Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:53:28 +1100 Subject: [PATCH 56/65] mtp chain: report graph reuse and process_ubatch host time per chain decode (LLAMA_MTP_DRAFT_TIMING) Diagnostics for the question whether the cached chain graph is reused in steady-state decoding (the chain's llama_decode took 1.9 ms at short context and 3.3-3.5 ms at 48K/115K). With LLAMA_MTP_DRAFT_TIMING=1, every 64 chain decodes print to stderr: mtp-chain-decode: 64 chain decodes, graph reused R, rebuilt M ( n, ...); per decode: reuse check or build+alloc X ms, set_inputs Y ms, compute submit Z ms Miss reasons come from llm_graph_result::reuse_miss_reason(): "new" (shape never built), "params" (allow_reuse), or the type name of the first input that refuses (attn_kv = mask n_kv, mtp_chain = a step's KV length). draft-timing "decode" minus X+Y+Z is the llama_decode time outside process_ubatch (batch split, slot search). No change without the env. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 505f7b8cd1ed8da05faf8fa9794654ffe289eb45 on Torchit1/llama.cpp arc-b580) --- src/llama-context.cpp | 41 +++++++++++++++++++++++++++++++++++++++++ src/llama-context.h | 10 ++++++++++ src/llama-graph.cpp | 17 +++++++++++++++++ src/llama-graph.h | 5 +++++ 4 files changed, 73 insertions(+) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index eb4f6ea76edc..a32df588d17e 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -1452,6 +1452,19 @@ llama_context::mtp_chain_entry * llama_context::mtp_chain_entry_for(uint32_t n_t return &(mtp_chain_cache[key] = std::move(e)); } +void llama_context::mtp_chain_reuse_stats::report() { + std::string why; + for (const auto & [k, v] : miss) { + why += (why.empty() ? "" : ", ") + k + " " + std::to_string(v); + } + fprintf(stderr, "mtp-chain-decode: %d chain decodes, graph reused %d, rebuilt %d (%s); per decode: reuse check or " + "build+alloc %.3f ms, set_inputs %.3f ms, compute submit %.3f ms\n", n, reused, n - reused, + why.empty() ? "-" : why.c_str(), build_ms / n, set_ms / n, compute_ms / n); + n = reused = 0; + build_ms = set_ms = compute_ms = 0; + miss.clear(); +} + const int32_t * llama_context::get_mtp_chain_ids() { synchronize(); @@ -1695,6 +1708,13 @@ llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, ll // in order to correctly reuse a graph, it's full topology has to be uniquely determined by these parameters const auto gparams = graph_params(res, ubatch, mctx, gtype); + static const bool chain_stats_on = [] { + const char * e = getenv("LLAMA_MTP_DRAFT_TIMING"); + return e && atoi(e) != 0; + }(); + const bool chain_stats = chain_stats_on && cparams.mtp_chain_n > 0; + const int64_t t_cs0 = chain_stats ? ggml_time_us() : 0; + if (!graph_reuse_disable && res->can_reuse(gparams)) { //LLAMA_LOG_DEBUG("%s: reusing previous graph\n", __func__); @@ -1706,7 +1726,14 @@ llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, ll } n_reused++; + if (chain_stats) { + mtp_chain_rs.reused++; + } } else { + if (chain_stats) { + const std::string why = graph_reuse_disable ? std::string("disabled") : res->reuse_miss_reason(gparams); + mtp_chain_rs.miss[why.empty() ? std::string("?") : why]++; + } res->reset(); ggml_backend_sched_reset(sched.get()); @@ -1731,6 +1758,8 @@ llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, ll } } + const int64_t t_cs1 = chain_stats ? ggml_time_us() : 0; + // set the input data for the input tensors { //const auto t_start_us = ggml_time_us(); @@ -1741,7 +1770,19 @@ llm_graph_result * llama_context::process_ubatch(const llama_ubatch & ubatch, ll //LLAMA_LOG_INFO("graph set inputs time: %.3f ms\n", (ggml_time_us() - t_start_us)/1000.0); } + const int64_t t_cs2 = chain_stats ? ggml_time_us() : 0; + const auto status = graph_compute(res->get_gf(), ubatch.n_tokens > 1); + + if (chain_stats) { + const int64_t t_cs3 = ggml_time_us(); + mtp_chain_rs.build_ms += (t_cs1 - t_cs0) / 1000.0; + mtp_chain_rs.set_ms += (t_cs2 - t_cs1) / 1000.0; + mtp_chain_rs.compute_ms += (t_cs3 - t_cs2) / 1000.0; + if (++mtp_chain_rs.n == 64) { + mtp_chain_rs.report(); + } + } if (status != GGML_STATUS_SUCCESS) { LLAMA_LOG_ERROR("%s: failed to compute graph, compute status: %d\n", __func__, status); ret = status; diff --git a/src/llama-context.h b/src/llama-context.h index 16f308b140b2..3fcb20cb1991 100644 --- a/src/llama-context.h +++ b/src/llama-context.h @@ -430,6 +430,16 @@ struct llama_context { mtp_chain_entry * mtp_chain_entry_for(uint32_t n_tokens); uint32_t mtp_chain_n_out = 0; + // ARC-LAB: LLAMA_MTP_DRAFT_TIMING=1: per chain decode, whether its cached graph was reused (else why not) and the + // host time of process_ubatch's parts; printed to stderr every 64 chain decodes + struct mtp_chain_reuse_stats { + int n = 0, reused = 0; + double build_ms = 0, set_ms = 0, compute_ms = 0; + std::map miss; + void report(); + }; + mtp_chain_reuse_stats mtp_chain_rs; + // host buffer for the model output (logits and embeddings) ggml_backend_buffer_ptr buf_output; diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 44a31198b323..75d3717667a3 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include // dedup helpers @@ -1489,6 +1490,22 @@ bool llm_graph_result::can_reuse(const llm_graph_params & params) { return res; } +std::string llm_graph_result::reuse_miss_reason(const llm_graph_params & params) { + if (inputs.empty()) { + return "new"; + } + if (!this->params.allow_reuse(params)) { + return "params"; + } + for (auto & input : inputs) { + if (!input->can_reuse(params)) { + const llm_graph_input_i & in = *input; + return typeid(in).name(); + } + } + return ""; +} + llm_graph_input_i * llm_graph_result::add_input(llm_graph_input_ptr input) { inputs.emplace_back(std::move(input)); return inputs.back().get(); diff --git a/src/llama-graph.h b/src/llama-graph.h index 7b1fd48989f8..e1be43b59627 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -11,6 +11,7 @@ #include #include #include +#include #include struct ggml_cgraph; @@ -1025,6 +1026,10 @@ class llm_graph_result { // return true if the graph was updated and can be reused bool can_reuse(const llm_graph_params & params); + // ARC-LAB: why can_reuse(params) refuses (diagnostics, LLAMA_MTP_DRAFT_TIMING): "new" (never built), "params" + // (allow_reuse), or the type name of the first input that refuses; "" if it would not refuse + std::string reuse_miss_reason(const llm_graph_params & params); + llm_graph_input_i * add_input(llm_graph_input_ptr input); void add_fused_node(llm_graph_fused_node result); From 0687925eadf6a58d7f8d43102b9885170952f605 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:28:56 +1100 Subject: [PATCH 57/65] sycl: GGML_SYCL_SYNC_TRACE=1 attributes every synchronize and blocking upload to its cause Each ggml_backend_sycl_synchronize and blocking buffer_set_tensor is counted under a cause, printed per 64 graphs of >= 2000 nodes (the target's graphs, so ~ per MTP cycle) with the totals per such graph: - the scheduler tags its copies (new ggml-base hook ggml_sync_trace_set_cause): "sched: split input " (computed inputs marked), "sched: backend switch"; - anything else is labelled with the first two named callers outside libggml (backtrace + dladdr), e.g. "llama_get_logits_ith <- common_sampler_sample". Lines: "sync-trace: N big graphs (+M smaller), per big graph: S synchronizes, U blocking uploads (B bytes)", then one line per cause. Off by default; costs a backtrace per sync when on (host timing is then not representative). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit b13c1b921e696e1d459d14df257b0f246d1fe183 on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-backend-impl.h | 5 ++ ggml/src/ggml-backend.cpp | 33 ++++++++++ ggml/src/ggml-sycl/ggml-sycl.cpp | 100 +++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+) diff --git a/ggml/src/ggml-backend-impl.h b/ggml/src/ggml-backend-impl.h index 9c56ec30c5f1..cd94ce4ad2d3 100644 --- a/ggml/src/ggml-backend-impl.h +++ b/ggml/src/ggml-backend-impl.h @@ -270,6 +270,11 @@ extern "C" { # define GGML_BACKEND_DL_SCORE_IMPL(score_fn) #endif + // ARC-LAB: sync/upload attribution (GGML_SYCL_SYNC_TRACE): the scheduler tags the blocking copies it makes with a + // cause (e.g. "split input "); a backend that counts synchronizes / uploads reads it. Thread-local; null = none. + GGML_API void ggml_sync_trace_set_cause(const char * cause); + GGML_API const char * ggml_sync_trace_get_cause(void); + #ifdef __cplusplus } #endif diff --git a/ggml/src/ggml-backend.cpp b/ggml/src/ggml-backend.cpp index 3d6310f3ffea..8dbe75408971 100644 --- a/ggml/src/ggml-backend.cpp +++ b/ggml/src/ggml-backend.cpp @@ -1591,6 +1591,26 @@ static bool ggml_backend_sched_alloc_splits(ggml_backend_sched_t sched) { return true; } +// ARC-LAB: see ggml-backend-impl.h +static thread_local char g_sync_trace_cause[128]; +static thread_local bool g_sync_trace_has_cause = false; + +void ggml_sync_trace_set_cause(const char * cause) { + g_sync_trace_has_cause = cause != nullptr; + if (cause) { + snprintf(g_sync_trace_cause, sizeof(g_sync_trace_cause), "%s", cause); + } +} + +const char * ggml_sync_trace_get_cause(void) { + return g_sync_trace_has_cause ? g_sync_trace_cause : nullptr; +} + +static bool ggml_sync_trace_on() { + static const bool on = getenv("GGML_SYCL_SYNC_TRACE") && atoi(getenv("GGML_SYCL_SYNC_TRACE")) != 0; + return on; +} + static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t sched) { GGML_ASSERT(sched); struct ggml_backend_sched_split * splits = sched->splits; @@ -1609,6 +1629,9 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s // ensure the previous split's async work has completed before we start // this split, the allocator may have reused buffer regions across splits if (split->n_inputs == 0 && prev_backend_id >= 0 && prev_backend_id != split_backend_id) { + if (ggml_sync_trace_on()) { + ggml_sync_trace_set_cause("sched: backend switch"); + } if (sched->events[prev_backend_id][sched->cur_copy] != NULL) { ggml_backend_event_synchronize(sched->events[prev_backend_id][sched->cur_copy]); } else { @@ -1621,6 +1644,12 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s ggml_backend_t input_backend = ggml_backend_sched_get_tensor_backend(sched, split->inputs[input_id]); struct ggml_tensor * input = split->inputs[input_id]; struct ggml_tensor * input_cpy = tensor_copy(input, split_backend_id, sched->cur_copy); + if (ggml_sync_trace_on()) { + char cause[128]; + snprintf(cause, sizeof(cause), "sched: split input %s%s", input->flags & GGML_TENSOR_FLAG_INPUT ? "" : "(computed) ", + input->name); + ggml_sync_trace_set_cause(cause); + } if (input->flags & GGML_TENSOR_FLAG_INPUT) { // inputs from the user must be copied immediately to prevent the user overwriting the data before the copy is done @@ -1739,6 +1768,10 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s } } + if (ggml_sync_trace_on()) { + ggml_sync_trace_set_cause(nullptr); + } + if (!sched->callback_eval) { enum ggml_status ec = ggml_backend_graph_compute_async(split_backend, &split->graph); if (ec != GGML_STATUS_SUCCESS) { diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 5ea6f2e7aaea..403e9dc17ce6 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -36,6 +36,9 @@ #include #include #include +#include +#include +#include #include #include @@ -677,6 +680,94 @@ catch (sycl::exception const &exc) { std::exit(1); } +// ARC-LAB: GGML_SYCL_SYNC_TRACE=1: every ggml_backend_sycl_synchronize and blocking buffer_set_tensor is counted under its +// cause: the scheduler's tag (ggml_sync_trace_set_cause: split input , backend switch), else the first named caller +// outside libggml (e.g. llama_get_logits_ith). Printed per 64 graphs of >= 2000 nodes (target graphs: ~ per MTP cycle). +struct ggml_sycl_sync_trace { + const bool on = getenv("GGML_SYCL_SYNC_TRACE") && atoi(getenv("GGML_SYCL_SYNC_TRACE")) != 0; + std::mutex mtx; + int big = 0, small = 0; + std::map syncs; + std::map> uploads; + std::unordered_map names; // return address -> caller label + + std::string caller() { + const char * c = ggml_sync_trace_get_cause(); + if (c) { + return c; + } + void * fr[24]; + const int n = backtrace(fr, 24); + std::string label, last; + int named = 0; + for (int i = 1; i < n && named < 2; ++i) { + auto it = names.find(fr[i]); + std::string nm; + if (it != names.end()) { + nm = it->second; + } else { + Dl_info di{}; + if (dladdr(fr[i], &di) && di.dli_sname && di.dli_fname && !strstr(di.dli_fname, "libggml")) { + int st = 0; + char * dm = abi::__cxa_demangle(di.dli_sname, nullptr, nullptr, &st); + nm = dm && st == 0 ? std::string(dm) : std::string(di.dli_sname); + free(dm); + const size_t paren = nm.find('('); + if (paren != std::string::npos) { + nm.resize(paren); + } + } + names[fr[i]] = nm; + } + if (!nm.empty() && nm != last) { + label += (label.empty() ? "" : " <- ") + nm; + last = nm; + named++; + } + } + return label.empty() ? std::string("?") : label; + } + void note_sync() { + std::lock_guard lock(mtx); + syncs[caller()]++; + } + void note_upload(size_t bytes) { + std::lock_guard lock(mtx); + auto & u = uploads[caller()]; + u.first++; + u.second += bytes; + } + void note_graph(const ggml_cgraph * g) { + std::lock_guard lock(mtx); + if (g->n_nodes < 2000) { + small++; + return; + } + if (++big < 64) { + return; + } + long ns = 0, nu = 0; + size_t nb = 0; + for (auto & kv : syncs) ns += kv.second; + for (auto & kv : uploads) { nu += kv.second.first; nb += kv.second.second; } + fprintf(stderr, "sync-trace: %d big graphs (+%d smaller), per big graph: %.1f synchronizes, %.1f blocking uploads (%.0f bytes)\n", + big, small, (double) ns / big, (double) nu / big, (double) nb / big); + std::vector> v; + for (auto & kv : syncs) v.push_back({ kv.second, kv.first }); + std::sort(v.rbegin(), v.rend()); + for (auto & e : v) fprintf(stderr, "sync-trace: sync %6.2f %s\n", (double) e.first / big, e.second.c_str()); + std::vector> w; + for (auto & kv : uploads) w.push_back({ kv.second.first, kv.first }); + std::sort(w.rbegin(), w.rend()); + for (auto & e : w) fprintf(stderr, "sync-trace: upload %6.2f %8.0f B %s\n", (double) e.first / big, + (double) uploads[e.second].second / big, e.second.c_str()); + big = small = 0; + syncs.clear(); + uploads.clear(); + } +}; +static ggml_sycl_sync_trace g_sync_trace; + // ARC-LAB: per-token accounting of synchronous buffer_set_tensor calls, printed with the GGML_SYCL_HOST_TIMING report static double g_st_ms = 0; static long g_st_calls = 0; static size_t g_st_bytes = 0; static void ggml_sycl_note_set_tensor(size_t bytes, double ms) { g_st_ms += ms; g_st_calls++; g_st_bytes += bytes; } @@ -696,6 +787,9 @@ static void ggml_backend_sycl_buffer_set_tensor(ggml_backend_buffer_t buffer, size_t bytes; std::chrono::steady_clock::time_point t0 = std::chrono::steady_clock::now(); ~st_timer() { ggml_sycl_note_set_tensor(bytes, std::chrono::duration(std::chrono::steady_clock::now() - t0).count()); } } st_guard{ size }; + if (g_sync_trace.on) { + g_sync_trace.note_upload(size); + } // ARC-LAB: GGML_SYCL_FAST_SET_TENSOR=1: small writes (<= 4 MB, i.e. per-token inputs, not weights) skip the device-wide // wait and the malloc'd bounce buffer (the mmap workaround is for weights loaded from disk): one pinned staging copy and // one wait on this queue. Still synchronous for the caller. @@ -6317,6 +6411,9 @@ static void ggml_backend_sycl_synchronize(ggml_backend_t backend) try { ggml_backend_sycl_context * sycl_ctx = (ggml_backend_sycl_context *)backend->context; const queue_ptr stream = sycl_ctx->stream(sycl_ctx->device, 0); const auto ht_t0 = sycl_host_timing::clk::now(); + if (g_sync_trace.on) { + g_sync_trace.note_sync(); + } SYCL_CHECK(CHECK_TRY_ERROR((stream)->wait())); g_upload_off = 0; if (g_ht.on) { @@ -7323,6 +7420,9 @@ static void ggml_backend_sycl_graph_compute_impl(ggml_backend_sycl_context * syc ot_prev = now; ot_op = next; }; + if (g_sync_trace.on) { + g_sync_trace.note_graph(cgraph); + } if (ot_on) { ot_mark(std::string()); // inputs and earlier work are not charged g_ot.nodes += cgraph->n_nodes; From 0cc9909ca2505cb594ee97f29a7be96b4ebebf63 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:32:01 +1100 Subject: [PATCH 58/65] llama: skip a context synchronize when nothing was queued since the last one (LLAMA_SYNC_TRIM bit 1) Every result getter (llama_get_logits_ith, the sampled-token getters, llama_get_embeddings_nextn_ith, ...) calls llama_context::synchronize, and an MTP cycle calls several per row: the server samples each verified row, then the MTP drafter reads the target's h_nextn row by row. Each is a full device wait on the one in-order queue every context shares. Only the first after a decode waits for this context's work; a later one can only wait for the other context's work queued in between (e.g. the draft graph), which this context does not need. Now the context tracks whether it queued anything (graph compute and the async output reads, decode/encode) since its last synchronize, and skips the device wait if not. Not with pipeline parallelism (its copy rotation runs in the sync). Results are unchanged: they were complete already. LLAMA_SYNC_TRIM is a new bit mask (default 3; 0 = upstream behaviour). Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 60ef575f3b84f421c8935358265bc6046e5192aa on Torchit1/llama.cpp arc-b580) --- src/llama-context.cpp | 22 ++++++++++++++++++++++ src/llama-context.h | 1 + src/llama-impl.h | 3 +++ 3 files changed, 26 insertions(+) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index a32df588d17e..6d1ef386012d 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -826,11 +826,29 @@ void llama_context::sched_reserve() { __func__, (t_end_us - t_start_us)/1000.0, ggml_backend_sched_get_n_copies(sched.get())); } +// ARC-LAB: LLAMA_SYNC_TRIM bit mask (default all on; 0 = upstream behaviour): 1 = skip a context synchronize when the +// context queued nothing since its last one, 2 = KV-cache rotation matrices kept on the device (llama-kv-cache.cpp) +int llama_sync_trim() { + static const int v = [] { + const char * e = getenv("LLAMA_SYNC_TRIM"); + return e ? atoi(e) : 3; + }(); + return v; +} + void llama_context::synchronize() { if (!sched) { return; } + // ARC-LAB: LLAMA_SYNC_TRIM bit 1 (default on): every getter (logits, sampled tokens, nextn rows, ...) synchronizes, + // and the device's one in-order queue is shared by all contexts. With nothing queued by this context since its + // last synchronize, its results are already complete: skip the device wait. + if ((llama_sync_trim() & 1) && !sync_pending && !cparams.pipeline_parallel) { + return; + } + sync_pending = false; + ggml_backend_sched_synchronize(sched.get()); // FIXME: if multiple single tokens are evaluated without a synchronization, @@ -1839,6 +1857,7 @@ int llama_context::encode(const llama_batch & batch_inp) { sched_reserve(); n_queued_tokens += n_tokens; + sync_pending = true; // reserve output buffer if (output_reserve(n_tokens) < n_tokens) { @@ -2145,6 +2164,7 @@ int llama_context::decode(const llama_batch & batch_inp) { t_compute_start_us = ggml_time_us(); } n_queued_tokens += n_tokens_all; + sync_pending = true; output_swaps.clear(); @@ -2992,6 +3012,7 @@ ggml_status llama_context::graph_compute( set_n_threads_fn.second(set_n_threads_fn.first, n_threads); } + sync_pending = true; auto status = ggml_backend_sched_graph_compute_async(sched.get(), gf); if (status != GGML_STATUS_SUCCESS) { LLAMA_LOG_ERROR("%s: ggml_backend_sched_graph_compute_async failed with error %d\n", __func__, status); @@ -3882,6 +3903,7 @@ void llama_context::opt_epoch_iter( const uint32_t n_tokens_all = balloc->get_n_tokens(); n_queued_tokens += n_tokens_all; + sync_pending = true; embd_seq.clear(); diff --git a/src/llama-context.h b/src/llama-context.h index 3fcb20cb1991..7ad06c4469c2 100644 --- a/src/llama-context.h +++ b/src/llama-context.h @@ -447,6 +447,7 @@ struct llama_context { std::map mem_storage; bool has_evaluated_once = false; + bool sync_pending = true; // ARC-LAB: work queued since the last synchronize (LLAMA_SYNC_TRIM) // env: LLAMA_GRAPH_REUSE_DISABLE bool graph_reuse_disable = false; diff --git a/src/llama-impl.h b/src/llama-impl.h index 4988b06d2ca0..997bb985c610 100644 --- a/src/llama-impl.h +++ b/src/llama-impl.h @@ -74,6 +74,9 @@ static inline ggml_tensor * llama_mul_mat_hadamard( return res; } +// ARC-LAB: LLAMA_SYNC_TRIM bit mask, see llama-context.cpp +int llama_sync_trim(); + struct time_meas { time_meas(int64_t & t_acc, bool disable = false); ~time_meas(); From 84dafe3c7c68917fb14460235b922e2d6b8ab975 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:32:02 +1100 Subject: [PATCH 59/65] kv-cache: keep the attention rotation matrices on the device (LLAMA_SYNC_TRIM bit 2) With a quantized KV cache every attention graph reads attn_inp_k_rot (256 x 256 f32 here) and attn_inp_v_rot (64 x 64) as graph inputs. They live in host memory, so each graph uploads them again: per graph 2 split inputs, i.e. 2 device synchronizes + 2 blocking copies (~272 KB), in the target's verify graph and in the MTP draft graph alike (upstream has a TODO for this). The cache now also allocates them once in its own device buffer (only when the cache has one non-host buffer type) and build_input_k_rot / v_rot return those tensors; set_input_k_rot / v_rot skip a non-input tensor. Same values (ggml_gen_hadamard, copied once): outputs identical. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 4ec75c3416006e44df935387b2ec514eb56f4e23 on Torchit1/llama.cpp arc-b580) --- src/llama-kv-cache.cpp | 41 +++++++++++++++++++++++++++++++++++++++++ src/llama-kv-cache.h | 6 ++++++ 2 files changed, 47 insertions(+) diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp index cb806c688166..2ac1dc5a01b9 100644 --- a/src/llama-kv-cache.cpp +++ b/src/llama-kv-cache.cpp @@ -361,6 +361,33 @@ llama_kv_cache::llama_kv_cache( ggml_gen_hadamard(tmp); } + + // ARC-LAB: keep them on the cache's device too (one buffer type only, and not host memory) + if ((llama_sync_trim() & 2) && ctxs_bufs.size() == 1 && !hparams.no_alloc) { + ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(ctxs_bufs[0].second.get()); + if (!ggml_backend_buft_is_host(buft)) { + ggml_init_params params = { + /* .mem_size = */ attn_rot_hadamard.size()*ggml_tensor_overhead(), + /* .mem_buffer = */ nullptr, + /* .no_alloc = */ true, + }; + attn_rot_dev_ctx.reset(ggml_init(params)); + for (const auto & [n, m] : attn_rot_hadamard) { + ggml_tensor * t = ggml_new_tensor_2d(attn_rot_dev_ctx.get(), GGML_TYPE_F32, n, n); + ggml_format_name(t, "attn_rot_dev_%d", (int) n); + attn_rot_dev[n] = t; + } + attn_rot_dev_buf.reset(ggml_backend_alloc_ctx_tensors_from_buft(attn_rot_dev_ctx.get(), buft)); + if (attn_rot_dev_buf) { + ggml_backend_buffer_set_usage(attn_rot_dev_buf.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS); + for (const auto & [n, t] : attn_rot_dev) { + ggml_backend_tensor_set(t, attn_rot_hadamard.at(n).data(), 0, ggml_nbytes(t)); + } + } else { + attn_rot_dev.clear(); + } + } + } } const char * LLAMA_KV_CACHE_DEBUG = getenv("LLAMA_KV_CACHE_DEBUG"); @@ -1654,6 +1681,10 @@ ggml_tensor * llama_kv_cache::build_input_k_rot(ggml_context * ctx) const { } while (n_embd_head_k_all % nrot == 0); nrot /= 2; + if (const auto it = attn_rot_dev.find(nrot); it != attn_rot_dev.end()) { + return it->second; + } + res = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nrot, nrot); ggml_set_input(res); ggml_set_name(res, "attn_inp_k_rot"); @@ -1674,6 +1705,10 @@ ggml_tensor * llama_kv_cache::build_input_v_rot(ggml_context * ctx) const { //} while (hparams.n_embd_head_v() % nrot == 0); //nrot /= 2; + if (const auto it = attn_rot_dev.find(nrot); it != attn_rot_dev.end()) { + return it->second; + } + res = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, nrot, nrot); ggml_set_input(res); ggml_set_name(res, "attn_inp_v_rot"); @@ -2012,6 +2047,9 @@ void llama_kv_cache::set_input_pos_bucket(ggml_tensor * dst, const llama_ubatch } void llama_kv_cache::set_input_k_rot(ggml_tensor * dst) const { + if (!(dst->flags & GGML_TENSOR_FLAG_INPUT)) { + return; // the device copy (attn_rot_dev), already filled + } GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer)); const auto n_rot = dst->ne[0]; @@ -2021,6 +2059,9 @@ void llama_kv_cache::set_input_k_rot(ggml_tensor * dst) const { } void llama_kv_cache::set_input_v_rot(ggml_tensor * dst) const { + if (!(dst->flags & GGML_TENSOR_FLAG_INPUT)) { + return; // the device copy (attn_rot_dev), already filled + } GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer)); const auto n_rot = dst->ne[0]; diff --git a/src/llama-kv-cache.h b/src/llama-kv-cache.h index 05add1b853c1..7f3d91bec362 100644 --- a/src/llama-kv-cache.h +++ b/src/llama-kv-cache.h @@ -280,6 +280,12 @@ class llama_kv_cache : public llama_memory_i { // pre-computed hadamard martrices std::unordered_map> attn_rot_hadamard; + // ARC-LAB: device copies of attn_rot_hadamard (LLAMA_SYNC_TRIM bit 2): graphs read these instead of an input that + // is uploaded (one sync + one blocking copy each) on every decode + ggml_context_ptr attn_rot_dev_ctx; + ggml_backend_buffer_ptr attn_rot_dev_buf; + std::unordered_map attn_rot_dev; + // env: LLAMA_KV_CACHE_DEBUG int debug = 0; From 880cc602a51fb67a846401e379cb0fdc4b4c8d5a Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:32:02 +1100 Subject: [PATCH 60/65] mtp chain: one index upload fewer twice over (LLAMA_MTP_CHAIN_TIGHT bit 8) Feature B round 2 (independent implementation of the technique described by llamADALL). Two chain inputs were uploaded separately though they need not be: - the output-row index (step 0 with catch-up rows) now rides at the end of the position input: one upload, sliced on the device; - V is stored with the K indices: for a non-transposed V cache set_input_v_idxs writes exactly what set_input_k_idxs writes, so the V index input is left out of the graph (llm_graph_input_attn_kv::set_input now skips an unallocated V index tensor). Per chain decode -1..2 split inputs (sync + blocking copy each). Same values in the same kernels: drafts unchanged. Default LLAMA_MTP_CHAIN_TIGHT is now 15. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit f3aa69d6a2c4ac3a3479576bf1d9fcada08eca44 on Torchit1/llama.cpp arc-b580) --- src/llama-graph.cpp | 5 ++++- src/models/qwen35.cpp | 34 +++++++++++++++++++++++----------- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 75d3717667a3..2020398f89cb 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -569,7 +569,10 @@ void llm_graph_input_attn_no_cache::set_input(const llama_ubatch * ubatch) { void llm_graph_input_attn_kv::set_input(const llama_ubatch * ubatch) { mctx->set_input_k_idxs(self_k_idxs, ubatch); - mctx->set_input_v_idxs(self_v_idxs, ubatch); + // ARC-LAB: left unallocated when the graph stores V with the K indices (MTP chain, non-transposed V) + if (self_v_idxs && self_v_idxs->buffer) { + mctx->set_input_v_idxs(self_v_idxs, ubatch); + } // the mask is left unallocated when the graph only stores K/V without attending // (e.g. DFlash's KV-injection pass) diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp index affb021fcd3c..f2e95cf41b6c 100644 --- a/src/models/qwen35.cpp +++ b/src/models/qwen35.cpp @@ -635,6 +635,7 @@ class llm_graph_input_mtp_chain : public llm_graph_input_i { int64_t n_tokens = 0; ggml_tensor * pos = nullptr; // I32 [n_tokens * n_pos_per_embd] ggml_tensor * out_idx = nullptr; // I32 [gathering steps]: nr - 1 (the step's last row produces its draft); null: none + bool out_in_pos = false; // the output rows follow the positions in pos (one upload), out_idx unused void set_input(const llama_ubatch * ubatch) override { GGML_ASSERT(ubatch->n_tokens == n_tokens); @@ -661,8 +662,11 @@ class llm_graph_input_mtp_chain : public llm_graph_input_i { o.push_back((int32_t) st.nr - 1); } } + if (out_in_pos) { + v.insert(v.end(), o.begin(), o.end()); + } ggml_backend_tensor_set(pos, v.data(), 0, v.size() * sizeof(int32_t)); - if (out_idx) { + if (out_idx && !out_in_pos) { ggml_backend_tensor_set(out_idx, o.data(), 0, o.size() * sizeof(int32_t)); } } @@ -986,16 +990,16 @@ void llama_model_qwen35::graph_mtp::build_chain(const llama_model & model) { // contiguous, only a multi-row step 0 narrower than the full mask needs the copy (one copy kernel, not n). // 4: a step picks its output row with a GET_ROWS only if it has several rows (step 0 with catch-up rows); a // 1-row step's row is already the output (no gather, no output-index slice). + // 8: fewer uploads: the output rows ride in the position input, and V is stored with the K indices (equal for a + // non-transposed V cache; the V index input is then unused and not uploaded). static const int tight = [] { const char * e = getenv("LLAMA_MTP_CHAIN_TIGHT"); - return e ? atoi(e) : 7; + return e ? atoi(e) : 15; }(); const uint32_t n_pos = hparams.n_pos_per_embd(); auto chain_inp = std::make_unique(n_pos); chain_inp->n_tokens = n_tokens; - chain_inp->pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens * n_pos); - ggml_set_input(chain_inp->pos); int64_t n_gather = 0; for (int64_t s = 0; s < n_steps; ++s) { llm_graph_input_mtp_chain::step st; @@ -1006,13 +1010,17 @@ void llama_model_qwen35::graph_mtp::build_chain(const llama_model & model) { n_gather += st.gather; chain_inp->steps.push_back(st); } - if (n_gather > 0) { + chain_inp->out_in_pos = (tight & 8) && n_gather > 0; + chain_inp->pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens * n_pos + (chain_inp->out_in_pos ? n_gather : 0)); + ggml_set_input(chain_inp->pos); + if (n_gather > 0 && !chain_inp->out_in_pos) { chain_inp->out_idx = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_gather); ggml_set_input(chain_inp->out_idx); } - const auto steps = chain_inp->steps; - ggml_tensor * pos_all = chain_inp->pos; - ggml_tensor * out_all = chain_inp->out_idx; + const auto steps = chain_inp->steps; + const bool out_in_pos = chain_inp->out_in_pos; + ggml_tensor * pos_all = chain_inp->pos; + ggml_tensor * out_all = chain_inp->out_idx; res->add_input(std::move(chain_inp)); // token embeddings: the host table for the ubatch's own tokens (catch-up tokens need not be in the draft @@ -1057,7 +1065,7 @@ void llama_model_qwen35::graph_mtp::build_chain(const llama_model & model) { ggml_tensor * tok_rows = ggml_get_rows(ctx0, tok_embd_w, ggml_view_1d(ctx0, tokens_all, n_lead, 0)); ggml_tensor * h_embd = ggml_view_2d(ctx0, h_all, h_all->ne[0], n_lead, h_all->nb[1], 0); ggml_tensor * k_all = inp_attn->get_k_idxs(); - ggml_tensor * v_all = inp_attn->get_v_idxs(); + ggml_tensor * v_all = (tight & 8) && !kvc->is_v_trans() ? k_all : inp_attn->get_v_idxs(); ggml_tensor * mask_all = inp_attn->get_kq_mask(); // the device holding the head's weights (none: CPU-only, the slices stay views of the host inputs) @@ -1087,11 +1095,12 @@ void llama_model_qwen35::graph_mtp::build_chain(const llama_model & model) { return c; }; pos_all = on_dev(pos_all, "mtp_chain_pos_dev"); - if (n_gather > 1) { + if (n_gather > 1 && !out_in_pos) { out_all = on_dev(out_all, "mtp_chain_out_dev"); } + const bool v_is_k = v_all == k_all; k_all = on_dev(k_all, "mtp_chain_k_idxs_dev"); - v_all = on_dev(v_all, "mtp_chain_v_idxs_dev"); + v_all = v_is_k ? k_all : on_dev(v_all, "mtp_chain_v_idxs_dev"); mask_all = on_dev(mask_all, "mtp_chain_kq_mask_dev"); } @@ -1100,6 +1109,9 @@ void llama_model_qwen35::graph_mtp::build_chain(const llama_model & model) { struct step_inputs { ggml_tensor * pos, * out, * k_idxs, * v_idxs, * kq_mask; }; + if (out_in_pos) { + out_all = ggml_view_1d(ctx0, pos_all, n_gather, n_tokens * n_pos * ggml_element_size(pos_all)); + } std::vector sin(n_steps); for (int64_t s = 0, g = 0; s < n_steps; ++s) { const auto & st = steps[s]; From d8e10e01faa54ea05b9bb68351dd2b15fcb8e0fb Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 10 Oct 2026 18:13:31 +1100 Subject: [PATCH 61/65] sycl: build on Windows again - the sync trace's caller lookup (cxxabi/dlfcn/execinfo) is Linux-only Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 135a7667b60c966925a0d0bee8de41942cbc481f on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/ggml-sycl.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 403e9dc17ce6..e102fe6e40b0 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -36,9 +36,11 @@ #include #include #include +#ifndef _WIN32 #include #include #include +#endif #include #include @@ -696,6 +698,9 @@ struct ggml_sycl_sync_trace { if (c) { return c; } +#ifdef _WIN32 + return std::string("?"); // no caller lookup on Windows +#else void * fr[24]; const int n = backtrace(fr, 24); std::string label, last; @@ -726,6 +731,7 @@ struct ggml_sycl_sync_trace { } } return label.empty() ? std::string("?") : label; +#endif } void note_sync() { std::lock_guard lock(mtx); From 1d7d3b241543471623ad6ef515a71b68366887c7 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 10 Oct 2026 18:37:20 +1100 Subject: [PATCH 62/65] sycl: rename the sync trace's small/big counters (the Windows SDK defines small as a macro) Co-Authored-By: Claude Opus 5.5 (cherry picked from commit b5a7db2682c295addc928b085b000b516b54aef6 on Torchit1/llama.cpp arc-b580) --- ggml/src/ggml-sycl/ggml-sycl.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index e102fe6e40b0..2cede321a1e7 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -688,7 +688,7 @@ catch (sycl::exception const &exc) { struct ggml_sycl_sync_trace { const bool on = getenv("GGML_SYCL_SYNC_TRACE") && atoi(getenv("GGML_SYCL_SYNC_TRACE")) != 0; std::mutex mtx; - int big = 0, small = 0; + int n_big = 0, n_small = 0; std::map syncs; std::map> uploads; std::unordered_map names; // return address -> caller label @@ -746,10 +746,10 @@ struct ggml_sycl_sync_trace { void note_graph(const ggml_cgraph * g) { std::lock_guard lock(mtx); if (g->n_nodes < 2000) { - small++; + n_small++; return; } - if (++big < 64) { + if (++n_big < 64) { return; } long ns = 0, nu = 0; @@ -757,17 +757,17 @@ struct ggml_sycl_sync_trace { for (auto & kv : syncs) ns += kv.second; for (auto & kv : uploads) { nu += kv.second.first; nb += kv.second.second; } fprintf(stderr, "sync-trace: %d big graphs (+%d smaller), per big graph: %.1f synchronizes, %.1f blocking uploads (%.0f bytes)\n", - big, small, (double) ns / big, (double) nu / big, (double) nb / big); + n_big, n_small, (double) ns / n_big, (double) nu / n_big, (double) nb / n_big); std::vector> v; for (auto & kv : syncs) v.push_back({ kv.second, kv.first }); std::sort(v.rbegin(), v.rend()); - for (auto & e : v) fprintf(stderr, "sync-trace: sync %6.2f %s\n", (double) e.first / big, e.second.c_str()); + for (auto & e : v) fprintf(stderr, "sync-trace: sync %6.2f %s\n", (double) e.first / n_big, e.second.c_str()); std::vector> w; for (auto & kv : uploads) w.push_back({ kv.second.first, kv.first }); std::sort(w.rbegin(), w.rend()); - for (auto & e : w) fprintf(stderr, "sync-trace: upload %6.2f %8.0f B %s\n", (double) e.first / big, - (double) uploads[e.second].second / big, e.second.c_str()); - big = small = 0; + for (auto & e : w) fprintf(stderr, "sync-trace: upload %6.2f %8.0f B %s\n", (double) e.first / n_big, + (double) uploads[e.second].second / n_big, e.second.c_str()); + n_big = n_small = 0; syncs.clear(); uploads.clear(); } From b49a95f779609aa33a619bc14a1a0b7067fdc5e8 Mon Sep 17 00:00:00 2001 From: Jesse Symons <155509533+Torchit1@users.noreply.github.com> Date: Sat, 10 Oct 2026 20:05:53 +1100 Subject: [PATCH 63/65] mtp chain: all draft ids in one tensor, read back with one copy (LLAMA_MTP_CHAIN_TIGHT bit 16) Feature B, idea 4 (independent implementation of the technique described by llamADALL). The chain read its n per-step argmax ids back with n separate 4-byte ggml_backend_tensor_get_async calls. On this SYCL/Level Zero stack each small device->host read costs ~0.2 ms (measured: reading only step 0's id cut the draft call from 6.07 to 5.16 ms). Now, after the last step, the ids are concatenated into one I32 [n] tensor (n-1 tiny CONCAT kernels at the end of the graph, after every step's compute nodes), marked as the only output, and llama_context reads it with one async copy into mtp_chain_out. The per-step argmax tensors are no longer graph outputs. get_mtp_chain_ids, the draft log and the ids are unchanged (the same integers, copied). Default LLAMA_MTP_CHAIN_TIGHT is now 31. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit b0376aec393cfe96e2652e9031bf0e5d7c410dd7 on Torchit1/llama.cpp arc-b580) --- src/llama-context.cpp | 10 +++++++++- src/llama-graph.cpp | 1 + src/llama-graph.h | 1 + src/models/qwen35.cpp | 20 ++++++++++++++++++-- 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 6d1ef386012d..9f5b27eef935 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -2266,7 +2266,15 @@ int llama_context::decode(const llama_batch & batch_inp) { if (chain_e) { // the per-step argmax rows, mapped to tokens in get_mtp_chain_ids() - if (res && !res->t_mtp_chain.empty()) { + if (res && res->t_mtp_chain_ids) { + // one copy of all the ids (LLAMA_MTP_CHAIN_TIGHT bit 16) + ggml_tensor * t = res->t_mtp_chain_ids; + ggml_backend_t b = ggml_backend_sched_get_tensor_backend(sched.get(), t); + GGML_ASSERT(b != nullptr && t->ne[0] == (int64_t) res->t_mtp_chain.size()); + mtp_chain_out.resize(res->t_mtp_chain.size()); + ggml_backend_tensor_get_async(b, t, mtp_chain_out.data(), 0, mtp_chain_out.size() * sizeof(int32_t)); + mtp_chain_n_out = (uint32_t) res->t_mtp_chain.size(); + } else if (res && !res->t_mtp_chain.empty()) { mtp_chain_out.resize(res->t_mtp_chain.size()); for (size_t s = 0; s < res->t_mtp_chain.size(); ++s) { ggml_tensor * t = res->t_mtp_chain[s]; diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index 2020398f89cb..c5e428af0e7b 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -1386,6 +1386,7 @@ void llm_graph_result::reset() { t_sampled.clear(); t_dspark_greedy.clear(); t_mtp_chain.clear(); + t_mtp_chain_ids = nullptr; t_sampled_probs.clear(); t_sampled_logits.clear(); t_candidates.clear(); diff --git a/src/llama-graph.h b/src/llama-graph.h index e1be43b59627..19dab8400cf6 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -1057,6 +1057,7 @@ class llm_graph_result { // ARC-LAB: chained MTP draft (cparams.mtp_chain_n): per step, the argmax row of the draft logits (I32 [1]); an // index into the draft head's vocabulary when it has one, else a token id std::vector t_mtp_chain; + ggml_tensor * t_mtp_chain_ids = nullptr; // I32 [n_steps]: the same ids in one tensor (one readback), or null std::vector t_sampled_probs; std::vector t_sampled_logits; std::vector t_candidates; diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp index f2e95cf41b6c..a8e6d608f7ab 100644 --- a/src/models/qwen35.cpp +++ b/src/models/qwen35.cpp @@ -992,9 +992,11 @@ void llama_model_qwen35::graph_mtp::build_chain(const llama_model & model) { // 1-row step's row is already the output (no gather, no output-index slice). // 8: fewer uploads: the output rows ride in the position input, and V is stored with the K indices (equal for a // non-transposed V cache; the V index input is then unused and not uploaded). + // 16: the n draft ids are concatenated into one I32 tensor at the end of the graph (n-1 tiny kernels), so the host + // reads them with one copy: each small device->host read costs ~0.2 ms on this stack. static const int tight = [] { const char * e = getenv("LLAMA_MTP_CHAIN_TIGHT"); - return e ? atoi(e) : 15; + return e ? atoi(e) : 31; }(); const uint32_t n_pos = hparams.n_pos_per_embd(); @@ -1262,7 +1264,9 @@ void llama_model_qwen35::graph_mtp::build_chain(const llama_model & model) { cb(logits, "result_output", -1); ggml_tensor * best = ggml_argmax(ctx0, logits); ggml_set_name(best, "mtp_chain_argmax"); - ggml_set_output(best); + if (!(tight & 16)) { + ggml_set_output(best); + } ggml_build_forward_expand(gf, best); res->t_mtp_chain.push_back(best); @@ -1284,4 +1288,16 @@ void llama_model_qwen35::graph_mtp::build_chain(const llama_model & model) { tok_embd = unrotate(e); h_embd = h_out; } + + if (tight & 16) { + // after the last step, so no node lands between a step's compute nodes + ggml_tensor * ids = res->t_mtp_chain[0]; + for (size_t s = 1; s < res->t_mtp_chain.size(); ++s) { + ids = ggml_concat(ctx0, ids, res->t_mtp_chain[s], 0); + } + ggml_set_name(ids, "mtp_chain_ids"); + ggml_set_output(ids); + ggml_build_forward_expand(gf, ids); + res->t_mtp_chain_ids = ids; + } } From 9c4763f57906ab4fca6203116f62b38e78b4a789 Mon Sep 17 00:00:00 2001 From: bri-prism <288398250+bri-prism@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:13:44 -0700 Subject: [PATCH 64/65] llama: default sync trim and the MTP K/V-only pass on only with a SYCL device LLAMA_SYNC_TRIM and LLAMA_MTP_KV_ONLY were on by default for every backend but were only measured on SYCL. They now default on only when a SYCL device is registered; both env vars still force either way. Other backends keep the previous behaviour unless asked. --- src/llama-context.cpp | 14 +++++++++++--- src/llama-impl.h | 3 +++ src/models/qwen35.cpp | 5 +++-- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 9f5b27eef935..3a37572122df 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -826,12 +826,20 @@ void llama_context::sched_reserve() { __func__, (t_end_us - t_start_us)/1000.0, ggml_backend_sched_get_n_copies(sched.get())); } -// ARC-LAB: LLAMA_SYNC_TRIM bit mask (default all on; 0 = upstream behaviour): 1 = skip a context synchronize when the +// ARC-LAB: LLAMA_SYNC_TRIM bit mask (default all on with a SYCL device, else 0; 0 = upstream behaviour): 1 = skip a context synchronize when the // context queued nothing since its last one, 2 = KV-cache rotation matrices kept on the device (llama-kv-cache.cpp) +bool llama_sycl_present() { + static const bool v = [] { + ggml_backend_reg_t reg = ggml_backend_reg_by_name("SYCL"); + return reg != nullptr && ggml_backend_reg_dev_count(reg) > 0; + }(); + return v; +} + int llama_sync_trim() { static const int v = [] { const char * e = getenv("LLAMA_SYNC_TRIM"); - return e ? atoi(e) : 3; + return e ? atoi(e) : (llama_sycl_present() ? 3 : 0); }(); return v; } @@ -841,7 +849,7 @@ void llama_context::synchronize() { return; } - // ARC-LAB: LLAMA_SYNC_TRIM bit 1 (default on): every getter (logits, sampled tokens, nextn rows, ...) synchronizes, + // ARC-LAB: LLAMA_SYNC_TRIM bit 1 (default on with SYCL): every getter (logits, sampled tokens, nextn rows, ...) synchronizes, // and the device's one in-order queue is shared by all contexts. With nothing queued by this context since its // last synchronize, its results are already complete: skip the device wait. if ((llama_sync_trim() & 1) && !sync_pending && !cparams.pipeline_parallel) { diff --git a/src/llama-impl.h b/src/llama-impl.h index 997bb985c610..c6bc214ccba0 100644 --- a/src/llama-impl.h +++ b/src/llama-impl.h @@ -77,6 +77,9 @@ static inline ggml_tensor * llama_mul_mat_hadamard( // ARC-LAB: LLAMA_SYNC_TRIM bit mask, see llama-context.cpp int llama_sync_trim(); +// true when a SYCL device is registered; the ARC-LAB defaults below were only measured there +bool llama_sycl_present(); + struct time_meas { time_meas(int64_t & t_acc, bool disable = false); ~time_meas(); diff --git a/src/models/qwen35.cpp b/src/models/qwen35.cpp index a8e6d608f7ab..914798657224 100644 --- a/src/models/qwen35.cpp +++ b/src/models/qwen35.cpp @@ -1,4 +1,5 @@ #include "models.h" +#include "llama-impl.h" #include "llama-memory-recurrent.h" #include "llama-kv-cache.h" @@ -758,14 +759,14 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr res->add_input(std::move(inp)); - // ARC-LAB: KV-only pass (LLAMA_MTP_KV_ONLY, default on; independent implementation of the technique described by + // ARC-LAB: KV-only pass (LLAMA_MTP_KV_ONLY, default on with a SYCL device; independent implementation of the technique described by // GRIMOIRE). A decode with no output rows (the head's prompt catch-up) is only there to fill this layer's K/V, so // build up to the cache store and stop: no query, attention, output projection, FFN or head. The K/V ops are the // full graph's, in the same order, so the cache rows are bit-identical. Unmasked nextn readers copy h for every // token, so they keep the full graph. static const bool kv_only_on = [] { const char * e = getenv("LLAMA_MTP_KV_ONLY"); - return e == nullptr || atoi(e) != 0; + return e ? atoi(e) != 0 : llama_sycl_present(); }(); const bool kv_only = kv_only_on && n_outputs == 0 && cparams.embeddings_nextn_masked; From 62f926b418bb5e025c10333eb513be9bfcfbd1be Mon Sep 17 00:00:00 2001 From: bri-prism <288398250+bri-prism@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:19:21 -0700 Subject: [PATCH 65/65] sycl: drop duplicate PTQ1_0 case labels left by the MTP chain merge The one-graph draft chain commit added PTQ1_0 to the supports_op type list and a bit-exact f32 get_rows case, but prism already had both labels, so the SYCL build failed (duplicate case value). Keep one label each; get_rows keeps prism's XMX-layout assert and uses the chain's f32 dequant. --- ggml/src/ggml-sycl/getrows.cpp | 6 +----- ggml/src/ggml-sycl/ggml-sycl.cpp | 1 - 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/ggml/src/ggml-sycl/getrows.cpp b/ggml/src/ggml-sycl/getrows.cpp index c6029be7215d..4e53476ba8f2 100644 --- a/ggml/src/ggml-sycl/getrows.cpp +++ b/ggml/src/ggml-sycl/getrows.cpp @@ -280,7 +280,7 @@ void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { GGML_ASSERT(!(dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.xmx_pq2) && "PTQ1_0 in the XMX layout reached get_rows"); - get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, + get_rows_sycl_f32(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, src1_i32, (float *)dst->data, ctx.stream()); break; case GGML_TYPE_PQ2_0: @@ -334,10 +334,6 @@ void ggml_sycl_op_get_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) { get_rows_sycl(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, src1_i32, (float *)dst->data, ctx.stream()); break; - case GGML_TYPE_PTQ1_0: - get_rows_sycl_f32(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, - src1_i32, (float *)dst->data, ctx.stream()); - break; case GGML_TYPE_Q2_K: get_rows_sycl_f32(ctx, dst->src[0], dst->src[1], dst, (const float *)dst->src[0]->data, src1_i32, (float *)dst->data, ctx.stream()); diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index 2cede321a1e7..91a4cd2adcdb 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -8030,7 +8030,6 @@ static bool do_ggml_backend_sycl_device_supports_op(ggml_backend_dev_t dev, cons case GGML_TYPE_BF16: case GGML_TYPE_F32: case GGML_TYPE_Q1_0: - case GGML_TYPE_PTQ1_0: case GGML_TYPE_PQ2_0: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: