From b8516a6f890dfd1a81420becdca2f9d7dfcb0500 Mon Sep 17 00:00:00 2001 From: Parker Loflin Date: Mon, 19 Oct 2026 18:26:38 -0400 Subject: [PATCH] crc: run crc32 alongside the folding in crc32_iscsi on Zen 5 On AMD Zen 5, crc32_iscsi_by16_10 is limited by vpclmulqdq, which issues one zmm instruction every two cycles there, while the crc32 instruction, which Zen 5 runs twice per cycle, sits idle. Add crc32_iscsi_by16_10_fused, built from the same source with FUSED_CRC32. For buffers of at least 4 KB, it splits each 3584 byte chunk into 1792 bytes for the folding loop and eight 224 byte streams for crc32, 32 crc32 per fold iteration. The stream crcs are shifted into place with two vpclmulqdq and xored into the second block after the streams like an initial crc, so the folding of one block hides their latency. Dispatch to it where crc32_iscsi_by16_10 would be used on AMD family 1Ah (base family Fh, extended family 0Bh), models 00h-2Fh, 40h-4Fh, 60h-7Fh and D0h-D7h, the Zen 5 models in Linux's amd.c. Zen 6 is also family 1Ah and keeps crc32_iscsi_by16_10 until someone measures it. Other CPUs keep it too: one crc32 per cycle, as on current Intel CPUs, cannot keep up with 32 per iteration. On a Ryzen 7 9800X3D, crc32_iscsi goes from 83 to 130 GB/s for 4 KB buffers, from about 82 to 137-148 GB/s for 16 KB to 256 KB, from 82 to 114 GB/s for 1 MB, and from 80 to 92 GB/s for 4 MB. Shorter buffers do not change. Signed-off-by: Parker Loflin --- Makefile.nmake | 1 + cmake/crc.cmake | 1 + crc/Makefile.am | 1 + crc/crc32_iscsi_by16_10.asm | 190 ++++++++++++++++++++++++++++++ crc/crc32_iscsi_by16_10_fused.asm | 35 ++++++ crc/crc_multibinary.asm | 3 +- include/multibinary.asm | 74 ++++++++++++ include/reg_sizes.asm | 10 ++ 8 files changed, 314 insertions(+), 1 deletion(-) create mode 100644 crc/crc32_iscsi_by16_10_fused.asm diff --git a/Makefile.nmake b/Makefile.nmake index 356becbf..7b663440 100644 --- a/Makefile.nmake +++ b/Makefile.nmake @@ -149,6 +149,7 @@ objs = \ bin\crc32_iscsi_01.obj \ bin\crc32_iscsi_by8_02.obj \ bin\crc32_iscsi_by16_10.obj \ + bin\crc32_iscsi_by16_10_fused.obj \ bin\crc32_iscsi_avx2.obj \ bin\crc_multibinary.obj \ bin\crc64_ecma_refl_by8.obj \ diff --git a/cmake/crc.cmake b/cmake/crc.cmake index b38db0d8..4ad93345 100644 --- a/cmake/crc.cmake +++ b/cmake/crc.cmake @@ -48,6 +48,7 @@ set(CRC_X86_64_SOURCES crc/crc32_iscsi_01.asm crc/crc32_iscsi_by8_02.asm crc/crc32_iscsi_by16_10.asm + crc/crc32_iscsi_by16_10_fused.asm crc/crc32_iscsi_avx2.asm crc/crc_multibinary.asm crc/crc64_ecma_refl_by8.asm diff --git a/crc/Makefile.am b/crc/Makefile.am index 3e482d5f..f341d47e 100644 --- a/crc/Makefile.am +++ b/crc/Makefile.am @@ -50,6 +50,7 @@ lsrc_x86_64 += \ crc/crc32_iscsi_01.asm \ crc/crc32_iscsi_by8_02.asm \ crc/crc32_iscsi_by16_10.asm \ + crc/crc32_iscsi_by16_10_fused.asm \ crc/crc32_iscsi_avx2.asm \ crc/crc_multibinary.asm \ crc/crc64_ecma_refl_by8.asm \ diff --git a/crc/crc32_iscsi_by16_10.asm b/crc/crc32_iscsi_by16_10.asm index 6ba8ee3e..4d999008 100644 --- a/crc/crc32_iscsi_by16_10.asm +++ b/crc/crc32_iscsi_by16_10.asm @@ -60,6 +60,12 @@ %define PREFETCH prefetcht0 %endif +;; Parameters of the fused folding and crc32 in FUSED_CRC32 builds +%define FUSED_ITERS 7 ; fold iterations per fused chunk +%define FUSED_VEC (256 * FUSED_ITERS) ; folded bytes per fused chunk +%define FUSED_STREAM (32 * FUSED_ITERS) ; bytes per crc32 stream per fused chunk +%define FUSED_CHUNK (FUSED_VEC + 8 * FUSED_STREAM) + [bits 64] default rel @@ -142,6 +148,12 @@ FUNCTION_NAME: vbroadcasti32x4 zmm16, [r10 + crc_fold_const_fold_16x128b] ;zmm16 has rk-1 and rk-2 sub arg3, 256 +%ifdef FUSED_CRC32 + ; with room for a fused chunk and two blocks after it, also use crc32 + cmp arg3, FUSED_CHUNK + jae .fused +%endif + %if fetch_dist != 0 ; check if there is at least 1.5KB (fetch distance) + 256B in the buffer cmp arg3, (fetch_dist + 256) @@ -199,6 +211,7 @@ align 16 sub arg3, 256 jge .fold_256_B_loop +.fold_256_B_end: ;; Fold 256 into 128 add arg2, 256 vpclmulqdq zmm1, zmm0, zmm10, 0x01 @@ -484,4 +497,181 @@ align 16 crc32 eax, byte[arg2] ; calculate 1 byte jmp .cleanup ; all done +%ifdef FUSED_CRC32 +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Fused folding and crc32 +;; +;; vpclmulqdq leaves the crc32 instruction idle, so for long buffers each +;; chunk of FUSED_CHUNK bytes is split into FUSED_VEC bytes for the folding +;; loop, followed by eight streams of FUSED_STREAM bytes for crc32. Each fold +;; iteration also takes 32 bytes of each stream. The folding treats the +;; streams as a gap: the first block after them is folded with constants for +;; 256 + 8 * FUSED_STREAM bytes. The stream crcs are shifted to the start of +;; the second block after the gap, combined, and xored into that block like an +;; initial crc, which gives their latency a block of folding to hide behind. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%define f_c0 rax +%define f_c1 rbx +%define f_c2 rbp +%define f_c3 r8 +%define f_c4 r11 +%define f_c5 r12 +%define f_c6 r13 +%define f_c7 r15 +%define f_sp r9 +%define f_tmp r14 + +;; Fold the accumulators with the constants in %1 into the block at arg2 + %2 +%macro FUSED_FOLD 2 + vpclmulqdq zmm1, zmm0, %1, 0x10 + vpclmulqdq zmm0, zmm0, %1, 0x01 + vpternlogq zmm0, zmm1, [arg2 + %2 + 64*0], 0x96 + vpclmulqdq zmm2, zmm4, %1, 0x10 + vpclmulqdq zmm4, zmm4, %1, 0x01 + vpternlogq zmm4, zmm2, [arg2 + %2 + 64*1], 0x96 + vpclmulqdq zmm3, zmm7, %1, 0x10 + vpclmulqdq zmm7, zmm7, %1, 0x01 + vpternlogq zmm7, zmm3, [arg2 + %2 + 64*2], 0x96 + vpclmulqdq zmm5, zmm8, %1, 0x10 + vpclmulqdq zmm8, zmm8, %1, 0x01 + vpternlogq zmm8, zmm5, [arg2 + %2 + 64*3], 0x96 +%endmacro + +;; crc32 of the 32 bytes at offset 32 * %1 of each stream +%macro FUSED_SCALAR 1 +%assign %%t 0 +%rep 4 + crc32 f_c0, qword [f_sp + 0*FUSED_STREAM + 32*%1 + 8*%%t] + crc32 f_c1, qword [f_sp + 1*FUSED_STREAM + 32*%1 + 8*%%t] + crc32 f_c2, qword [f_sp + 2*FUSED_STREAM + 32*%1 + 8*%%t] + crc32 f_c3, qword [f_sp + 3*FUSED_STREAM + 32*%1 + 8*%%t] + crc32 f_c4, qword [f_sp + 4*FUSED_STREAM + 32*%1 + 8*%%t] + crc32 f_c5, qword [f_sp + 5*FUSED_STREAM + 32*%1 + 8*%%t] + crc32 f_c6, qword [f_sp + 6*FUSED_STREAM + 32*%1 + 8*%%t] + crc32 f_c7, qword [f_sp + 7*FUSED_STREAM + 32*%1 + 8*%%t] +%assign %%t %%t+1 +%endrep +%endmacro + +;; Start the streams of the chunk at arg2 +%macro FUSED_START 0 + lea f_sp, [arg2 + FUSED_VEC] + xor f_c0 %+ d, f_c0 %+ d + xor f_c1 %+ d, f_c1 %+ d + xor f_c2 %+ d, f_c2 %+ d + xor f_c3 %+ d, f_c3 %+ d + xor f_c4 %+ d, f_c4 %+ d + xor f_c5 %+ d, f_c5 %+ d + xor f_c6 %+ d, f_c6 %+ d + xor f_c7 %+ d, f_c7 %+ d +%endmacro + +;; Combine the stream crcs into xmm19. Stream i is shifted over the +;; (7 - i) * FUSED_STREAM + 256 bytes before the second block after the +;; streams: crc32(0, xor of clmul(crc_i, x^(8 * shift_i - 33) mod P)). +;; zmm20 holds the eight crcs, and two vpclmulqdq multiply them all. +%macro FUSED_COMBINE 0 + vmovq xmm20, f_c0 + vpinsrq xmm20, xmm20, f_c1, 1 + vmovq xmm21, f_c2 + vpinsrq xmm21, xmm21, f_c3, 1 + vmovq xmm22, f_c4 + vpinsrq xmm22, xmm22, f_c5, 1 + vmovq xmm23, f_c6 + vpinsrq xmm23, xmm23, f_c7, 1 + vinserti32x4 ymm20, ymm20, xmm21, 1 + vinserti32x4 ymm22, ymm22, xmm23, 1 + vinserti64x4 zmm20, zmm20, ymm22, 1 + vpclmulqdq zmm21, zmm20, zmm18, 0x00 + vpclmulqdq zmm20, zmm20, zmm18, 0x11 + vpxorq zmm20, zmm20, zmm21 + vextracti64x4 ymm21, zmm20, 1 + vpxorq ymm20, ymm20, ymm21 + vextracti32x4 xmm21, ymm20, 1 + vpxorq xmm20, xmm20, xmm21 + vmovq f_tmp, xmm20 + xor eax, eax + crc32 rax, f_tmp + vmovd xmm19, eax +%endmacro + +align 32 +.fused: + push rbx + push rbp + push r12 + push r13 + push r14 + push r15 + vbroadcasti32x4 zmm17, [rel fused_fold_gap] + vmovdqu64 zmm18, [rel fused_shift] + + ;; The first block of the first chunk is already in the accumulators + FUSED_START + FUSED_SCALAR 0 +%assign i 1 +%rep FUSED_ITERS - 1 + FUSED_FOLD zmm16, 256*i + FUSED_SCALAR i +%assign i i+1 +%endrep + FUSED_COMBINE + add arg2, FUSED_CHUNK + sub arg3, FUSED_CHUNK + cmp arg3, FUSED_CHUNK + jb .fused_end + +align 32 +.fused_loop: + FUSED_START + FUSED_FOLD zmm17, 0 + FUSED_SCALAR 0 + FUSED_FOLD zmm16, 256 + vpxorq zmm0, zmm0, zmm19 + FUSED_SCALAR 1 +%assign i 2 +%rep FUSED_ITERS - 2 + FUSED_FOLD zmm16, 256*i + FUSED_SCALAR i +%assign i i+1 +%endrep + FUSED_COMBINE + add arg2, FUSED_CHUNK + sub arg3, FUSED_CHUNK + cmp arg3, FUSED_CHUNK + jae .fused_loop + +.fused_end: + ;; The two blocks after the last streams take the gap and their crc + FUSED_FOLD zmm17, 0 + FUSED_FOLD zmm16, 256 + vpxorq zmm0, zmm0, zmm19 + + pop r15 + pop r14 + pop r13 + pop r12 + pop rbp + pop rbx + + ;; Continue in the folding loop after the second block + add arg2, 256 + sub arg3, 256 + jge .fold_256_B_loop + jmp .fold_256_B_end + +section .rodata +align 64 +;; x^(8 * ((7 - i) * FUSED_STREAM + 256) - 33) mod P for stream i +fused_shift: + dq 0x00000000ded288f8, 0x000000002664fd8b + dq 0x0000000093e106a4, 0x0000000086d8e4d2 + dq 0x000000006956fc3b, 0x00000000e6fc4e6a + dq 0x00000000b3e32c28, 0x00000000b9e02b86 +;; Folding constants for 256 + 8 * FUSED_STREAM bytes +fused_fold_gap: + dq 0x00000001a0f717c4, 0x00000000f7506984 +%endif ; FUSED_CRC32 + %include "crc_const_extern.asm" diff --git a/crc/crc32_iscsi_by16_10_fused.asm b/crc/crc32_iscsi_by16_10_fused.asm new file mode 100644 index 00000000..67f7ea4c --- /dev/null +++ b/crc/crc32_iscsi_by16_10_fused.asm @@ -0,0 +1,35 @@ +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; Copyright(c) 2026 Intel Corporation All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions +; are met: +; * Redistributions of source code must retain the above copyright +; notice, this list of conditions and the following disclaimer. +; * Redistributions in binary form must reproduce the above copyright +; notice, this list of conditions and the following disclaimer in +; the documentation and/or other materials provided with the +; distribution. +; * Neither the name of Intel Corporation nor the names of its +; contributors may be used to endorse or promote products derived +; from this software without specific prior written permission. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; crc32_iscsi_by16_10 that also runs the crc32 instruction on part of long +; buffers, for CPUs that execute crc32 alongside vpclmulqdq at a high rate + +%define FUNCTION_NAME crc32_iscsi_by16_10_fused +%define FUSED_CRC32 +%include "crc32_iscsi_by16_10.asm" diff --git a/crc/crc_multibinary.asm b/crc/crc_multibinary.asm index f5b5ab19..f5fbc5ab 100644 --- a/crc/crc_multibinary.asm +++ b/crc/crc_multibinary.asm @@ -57,6 +57,7 @@ extern crc16_t10dif_copy_base extern crc32_gzip_refl_by16_10 extern crc32_ieee_by16_10 extern crc32_iscsi_by16_10 +extern crc32_iscsi_by16_10_fused extern crc16_t10dif_by16_10 extern crc64_ecma_refl_by8 @@ -102,7 +103,7 @@ extern crc64_rocksoft_norm_by16_10 mbin_interface crc32_iscsi -mbin_dispatch_init_clmul crc32_iscsi, crc32_iscsi_base, crc32_iscsi_01, crc32_iscsi_by8_02, crc32_iscsi_avx2, crc32_iscsi_by16_10 +mbin_dispatch_init_clmul_fused crc32_iscsi, crc32_iscsi_base, crc32_iscsi_01, crc32_iscsi_by8_02, crc32_iscsi_avx2, crc32_iscsi_by16_10, crc32_iscsi_by16_10_fused mbin_interface crc32_gzip_refl mbin_dispatch_init_clmul crc32_gzip_refl, crc32_gzip_refl_base, crc32_gzip_refl_by8, crc32_gzip_refl_by8, crc32_gzip_refl_avx2, crc32_gzip_refl_by16_10 diff --git a/include/multibinary.asm b/include/multibinary.asm index 765bb852..9bb687ff 100644 --- a/include/multibinary.asm +++ b/include/multibinary.asm @@ -136,6 +136,80 @@ pop rsi ret %endmacro +;;;;; +; mbin_dispatch_init_clmul_fused 7 parameters +; Like mbin_dispatch_init_clmul, with a function for AMD Zen 5 +; 1-> function name +; 2-> base function +; 3-> SSE4_1 and CLMUL optimized function +; 4-> AVX/02 opt func +; 5-> AVX2 opt func +; 6-> AVX512/10 opt func +; 7-> AVX512/10 opt func for AMD Zen 5 +;;;;; +%macro mbin_dispatch_init_clmul_fused 7 + section .data + align 8 + %1_generic_dispatched: + dq 0 + + mbin_dispatch_init_clmul %1_generic, %2, %3, %4, %5, %6 + section .text + %1_dispatch_init: + push rax + push rbx + push rcx + push rdx + call %1_generic_dispatch_init + mov rax, [%1_generic_dispatched] + mov [%1_dispatched], rax + + ;; Use the Zen 5 function where the AVX512/10 function was chosen + lea rbx, [%6 WRT_OPT] + cmp rax, rbx + jne _%1_init_done + + xor eax, eax + cpuid + cmp ebx, FLAG_CPUID0_EBX_AMD + jne _%1_init_done + cmp edx, FLAG_CPUID0_EDX_AMD + jne _%1_init_done + cmp ecx, FLAG_CPUID0_ECX_AMD + jne _%1_init_done + + ;; Family 1Ah is base family Fh with extended family 0Bh + mov eax, 1 + cpuid + mov ecx, eax + and eax, CPUID1_EAX_FAMILY_MASK + cmp eax, CPUID1_EAX_AMD_ZEN5_FAMILY + jne _%1_init_done + + ;; Family 1Ah also has Zen 6 models, so check the model, which is + ;; the base model plus the extended model times 16 + mov eax, ecx + shr eax, 4 + and eax, 0xf + shr ecx, 12 + and ecx, 0xf0 + or eax, ecx + shr eax, 3 + mov ebx, CPUID1_EAX_AMD_ZEN5_MODELS + bt ebx, eax + jnc _%1_init_done + + lea rax, [%7 WRT_OPT] + mov [%1_dispatched], rax + + _%1_init_done: + pop rdx + pop rcx + pop rbx + pop rax + ret +%endmacro + ;;;;; ; mbin_dispatch_init5 parameters ; 1-> function name diff --git a/include/reg_sizes.asm b/include/reg_sizes.asm index 16def226..7217c86e 100644 --- a/include/reg_sizes.asm +++ b/include/reg_sizes.asm @@ -78,6 +78,16 @@ %define FLAG_CPUID0_EDX_HYGON 0x6e65476e ;"neGn" %define FLAG_CPUID0_ECX_HYGON 0x656e6975 ;"eniu" +%define FLAG_CPUID0_EBX_AMD 0x68747541 ;"htuA" +%define FLAG_CPUID0_EDX_AMD 0x69746e65 ;"itne" +%define FLAG_CPUID0_ECX_AMD 0x444d4163 ;"DMAc" +; Family 1Ah: base family Fh and extended family 0Bh in CPUID 1 EAX +%define CPUID1_EAX_FAMILY_MASK 0x0ff00f00 +%define CPUID1_EAX_AMD_ZEN5_FAMILY 0x00b00f00 +; Bit n is set if family 1Ah models 8n to 8n+7 are Zen 5: models 00h-2Fh, +; 40h-4Fh, 60h-7Fh and D0h-D7h, as listed in Linux arch/x86/kernel/cpu/amd.c +%define CPUID1_EAX_AMD_ZEN5_MODELS 0x0400f33f + ; define d and w variants for registers %define raxd eax