diff --git a/Makefile.nmake b/Makefile.nmake index 356becbf..7b663440 100644 --- a/Makefile.nmake +++ b/Makefile.nmake @@ -149,6 +149,7 @@ objs = \ bin\crc32_iscsi_01.obj \ bin\crc32_iscsi_by8_02.obj \ bin\crc32_iscsi_by16_10.obj \ + bin\crc32_iscsi_by16_10_fused.obj \ bin\crc32_iscsi_avx2.obj \ bin\crc_multibinary.obj \ bin\crc64_ecma_refl_by8.obj \ diff --git a/cmake/crc.cmake b/cmake/crc.cmake index b38db0d8..4ad93345 100644 --- a/cmake/crc.cmake +++ b/cmake/crc.cmake @@ -48,6 +48,7 @@ set(CRC_X86_64_SOURCES crc/crc32_iscsi_01.asm crc/crc32_iscsi_by8_02.asm crc/crc32_iscsi_by16_10.asm + crc/crc32_iscsi_by16_10_fused.asm crc/crc32_iscsi_avx2.asm crc/crc_multibinary.asm crc/crc64_ecma_refl_by8.asm diff --git a/crc/Makefile.am b/crc/Makefile.am index 3e482d5f..f341d47e 100644 --- a/crc/Makefile.am +++ b/crc/Makefile.am @@ -50,6 +50,7 @@ lsrc_x86_64 += \ crc/crc32_iscsi_01.asm \ crc/crc32_iscsi_by8_02.asm \ crc/crc32_iscsi_by16_10.asm \ + crc/crc32_iscsi_by16_10_fused.asm \ crc/crc32_iscsi_avx2.asm \ crc/crc_multibinary.asm \ crc/crc64_ecma_refl_by8.asm \ diff --git a/crc/crc32_iscsi_by16_10.asm b/crc/crc32_iscsi_by16_10.asm index 6ba8ee3e..4d999008 100644 --- a/crc/crc32_iscsi_by16_10.asm +++ b/crc/crc32_iscsi_by16_10.asm @@ -60,6 +60,12 @@ %define PREFETCH prefetcht0 %endif +;; Parameters of the fused folding and crc32 in FUSED_CRC32 builds +%define FUSED_ITERS 7 ; fold iterations per fused chunk +%define FUSED_VEC (256 * FUSED_ITERS) ; folded bytes per fused chunk +%define FUSED_STREAM (32 * FUSED_ITERS) ; bytes per crc32 stream per fused chunk +%define FUSED_CHUNK (FUSED_VEC + 8 * FUSED_STREAM) + [bits 64] default rel @@ -142,6 +148,12 @@ FUNCTION_NAME: vbroadcasti32x4 zmm16, [r10 + crc_fold_const_fold_16x128b] ;zmm16 has rk-1 and rk-2 sub arg3, 256 +%ifdef FUSED_CRC32 + ; with room for a fused chunk and two blocks after it, also use crc32 + cmp arg3, FUSED_CHUNK + jae .fused +%endif + %if fetch_dist != 0 ; check if there is at least 1.5KB (fetch distance) + 256B in the buffer cmp arg3, (fetch_dist + 256) @@ -199,6 +211,7 @@ align 16 sub arg3, 256 jge .fold_256_B_loop +.fold_256_B_end: ;; Fold 256 into 128 add arg2, 256 vpclmulqdq zmm1, zmm0, zmm10, 0x01 @@ -484,4 +497,181 @@ align 16 crc32 eax, byte[arg2] ; calculate 1 byte jmp .cleanup ; all done +%ifdef FUSED_CRC32 +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Fused folding and crc32 +;; +;; vpclmulqdq leaves the crc32 instruction idle, so for long buffers each +;; chunk of FUSED_CHUNK bytes is split into FUSED_VEC bytes for the folding +;; loop, followed by eight streams of FUSED_STREAM bytes for crc32. Each fold +;; iteration also takes 32 bytes of each stream. The folding treats the +;; streams as a gap: the first block after them is folded with constants for +;; 256 + 8 * FUSED_STREAM bytes. The stream crcs are shifted to the start of +;; the second block after the gap, combined, and xored into that block like an +;; initial crc, which gives their latency a block of folding to hide behind. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +%define f_c0 rax +%define f_c1 rbx +%define f_c2 rbp +%define f_c3 r8 +%define f_c4 r11 +%define f_c5 r12 +%define f_c6 r13 +%define f_c7 r15 +%define f_sp r9 +%define f_tmp r14 + +;; Fold the accumulators with the constants in %1 into the block at arg2 + %2 +%macro FUSED_FOLD 2 + vpclmulqdq zmm1, zmm0, %1, 0x10 + vpclmulqdq zmm0, zmm0, %1, 0x01 + vpternlogq zmm0, zmm1, [arg2 + %2 + 64*0], 0x96 + vpclmulqdq zmm2, zmm4, %1, 0x10 + vpclmulqdq zmm4, zmm4, %1, 0x01 + vpternlogq zmm4, zmm2, [arg2 + %2 + 64*1], 0x96 + vpclmulqdq zmm3, zmm7, %1, 0x10 + vpclmulqdq zmm7, zmm7, %1, 0x01 + vpternlogq zmm7, zmm3, [arg2 + %2 + 64*2], 0x96 + vpclmulqdq zmm5, zmm8, %1, 0x10 + vpclmulqdq zmm8, zmm8, %1, 0x01 + vpternlogq zmm8, zmm5, [arg2 + %2 + 64*3], 0x96 +%endmacro + +;; crc32 of the 32 bytes at offset 32 * %1 of each stream +%macro FUSED_SCALAR 1 +%assign %%t 0 +%rep 4 + crc32 f_c0, qword [f_sp + 0*FUSED_STREAM + 32*%1 + 8*%%t] + crc32 f_c1, qword [f_sp + 1*FUSED_STREAM + 32*%1 + 8*%%t] + crc32 f_c2, qword [f_sp + 2*FUSED_STREAM + 32*%1 + 8*%%t] + crc32 f_c3, qword [f_sp + 3*FUSED_STREAM + 32*%1 + 8*%%t] + crc32 f_c4, qword [f_sp + 4*FUSED_STREAM + 32*%1 + 8*%%t] + crc32 f_c5, qword [f_sp + 5*FUSED_STREAM + 32*%1 + 8*%%t] + crc32 f_c6, qword [f_sp + 6*FUSED_STREAM + 32*%1 + 8*%%t] + crc32 f_c7, qword [f_sp + 7*FUSED_STREAM + 32*%1 + 8*%%t] +%assign %%t %%t+1 +%endrep +%endmacro + +;; Start the streams of the chunk at arg2 +%macro FUSED_START 0 + lea f_sp, [arg2 + FUSED_VEC] + xor f_c0 %+ d, f_c0 %+ d + xor f_c1 %+ d, f_c1 %+ d + xor f_c2 %+ d, f_c2 %+ d + xor f_c3 %+ d, f_c3 %+ d + xor f_c4 %+ d, f_c4 %+ d + xor f_c5 %+ d, f_c5 %+ d + xor f_c6 %+ d, f_c6 %+ d + xor f_c7 %+ d, f_c7 %+ d +%endmacro + +;; Combine the stream crcs into xmm19. Stream i is shifted over the +;; (7 - i) * FUSED_STREAM + 256 bytes before the second block after the +;; streams: crc32(0, xor of clmul(crc_i, x^(8 * shift_i - 33) mod P)). +;; zmm20 holds the eight crcs, and two vpclmulqdq multiply them all. +%macro FUSED_COMBINE 0 + vmovq xmm20, f_c0 + vpinsrq xmm20, xmm20, f_c1, 1 + vmovq xmm21, f_c2 + vpinsrq xmm21, xmm21, f_c3, 1 + vmovq xmm22, f_c4 + vpinsrq xmm22, xmm22, f_c5, 1 + vmovq xmm23, f_c6 + vpinsrq xmm23, xmm23, f_c7, 1 + vinserti32x4 ymm20, ymm20, xmm21, 1 + vinserti32x4 ymm22, ymm22, xmm23, 1 + vinserti64x4 zmm20, zmm20, ymm22, 1 + vpclmulqdq zmm21, zmm20, zmm18, 0x00 + vpclmulqdq zmm20, zmm20, zmm18, 0x11 + vpxorq zmm20, zmm20, zmm21 + vextracti64x4 ymm21, zmm20, 1 + vpxorq ymm20, ymm20, ymm21 + vextracti32x4 xmm21, ymm20, 1 + vpxorq xmm20, xmm20, xmm21 + vmovq f_tmp, xmm20 + xor eax, eax + crc32 rax, f_tmp + vmovd xmm19, eax +%endmacro + +align 32 +.fused: + push rbx + push rbp + push r12 + push r13 + push r14 + push r15 + vbroadcasti32x4 zmm17, [rel fused_fold_gap] + vmovdqu64 zmm18, [rel fused_shift] + + ;; The first block of the first chunk is already in the accumulators + FUSED_START + FUSED_SCALAR 0 +%assign i 1 +%rep FUSED_ITERS - 1 + FUSED_FOLD zmm16, 256*i + FUSED_SCALAR i +%assign i i+1 +%endrep + FUSED_COMBINE + add arg2, FUSED_CHUNK + sub arg3, FUSED_CHUNK + cmp arg3, FUSED_CHUNK + jb .fused_end + +align 32 +.fused_loop: + FUSED_START + FUSED_FOLD zmm17, 0 + FUSED_SCALAR 0 + FUSED_FOLD zmm16, 256 + vpxorq zmm0, zmm0, zmm19 + FUSED_SCALAR 1 +%assign i 2 +%rep FUSED_ITERS - 2 + FUSED_FOLD zmm16, 256*i + FUSED_SCALAR i +%assign i i+1 +%endrep + FUSED_COMBINE + add arg2, FUSED_CHUNK + sub arg3, FUSED_CHUNK + cmp arg3, FUSED_CHUNK + jae .fused_loop + +.fused_end: + ;; The two blocks after the last streams take the gap and their crc + FUSED_FOLD zmm17, 0 + FUSED_FOLD zmm16, 256 + vpxorq zmm0, zmm0, zmm19 + + pop r15 + pop r14 + pop r13 + pop r12 + pop rbp + pop rbx + + ;; Continue in the folding loop after the second block + add arg2, 256 + sub arg3, 256 + jge .fold_256_B_loop + jmp .fold_256_B_end + +section .rodata +align 64 +;; x^(8 * ((7 - i) * FUSED_STREAM + 256) - 33) mod P for stream i +fused_shift: + dq 0x00000000ded288f8, 0x000000002664fd8b + dq 0x0000000093e106a4, 0x0000000086d8e4d2 + dq 0x000000006956fc3b, 0x00000000e6fc4e6a + dq 0x00000000b3e32c28, 0x00000000b9e02b86 +;; Folding constants for 256 + 8 * FUSED_STREAM bytes +fused_fold_gap: + dq 0x00000001a0f717c4, 0x00000000f7506984 +%endif ; FUSED_CRC32 + %include "crc_const_extern.asm" diff --git a/crc/crc32_iscsi_by16_10_fused.asm b/crc/crc32_iscsi_by16_10_fused.asm new file mode 100644 index 00000000..67f7ea4c --- /dev/null +++ b/crc/crc32_iscsi_by16_10_fused.asm @@ -0,0 +1,35 @@ +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; Copyright(c) 2026 Intel Corporation All rights reserved. +; +; Redistribution and use in source and binary forms, with or without +; modification, are permitted provided that the following conditions +; are met: +; * Redistributions of source code must retain the above copyright +; notice, this list of conditions and the following disclaimer. +; * Redistributions in binary form must reproduce the above copyright +; notice, this list of conditions and the following disclaimer in +; the documentation and/or other materials provided with the +; distribution. +; * Neither the name of Intel Corporation nor the names of its +; contributors may be used to endorse or promote products derived +; from this software without specific prior written permission. +; +; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; crc32_iscsi_by16_10 that also runs the crc32 instruction on part of long +; buffers, for CPUs that execute crc32 alongside vpclmulqdq at a high rate + +%define FUNCTION_NAME crc32_iscsi_by16_10_fused +%define FUSED_CRC32 +%include "crc32_iscsi_by16_10.asm" diff --git a/crc/crc_multibinary.asm b/crc/crc_multibinary.asm index f5b5ab19..f5fbc5ab 100644 --- a/crc/crc_multibinary.asm +++ b/crc/crc_multibinary.asm @@ -57,6 +57,7 @@ extern crc16_t10dif_copy_base extern crc32_gzip_refl_by16_10 extern crc32_ieee_by16_10 extern crc32_iscsi_by16_10 +extern crc32_iscsi_by16_10_fused extern crc16_t10dif_by16_10 extern crc64_ecma_refl_by8 @@ -102,7 +103,7 @@ extern crc64_rocksoft_norm_by16_10 mbin_interface crc32_iscsi -mbin_dispatch_init_clmul crc32_iscsi, crc32_iscsi_base, crc32_iscsi_01, crc32_iscsi_by8_02, crc32_iscsi_avx2, crc32_iscsi_by16_10 +mbin_dispatch_init_clmul_fused crc32_iscsi, crc32_iscsi_base, crc32_iscsi_01, crc32_iscsi_by8_02, crc32_iscsi_avx2, crc32_iscsi_by16_10, crc32_iscsi_by16_10_fused mbin_interface crc32_gzip_refl mbin_dispatch_init_clmul crc32_gzip_refl, crc32_gzip_refl_base, crc32_gzip_refl_by8, crc32_gzip_refl_by8, crc32_gzip_refl_avx2, crc32_gzip_refl_by16_10 diff --git a/include/multibinary.asm b/include/multibinary.asm index 765bb852..9bb687ff 100644 --- a/include/multibinary.asm +++ b/include/multibinary.asm @@ -136,6 +136,80 @@ pop rsi ret %endmacro +;;;;; +; mbin_dispatch_init_clmul_fused 7 parameters +; Like mbin_dispatch_init_clmul, with a function for AMD Zen 5 +; 1-> function name +; 2-> base function +; 3-> SSE4_1 and CLMUL optimized function +; 4-> AVX/02 opt func +; 5-> AVX2 opt func +; 6-> AVX512/10 opt func +; 7-> AVX512/10 opt func for AMD Zen 5 +;;;;; +%macro mbin_dispatch_init_clmul_fused 7 + section .data + align 8 + %1_generic_dispatched: + dq 0 + + mbin_dispatch_init_clmul %1_generic, %2, %3, %4, %5, %6 + section .text + %1_dispatch_init: + push rax + push rbx + push rcx + push rdx + call %1_generic_dispatch_init + mov rax, [%1_generic_dispatched] + mov [%1_dispatched], rax + + ;; Use the Zen 5 function where the AVX512/10 function was chosen + lea rbx, [%6 WRT_OPT] + cmp rax, rbx + jne _%1_init_done + + xor eax, eax + cpuid + cmp ebx, FLAG_CPUID0_EBX_AMD + jne _%1_init_done + cmp edx, FLAG_CPUID0_EDX_AMD + jne _%1_init_done + cmp ecx, FLAG_CPUID0_ECX_AMD + jne _%1_init_done + + ;; Family 1Ah is base family Fh with extended family 0Bh + mov eax, 1 + cpuid + mov ecx, eax + and eax, CPUID1_EAX_FAMILY_MASK + cmp eax, CPUID1_EAX_AMD_ZEN5_FAMILY + jne _%1_init_done + + ;; Family 1Ah also has Zen 6 models, so check the model, which is + ;; the base model plus the extended model times 16 + mov eax, ecx + shr eax, 4 + and eax, 0xf + shr ecx, 12 + and ecx, 0xf0 + or eax, ecx + shr eax, 3 + mov ebx, CPUID1_EAX_AMD_ZEN5_MODELS + bt ebx, eax + jnc _%1_init_done + + lea rax, [%7 WRT_OPT] + mov [%1_dispatched], rax + + _%1_init_done: + pop rdx + pop rcx + pop rbx + pop rax + ret +%endmacro + ;;;;; ; mbin_dispatch_init5 parameters ; 1-> function name diff --git a/include/reg_sizes.asm b/include/reg_sizes.asm index 16def226..7217c86e 100644 --- a/include/reg_sizes.asm +++ b/include/reg_sizes.asm @@ -78,6 +78,16 @@ %define FLAG_CPUID0_EDX_HYGON 0x6e65476e ;"neGn" %define FLAG_CPUID0_ECX_HYGON 0x656e6975 ;"eniu" +%define FLAG_CPUID0_EBX_AMD 0x68747541 ;"htuA" +%define FLAG_CPUID0_EDX_AMD 0x69746e65 ;"itne" +%define FLAG_CPUID0_ECX_AMD 0x444d4163 ;"DMAc" +; Family 1Ah: base family Fh and extended family 0Bh in CPUID 1 EAX +%define CPUID1_EAX_FAMILY_MASK 0x0ff00f00 +%define CPUID1_EAX_AMD_ZEN5_FAMILY 0x00b00f00 +; Bit n is set if family 1Ah models 8n to 8n+7 are Zen 5: models 00h-2Fh, +; 40h-4Fh, 60h-7Fh and D0h-D7h, as listed in Linux arch/x86/kernel/cpu/amd.c +%define CPUID1_EAX_AMD_ZEN5_MODELS 0x0400f33f + ; define d and w variants for registers %define raxd eax