From 0683445680e5d8eab425b4bbc21e8f529a2aa4d3 Mon Sep 17 00:00:00 2001 From: deadcafe Date: Sat, 10 Oct 2026 13:29:13 +0530 Subject: [PATCH] FSE: fuse symbol spreading with table building on the fast path FSE_buildCTable_wksp() and ZSTD_buildFSETable_body() spread symbols into the table in one pass, then walk the table in position order to build the state transitions, reading each symbol back. The spread step, tableSize/2 + tableSize/8 + 3, is odd for every valid tableLog, so the spread is an invertible permutation: position u holds the symbol at sorted index (u * inv) & (tableSize-1), where inv is the inverse of step modulo tableSize. Walking positions in order and reading symbols straight from the sorted layout fuses the two passes, removing the scattered writes into the table and the read-back. Only the fast path (no low-probability symbols) changes. The low-probability spread skips positions and is left as is. Tables are bit-identical, so compressed output is unchanged. Fast-path table build time, ns/table, mixed tableLog 5-9: FSE_buildCTable_wksp ZSTD_buildFSETable Apple M1, clang 21 449 -> 365 509 -> 389 Xeon 8481C, gcc 13 677 -> 512 600 -> 524 EPYC 9B14, gcc 13 547 -> 460 628 -> 511 End-to-end decompression of independent 4 KB to 32 KB chunks improves by 1-2% on all three machines; 64 KB and larger blocks are unchanged. --- lib/common/fse.h | 19 ++++ lib/compress/fse_compress.c | 40 ++++--- lib/decompress/zstd_decompress_block.c | 61 +++++------ tests/fuzzer.c | 143 +++++++++++++++++++++++++ 4 files changed, 212 insertions(+), 51 deletions(-) diff --git a/lib/common/fse.h b/lib/common/fse.h index b6c2a3e9ccc..f80442796bf 100644 --- a/lib/common/fse.h +++ b/lib/common/fse.h @@ -622,4 +622,23 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) #define FSE_TABLESTEP(tableSize) (((tableSize)>>1) + ((tableSize)>>3) + 3) +/* FSE_invTableStep() : + * Multiplicative inverse of FSE_TABLESTEP(tableSize) modulo tableSize. + * FSE_TABLESTEP() is odd for any tableSize >= 16, hence for any valid tableLog + * (>= FSE_MIN_TABLELOG), so the inverse exists. + * Symbol spreading places the i-th symbol (in symbol order) at position (i*step) & (tableSize-1), + * hence position u holds the symbol at index (u*inv) & (tableSize-1). + * This allows walking the table in position order while reading symbols + * from their sorted layout, instead of scattering them and reading them back. */ +MEM_STATIC U32 FSE_invTableStep(U32 tableSize) +{ + U32 const step = FSE_TABLESTEP(tableSize); + U32 inv = step; /* odd*odd == 1 (mod 8) : 3 correct low bits */ + inv *= 2 - step * inv; /* each Newton iteration doubles the nb of correct bits : 6 */ + inv *= 2 - step * inv; /* 12 */ + inv *= 2 - step * inv; /* 24 >= FSE_TABLELOG_ABSOLUTE_MAX */ + assert(((step * inv) & (tableSize - 1)) == 1); + return inv & (tableSize - 1); +} + #endif /* FSE_STATIC_LINKING_ONLY */ diff --git a/lib/compress/fse_compress.c b/lib/compress/fse_compress.c index 1ce3cf16ac1..8136553b4ae 100644 --- a/lib/compress/fse_compress.c +++ b/lib/compress/fse_compress.c @@ -133,23 +133,21 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, pos += (size_t)n; } } - /* Spread symbols across the table. Lack of lowprob symbols means that - * we don't need variable sized inner loop, so we can unroll the loop and - * reduce branch misses. + /* Build the table directly, walking positions in order. + * Position u holds spread[(u * invStep) & tableMask], where invStep + * is the inverse of step modulo tableSize (see FSE_invTableStep()). + * This fuses symbol spreading with table building into a single pass, + * so symbols are never scattered into tableSymbol[] and read back. */ - { size_t position = 0; - size_t s; - size_t const unroll = 2; /* Experimentally determined optimal unroll */ - assert(tableSize % unroll == 0); /* FSE_MIN_TABLELOG is 5 */ - for (s = 0; s < (size_t)tableSize; s += unroll) { - size_t u; - for (u = 0; u < unroll; ++u) { - size_t const uPosition = (position + (u * step)) & tableMask; - tableSymbol[uPosition] = spread[s + u]; - } - position = (position + (unroll * step)) & tableMask; + { size_t const invStep = FSE_invTableStep(tableSize); + size_t s = 0; + U32 u; + for (u=0; u highThreshold)) position = (position + step) & tableMask; /* lowprob area */ } } assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ - } - /* Build Decoding table */ - { - U32 u; - for (u=0; u 1; s++) { + if ((FUZ_rand(seed) & 3) == 0) { norm[s] = -1; remaining--; } + } } + while (remaining > 0) { + s = FUZ_rand(seed) % maxSV1; + if (norm[s] >= 0) { norm[s]++; remaining--; } + } +} + +/* Reference spreading : low probability symbols at the end of the table, + * then each symbol repeated norm[s] times at FSE_TABLESTEP() intervals. */ +static void FUZ_refSpreadSymbols(BYTE* tableSymbol, const short* norm, unsigned maxSV1, unsigned tableLog) +{ + U32 const tableSize = 1u << tableLog; + U32 const tableMask = tableSize - 1; + U32 const step = FSE_TABLESTEP(tableSize); + U32 highThreshold = tableSize - 1; + U32 s, position = 0; + for (s = 0; s < maxSV1; s++) { + if (norm[s] == -1) tableSymbol[highThreshold--] = (BYTE)s; + } + for (s = 0; s < maxSV1; s++) { + int i; + for (i = 0; i < norm[s]; i++) { + tableSymbol[position] = (BYTE)s; + position = (position + step) & tableMask; + while (position > highThreshold) position = (position + step) & tableMask; + } } + assert(position == 0); +} + +/* Reference for the header and state table of an FSE_CTable : the first 2 + tableSize U16. + * The symbol transformation table that follows does not depend on the spreading. */ +static void FUZ_refBuildCStateTable(U16* tableU16, const BYTE* tableSymbol, const short* norm, unsigned maxSV1, unsigned tableLog) +{ + U32 const tableSize = 1u << tableLog; + U16 cumul[MaxSeq + 2]; + U32 u; + tableU16[0] = (U16)tableLog; + tableU16[1] = (U16)(maxSV1 - 1); + cumul[0] = 0; + for (u = 1; u <= maxSV1; u++) { + cumul[u] = (U16)(cumul[u-1] + (norm[u-1] == -1 ? 1 : norm[u-1])); + } + for (u = 0; u < tableSize; u++) { + tableU16[2 + cumul[tableSymbol[u]]++] = (U16)(tableSize + u); + } +} + +/* Reference for a complete sequence decoding table, header included. */ +static void FUZ_refBuildFSETable(ZSTD_seqSymbol* dt, const BYTE* tableSymbol, const short* norm, unsigned maxSV1, + const U32* baseValue, const U8* nbAdditionalBits, unsigned tableLog) +{ + ZSTD_seqSymbol* const tableDecode = dt + 1; + U32 const tableSize = 1u << tableLog; + S16 const largeLimit = (S16)(1 << (tableLog - 1)); + U16 symbolNext[MaxSeq + 1]; + ZSTD_seqSymbol_header DTableH; + U32 s, u; + DTableH.tableLog = tableLog; + DTableH.fastMode = 1; + for (s = 0; s < maxSV1; s++) { + if (norm[s] == -1) { + symbolNext[s] = 1; + } else { + if (norm[s] >= largeLimit) DTableH.fastMode = 0; + symbolNext[s] = (U16)norm[s]; + } } + memcpy(dt, &DTableH, sizeof(DTableH)); + for (u = 0; u < tableSize; u++) { + U32 const symbol = tableSymbol[u]; + U32 const nextState = symbolNext[symbol]++; + BYTE const nbBits = (BYTE)(tableLog - ZSTD_highbit32(nextState)); + tableDecode[u].nbBits = nbBits; + tableDecode[u].nextState = (U16)((nextState << nbBits) - tableSize); + tableDecode[u].nbAdditionalBits = nbAdditionalBits[symbol]; + tableDecode[u].baseValue = baseValue[symbol]; + } +} + /* ============================================================= */ static int basicUnitTests(U32 const seed, double compressibility) @@ -4771,6 +4870,50 @@ static int basicUnitTests(U32 const seed, double compressibility) } DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : FSE table construction matches reference spreading : ", testNb++); + { /* Both builders take a fused single-pass path when no symbol has a -1 count, + * and the plain spread-then-read-back path otherwise. + * Check that each produces the reference table for every tableLog. */ + U32 rSeed = seed; + short norm[MaxSeq + 1]; + U32 baseValue[MaxSeq + 1]; + U8 nbAdditionalBits[MaxSeq + 1]; + BYTE tableSymbol[FSE_MAX_TABLESIZE]; + FSE_CTable ct[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, MaxSeq)]; + U16 refCt[2 + FSE_MAX_TABLESIZE]; + U32 cwksp[FSE_BUILD_CTABLE_WORKSPACE_SIZE_U32(MaxSeq, FSE_MAX_TABLELOG)]; + ZSTD_seqSymbol dt[SEQSYMBOL_TABLE_SIZE(MaxFSELog)]; + ZSTD_seqSymbol refDt[SEQSYMBOL_TABLE_SIZE(MaxFSELog)]; + U32 dwksp[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32]; + unsigned tableLog, s; + for (s = 0; s <= MaxSeq; s++) { + baseValue[s] = 1 + s * s; + nbAdditionalBits[s] = (U8)(s % 32); + } + for (tableLog = FSE_MIN_TABLELOG; tableLog <= FSE_MAX_TABLELOG; tableLog++) { + U32 const tableSize = 1u << tableLog; + int lowProb; + for (lowProb = 0; lowProb < 2; lowProb++) { + int n; + for (n = 0; n < 16; n++) { + unsigned const maxSV1 = 2 + FUZ_rand(&rSeed) % MaxSeq; + FUZ_genNormalizedCount(norm, maxSV1, tableLog, lowProb, &rSeed); + FUZ_refSpreadSymbols(tableSymbol, norm, maxSV1, tableLog); + + CHECK_Z( FSE_buildCTable_wksp(ct, norm, maxSV1 - 1, tableLog, cwksp, sizeof(cwksp)) ); + FUZ_refBuildCStateTable(refCt, tableSymbol, norm, maxSV1, tableLog); + if (memcmp(ct, refCt, (2 + tableSize) * sizeof(U16)) != 0) goto _output_error; + + if (tableLog <= MaxFSELog) { + memset(dt, 0, sizeof(dt)); + memset(refDt, 0, sizeof(refDt)); + ZSTD_buildFSETable(dt, norm, maxSV1 - 1, baseValue, nbAdditionalBits, tableLog, dwksp, sizeof(dwksp), 0); + FUZ_refBuildFSETable(refDt, tableSymbol, norm, maxSV1, baseValue, nbAdditionalBits, tableLog); + if (memcmp(dt, refDt, (1 + tableSize) * sizeof(ZSTD_seqSymbol)) != 0) goto _output_error; + } } } } + } + DISPLAYLEVEL(3, "OK \n"); + DISPLAYLEVEL(3, "test%3i : testing bitwise intrinsics PR#3045: ", testNb++); { U32 seed_copy = seed; /* need non-const seed to avoid compiler warning for FUZ_rand(&seed) */