From 945a9e4b90487c4f416ff4fe9fb8a557b7c75fc2 Mon Sep 17 00:00:00 2001 From: TCMalloc Team Date: Thu, 20 Aug 2026 02:16:15 -0700 Subject: [PATCH] Require asm goto support unconditionally for rseq. Clang 11 (which added asm goto support) was released almost six years ago, and GCC has supported this since about 2010. Clean it up so that the code is easier to read. PiperOrigin-RevId: 967677836 --- docs/platforms.md | 6 +- tcmalloc/internal/percpu_tcmalloc.h | 124 ---------------------------- 2 files changed, 3 insertions(+), 127 deletions(-) diff --git a/docs/platforms.md b/docs/platforms.md index 8d4250d69..6f01c1043 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -13,7 +13,7 @@ We guarantee that our code will compile under the following compilation flags: Linux: -* gcc 9.2+, clang 9.0+: `-std=c++17` +* gcc 9.2+, clang 11.0+: `-std=c++17` (TL;DR; All code at this time must be built under C++17. We will update this list if circumstances change.) @@ -42,7 +42,7 @@ Architecture, Specific Compiler, and Standard Library implementation. Linux little-endian, 64-bit x86, AArch64 - gcc 9.2+
clang 9.0+ + gcc 9.2+
clang 11.0+ libstdc++
libc++ @@ -68,7 +68,7 @@ release should also work. Linux little-endian, 64-bit PPC - gcc 9.2+
clang 9.0+ + gcc 9.2+
clang 11.0+ libstdc++
libc++ diff --git a/tcmalloc/internal/percpu_tcmalloc.h b/tcmalloc/internal/percpu_tcmalloc.h index 26f12d68d..b65c0089e 100644 --- a/tcmalloc/internal/percpu_tcmalloc.h +++ b/tcmalloc/internal/percpu_tcmalloc.h @@ -46,10 +46,6 @@ #include "tcmalloc/internal/prefetch.h" #include "tcmalloc/internal/sysinfo.h" -#if __clang_major__ >= 11 -#define TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT 1 -#endif - GOOGLE_MALLOC_SECTION_BEGIN namespace tcmalloc { namespace tcmalloc_internal { @@ -655,55 +651,30 @@ inline ABSL_ATTRIBUTE_ALWAYS_INLINE void PrefetchSlabMemory(uintptr_t ptr) { static inline ABSL_ATTRIBUTE_ALWAYS_INLINE bool TcmallocSlab_Internal_Push( size_t size_class, void* item) { uintptr_t scratch, current; -#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT asm goto( -#else - bool overflow; - asm volatile( -#endif TCMALLOC_RSEQ_PROLOGUE(TcmallocSlab_Internal_Push) // scratch = tcmalloc_slabs; "movq %[rseq_slabs_addr], %[scratch]\n" // if (scratch & TCMALLOC_CACHED_SLABS_MASK>) goto overflow_label; // scratch &= ~TCMALLOC_CACHED_SLABS_MASK; "btrq $%c[cached_slabs_bit], %[scratch]\n" -#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT "jnc %l[overflow_label]\n" -#else - "jae 5f\n" // ae==c -#endif // current = slabs->current; "movzwq (%[scratch], %[size_class], 4), %[current]\n" // if (ABSL_PREDICT_FALSE(current >= slabs->end)) { goto overflow_label; } "cmp 2(%[scratch], %[size_class], 4), %w[current]\n" -#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT "jae %l[overflow_label]\n" -#else - "jae 5f\n" - // Important! code below this must not affect any flags (i.e.: ccae) - // If so, the above code needs to explicitly set a ccae return value. -#endif "mov %[item], (%[scratch], %[current], 8)\n" "lea 1(%[current]), %[current]\n" "mov %w[current], (%[scratch], %[size_class], 4)\n" // Commit "5:\n" : -#if !TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT - [overflow] "=@ccae"(overflow), -#endif [scratch] "=&r"(scratch), [current] "=&r"(current) : TCMALLOC_RSEQ_INPUTS, [size_class] "r"(size_class), [item] "r"(item) : "cc", "memory" -#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT : overflow_label -#endif ); -#if !TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT - if (ABSL_PREDICT_FALSE(overflow)) { - goto overflow_label; - } -#endif // Current now points to the slot we are going to push to next. PrefetchSlabMemory(scratch + current * sizeof(void*)); return true; @@ -716,24 +687,14 @@ static inline ABSL_ATTRIBUTE_ALWAYS_INLINE bool TcmallocSlab_Internal_Push( static inline ABSL_ATTRIBUTE_ALWAYS_INLINE bool TcmallocSlab_Internal_Push( size_t size_class, void* item) { uintptr_t region_start, scratch, scratch_ptr, end; -#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT asm goto( -#else - bool overflow; - asm volatile( -#endif TCMALLOC_RSEQ_PROLOGUE(TcmallocSlab_Internal_Push) // region_start = tcmalloc_slabs; "ldr %[region_start], [%[sampler_addr], #%c[sampler_slabs_offset]]\n" // if (region_start & TCMALLOC_CACHED_SLABS_MASK) goto overflow_label; // region_start is unmasked on aarch64, since Top Byte Ignore is enabled in // user space by default -#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT "tbz %[region_start], #%c[cached_slabs_bit], %l[overflow_label]\n" -#else - "subs %[region_start], %[region_start], %[cached_slabs_mask]\n" - "b.ls 5f\n" -#endif // scratch = slab_headers[size_class]->current (current index) "add %[scratch_ptr], %[region_start], %[size_class], LSL #2\n" "ldrh %w[scratch], [%[scratch_ptr]]\n" @@ -741,13 +702,7 @@ static inline ABSL_ATTRIBUTE_ALWAYS_INLINE bool TcmallocSlab_Internal_Push( "ldrh %w[end], [%[scratch_ptr], #2]\n" // if (ABSL_PREDICT_FALSE(end <= scratch)) { goto overflow_label; } "cmp %[end], %[scratch]\n" -#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT "b.ls %l[overflow_label]\n" -#else - "b.ls 5f\n" - // Important! code below this must not affect any flags (i.e.: ccls) - // If so, the above code needs to explicitly set a ccls return value. -#endif "str %[item], [%[region_start], %[scratch], LSL #3]\n" "add %w[scratch], %w[scratch], #1\n" "strh %w[scratch], [%[scratch_ptr]]\n" @@ -755,27 +710,14 @@ static inline ABSL_ATTRIBUTE_ALWAYS_INLINE bool TcmallocSlab_Internal_Push( "5:\n" : [scratch_ptr] "=&r"(scratch_ptr), [scratch] "=&r"(scratch), [end] "=&r"(end), -#if !TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT - [overflow] "=@ccls"(overflow), -#endif [region_start] "=&r"(region_start) : TCMALLOC_RSEQ_INPUTS, -#if !TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT - [cached_slabs_mask] "r"(TCMALLOC_CACHED_SLABS_MASK), -#endif [size_class] "r"(size_class), [item] "r"(item) : TCMALLOC_RSEQ_CLOBBER, "memory" -#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT , "cc" : overflow_label -#endif ); -#if !TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT - if (ABSL_PREDICT_FALSE(overflow)) { - goto overflow_label; - } -#endif PrefetchSlabMemory(reinterpret_cast(region_start) + scratch * sizeof(void*)); @@ -833,63 +775,32 @@ inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* TcmallocSlab::Pop( void* result; uintptr_t scratch, current; -#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT asm goto( -#else - bool underflow; - asm( -#endif TCMALLOC_RSEQ_PROLOGUE(TcmallocSlab_Internal_Pop) // scratch = tcmalloc_slabs; "movq %[rseq_slabs_addr], %[scratch]\n" // if (scratch & TCMALLOC_CACHED_SLABS_MASK) goto overflow_label; // scratch &= ~TCMALLOC_CACHED_SLABS_MASK; "btrq $%c[cached_slabs_bit], %[scratch]\n" -#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT "jnc %l[underflow_path]\n" -#else - "cmc\n" - "jc 5f\n" -#endif // current = scratch->header[size_class].current; "movzwq (%[scratch], %[size_class], 4), %[current]\n" "movq -8(%[scratch], %[current], 8), %[result]\n" -#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT "testb $%c[begin_mark_mask], %b[result]\n" "jnz %l[underflow_path]\n" -#else - "btq $%c[begin_mark_bit], %[result]\n" - "jc 5f\n" - // Important! code below this must not affect any flags (i.e.: ccc) - // If so, the above code needs to explicitly set a ccc return value. -#endif "movq -16(%[scratch], %[current], 8), %[next]\n" "lea -1(%[current]), %[current]\n" "movw %w[current], (%[scratch], %[size_class], 4)\n" // Commit "5:\n" : [result] "=&r"(result), -#if !TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT - [underflow] "=@ccc"(underflow), -#endif [scratch] "=&r"(scratch), [current] "=&r"(current), [next] "=&r"(next) : TCMALLOC_RSEQ_INPUTS, -#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT [begin_mark_mask] "n"(kBeginMark), -#else - [begin_mark_bit] "n"(absl::countr_zero(kBeginMark)), -#endif [size_class] "r"(size_class) : "cc", "memory" -#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT : underflow_path -#endif ); -#if !TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT - if (ABSL_PREDICT_FALSE(underflow)) { - goto underflow_path; - } -#endif TC_ASSERT(next); TC_ASSERT(result); TSANAcquire(result); @@ -914,13 +825,7 @@ inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* TcmallocSlab::Pop( void* prefetch; uintptr_t scratch; uintptr_t previous; -#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT asm goto( -#else - uintptr_t scratch2; - bool underflow; - asm( -#endif TCMALLOC_RSEQ_PROLOGUE(TcmallocSlab_Internal_Pop) // region_start = tcmalloc_slabs; @@ -928,12 +833,7 @@ inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* TcmallocSlab::Pop( // if (region_start & TCMALLOC_CACHED_SLABS_MASK) goto overflow_label; // region_start is unmasked on aarch64, since Top Byte Ignore is enabled in // user space by default -#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT "tbz %[region_start], #%c[cached_slabs_bit], %l[underflow_path]\n" -#else - "tst %[region_start], %[cached_slabs_mask]\n" - "b.eq 5f\n" -#endif // scratch = slab_headers[size_class]->current (current index) "ldrh %w[scratch], [%[region_start], %[size_class_lsl2]]\n" // previous = scratch - 1 @@ -942,48 +842,24 @@ inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* TcmallocSlab::Pop( // Note we are using a pre-indexed ldp. After this instruction, scratch // points to the next location in slab memory, which we will prefetch. "ldp %[prefetch], %[result], [%[scratch], #-16]!\n" -#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT "tbnz %[result], #%c[begin_mark_bit], %l[underflow_path]\n" -#else - // Temporary use %[scratch2] to store %[result] with inverted mark bit. - "eor %[scratch2], %[result], #%c[begin_mark_mask]\n" - "tst %[scratch2], #%c[begin_mark_mask]\n" - "b.eq 5f\n" - // Important! code below this must not affect any flags (i.e.: cceq) - // If so, the above code needs to explicitly set a cceq return value. -#endif "strh %w[previous], [%[region_start], %[size_class_lsl2]]\n" // Commit "5:\n" : -#if !TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT - [underflow] "=@cceq"(underflow), -#endif [result] "=&r"(result), [prefetch] "=&r"(prefetch), // Temps [region_start] "=&r"(region_start), [previous] "=&r"(previous), [scratch] "=&r"(scratch) // Real inputs : TCMALLOC_RSEQ_INPUTS, -#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT [begin_mark_bit] "n"(absl::countr_zero(kBeginMark)), -#else - [cached_slabs_mask] "r"(TCMALLOC_CACHED_SLABS_MASK), - [begin_mark_mask] "n"(kBeginMark), [scratch2] "=&r"(scratch2), -#endif [size_class_lsl2] "r"(size_class << 2) : TCMALLOC_RSEQ_CLOBBER, "memory" -#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT , "cc" : underflow_path -#endif ); -#if !TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT - if (ABSL_PREDICT_FALSE(underflow)) { - goto underflow_path; - } -#endif TSANAcquire(result); PrefetchSlabMemory(scratch); PrefetchNextObject(prefetch);