diff --git a/docs/platforms.md b/docs/platforms.md
index 8d4250d69..6f01c1043 100644
--- a/docs/platforms.md
+++ b/docs/platforms.md
@@ -13,7 +13,7 @@ We guarantee that our code will compile under the following compilation flags:
Linux:
-* gcc 9.2+, clang 9.0+: `-std=c++17`
+* gcc 9.2+, clang 11.0+: `-std=c++17`
(TL;DR; All code at this time must be built under C++17. We will update this
list if circumstances change.)
@@ -42,7 +42,7 @@ Architecture, Specific Compiler, and Standard Library implementation.
Linux |
little-endian, 64-bit |
x86, AArch64 |
- gcc 9.2+ clang 9.0+ |
+ gcc 9.2+ clang 11.0+ |
libstdc++ libc++ |
@@ -68,7 +68,7 @@ release should also work.
Linux |
little-endian, 64-bit |
PPC |
- gcc 9.2+ clang 9.0+ |
+ gcc 9.2+ clang 11.0+ |
libstdc++ libc++ |
diff --git a/tcmalloc/internal/percpu_tcmalloc.h b/tcmalloc/internal/percpu_tcmalloc.h
index 26f12d68d..b65c0089e 100644
--- a/tcmalloc/internal/percpu_tcmalloc.h
+++ b/tcmalloc/internal/percpu_tcmalloc.h
@@ -46,10 +46,6 @@
#include "tcmalloc/internal/prefetch.h"
#include "tcmalloc/internal/sysinfo.h"
-#if __clang_major__ >= 11
-#define TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT 1
-#endif
-
GOOGLE_MALLOC_SECTION_BEGIN
namespace tcmalloc {
namespace tcmalloc_internal {
@@ -655,55 +651,30 @@ inline ABSL_ATTRIBUTE_ALWAYS_INLINE void PrefetchSlabMemory(uintptr_t ptr) {
static inline ABSL_ATTRIBUTE_ALWAYS_INLINE bool TcmallocSlab_Internal_Push(
size_t size_class, void* item) {
uintptr_t scratch, current;
-#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
asm goto(
-#else
- bool overflow;
- asm volatile(
-#endif
TCMALLOC_RSEQ_PROLOGUE(TcmallocSlab_Internal_Push)
// scratch = tcmalloc_slabs;
"movq %[rseq_slabs_addr], %[scratch]\n"
// if (scratch & TCMALLOC_CACHED_SLABS_MASK>) goto overflow_label;
// scratch &= ~TCMALLOC_CACHED_SLABS_MASK;
"btrq $%c[cached_slabs_bit], %[scratch]\n"
-#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
"jnc %l[overflow_label]\n"
-#else
- "jae 5f\n" // ae==c
-#endif
// current = slabs->current;
"movzwq (%[scratch], %[size_class], 4), %[current]\n"
// if (ABSL_PREDICT_FALSE(current >= slabs->end)) { goto overflow_label; }
"cmp 2(%[scratch], %[size_class], 4), %w[current]\n"
-#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
"jae %l[overflow_label]\n"
-#else
- "jae 5f\n"
- // Important! code below this must not affect any flags (i.e.: ccae)
- // If so, the above code needs to explicitly set a ccae return value.
-#endif
"mov %[item], (%[scratch], %[current], 8)\n"
"lea 1(%[current]), %[current]\n"
"mov %w[current], (%[scratch], %[size_class], 4)\n"
// Commit
"5:\n"
:
-#if !TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
- [overflow] "=@ccae"(overflow),
-#endif
[scratch] "=&r"(scratch), [current] "=&r"(current)
: TCMALLOC_RSEQ_INPUTS, [size_class] "r"(size_class), [item] "r"(item)
: "cc", "memory"
-#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
: overflow_label
-#endif
);
-#if !TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
- if (ABSL_PREDICT_FALSE(overflow)) {
- goto overflow_label;
- }
-#endif
// Current now points to the slot we are going to push to next.
PrefetchSlabMemory(scratch + current * sizeof(void*));
return true;
@@ -716,24 +687,14 @@ static inline ABSL_ATTRIBUTE_ALWAYS_INLINE bool TcmallocSlab_Internal_Push(
static inline ABSL_ATTRIBUTE_ALWAYS_INLINE bool TcmallocSlab_Internal_Push(
size_t size_class, void* item) {
uintptr_t region_start, scratch, scratch_ptr, end;
-#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
asm goto(
-#else
- bool overflow;
- asm volatile(
-#endif
TCMALLOC_RSEQ_PROLOGUE(TcmallocSlab_Internal_Push)
// region_start = tcmalloc_slabs;
"ldr %[region_start], [%[sampler_addr], #%c[sampler_slabs_offset]]\n"
// if (region_start & TCMALLOC_CACHED_SLABS_MASK) goto overflow_label;
// region_start is unmasked on aarch64, since Top Byte Ignore is enabled in
// user space by default
-#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
"tbz %[region_start], #%c[cached_slabs_bit], %l[overflow_label]\n"
-#else
- "subs %[region_start], %[region_start], %[cached_slabs_mask]\n"
- "b.ls 5f\n"
-#endif
// scratch = slab_headers[size_class]->current (current index)
"add %[scratch_ptr], %[region_start], %[size_class], LSL #2\n"
"ldrh %w[scratch], [%[scratch_ptr]]\n"
@@ -741,13 +702,7 @@ static inline ABSL_ATTRIBUTE_ALWAYS_INLINE bool TcmallocSlab_Internal_Push(
"ldrh %w[end], [%[scratch_ptr], #2]\n"
// if (ABSL_PREDICT_FALSE(end <= scratch)) { goto overflow_label; }
"cmp %[end], %[scratch]\n"
-#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
"b.ls %l[overflow_label]\n"
-#else
- "b.ls 5f\n"
- // Important! code below this must not affect any flags (i.e.: ccls)
- // If so, the above code needs to explicitly set a ccls return value.
-#endif
"str %[item], [%[region_start], %[scratch], LSL #3]\n"
"add %w[scratch], %w[scratch], #1\n"
"strh %w[scratch], [%[scratch_ptr]]\n"
@@ -755,27 +710,14 @@ static inline ABSL_ATTRIBUTE_ALWAYS_INLINE bool TcmallocSlab_Internal_Push(
"5:\n"
: [scratch_ptr] "=&r"(scratch_ptr), [scratch] "=&r"(scratch),
[end] "=&r"(end),
-#if !TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
- [overflow] "=@ccls"(overflow),
-#endif
[region_start] "=&r"(region_start)
: TCMALLOC_RSEQ_INPUTS,
-#if !TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
- [cached_slabs_mask] "r"(TCMALLOC_CACHED_SLABS_MASK),
-#endif
[size_class] "r"(size_class), [item] "r"(item)
: TCMALLOC_RSEQ_CLOBBER, "memory"
-#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
,
"cc"
: overflow_label
-#endif
);
-#if !TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
- if (ABSL_PREDICT_FALSE(overflow)) {
- goto overflow_label;
- }
-#endif
PrefetchSlabMemory(reinterpret_cast(region_start) +
scratch * sizeof(void*));
@@ -833,63 +775,32 @@ inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* TcmallocSlab::Pop(
void* result;
uintptr_t scratch, current;
-#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
asm goto(
-#else
- bool underflow;
- asm(
-#endif
TCMALLOC_RSEQ_PROLOGUE(TcmallocSlab_Internal_Pop)
// scratch = tcmalloc_slabs;
"movq %[rseq_slabs_addr], %[scratch]\n"
// if (scratch & TCMALLOC_CACHED_SLABS_MASK) goto overflow_label;
// scratch &= ~TCMALLOC_CACHED_SLABS_MASK;
"btrq $%c[cached_slabs_bit], %[scratch]\n"
-#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
"jnc %l[underflow_path]\n"
-#else
- "cmc\n"
- "jc 5f\n"
-#endif
// current = scratch->header[size_class].current;
"movzwq (%[scratch], %[size_class], 4), %[current]\n"
"movq -8(%[scratch], %[current], 8), %[result]\n"
-#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
"testb $%c[begin_mark_mask], %b[result]\n"
"jnz %l[underflow_path]\n"
-#else
- "btq $%c[begin_mark_bit], %[result]\n"
- "jc 5f\n"
- // Important! code below this must not affect any flags (i.e.: ccc)
- // If so, the above code needs to explicitly set a ccc return value.
-#endif
"movq -16(%[scratch], %[current], 8), %[next]\n"
"lea -1(%[current]), %[current]\n"
"movw %w[current], (%[scratch], %[size_class], 4)\n"
// Commit
"5:\n"
: [result] "=&r"(result),
-#if !TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
- [underflow] "=@ccc"(underflow),
-#endif
[scratch] "=&r"(scratch), [current] "=&r"(current), [next] "=&r"(next)
: TCMALLOC_RSEQ_INPUTS,
-#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
[begin_mark_mask] "n"(kBeginMark),
-#else
- [begin_mark_bit] "n"(absl::countr_zero(kBeginMark)),
-#endif
[size_class] "r"(size_class)
: "cc", "memory"
-#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
: underflow_path
-#endif
);
-#if !TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
- if (ABSL_PREDICT_FALSE(underflow)) {
- goto underflow_path;
- }
-#endif
TC_ASSERT(next);
TC_ASSERT(result);
TSANAcquire(result);
@@ -914,13 +825,7 @@ inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* TcmallocSlab::Pop(
void* prefetch;
uintptr_t scratch;
uintptr_t previous;
-#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
asm goto(
-#else
- uintptr_t scratch2;
- bool underflow;
- asm(
-#endif
TCMALLOC_RSEQ_PROLOGUE(TcmallocSlab_Internal_Pop)
// region_start = tcmalloc_slabs;
@@ -928,12 +833,7 @@ inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* TcmallocSlab::Pop(
// if (region_start & TCMALLOC_CACHED_SLABS_MASK) goto overflow_label;
// region_start is unmasked on aarch64, since Top Byte Ignore is enabled in
// user space by default
-#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
"tbz %[region_start], #%c[cached_slabs_bit], %l[underflow_path]\n"
-#else
- "tst %[region_start], %[cached_slabs_mask]\n"
- "b.eq 5f\n"
-#endif
// scratch = slab_headers[size_class]->current (current index)
"ldrh %w[scratch], [%[region_start], %[size_class_lsl2]]\n"
// previous = scratch - 1
@@ -942,48 +842,24 @@ inline ABSL_ATTRIBUTE_ALWAYS_INLINE void* TcmallocSlab::Pop(
// Note we are using a pre-indexed ldp. After this instruction, scratch
// points to the next location in slab memory, which we will prefetch.
"ldp %[prefetch], %[result], [%[scratch], #-16]!\n"
-#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
"tbnz %[result], #%c[begin_mark_bit], %l[underflow_path]\n"
-#else
- // Temporary use %[scratch2] to store %[result] with inverted mark bit.
- "eor %[scratch2], %[result], #%c[begin_mark_mask]\n"
- "tst %[scratch2], #%c[begin_mark_mask]\n"
- "b.eq 5f\n"
- // Important! code below this must not affect any flags (i.e.: cceq)
- // If so, the above code needs to explicitly set a cceq return value.
-#endif
"strh %w[previous], [%[region_start], %[size_class_lsl2]]\n"
// Commit
"5:\n"
:
-#if !TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
- [underflow] "=@cceq"(underflow),
-#endif
[result] "=&r"(result), [prefetch] "=&r"(prefetch),
// Temps
[region_start] "=&r"(region_start), [previous] "=&r"(previous),
[scratch] "=&r"(scratch)
// Real inputs
: TCMALLOC_RSEQ_INPUTS,
-#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
[begin_mark_bit] "n"(absl::countr_zero(kBeginMark)),
-#else
- [cached_slabs_mask] "r"(TCMALLOC_CACHED_SLABS_MASK),
- [begin_mark_mask] "n"(kBeginMark), [scratch2] "=&r"(scratch2),
-#endif
[size_class_lsl2] "r"(size_class << 2)
: TCMALLOC_RSEQ_CLOBBER, "memory"
-#if TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
,
"cc"
: underflow_path
-#endif
);
-#if !TCMALLOC_INTERNAL_PERCPU_USE_RSEQ_ASM_GOTO_OUTPUT
- if (ABSL_PREDICT_FALSE(underflow)) {
- goto underflow_path;
- }
-#endif
TSANAcquire(result);
PrefetchSlabMemory(scratch);
PrefetchNextObject(prefetch);