perf(gc): survival-aware nursery pacing, 4 MB floor on the existing influx ladder (includes #11612)

changelog.d/11612-regex-scratch-not-gc-pressure.md

@@ -0,0 +1 @@
+perf(regex): regex operation scratch is no longer reported to the collector as external side bytes (#11549). The owned search path's per-call match buffers, compile scratch, KMP tables and replacer argument slots are freed by the operation and bounded by its `MemoryBudget`, but each release fed `GC_EXTERNAL_SIDE_DRAINED_SINCE_FULL` and paced loops over >32-register programs (dotenv's `LINE`) into a full mark-sweep every few hundred calls. The lent per-thread scratch cell now grows its registers on demand up to 1024 (8 KiB per thread), so such programs stop building per-call buffers. Subject-proportional replace storage is still reported. Instructions: dotenv/parse −30.1%, moment/parse_format −13.8%, validator/batch −1.9%; peak RSS rises on moment/parse_format (+29%) because those loops now pace by the 16 MB nursery like other allocating programs — held pending the young-generation pacing half of #11549.

changelog.d/11645-gc-nursery-pacing.md

@@ -0,0 +1 @@
+- **gc: survival-aware nursery pacing (Part of #11549).** The scavenge nursery cap's influx ladder now starts at a quarter of the 16 MB base (4 MB). It climbs toward the unchanged 64 MB top only while survivor influx stays above 4% of the cap. A ladder move takes every step the one-step rule would take on the same reading, so survivor-heavy programs settle where they used to, two minors in. The survivor target, the promoted-cohort floor and the census seed point stay on the 16 MB base. Measured against main `7fa094cb4` with THP off: dotenv −30.5% instructions and −20.3% peak RSS; moment −20.6% instructions; validator −26.0% RSS; date-fns −30.0% RSS; jsonwebtoken −22.0% RSS; uuid −21.0% RSS; gc_ratchet 01 −25.5% RSS. **Owner-accepted trade, 2026-09-30, tracked in #11699** (re-measured on main `034b1ea38`, with #11676): a program that holds a large live structure and exits after 4–16 MB of allocation now runs a minor that main never ran (binary-trees n=3/6/10: +498/+433/+369% instructions, +60/+55/+49% RSS); gc_ratchet 02 is +19.8% instructions, 12_large_live_set +2.5% RSS, moment +13.8% RSS (an unexplained shift that #11699 tracks), qs parse +0.75% instructions, and the allocation loop +0.57%. Also classifies `GC_EXTERNAL_SIDE_*` as byte counters in `scripts/gc_runtime_root_holders.json`, because #11612's commits turned that gate red.

crates/perry-runtime/src/gc/tests/copying/adaptive_tenuring.rs

@@ -196,10 +196,13 @@ fn allocation_census_seeds_the_first_cap_before_any_minor() {
         seeded,
         crate::gc::tenuring::NURSERY_CAP_REFERENCE_OBJECT_BYTES
     );
-    // ...and the first cap already reflects it — before any collection.
+    // ...and the first cap already reflects it — before any collection. The
+    // first cap is the ladder's floor (#11549), a quarter of the base.
     assert_eq!(
         crate::gc::tenuring::influx_driven_nursery_cap_bytes(),
-        base * crate::gc::tenuring::nursery_cap_object_scale_permille(seeded) / 1000
+        crate::gc::tenuring::nursery_cap_floor_bytes()
+            * crate::gc::tenuring::nursery_cap_object_scale_permille(seeded)
+            / 1000
     );
 
     // One-shot: a different population allocated afterwards does not move

crates/perry-runtime/src/gc/tests/runtime_roots.rs

@@ -47,6 +47,8 @@ mod perex_replace_direct;
 #[cfg(feature = "regex-engine")]
 mod perex_reuse;
 #[cfg(feature = "regex-engine")]
+mod perex_scratch_pressure;
+#[cfg(feature = "regex-engine")]
 mod perex_split;
 #[cfg(feature = "regex-engine")]
 mod perex_strings;

crates/perry-runtime/src/gc/tests/runtime_roots/perex_execution.rs

@@ -41,20 +41,24 @@ fn compile<'s>(
 fn perex_host_buffers_account_overlap_failure_and_unwind() {
     let _guard = CopyingNurseryTestGuard::new(0);
     let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers();
+    // #11549: operation scratch is bounded by its budget and released by the
+    // operation, so the budget accounts the overlap and the collector's
+    // external-side reading never moves.
     let before = external_side_live_bytes();
     let memory = MemoryBudget::new(128);
     {
         let first = Buffer::<u8>::new(&memory, 64).unwrap();
-        assert_eq!(external_side_live_bytes(), before + memory.live_bytes());
+        assert_eq!(memory.live_bytes(), 64);
         assert!(matches!(
             Buffer::<u8>::new(&memory, 65),
             Err(StorageError::Limit)
         ));
         let replacement = Buffer::<u8>::new(&memory, 64).unwrap();
         assert_eq!(memory.peak_bytes(), 128);
-        assert_eq!(external_side_live_bytes(), before + 128);
+        assert_eq!(memory.live_bytes(), 128);
+        assert_eq!(external_side_live_bytes(), before);
         drop(first);
-        assert_eq!(external_side_live_bytes(), before + 64);
+        assert_eq!(memory.live_bytes(), 64);
         drop(replacement);
     }
     let unwind = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {

crates/perry-runtime/src/gc/tests/runtime_roots/perex_scratch_pressure.rs

@@ -0,0 +1,140 @@
+//! #11549: a regex operation's own scratch is not collector pressure.
+//!
+//! Operation scratch (`perex_memory::Buffer`, the owned search path's match
+//! buffers, the lent cell) is freed by the operation, never by a collection.
+//! Reporting it as external side bytes put every per-call buffer's release into
+//! `GC_EXTERNAL_SIDE_DRAINED_SINCE_FULL`, which old-reclaim holds as pressure
+//! until the next full: dotenv's `LINE` (42 registers, past the lent cell's old
+//! fixed 32) ran a budgeted full mark-sweep every ~250 parses on phantom bytes.
+use super::*;
+use crate::gc::policy::{external_side_live_bytes, GC_EXTERNAL_SIDE_DRAINED_SINCE_FULL};
+use crate::regex::perex_api as api;
+use crate::regex::perex_memory::{Buffer, MemoryBudget};
+use crate::regex::perex_runtime::{EngineError, OWNED_SEARCHES};
+use crate::regex::RegExpHeader;
+use crate::string::StringHeader;
+
+fn text<'s>(scope: &'s RuntimeHandleScope, bytes: &[u8]) -> RuntimeHandle<'s> {
+    scope.root_string_ptr(crate::string::js_string_from_bytes(
+        bytes.as_ptr(),
+        bytes.len() as u32,
+    ))
+}
+
+fn regex<'s>(scope: &'s RuntimeHandleScope, pattern: &str) -> RuntimeHandle<'s> {
+    let pattern = text(scope, pattern.as_bytes());
+    let flags = text(scope, b"");
+    scope.root_raw_mut_ptr(pattern.with_const_ptr::<StringHeader, _>(|pattern| {
+        flags.with_const_ptr::<StringHeader, _>(|flags| crate::regex::js_regexp_new(pattern, flags))
+    }))
+}
+
+fn search(
+    re: &RuntimeHandle<'_>,
+    input: &RuntimeHandle<'_>,
+) -> Result<Option<(usize, usize)>, EngineError> {
+    re.with_mut_ptr::<RegExpHeader, _>(|re| {
+        input.with_const_ptr::<StringHeader, _>(|input| {
+            api::execute(re, input, false, &mut || Ok(()))
+                .map(|found| found.map(|m| (m.full.start(), m.full.end())))
+        })
+    })
+}
+
+/// `groups` capturing groups of one `a` each: `(a)(a)...`. A program's register
+/// count is at least twice its capture count (group zero included), so this
+/// needs at least `2 * (groups + 1)` registers.
+fn groups_pattern(groups: usize) -> String {
+    "(a)".repeat(groups)
+}
+
+fn external_readings() -> (usize, usize) {
+    (
+        external_side_live_bytes(),
+        GC_EXTERNAL_SIDE_DRAINED_SINCE_FULL.with(TriggerInput::get),
+    )
+}
+
+fn owned_searches() -> usize {
+    OWNED_SEARCHES.with(std::cell::Cell::get)
+}
+
+#[test]
+fn perex_operation_buffer_is_not_reported_as_external_side_bytes() {
+    let _guard = CopyingNurseryTestGuard::new(0);
+    let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers();
+    let before = external_readings();
+    let memory = MemoryBudget::new(1 << 20);
+    {
+        let buffer = Buffer::<usize>::new(&memory, 4096).expect("fits the budget");
+        assert_eq!(buffer.len(), 4096);
+        assert_eq!(memory.live_bytes(), 4096 * 8, "the budget still sees it");
+        assert_eq!(external_readings(), before, "allocation noted nothing");
+    }
+    assert_eq!(memory.live_bytes(), 0);
+    assert_eq!(memory.peak_bytes(), 4096 * 8);
+    assert_eq!(
+        external_readings(),
+        before,
+        "a released operation buffer is not drained external pressure (#11549)"
+    );
+    // The budget remains the bound: past it, nothing is allocated.
+    assert!(Buffer::<usize>::new(&memory, (1 << 20) / 8 + 1).is_err());
+}
+
+#[test]
+fn perex_search_past_32_registers_uses_the_lent_cell_and_notes_nothing() {
+    let _guard = CopyingNurseryTestGuard::new(0);
+    let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers();
+    super::perex_public::register_host_roots();
+    let scope = RuntimeHandleScope::new();
+    // 20 groups: 42+ registers, which the lent cell's old fixed 32 could not
+    // hold, so every call built and dropped owned match buffers.
+    let re = regex(&scope, &groups_pattern(20));
+    let input = text(&scope, "a".repeat(25).as_bytes());
+    // The first call may grow the thread's cell; every later one must not
+    // build anything.
+    assert_eq!(search(&re, &input).unwrap(), Some((0, 20)));
+    let before = external_readings();
+    let owned = owned_searches();
+    for _ in 0..64 {
+        assert_eq!(search(&re, &input).unwrap(), Some((0, 20)));
+    }
+    assert_eq!(
+        owned_searches(),
+        owned,
+        "a program within the lent bound searches on the lent cell"
+    );
+    assert_eq!(
+        external_readings(),
+        before,
+        "and reports no scratch to the collector"
+    );
+}
+
+#[test]
+fn perex_search_past_the_lent_bound_takes_the_owned_path_and_notes_nothing() {
+    let _guard = CopyingNurseryTestGuard::new(0);
+    let _triggers = GcTriggerThresholdTestGuard::suppress_automatic_triggers();
+    super::perex_public::register_host_roots();
+    let scope = RuntimeHandleScope::new();
+    // 600 groups: 1,202+ registers, past what the lent cell may retain. The
+    // cell must not grow to it; the operation builds, and frees, its own.
+    let re = regex(&scope, &groups_pattern(600));
+    let input = text(&scope, "a".repeat(600).as_bytes());
+    let before = external_readings();
+    let owned = owned_searches();
+    for _ in 0..4 {
+        assert_eq!(search(&re, &input).unwrap(), Some((0, 600)));
+    }
+    assert_eq!(
+        owned_searches(),
+        owned + 4,
+        "a program past the lent bound runs the owned path every call"
+    );
+    assert_eq!(
+        external_readings(),
+        before,
+        "owned match buffers are released by the search, not drained into GC pressure"
+    );
+}

crates/perry-runtime/src/gc/tests/triggers.rs

@@ -753,7 +753,11 @@ fn test_effective_arena_trigger_respects_armed_values() {
     // the cap's own basis.
     let nursery_capped = gc_moving_loop_polls_enabled();
     let ceiling = gc_trigger_absolute_ceiling_bytes();
-    let nursery_cap = gc_scavenge_nursery_cap_bytes();
+    // The cap the clamp applies is the EFFECTIVE one — the survival-driven
+    // ladder's current size (#11549: a fresh thread starts at a quarter of the
+    // base), not the configured base.
+    let nursery_cap = crate::gc::tenuring::scavenge_nursery_cap_effective_bytes();
+    assert!(nursery_cap <= gc_scavenge_nursery_cap_bytes() * 4);
 
     let prev_trigger = GC_NEXT_TRIGGER_BYTES.with(|c| c.get());
     let prev_armed = GC_TRIGGER_ARMED.with(|c| c.get());

crates/perry-runtime/src/regex/perex_api.rs

@@ -48,9 +48,7 @@ fn raise(error: EngineError) -> ! {
     if let EngineError::Abrupt(value) = error {
         crate::exception::js_throw(value);
     }
-    if let EngineError::Build(BuildError::Abrupt(bits))
-    | EngineError::Storage(StorageError::Abrupt(bits)) = error
-    {
+    if let EngineError::Build(BuildError::Abrupt(bits)) = error {
         crate::exception::js_throw(f64::from_bits(bits));
     }
     let type_error = matches!(error, EngineError::Type(_));

crates/perry-runtime/src/regex/perex_memory.rs

@@ -1,6 +1,22 @@
-//! Operation-owned native scratch, charged to Perry's external-byte budget.
-//! No GC pointer may be stored in these buffers. Allocation/accounting can
-//! collect, so callers must release all program/subject views first.
+//! Operation-owned native scratch, bounded by the operation's `MemoryBudget`.
+//! No GC pointer may be stored in these buffers.
+//!
+//! This scratch is NOT reported to the collector as external side bytes
+//! (#11549). Everything here is freed by the operation that allocated it,
+//! when that operation returns or unwinds; no collection can ever reclaim a
+//! byte of it, and no collection is needed for it to be released. Reporting it
+//! told the old-reclaim pacing the opposite: every per-call buffer's release
+//! landed in `GC_EXTERNAL_SIDE_DRAINED_SINCE_FULL`, which is held as pressure
+//! until the next FULL collection, so a regex loop that took the owned search
+//! path (a program with more registers than the lent cell holds) was paced by
+//! phantom bytes into a budgeted full mark-sweep every few hundred calls.
+//!
+//! What bounds it instead is the budget: every buffer, inline charge and
+//! reservation is checked against the operation's hard limit
+//! (`perex_api::SCRATCH_BYTES`) before it exists, so one operation can never
+//! hold more than that. Storage whose size follows the SUBJECT rather than
+//! that limit (`perex_replace_direct::Spans`, `perex_replace_storage`'s
+//! native piece records) is not scratch in this sense and is still reported.
 
 use std::cell::Cell;
 use std::ops::{Deref, DerefMut};
@@ -9,7 +25,6 @@ use std::ops::{Deref, DerefMut};
 pub(crate) enum StorageError {
     Limit,
     Allocation,
-    Abrupt(u64),
 }
 
 /// One operation's hard scratch/result-metadata limit. Simultaneous old/new
@@ -69,37 +84,31 @@ impl Drop for Charge<'_> {
     }
 }
 
-/// Account a stable native allocation whose GC-bearing slots are separately
-/// registered with the host's mutable root scanner before this can collect.
+/// Charge a stable native allocation the caller owns, such as a replacer
+/// call's argument slots, to the operation's limit. Its GC-bearing slots are
+/// registered with the shadow stack separately; like every other buffer here
+/// it is released by the operation, not by a collection, so the collector is
+/// not told about it.
 pub(super) struct Reservation<'a> {
     budget: &'a MemoryBudget,
     bytes: usize,
 }
 impl<'a> Reservation<'a> {
     pub(super) fn new(budget: &'a MemoryBudget, bytes: usize) -> Result<Self, StorageError> {
         let live = budget.check(bytes)?;
-        let owned = Self { budget, bytes };
         budget.live.set(live);
         budget.peak.set(budget.peak.get().max(live));
-        if bytes != 0 {
-            crate::exception::catch_js_throw(|| crate::gc::gc_note_external_side_alloc(bytes))
-                .map_err(|value| StorageError::Abrupt(value.to_bits()))?;
-        }
-        Ok(owned)
+        Ok(Self { budget, bytes })
     }
 }
 impl Drop for Reservation<'_> {
     fn drop(&mut self) {
         self.budget.live.set(self.budget.live.get() - self.bytes);
-        if self.bytes != 0 {
-            crate::gc::gc_note_external_side_free(self.bytes);
-        }
     }
 }
 
-/// Stable initialized native allocation. Its accounting owner is established
-/// before notifying the collector, so a collecting/unwinding notification
-/// cannot strand a buffer or leave its bytes charged.
+/// Stable initialized native allocation, charged to the operation's limit for
+/// as long as it lives. Creating one never collects.
 pub(crate) struct Buffer<'a, T: Copy + Default> {
     data: Vec<T>,
     budget: &'a MemoryBudget,
@@ -121,18 +130,13 @@ impl<'a, T: Copy + Default> Buffer<'a, T> {
             .ok_or(StorageError::Limit)?;
         let live = budget.check(bytes)?;
         data.resize(count, T::default());
-        let owned = Self {
+        budget.live.set(live);
+        budget.peak.set(budget.peak.get().max(live));
+        Ok(Self {
             data,
             budget,
             bytes,
-        };
-        budget.live.set(live);
-        budget.peak.set(budget.peak.get().max(live));
-        if bytes != 0 {
-            crate::exception::catch_js_throw(|| crate::gc::gc_note_external_side_alloc(bytes))
-                .map_err(|value| StorageError::Abrupt(value.to_bits()))?;
-        }
-        Ok(owned)
+        })
     }
 }
 
@@ -152,8 +156,5 @@ impl<T: Copy + Default> DerefMut for Buffer<'_, T> {
 impl<T: Copy + Default> Drop for Buffer<'_, T> {
     fn drop(&mut self) {
         self.budget.live.set(self.budget.live.get() - self.bytes);
-        if self.bytes != 0 {
-            crate::gc::gc_note_external_side_free(self.bytes);
-        }
     }
 }