From bbad8fbf086515695d9ad0fffe0d74ca630becdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 4 Sep 2026 11:53:34 +0200 Subject: [PATCH 1/5] perf(runtime): store shape descriptors in an id-indexed slab and retire owned growth history Closes #9706. The agent-local shape table kept a PtrHashMap> beside two Vec-valued reverse maps (exact facts, keys address). On the compiled claude-code TUI at idle that cost ~330 bytes per live descriptor. Replace the by-id map with a chunked slab indexed by ShapeId (32-byte packed records, stable addresses, lazily allocated chunks released at major GC), and both reverse maps with one keys-address family index that exact-facts interning walks. Retire an OWNED keys array's same-address growth history behind the version its single owner carries, except for Array-subclass receivers whose tail-transition cache reinstalls it. Claude-Session: https://claude.ai/code/session_016TiA2Y98uX79JSsY3eV1DS --- crates/perry-runtime/src/fast_hash.rs | 14 +- crates/perry-runtime/src/gc/census.rs | 18 +- .../gc/tests/shape_keys_descriptor_edge.rs | 50 + crates/perry-runtime/src/object/shapes.rs | 952 ++++++++++-------- .../src/object/shapes_reverse_indices.rs | 119 --- .../src/object/shapes_slot_list.rs | 261 +++-- .../perry-runtime/src/object/shapes_store.rs | 776 ++++++++++++++ .../src/object/shapes_test_support.rs | 73 +- .../perry-runtime/src/object/shapes_tests.rs | 207 ++-- scripts/shape_descriptor_census.py | 112 ++- 10 files changed, 1749 insertions(+), 833 deletions(-) delete mode 100644 crates/perry-runtime/src/object/shapes_reverse_indices.rs create mode 100644 crates/perry-runtime/src/object/shapes_store.rs diff --git a/crates/perry-runtime/src/fast_hash.rs b/crates/perry-runtime/src/fast_hash.rs index f020387b82..3d021d1f17 100644 --- a/crates/perry-runtime/src/fast_hash.rs +++ b/crates/perry-runtime/src/fast_hash.rs @@ -197,13 +197,13 @@ impl Hasher for FastKeyHasherImpl { // Integer writes fold one word per call instead of falling into `Hasher`'s // default `write_uN` -> `write(&n.to_ne_bytes())` byte loop. // - // This is what the shape table's `ids_by_facts` key pays for: `ShapeFacts` - // is six integer fields (two `u64`, three `u32`, one enum discriminant, an - // `isize`), so the derived `Hash` fed ~36 bytes -- ~36 serial multiplies -- - // through the byte loop for a key that six folds mix just as well. That - // lookup runs on every shape publish (`shape_descriptor_ensure_with_holes` - // probes `ids_by_facts` before minting an id), i.e. on every object - // property add/delete that transitions a shape. + // This was written for the shape table's `ShapeFacts` key (six integer + // fields: two `u64`, three `u32`, one enum discriminant, an `isize`), whose + // derived `Hash` fed ~36 bytes -- ~36 serial multiplies -- through the byte + // loop for a key that six folds mix just as well. #9706 replaced that map + // with a pre-folded `u64` key (`object/shapes_store.rs::facts_key`), but + // the same shape of key remains on this hasher: `RegisteredTypedShapeKey` + // (`gc/layout/typed_shape.rs`) and the `(usize, String)` descriptor keys. // // `write_u8` is deliberately included even though it is exactly equivalent // to the byte path for a single byte: routing it here keeps every integer diff --git a/crates/perry-runtime/src/gc/census.rs b/crates/perry-runtime/src/gc/census.rs index 63b8b5c8ac..5aeaad82a0 100644 --- a/crates/perry-runtime/src/gc/census.rs +++ b/crates/perry-runtime/src/gc/census.rs @@ -346,6 +346,10 @@ struct Census { clo_captures: u64, // objects obj_meta: u64, + /// ShapeId stamp of every live shaped object (duplicates included; sorted + /// and deduplicated once the walk is over). Feeds the shape table's + /// "carried by a live object" rows (#9706). + live_shape_ids: Vec, } const SPACE_NAMES: [&str; 6] = [ @@ -432,7 +436,11 @@ impl Census { self.obj_meta += 1; } let live = match crate::object::shapes::object_shape_descriptor(obj) { - Some(d) => (d.live_inline_slot_count as usize).min(slot_capacity), + Some(d) => { + self.live_shape_ids + .push(crate::object::shapes::object_shape_stamp(obj)); + (d.live_inline_slot_count as usize).min(slot_capacity) + } None => { entry.unshaped += 1; 0 @@ -764,7 +772,13 @@ fn take_census(label: &str, pass1: Option>) { }) .collect(); - let side: Vec = side_tables() + c.live_shape_ids.sort_unstable(); + c.live_shape_ids.dedup(); + let mut side_rows = side_tables(); + side_rows.extend(crate::object::shapes::shape_table_liveness_census( + &c.live_shape_ids, + )); + let side: Vec = side_rows .into_iter() .map(|(n, e, b)| serde_json::json!({"table": n, "entries": e, "bytes": b})) .collect(); diff --git a/crates/perry-runtime/src/gc/tests/shape_keys_descriptor_edge.rs b/crates/perry-runtime/src/gc/tests/shape_keys_descriptor_edge.rs index ac7adaf2c1..856a21dc62 100644 --- a/crates/perry-runtime/src/gc/tests/shape_keys_descriptor_edge.rs +++ b/crates/perry-runtime/src/gc/tests/shape_keys_descriptor_edge.rs @@ -212,6 +212,56 @@ fn a_keys_array_reachable_only_through_the_descriptor_survives_and_is_rewritten( ); } +/// #9706: the reverse indices are keyed by the keys ADDRESS. After a copying +/// minor moves the keys array, the metadata scan must re-key the family and +/// the exact-facts accelerator, so that interning the moved facts answers the +/// SAME id (a fresh id would be a duplicate descriptor per collection) and +/// the stale address answers nothing. +#[test] +fn the_reverse_indices_follow_a_moved_keys_array() { + let _guard = CopyingNurseryTestGuard::new(2); + // The record rewrite comes from the receiver's own edge; the re-keying + // is the metadata scanner's job, which production registers at gc init + // and a unit test must register itself (as the recycled-keys fixtures do). + gc_register_mutable_root_scanner(shapes::scan_shape_table_rekey_mut); + let (before, after) = collect_and_report(false) + .expect("#9706: the receiver must move for this cycle to be discriminating"); + assert_ne!( + after.keys, before.keys, + "test premise: the keys array moved" + ); + let obj = (js_shadow_slot_get(0) & POINTER_MASK) as *mut crate::ObjectHeader; + let id = unsafe { shapes::object_shape_stamp(obj) }; + assert!(shapes::is_shape_id(id), "the receiver stays stamped"); + assert_eq!( + shapes::test_shape_ids_for_keys(after.keys as usize), + vec![id], + "#9706: the family index must be re-keyed to the forwarded address \ + (before={:#x} after={:#x} under-before={:?} record-keys={:#x})", + before.keys, + after.keys, + shapes::test_shape_ids_for_keys(before.keys as usize), + unsafe { shapes::object_shape_descriptor(obj) } + .map(|d| d.keys) + .unwrap_or(0) + ); + assert!( + shapes::test_shape_ids_for_keys(before.keys as usize).is_empty(), + "#9706: nothing may stay indexed under the from-space address" + ); + let descriptor = unsafe { shapes::object_shape_descriptor(obj) }.expect("published"); + assert_eq!(descriptor.keys, after.keys); + assert_eq!( + shapes::shape_descriptor_ensure( + after.keys as usize as *const crate::ArrayHeader, + descriptor.logical_key_count, + descriptor.live_inline_slot_count, + ), + Ok(id), + "#9706: interning the moved facts must answer the existing id, not mint a duplicate" + ); +} + #[test] fn keys_edge_sabotage_is_detected() { let _guard = CopyingNurseryTestGuard::new(2); diff --git a/crates/perry-runtime/src/object/shapes.rs b/crates/perry-runtime/src/object/shapes.rs index 95b6754bd5..5ec883968c 100644 --- a/crates/perry-runtime/src/object/shapes.rs +++ b/crates/perry-runtime/src/object/shapes.rs @@ -34,23 +34,23 @@ use crate::array::ArrayHeader; use std::cell::RefCell; -#[path = "shapes_reverse_indices.rs"] -mod shapes_reverse_indices; #[path = "shapes_slot_list.rs"] mod shapes_slot_list; -use shapes_reverse_indices::{ - descriptor_facts, insert_descriptor_id_sorted, remove_descriptor_and_reverse_indices, - remove_descriptor_id_from_facts_index, sync_descriptor_reverse_indices, -}; +#[path = "shapes_store.rs"] +mod shapes_store; #[cfg(test)] pub(crate) use shapes_slot_list::shape_descriptor_keys_slot; pub(crate) use shapes_slot_list::shape_id_owns_keys_slot; pub(crate) use shapes_slot_list::{ - object_shape_hole_count, publish_object_shape_holes, record_shape_scan_outcome, + object_shape_hole_count, publish_object_shape_holes, rekey_stable_tombstone_shape_after_squeeze, retire_owned_shape_history, shape_index_migrate_after_delete, shape_index_shift_in_place, try_update_stable_tombstone_shape, try_update_stable_tombstone_shape_cached, SlotList, }; +use shapes_store::{ + IdList, ShapeRecord, ShapeSlab, RECORD_FLAG_CACHE_CARRIER, RECORD_FLAG_FACTS_INDEXED, + RECORD_FLAG_OLD_CARRIER, RECORD_FLAG_OLD_CARRIER_SEEN, +}; #[derive(Clone)] pub(crate) struct ShapeIndex { @@ -70,39 +70,27 @@ pub(crate) struct ShapeIndex { slots: crate::fast_hash::PtrHashMap, } -/// Immutable facts named by one ShapeId. +/// Immutable facts named by one ShapeId, copied out of the table. /// -/// #8112: `keys` is the AUTHORITATIVE ordered-keys edge — the collector marks -/// it and rewrites it in place. Before #8112 the header word was the sole -/// strong edge and this field a weak copy that a post-visit callback repaired. -/// The inversion is what #8047 needs, because deleting the header word must -/// not unroot anything. +/// #8112: the table record's `keys` is the AUTHORITATIVE ordered-keys edge — +/// the collector marks it and rewrites it in place. Before #8112 the header +/// word was the sole strong edge and this field a weak copy that a post-visit +/// callback repaired. The inversion is what #8047 needs, because deleting the +/// header word must not unroot anything. /// -/// The table is a rehashing `PtrHashMap`, so the bucket address is NOT stable -/// across descriptor insertion — and the incremental collector retains -/// enumerated slot addresses across budgeted resumptions. Descriptors are -/// therefore BOXED (`ShapeTableInner::descriptors`), which makes each record's -/// address fixed for its lifetime, and `record` carries the address of THIS -/// boxed descriptor so a traced receiver can hand the collector a rewritable +/// The table stores a packed [`ShapeRecord`] in a chunked slab whose record +/// addresses never move (#9706, `shapes_store.rs`); the incremental collector +/// retains enumerated slot addresses across budgeted resumptions, so that +/// stability is load-bearing. This value is the UNPACKED copy the rest of the +/// runtime consumes, and `record` carries the address of the slab record it +/// was lifted from so a traced receiver can hand the collector a rewritable /// `keys` location without a second table probe (#8122's one-probe rule). #[derive(Clone, Copy, Debug)] pub(crate) struct ShapeDescriptor { /// Raw ArrayHeader address in Perry's fixed-width heap-word ABI. Keeping /// this u64 preserves identical representation on ILP32/LP64. pub(crate) keys: u64, - /// The keys address currently represented in the reverse indices. The - /// collector may rewrite `keys` directly through a raw - /// slot before the metadata scanner runs; retaining the indexed address - /// lets that scanner repair exactly this descriptor instead of rebuilding - /// and sorting both reverse maps for every shape in the agent. - /// Never part of shape identity. - indexed_keys: u64, - /// Whether this descriptor participates in exact-facts interning. A - /// private stable-tombstone epoch mutates its counts in place and detaches - /// from `ids_by_facts`; it remains in `ids_by_keys` for GC relocation and - /// deterministic retirement at squeeze. - facts_indexed: bool, - /// Address of the BOXED record this value was lifted from, or 0 for a + /// Address of the slab record this value was lifted from, or 0 for a /// descriptor built outside the table (equality comparisons, tests). /// Never part of shape IDENTITY — see the hand-written `PartialEq` below. pub(crate) record: usize, @@ -116,10 +104,12 @@ pub(crate) struct ShapeDescriptor { /// on. It is sticky within an epoch and recomputed by every full trace, so /// it over-approximates by at most one full collection: exactly the /// generational contract, and never unconditional rooting. + /// + /// The record also keeps the notes accumulated since the last full trace + /// (`RECORD_FLAG_OLD_CARRIER_SEEN`), adopted into this bit by + /// [`rotate_old_carrier_epoch_after_full_trace`]; the copy carries only + /// the adopted gate. pub(crate) old_carrier: bool, - /// Notes accumulated since the last full trace; adopted into `old_carrier` - /// by [`rotate_old_carrier_epoch_after_full_trace`]. - pub(crate) old_carrier_seen: bool, /// A runtime optimization cache can reinstall this historical shape even /// while no live object currently carries it. Such a cache is an explicit /// strong metadata owner, so collection must root and rewrite `keys` before @@ -144,22 +134,30 @@ pub(crate) struct ShapeDescriptor { } /// Shape identity is the FACTS, never the storage address. A descriptor value -/// lifted out of the table compares equal to the boxed record it came from. +/// lifted out of the table compares equal to the record it came from. impl ShapeDescriptor { /// The one `keys` word the collector rewrites for this shape, or `None` /// for a descriptor value that was never lifted out of the table. + /// + /// `keys` is the first field of the `#[repr(C)]` slab record, so the + /// record address IS the slot address. #[inline] pub(crate) fn keys_slot(&self) -> Option<*mut u64> { if self.record == 0 { return None; } - Some(unsafe { std::ptr::addr_of_mut!((*(self.record as *mut ShapeDescriptor)).keys) }) + Some(self.record as *mut u64) } } impl PartialEq for ShapeDescriptor { fn eq(&self, other: &Self) -> bool { - descriptor_facts(*self) == descriptor_facts(*other) + self.keys == other.keys + && self.logical_key_count == other.logical_key_count + && self.live_inline_slot_count == other.live_inline_slot_count + && self.semantic_generation == other.semantic_generation + && self.object_kind == other.object_kind + && self.hole_count == other.hole_count } } @@ -226,121 +224,128 @@ fn clear_shape_object_kind_cache() { cache.fill(0); } -#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] -struct ShapeFacts { - keys: u64, - logical_key_count: u32, - live_inline_slot_count: u32, - semantic_generation: u64, - object_kind: ShapeObjectKind, - /// Tombstoned key slots in the keys array (`TAG_HOLE` markers left by - /// O(1) deletes). Ordinarily part of identity. A private ordinary receiver - /// in the stable-tombstone epoch updates this fact in place; its ICs - /// validate the cached value slot against `TAG_HOLE` instead of relying on - /// token churn. - hole_count: u32, -} - struct ShapeTableInner { indices: crate::fast_hash::PtrHashMap, - /// #8125: `PtrHashMap`, not the SipHash default. - /// - /// This is the map `shape_descriptor_by_id` probes, and that probe is the - /// single hottest runtime lookup_ways in the object model: `object_is_regular` - /// runs it once per array element-shape test (3 M times on the `retain` - /// bench, 20 M on `churn`) and, since #8113 deleted - /// `ObjectHeader::field_count`, `object_live_slot_count` runs it on every - /// indexed field get/set. A symbol profile of the `shapes` bench - /// (`PERRY_DEBUG_SYMBOLS=1` + `sample`) put `RandomState::hash_one` at the - /// TOP of self time with `shape_descriptor_by_id` fourth — together ~22% of - /// the program, nearly all of it SipHash on a bare `u32`. - /// - /// The key is a ShapeId minted by this process from a monotonic counter. - /// No external input reaches it, so hash-flooding resistance buys nothing - /// here for the same reason it buys nothing on the pointer-keyed - /// registries `fast_hash` already serves. - /// BOXED (#8112): the collector enumerates `&mut record.keys` as an - /// ordinary GC slot, so the record's address must survive every descriptor - /// insertion that can happen while a budgeted scan holds it. A `Box` keeps - /// the payload put when the map rehashes; the map only ever moves the - /// eight-byte owning pointer. - descriptors: crate::fast_hash::PtrHashMap>, - /// Exact-facts reverse index. More than one id is legal when a worker - /// minted a local descriptor before a process-global module id arrived. + /// Exact-facts accelerator (#9706): the 64-bit fold of a descriptor's six + /// identity facts (`shapes_store::facts_key`) -> the ids carrying those + /// facts. Almost always one id; more than one is legal when a worker + /// minted a local descriptor before a process-global module id arrived, + /// or on a 64-bit collision — every hit re-validates the slab record, so + /// a collision costs a second record read, never a wrong answer. This + /// replaces the `ShapeFacts`-keyed map whose 32-byte key and 24-byte + /// `Vec` value made it the largest of the old reverse indices. /// - /// Deliberately NOT a `PtrHashMap`: `PtrHasher`'s `write_*` methods - /// OVERWRITE the accumulator instead of folding it, which is exactly right - /// for a single-word key and wrong for this five-field one — every - /// `ShapeFacts` would hash to its last field alone. + /// The key is a fold of internal shape state (never program input), so + /// `PtrHasher` (#8125) is the right hasher: the word is already mixed. + by_facts: crate::fast_hash::PtrHashMap, + /// Keys-array address -> every descriptor id currently indexed under it. + /// Same-address retirement, squeeze rekeying and GC relocation all work + /// per family instead of per descriptor, and the family is what the + /// metadata scan probes ONCE per keys array. /// - /// It is a `FastKeyHashMap` rather than the SipHash default, though: that - /// objection is to `PtrHasher` specifically, and leaving std's - /// `RandomState` here made this the only SipHash map left on the shape - /// path. Profiling `claude -p` showed `RandomState::hash_one` at 17 - /// self-samples inside `shapes::` alone (57 across the process) — pure - /// hashing overhead on a lookup_ways that runs on every descriptor - /// install/retire. + /// A family is small by construction for a SHARED keys array, which is + /// immutable (mutation forks a private clone): its descriptors differ only + /// in the birth bound, a semantic generation, the class kind, or a + /// tombstone count. An OWNED array grows in place, and every same-address + /// publish retires the predecessor it just superseded + /// (`retire_owned_shape_siblings`), so its family holds the current + /// version plus at most the cache-carried ones. Without that retirement a + /// dictionary built by ten thousand appends kept ten thousand prefix + /// descriptors alive until the array died. /// - /// `FastKeyHasher` is the right third option: it implements only `write`, - /// so every `write_u32` / `write_u64` from the derived `Hash` forwards - /// there and FOLDS with FNV-1a. All five fields reach the accumulator, - /// which is exactly the property `PtrHasher` lacks. The key is built from - /// internal shape state (never program input), so DoS-resistant hashing - /// buys nothing here — the same rationale already applied to the - /// descriptor side tables and to `indices` (#8125). - ids_by_facts: crate::fast_hash::FastKeyHashMap>, - /// Keys-array address -> every descriptor id that currently names it. - /// Same-address key-count retirement uses this index instead of scanning - /// every shape ever observed by the agent. Single-word key, so `PtrHasher` - /// (#8125). - ids_by_keys: crate::fast_hash::PtrHashMap>, -} - -/// Ways in the direct-mapped shape-descriptor lookup_ways cache. Power of two so -/// the index is a mask. 256 x 16 bytes = 4 KiB per thread. -const SHAPE_LOOKUP_WAYS: usize = 256; - -/// One way: `(shape_id, boxed record address, epoch)`. `shape_id == 0` is the -/// empty sentinel — a real id is always >= `SHAPE_ID_BASE`. -type ShapeLookupWay = std::cell::Cell<(u32, usize, u32)>; + /// Single-word key, so `PtrHasher` (#8125). + families: crate::fast_hash::PtrHashMap, +} + +impl ShapeTableInner { + #[inline] + fn family_push_back(&mut self, keys: u64, id: u32) { + self.families.entry(keys).or_default().push_back(id); + } + + #[inline] + fn family_push_front(&mut self, keys: u64, id: u32) { + self.families.entry(keys).or_default().push_front(id); + } + + /// Drop `id` from the family under `keys`, removing an emptied family. + #[inline] + fn family_remove(&mut self, keys: u64, id: u32) -> bool { + let Some(ids) = self.families.get_mut(&keys) else { + return false; + }; + let removed = ids.remove(id); + if ids.is_empty() { + self.families.remove(&keys); + } + removed + } + + #[inline] + fn facts_push_back(&mut self, facts: u64, id: u32) { + self.by_facts.entry(facts).or_default().push_back(id); + } + + #[inline] + fn facts_push_front(&mut self, facts: u64, id: u32) { + self.by_facts.entry(facts).or_default().push_front(id); + } + + /// Drop `id` from the accelerator bucket `facts`, removing it if emptied. + #[inline] + fn facts_remove(&mut self, facts: u64, id: u32) -> bool { + let Some(ids) = self.by_facts.get_mut(&facts) else { + return false; + }; + let removed = ids.remove(id); + if ids.is_empty() { + self.by_facts.remove(&facts); + } + removed + } +} pub(crate) struct ShapeTable { + /// The by-id store, outside the `RefCell` on purpose: `shape_descriptor_by_id` + /// is on the hot property path (profiling a dynamic-property loop put it + /// and `shape_descriptor_ensure_with_generation` at ~13% of main-thread + /// samples between them), and the collector reads and writes records + /// through raw pointers from inside walks that hold `inner` borrowed. + /// Records are cells; every access goes through a short-lived pointer. + slab: std::cell::UnsafeCell, inner: RefCell, - /// Direct-mapped cache in front of `inner.descriptors`. - /// - /// `shape_descriptor_by_id` is on the hot property path — profiling a - /// dynamic-property loop put it and `shape_descriptor_ensure_with_generation` - /// at ~13% of main-thread samples between them — and each call paid a - /// `RefCell` borrow plus a hash probe to reach a record whose address never - /// moves. `Box` is stable across rehash, so a way can hold - /// the record's address directly and a hit is: mask, compare, deref. - /// - /// Deliberately NOT holding a copy of the descriptor. The record is mutated - /// in place (`old_carrier`, `cache_carrier`, `keys` after evacuation), and a - /// cached copy would go quietly stale. Holding the address means a hit - /// always reads current data. - lookup_ways: [ShapeLookupWay; SHAPE_LOOKUP_WAYS], - /// Bumped whenever a record's ADDRESS can change under an id that is still - /// in use: removal, and the one insert path that can replace an existing id - /// with a fresh `Box`. A fresh-id insert cannot invalidate an existing way, - /// so it deliberately does not bump — otherwise ordinary shape creation - /// would flush the cache continuously. - lookup_epoch: std::cell::Cell, } impl ShapeTable { pub(crate) fn new() -> Self { ShapeTable { - lookup_ways: std::array::from_fn(|_| std::cell::Cell::new((0, 0, 0))), - lookup_epoch: std::cell::Cell::new(1), + slab: std::cell::UnsafeCell::new(ShapeSlab::new()), inner: RefCell::new(ShapeTableInner { indices: crate::fast_hash::new_ptr_hash_map(), - descriptors: crate::fast_hash::new_ptr_hash_map(), - ids_by_facts: crate::fast_hash::new_fast_key_hash_map(), - ids_by_keys: crate::fast_hash::new_ptr_hash_map(), + by_facts: crate::fast_hash::new_ptr_hash_map(), + families: crate::fast_hash::new_ptr_hash_map(), }), } } + + /// Shared view of the slab. Sound under the single-threaded agent + /// discipline the whole table relies on; mutation happens only through + /// [`Self::slab_mut`] in code that holds no other slab reference. + #[inline] + fn slab(&self) -> &ShapeSlab { + // SAFETY: see the field docs — one agent, one thread, no reference + // held across a call that can insert or remove. + unsafe { &*self.slab.get() } + } + + /// # Safety + /// + /// The caller holds no other reference into the slab for the duration. + #[inline] + #[allow(clippy::mut_from_ref)] + unsafe fn slab_mut(&self) -> &mut ShapeSlab { + &mut *self.slab.get() + } } /// #6759 C3c: ShapeIds live in their own u32 range, disjoint from every @@ -450,47 +455,59 @@ pub(crate) fn shape_descriptor_ensure_with_holes( object_kind: ShapeObjectKind, hole_count: u32, ) -> Result { - let keys_id = keys as usize; + let keys_id = keys as usize as u64; if keys_id == 0 && logical_key_count != 0 { return Err(ShapeDescriptorError::InvalidFacts); } - let facts = ShapeFacts { - keys: keys_id as u64, + let facts = shapes_store::facts_key( + keys_id, logical_key_count, live_inline_slot_count, semantic_generation, object_kind, hole_count, - }; - let mut inner = crate::state::state().shapes.inner.borrow_mut(); - if let Some(id) = inner - .ids_by_facts - .get(&facts) - .and_then(|ids| ids.first().copied()) - { - return Ok(id); + ); + let table = &crate::state::state().shapes; + let mut inner = table.inner.borrow_mut(); + if let Some(ids) = inner.by_facts.get(&facts) { + let slab = table.slab(); + for &id in ids.as_slice() { + let Some(record) = slab.record_ptr(id) else { + continue; + }; + // SAFETY: live slab record, read immediately. + let record = unsafe { *record }; + // The bucket is a 64-bit fold: validate the facts on every hit. + if record.has(RECORD_FLAG_FACTS_INDEXED) + && record.facts_match( + keys_id, + logical_key_count, + live_inline_slot_count, + semantic_generation, + object_kind, + hole_count, + ) + { + return Ok(id); + } + } } let id = alloc_shape_id().map_err(|_| ShapeDescriptorError::IdExhausted)?; - let descriptor = ShapeDescriptor { - keys: keys_id as u64, - indexed_keys: keys_id as u64, - facts_indexed: true, - record: 0, - old_carrier: false, - old_carrier_seen: false, - cache_carrier: false, + let record = ShapeRecord::new( + keys_id, logical_key_count, live_inline_slot_count, semantic_generation, object_kind, hole_count, - }; - // Publish by-id first, then the reverse accelerator. An ObjectHeader is + ); + // Publish by-id first, then the reverse accelerators. An ObjectHeader is // stamped only after this function returns, so a visible id always has a // complete descriptor. - inner.descriptors.insert(id, box_descriptor(descriptor)); - inner.ids_by_facts.entry(facts).or_default().push(id); - inner.ids_by_keys.entry(facts.keys).or_default().push(id); + // SAFETY: no slab reference is held; `slab()` above went out of scope. + unsafe { table.slab_mut().insert(id, record) }; + inner.facts_push_back(facts, id); + inner.family_push_back(keys_id, id); Ok(id) } @@ -547,34 +564,15 @@ pub(crate) fn shape_id_for_keys_ensure(keys: *const ArrayHeader, key_count: u32) /// One FIELD of a shape's descriptor, without lifting the whole record. /// /// [`shape_descriptor_by_id`] returns `ShapeDescriptor` **by value**, so every -/// caller that wants a single `u32` still copies the entire ~48-byte record -/// out of the table. That is most of them: `object_live_slot_count` — the slot -/// bound consulted on essentially every property read and write — throws away -/// all of it but `live_inline_slot_count`. -/// -/// This shares the way-cache probe with `shape_descriptor_by_id` and reads the -/// field through the record pointer instead. Same lookup, same validation, -/// four bytes instead of forty-eight. +/// caller that wants a single `u32` still copies the entire record out of the +/// table. That is most of them: `object_live_slot_count` — the slot bound +/// consulted on essentially every property read and write — throws away all +/// of it but `live_inline_slot_count`. #[inline] -fn shape_descriptor_field_by_id( - shape_id: u32, - read: impl Fn(&ShapeDescriptor) -> T, -) -> Option { - if !is_shape_id(shape_id) { - return None; - } - let table = &crate::state::state().shapes; - let epoch = table.lookup_epoch.get(); - let way = &table.lookup_ways[(shape_id as usize) & (SHAPE_LOOKUP_WAYS - 1)]; - let (cached_id, record, cached_epoch) = way.get(); - if cached_id == shape_id && cached_epoch == epoch && record != 0 { - // SAFETY: identical to `shape_descriptor_by_id`'s hit arm — the way is - // only filled from a live `Box` and the epoch is - // bumped whenever a record's address can change under an id still in - // use, so a matching epoch means this address is the table's record. - return Some(read(unsafe { &*(record as *const ShapeDescriptor) })); - } - shape_descriptor_by_id(shape_id).map(|d| read(&d)) +fn shape_descriptor_field_by_id(shape_id: u32, read: impl Fn(&ShapeRecord) -> T) -> Option { + let record = crate::state::state().shapes.slab().record_ptr(shape_id)?; + // SAFETY: `record_ptr` only returns a live slab record. + Some(read(unsafe { &*record })) } /// The live inline-slot bound for `shape_id`, without copying its descriptor. @@ -582,45 +580,16 @@ pub(crate) fn shape_live_inline_slot_count_by_id(shape_id: u32) -> Option { shape_descriptor_field_by_id(shape_id, |d| d.live_inline_slot_count) } -pub(crate) fn shape_descriptor_by_id(shape_id: u32) -> Option { - if !is_shape_id(shape_id) { - return None; - } - let table = &crate::state::state().shapes; - let epoch = table.lookup_epoch.get(); - let way = &table.lookup_ways[(shape_id as usize) & (SHAPE_LOOKUP_WAYS - 1)]; - - // Hit: mask, compare, deref. No RefCell borrow, no hash probe. - let (cached_id, record, cached_epoch) = way.get(); - if cached_id == shape_id && cached_epoch == epoch && record != 0 { - // SAFETY: the way is only filled from a live `Box`, - // and the epoch is bumped whenever a record's address can change under - // an id still in use, so a matching epoch means this address is the - // one the table holds for `shape_id`. - return Some(unsafe { *(record as *const ShapeDescriptor) }); - } - - let inner = table.inner.borrow(); - let record = inner.descriptors.get(&shape_id)?; - // `descriptor.record` is the box's own address (self-referential, #8112), - // so it is exactly the stable pointer the cache wants. - way.set((shape_id, record.record, epoch)); - Some(lift_descriptor(record)) -} - -/// Invalidate the whole lookup_ways cache. +/// The descriptor named by `shape_id`, or `None` when the id names no +/// descriptor in this agent. /// -/// Called where a record's ADDRESS can change while its id stays in use: -/// removal, and the insert path that can replace an existing id with a fresh -/// `Box`. A fresh-id insert deliberately does NOT bump — it cannot invalidate -/// an existing way, and bumping there would flush the cache on every shape -/// creation, which is precisely the workload that has one. +/// #9706: a slab probe — range check, chunk index, record — with no hash, +/// no `RefCell` borrow and no invalidation epoch. The direct-mapped way cache +/// that used to front the hash map is gone because the slab IS that cache: +/// a hit was "mask, compare, deref" and a probe is "shift, index, deref". #[inline] -fn invalidate_shape_lookup_cache() { - let table = &crate::state::state().shapes; - table - .lookup_epoch - .set(table.lookup_epoch.get().wrapping_add(1)); +pub(crate) fn shape_descriptor_by_id(shape_id: u32) -> Option { + crate::state::state().shapes.slab().lift(shape_id) } /// Immutable ordinary-vs-class fact with a pointer-free, per-agent direct @@ -636,31 +605,6 @@ pub(crate) fn shape_object_kind_by_id(shape_id: u32) -> Option Some(kind) } -/// Box a descriptor and stamp the record with its OWN address (#8112). -/// -/// Self-referential on purpose. The alternative — deriving the address in -/// `lift_descriptor` from the `&ShapeDescriptor` a shared table borrow yields — -/// would hand the collector a pointer with SHARED provenance and then write -/// through it. Taking it from the box while it is still uniquely owned keeps -/// the write well-formed. -fn box_descriptor(descriptor: ShapeDescriptor) -> Box { - let mut boxed = Box::new(descriptor); - boxed.record = std::ptr::addr_of_mut!(*boxed) as usize; - boxed -} - -/// Copy a boxed record out of the table (#8112). -/// -/// The copy's `keys` is a snapshot; `record` — stamped by [`box_descriptor`] — -/// names the one storage the collector rewrites. A caller that only reads facts -/// uses the snapshot; the GC hands `keys_slot()` to the slot visitor, so a -/// moved keys array lands back in the table with no second probe and no -/// write-back callback. -#[inline] -fn lift_descriptor(record: &ShapeDescriptor) -> ShapeDescriptor { - *record -} - /// Record that a shape is carried by an OLD-generation receiver. /// /// Called from the collector's slot visitor, which resolved the descriptor for @@ -672,10 +616,11 @@ fn lift_descriptor(record: &ShapeDescriptor) -> ShapeDescriptor { /// /// # Safety /// -/// `descriptor.record`, when non-zero, is the address of a live boxed record -/// owned by this agent's shape table. Records are freed only by -/// `prune_dead_shape_keys`, which runs at sweep — after every enumeration of -/// the cycle that produced this descriptor. +/// `descriptor.record`, when non-zero, is the address of a live slab record +/// owned by this agent's shape table. Records are retired only by the table's +/// own retirement paths, and their chunk is released by +/// `shrink_shape_tables` at the end of a major collection — after every +/// enumeration of the cycle that produced this descriptor. #[inline] pub(crate) unsafe fn note_old_generation_carrier(descriptor: Option) { let Some(descriptor) = descriptor else { @@ -684,10 +629,9 @@ pub(crate) unsafe fn note_old_generation_carrier(descriptor: Option) { if descriptor.record == 0 { return; } - let record = descriptor.record as *mut ShapeDescriptor; - // GC_STORE_AUDIT(POINTER_FREE): liveness bookkeeping byte, never a heap reference. - (*record).cache_carrier = true; + let record = descriptor.record as *mut ShapeRecord; + // GC_STORE_AUDIT(POINTER_FREE): liveness bookkeeping bit, never a heap reference. + (*record).set(RECORD_FLAG_CACHE_CARRIER, true); } /// The post-birth publication point for a ShapeId into a receiver's header @@ -747,10 +691,10 @@ pub(crate) unsafe fn stamp_object_shape_id_with_carrier_note( /// Clear every `cache_carrier` bit ahead of the post-full-trace recompute. pub(crate) fn clear_all_cache_carriers() { - let mut inner = crate::state::state().shapes.inner.borrow_mut(); - for record in inner.descriptors.values_mut() { - record.cache_carrier = false; - } + crate::state::state().shapes.slab().for_each(|_, record| { + // SAFETY: live slab record, single-threaded agent. + unsafe { (*record).set(RECORD_FLAG_CACHE_CARRIER, false) }; + }); } /// Recompute the old-carrier gate from the trace that just finished. @@ -761,11 +705,14 @@ pub(crate) fn clear_all_cache_carriers() { /// trace to shed a shape whose last old carrier died — the same rule that /// governs every other old-generation reclamation. pub(crate) fn rotate_old_carrier_epoch_after_full_trace() { - let mut inner = crate::state::state().shapes.inner.borrow_mut(); - for record in inner.descriptors.values_mut() { - record.old_carrier = record.old_carrier_seen; - record.old_carrier_seen = false; - } + crate::state::state().shapes.slab().for_each(|_, record| { + // SAFETY: live slab record, single-threaded agent. + unsafe { + let seen = (*record).has(RECORD_FLAG_OLD_CARRIER_SEEN); + (*record).set(RECORD_FLAG_OLD_CARRIER, seen); + (*record).set(RECORD_FLAG_OLD_CARRIER_SEEN, false); + } + }); } /// Mint (or retrieve) the ShapeId paired with canonical keys and equal @@ -1264,6 +1211,7 @@ pub(crate) unsafe fn publish_object_shape_from( // shared array must have cloned before push; otherwise siblings already // observe mutated bytes and no descriptor can make that state sound. let old_id = object_shape_stamp(obj); + let mut retire_owned_history = false; if let Some(old) = shape_descriptor_by_id(old_id) { // #9064: an owned ordinary receiver that already entered stable- // tombstone mode keeps its id across same-allocation tail appends and @@ -1301,7 +1249,14 @@ pub(crate) unsafe fn publish_object_shape_from( if shared { return old_id; } - retain_key_count_versions(keys as u64); + // An Array-subclass receiver is the one owner whose history IS + // reinstalled: `array_tail_transition` learns the (predecessor, + // successor) pair right after this publish returns and its + // reverse edge stamps the predecessor back on `pop`. That cache + // takes ownership through `cache_carrier`, but only once the + // learner has run, so the gate here is the receiver kind the + // learner is scoped to (`record_array_tail` in the append tail). + retire_owned_history = !crate::array::is_array_subclass_class_id((*obj).class_id); } } @@ -1332,10 +1287,55 @@ pub(crate) unsafe fn publish_object_shape_from( hole_count, )); stamp_object_shape_id_with_carrier_note(obj, id); + if retire_owned_history { + // #9706: the array is OWNED, so this receiver was the only carrier of + // every earlier same-address version, and the stamp above just + // superseded the last of them. Retire the growth history now rather + // than leaving one prefix descriptor per append alive until the + // array itself dies: on the compiled claude-code TUI that history was + // most of the descriptor table. Ordered after the stamp for the same + // reason as the tombstone publish (#9200) — the successor must be + // armed before the armed predecessor goes. + retire_owned_shape_siblings(keys as u64, id); + } debug_assert_object_shape_parity_for_keys(obj, keys); id } +/// Retire every descriptor of an OWNED keys array other than `keep`. +/// +/// Sound because `GC_FLAG_SHAPE_SHARED` is sticky: an array without it has +/// had exactly one owner for its whole life, and that owner now carries +/// `keep`. A stale IC token already misses on the stamp compare and +/// `shape_descriptor_by_id` of a retired id is `None`, so nothing can observe +/// the retired versions — with one exception: a descriptor an optimization +/// cache permanently owns (`cache_carrier`) may be reinstalled by that cache +/// while no live object carries it, so it stays. +fn retire_owned_shape_siblings(keys: u64, keep: u32) { + let table = &crate::state::state().shapes; + let mut inner = table.inner.borrow_mut(); + let stale: Vec = inner + .families + .get(&keys) + .map(|ids| { + ids.as_slice() + .iter() + .copied() + .filter(|&id| { + id != keep + && table + .slab() + .get(id) + .is_some_and(|record| !record.has(RECORD_FLAG_CACHE_CARRIER)) + }) + .collect() + }) + .unwrap_or_default(); + for id in stale { + remove_descriptor_and_reverse_indices(&mut inner, id); + } +} + /// Mint an exact successor for a descriptor/prototype semantic transition. /// The structural facts remain unchanged, but the process-unique generation /// prevents a cache trained before the transition from comparing equal after @@ -1433,37 +1433,36 @@ pub(crate) unsafe fn object_shape_id(obj: *const crate::object::ObjectHeader) -> .unwrap_or(0) } -fn retain_key_count_versions(keys: u64) { - let mut inner = crate::state::state().shapes.inner.borrow_mut(); - let Some(ids) = inner.ids_by_keys.remove(&keys) else { +/// Retire `id` from the by-id store and the family index (#9706). +/// +/// The family index is keyed by the address the descriptor was indexed under. +/// Between a live receiver rewriting the record's `keys` word and the +/// metadata scan moving the family, the two can name different addresses; a +/// removal in that window leaves the id in the stale family, where every +/// walk skips it (`record_ptr` is `None`) and the next scan drops it. +fn remove_descriptor_and_reverse_indices(inner: &mut ShapeTableInner, id: u32) { + let table = &crate::state::state().shapes; + let Some(indexed) = table.slab().get(id).map(|record| record.keys) else { return; }; - let mut current_ids = Vec::with_capacity(ids.len()); - for id in ids { - let Some(descriptor) = inner.descriptors.get(&id).map(|record| **record) else { - continue; - }; - debug_assert_eq!( - descriptor.keys, keys, - "keys index contains a foreign descriptor" - ); - if descriptor.keys != keys { - let correct_ids = inner.ids_by_keys.entry(descriptor.keys).or_default(); - if !correct_ids.contains(&id) { - correct_ids.push(id); - } - } else { - // Keep immutable historical descriptors addressable by id. An - // append under an owned keys allocation preserves the old prefix, - // and a stale cache/object may still carry either a local or an - // equivalent external id. Dead-key pruning reclaims the whole - // lineage once no live owner reaches the keys allocation. - current_ids.push(id); - } - } - if !current_ids.is_empty() { - inner.ids_by_keys.insert(keys, current_ids); + remove_descriptor_indexed_under(inner, id, indexed); +} + +/// [`remove_descriptor_and_reverse_indices`] for a caller that knows the +/// address the id is indexed under — the metadata scan, which retires a +/// family whose keys address was recycled while a live edge may already have +/// rewritten the records to the forwarded address. +fn remove_descriptor_indexed_under(inner: &mut ShapeTableInner, id: u32, indexed: u64) { + let table = &crate::state::state().shapes; + // SAFETY: no slab reference is held by the caller across this call. + let Some(record) = (unsafe { table.slab_mut().remove(id) }) else { + return; + }; + retire_cached_shape_object_kind(id); + if record.has(RECORD_FLAG_FACTS_INDEXED) { + inner.facts_remove(record.facts_key_with_keys(indexed), id); } + inner.family_remove(indexed, id); } /// Exact-facts test for a candidate id against the receiver's authoritative @@ -1760,7 +1759,8 @@ fn shape_keys_address_is_recycled(addr: usize) -> bool { /// descriptor removed here cannot be named by a live object. Correctness fails /// closed on a missing lookup_ways, independently of pruning. pub(crate) fn prune_dead_shape_keys(is_dead_owner: &dyn Fn(usize) -> bool) { - let mut inner = crate::state::state().shapes.inner.borrow_mut(); + let table = &crate::state::state().shapes; + let mut inner = table.inner.borrow_mut(); // A shape keys entry is keyed by the address of its keys array — a // `GC_TYPE_ARRAY` (or `GC_TYPE_LAZY_ARRAY`). When the keys array dies // and the arena recycles its address for a different object type @@ -1776,132 +1776,128 @@ pub(crate) fn prune_dead_shape_keys(is_dead_owner: &dyn Fn(usize) -> bool) { !is_dead_owner(*keys_id) && !shape_keys_address_is_recycled(*keys_id) }); } - let stale: Vec = inner - .descriptors - .iter() - .filter_map(|(&id, descriptor)| { - let keys = descriptor.keys as usize; - (is_dead_owner(descriptor.keys as usize) || shape_keys_address_is_recycled(keys)) - .then_some(id) - }) - .collect(); - if !stale.is_empty() { - for id in stale { - remove_descriptor_and_reverse_indices(&mut inner, id); + let mut stale: Vec = Vec::new(); + table.slab().for_each(|id, record| { + // SAFETY: live slab record, read immediately. + let descriptor = unsafe { *record }; + let keys = descriptor.keys as usize; + if is_dead_owner(descriptor.keys as usize) || shape_keys_address_is_recycled(keys) { + stale.push(id); } + }); + for id in stale { + remove_descriptor_and_reverse_indices(&mut inner, id); } } -crate::perry_thread_local! { - /// Scratch memo for [`scan_shape_table_rekey_mut`]'s per-address probe, - /// reused across collections so the scan allocates nothing. - /// `PtrHashMap`, NOT std's SipHash default: perf on the dynamic-property - /// benchmark put `RandomState::hash_one::<&(usize, bool)>` at **7.0% of - /// total samples** — pure hashing overhead inside the GC scan this memo - /// exists to make cheaper. The key is folded to one word (`addr ^ carrier` - /// in bit 0; addresses are >= 8-aligned so bit 0 is free), which is the - /// single-word shape `PtrHasher` is built for. - static PROBE_MEMO: std::cell::RefCell> = - std::cell::RefCell::new(crate::fast_hash::new_ptr_hash_map()); -} - /// Metadata-only forwarding repair for the weak descriptor table and /// pointer-keyed slot indices. Mark/copy mode does not root anything; live /// object scans provide descriptor reachability, and post-copy rewrite follows /// only forwarding records those live edges already created. +/// +/// #9706: the walk is per keys-array FAMILY, not per descriptor. Every +/// descriptor of a family shares one keys address, so one probe answers for +/// all of them — the per-address memo the descriptor walk used to keep +/// (`PROBE_MEMO`, a persistent map sized to every distinct address in the +/// table) is now simply the family index itself. A family is probed with the +/// MARKING visit when any of its descriptors is a carrier, which is exactly +/// the rooting duty the #8112 gate assigns: the keys array must survive while +/// an old receiver or a cache still names one of its shapes. pub(crate) fn scan_shape_table_rekey_mut(visitor: &mut crate::gc::RuntimeRootVisitor<'_>) { - let mut inner = crate::state::state().shapes.inner.borrow_mut(); - // TEMPORARY (#6759 phase 2 measurement): this scanner is 53.3% of all - // root-scanner time on `claude -p`. Report what it is actually walking so - // the fix targets the real term instead of a guess. - let mut descriptor_rekeys: Vec = Vec::new(); - let mut dead_descriptor_ids: Vec = Vec::new(); - - // #6759 phase 2: probe each distinct keys-array address ONCE. - // - // The per-descriptor probe is the expensive part of this scanner — 89.6% of - // its time is the rewrite phase, and each probe runs - // `classify_heap_space_in_range` and then reads the GC header at a - // scattered address (two likely cache misses). Shapes share keys arrays at - // a measured, stable 2.5:1, so the unmemoised loop paid that ~2.5 times per - // distinct address. - // - // Memoising is sound by construction: the same addresses are visited, just - // once each, and forwarding is a pure function of the address within one - // pass. Carriers take a different visit (`visit_usize_slot`, which MARKS in - // mark modes) than non-carriers, so the carrier flag is part of the key — - // otherwise a non-carrier hit could satisfy a carrier's marking duty. - // Reused across collections rather than allocated per scan: at ~300k - // entries a fresh map every GC is exactly the kind of churn the - // memory-parity work is trying to remove. `clear()` keeps the capacity. - PROBE_MEMO.with(|memo| { - let mut probe_memo = memo.borrow_mut(); - probe_memo.clear(); - - for (id, descriptor) in inner.descriptors.iter_mut() { - let mut addr = descriptor.keys as usize; - // #8112 ephemeron gate. A shape with an OLD carrier is rooted here: - // the minor that has to keep its keys array alive never enumerates the - // object that carries it. A shape with only young carriers is NOT — - // those receivers are traced, and each one emits the edge itself, so - // rooting them from the table would make every keys array ever minted - // immortal and turn `prune_dead_shape_keys`'s "is the keys array - // dead?" into a question it asks of itself. - let is_carrier = descriptor.old_carrier || descriptor.cache_carrier; - // Addresses are 8-aligned, so bit 0 is free to carry the carrier - // duty (carriers use a MARKING visit; the answers must not mix). - let memo_key = addr | usize::from(is_carrier); - if let Some(&(prev_moved, prev_addr)) = probe_memo.get(&memo_key) { - // Already probed this exact (address, carrier-duty) pair in this - // pass — reuse the answer instead of paying the walk again. - let moved = prev_moved; - addr = prev_addr; - record_shape_scan_outcome( - visitor, - id, - descriptor, - addr, - moved, - &mut dead_descriptor_ids, - &mut descriptor_rekeys, - ); - continue; + let table = &crate::state::state().shapes; + let mut inner = table.inner.borrow_mut(); + let rewrite_phase = visitor.is_metadata_rewrite_phase(); + let mut moved_families: Vec<(u64, u64)> = Vec::new(); + let mut dead_descriptor_ids: Vec<(u32, u64)> = Vec::new(); + // The shared slab view is scoped to the probe loop: retirement below + // takes the slab mutably, and nothing after the loop may still hold it. + let slab = table.slab(); + for (&indexed, ids) in inner.families.iter() { + if indexed == 0 { + // Keyless shapes hold no edge. + continue; + } + // #8112 ephemeron gate. A shape with an OLD carrier is rooted here: + // the minor that has to keep its keys array alive never enumerates the + // object that carries it. A shape with only young carriers is NOT — + // those receivers are traced, and each one emits the edge itself, so + // rooting them from the table would make every keys array ever minted + // immortal and turn `prune_dead_shape_keys`'s "is the keys array + // dead?" into a question it asks of itself. + // + // One descriptor stands for the family: a carrier if the family has + // one (its duty is the strongest), else any present member. + let mut descriptor: Option = None; + for &id in ids.as_slice() { + if let Some(lifted) = slab.lift(id) { + if lifted.old_carrier || lifted.cache_carrier { + descriptor = Some(lifted); + break; + } + descriptor.get_or_insert(lifted); } - let probe_addr = addr; - // Written out rather than reusing `is_carrier` on purpose: the - // census gate (`scripts/shape_descriptor_census.py`) pins this exact - // two-armed expression so that a sabotage which widens the gate or - // swaps the arms is red, and its own self-test sabotages this very - // literal. `is_carrier` above is the same predicate, and is what - // keys the memo. - let moved = if descriptor.old_carrier || descriptor.cache_carrier { - visitor.visit_usize_slot(&mut addr) - } else { - visitor.visit_metadata_usize_slot(&mut addr) - }; - probe_memo.insert(probe_addr | usize::from(is_carrier), (moved, addr)); - record_shape_scan_outcome( - visitor, - id, - descriptor, - addr, - moved, - &mut dead_descriptor_ids, - &mut descriptor_rekeys, - ); } - }); - // Remove descriptors whose keys array was recycled. - if !dead_descriptor_ids.is_empty() { - for id in &dead_descriptor_ids { - remove_descriptor_and_reverse_indices(&mut inner, *id); + let Some(descriptor) = descriptor else { + // Every id retired under a stale address; the family is empty. + moved_families.push((indexed, 0)); + continue; + }; + let mut addr = indexed as usize; + // The census gate (`scripts/shape_descriptor_census.py`) pins this + // exact two-armed expression so that a sabotage which widens the gate + // or swaps the arms is red, and its own self-test sabotages this very + // literal. + let moved = if descriptor.old_carrier || descriptor.cache_carrier { + visitor.visit_usize_slot(&mut addr) + } else { + visitor.visit_metadata_usize_slot(&mut addr) + }; + // Validate the POST-visit address. A stale shape key can follow the + // forwarding record of the non-array tenant that recycled its address; + // checking only an unmoved old address misses that case. + if rewrite_phase && shape_keys_address_is_recycled(addr) { + dead_descriptor_ids.extend(ids.as_slice().iter().map(|&id| (id, indexed))); + continue; + } + if moved { + for &id in ids.as_slice() { + if let Some(record) = slab.record_ptr(id) { + // SAFETY: live slab record, single-threaded agent. A live + // receiver's edge may already have written the same + // forwarded address here; the store is idempotent. + unsafe { (*record).keys = addr as u64 }; + } + } } + if addr as u64 != indexed { + moved_families.push((indexed, addr as u64)); + } + } + for (id, indexed) in dead_descriptor_ids { + remove_descriptor_indexed_under(&mut inner, id, indexed); } - for id in descriptor_rekeys { - sync_descriptor_reverse_indices(&mut inner, id); + for (old, new) in moved_families { + let Some(ids) = inner.families.remove(&old) else { + continue; + }; + if new == 0 { + continue; + } + for &id in ids.as_slice() { + let Some(record) = table.slab().get(id) else { + continue; + }; + // The accelerator was keyed with the OLD address; the other five + // facts never change under the collector. + if record.has(RECORD_FLAG_FACTS_INDEXED) { + inner.facts_remove(record.facts_key_with_keys(old), id); + inner.facts_push_back(record.facts_key_with_keys(new), id); + } + inner.family_push_back(new, id); + } } - if !visitor.is_metadata_rewrite_phase() || inner.indices.is_empty() { + if !rewrite_phase || inner.indices.is_empty() { return; } let moved: Vec<(usize, usize)> = inner @@ -1973,47 +1969,43 @@ mod shapes_tests; /// `prune_dead_shape_keys` had already discarded. /// /// Called once per MAJOR collection, right after the prune, where a rehash is -/// already amortized against a full heap walk. `shrink_to(2 * len)` rather -/// than `shrink_to_fit()` keeps one doubling of headroom so a table that is -/// merely oscillating does not re-grow on the next insert. +/// already amortized against a full heap walk. #9706: the by-id store is a +/// slab now, so this also releases its all-dead chunks; the family and slot +/// index maps are shrunk to `len + len / 4`, one growth step of headroom. pub(crate) fn shrink_shape_tables() { - fn worth_shrinking(len: usize, capacity: usize, _: &T) -> bool { - // Only when the table is holding at least 1 MB-ish of slack and is - // less than half used; a small or well-packed table is left alone. + fn worth_shrinking(len: usize, capacity: usize) -> bool { + // Only when the table is holding real slack and is less than half + // used; a small or well-packed table is left alone. capacity > 4096 && capacity > len.saturating_mul(2) } - let mut inner = crate::state::state().shapes.inner.borrow_mut(); - if worth_shrinking(inner.descriptors.len(), inner.descriptors.capacity(), &()) { - let target = inner.descriptors.len().saturating_mul(2); - inner.descriptors.shrink_to(target); - } - if worth_shrinking(inner.indices.len(), inner.indices.capacity(), &()) { - let target = inner.indices.len().saturating_mul(2); + let table = &crate::state::state().shapes; + let mut inner = table.inner.borrow_mut(); + if worth_shrinking(inner.indices.len(), inner.indices.capacity()) { + let target = inner.indices.len() + inner.indices.len() / 4; inner.indices.shrink_to(target); } - if worth_shrinking(inner.ids_by_facts.len(), inner.ids_by_facts.capacity(), &()) { - let target = inner.ids_by_facts.len().saturating_mul(2); - inner.ids_by_facts.shrink_to(target); + if worth_shrinking(inner.by_facts.len(), inner.by_facts.capacity()) { + let target = inner.by_facts.len() + inner.by_facts.len() / 4; + inner.by_facts.shrink_to(target); } - if worth_shrinking(inner.ids_by_keys.len(), inner.ids_by_keys.capacity(), &()) { - let target = inner.ids_by_keys.len().saturating_mul(2); - inner.ids_by_keys.shrink_to(target); + if worth_shrinking(inner.families.len(), inner.families.capacity()) { + let target = inner.families.len() + inner.families.len() / 4; + inner.families.shrink_to(target); } + // SAFETY: the prune that precedes this call holds no slab reference, and + // neither does anything else while the major collection owns the agent. + unsafe { table.slab_mut().release_empty_chunks() }; } -/// `PERRY_GC_CENSUS`: the shape table's four maps plus the boxed -/// descriptors and per-shape key indices they own. +/// `PERRY_GC_CENSUS`: the by-id slab, the per-shape key indices, the +/// exact-facts accelerator and the keys-address family index. pub(crate) fn shape_table_census() -> Vec { - use crate::gc::census::{hash_table_bytes, map_bytes, vec_bytes}; + use crate::gc::census::{hash_table_bytes, map_bytes}; let table = &crate::state::state().shapes; let inner = table.inner.borrow(); + let slab = table.slab(); let mut rows = Vec::new(); - rows.push(( - "shapes.descriptors", - inner.descriptors.len(), - map_bytes(&inner.descriptors) - + inner.descriptors.len() * (std::mem::size_of::() + 16), - )); + rows.push(("shapes.descriptors", slab.len(), slab.estimated_bytes())); let index_inner: usize = inner .indices .values() @@ -2024,17 +2016,87 @@ pub(crate) fn shape_table_census() -> Vec { inner.indices.len(), map_bytes(&inner.indices) + index_inner, )); - let facts_inner: usize = inner.ids_by_facts.values().map(vec_bytes).sum(); + let facts_inner: usize = inner.by_facts.values().map(IdList::heap_bytes).sum(); rows.push(( - "shapes.ids_by_facts", - inner.ids_by_facts.len(), - map_bytes(&inner.ids_by_facts) + facts_inner, + "shapes.by_facts", + inner.by_facts.len(), + map_bytes(&inner.by_facts) + facts_inner, )); - let keys_inner: usize = inner.ids_by_keys.values().map(vec_bytes).sum(); + let families_inner: usize = inner.families.values().map(IdList::heap_bytes).sum(); rows.push(( - "shapes.ids_by_keys", - inner.ids_by_keys.len(), - map_bytes(&inner.ids_by_keys) + keys_inner, + "shapes.families", + inner.families.len(), + map_bytes(&inner.families) + families_inner, )); + // Ids ever minted by this process: the slab is indexed by id, so the gap + // between this and `shapes.descriptors` is what chunk release reclaims. + let minted = SHAPE_ID_NEXT.load(std::sync::atomic::Ordering::Relaxed) - SHAPE_ID_BASE; + rows.push(("shapes.ids_minted(process)", minted as usize, 0)); rows } + +/// `PERRY_GC_CENSUS`: how the descriptor population relates to the live heap +/// (#9706). `live_ids` is the sorted, deduplicated set of ShapeIds stamped on +/// live shaped objects, collected by the census walk. +/// +/// * `shapes.descriptors.carried` — descriptors some live object is stamped +/// with: the population V8's "object shape" bucket corresponds to. +/// * `shapes.descriptors.uncarried` — descriptors no live object carries: +/// transition history a cache may reinstall (`cache_carrier`), versions +/// kept for an old receiver since the last full trace, and shapes whose +/// keys array is still alive on some other descriptor. +/// * `shapes.families.multi` — keys arrays with more than one descriptor, +/// and the descriptors they hold beyond the first: the duplication the +/// family walk pays for. +pub(crate) fn shape_table_liveness_census( + live_ids: &[u32], +) -> Vec { + let table = &crate::state::state().shapes; + let inner = table.inner.borrow(); + let slab = table.slab(); + let mut carried = 0usize; + let mut uncarried = 0usize; + let mut uncarried_cache = 0usize; + let mut uncarried_old = 0usize; + slab.for_each(|id, record| { + if live_ids.binary_search(&id).is_ok() { + carried += 1; + return; + } + uncarried += 1; + // SAFETY: live slab record, read immediately. + let record = unsafe { *record }; + if record.has(RECORD_FLAG_CACHE_CARRIER) { + uncarried_cache += 1; + } else if record.has(RECORD_FLAG_OLD_CARRIER) { + uncarried_old += 1; + } + }); + let mut multi_families = 0usize; + let mut multi_extra = 0usize; + let mut largest = 0usize; + for ids in inner.families.values() { + let n = ids.len(); + largest = largest.max(n); + if n > 1 { + multi_families += 1; + multi_extra += n - 1; + } + } + vec![ + ("shapes.descriptors.carried(live objects)", carried, 0), + ("shapes.descriptors.uncarried", uncarried, 0), + ( + "shapes.descriptors.uncarried.cache_carrier", + uncarried_cache, + 0, + ), + ("shapes.descriptors.uncarried.old_carrier", uncarried_old, 0), + ( + "shapes.families.multi(families,extra descriptors)", + multi_families, + multi_extra, + ), + ("shapes.families.largest", largest, 0), + ] +} diff --git a/crates/perry-runtime/src/object/shapes_reverse_indices.rs b/crates/perry-runtime/src/object/shapes_reverse_indices.rs deleted file mode 100644 index 59b475fc16..0000000000 --- a/crates/perry-runtime/src/object/shapes_reverse_indices.rs +++ /dev/null @@ -1,119 +0,0 @@ -//! Reverse-index maintenance for the shape descriptor table. -//! -//! Descriptors are indexed both by exact semantic facts and by their keys -//! allocation. Stable-tombstone epochs deliberately leave exact-facts -//! interning while retaining the keys index, so all insertion, relocation, -//! and retirement bookkeeping lives together here. - -use super::{ - invalidate_shape_lookup_cache, retire_cached_shape_object_kind, ShapeDescriptor, ShapeFacts, - ShapeTableInner, -}; - -#[inline] -pub(super) fn descriptor_facts(descriptor: ShapeDescriptor) -> ShapeFacts { - ShapeFacts { - keys: descriptor.keys, - logical_key_count: descriptor.logical_key_count, - live_inline_slot_count: descriptor.live_inline_slot_count, - semantic_generation: descriptor.semantic_generation, - object_kind: descriptor.object_kind, - hole_count: descriptor.hole_count, - } -} - -fn descriptor_facts_with_keys(descriptor: ShapeDescriptor, keys: u64) -> ShapeFacts { - ShapeFacts { - keys, - logical_key_count: descriptor.logical_key_count, - live_inline_slot_count: descriptor.live_inline_slot_count, - semantic_generation: descriptor.semantic_generation, - object_kind: descriptor.object_kind, - hole_count: descriptor.hole_count, - } -} - -pub(super) fn remove_descriptor_id_from_facts_index( - inner: &mut ShapeTableInner, - facts: ShapeFacts, - id: u32, -) { - let remove_entry = if let Some(ids) = inner.ids_by_facts.get_mut(&facts) { - if let Ok(index) = ids.binary_search(&id) { - ids.remove(index); - } else { - ids.retain(|&candidate| candidate != id); - } - ids.is_empty() - } else { - false - }; - if remove_entry { - inner.ids_by_facts.remove(&facts); - } -} - -fn remove_descriptor_id_from_keys_index(inner: &mut ShapeTableInner, keys: u64, id: u32) { - let remove_entry = if let Some(ids) = inner.ids_by_keys.get_mut(&keys) { - if let Ok(index) = ids.binary_search(&id) { - ids.remove(index); - } else { - ids.retain(|&candidate| candidate != id); - } - ids.is_empty() - } else { - false - }; - if remove_entry { - inner.ids_by_keys.remove(&keys); - } -} - -#[inline] -pub(super) fn insert_descriptor_id_sorted(ids: &mut Vec, id: u32) { - if let Err(index) = ids.binary_search(&id) { - ids.insert(index, id); - } -} - -/// Repair one descriptor after its collector-owned `keys` slot moved. -/// -/// `indexed_keys` records the address under which the id is still indexed, so -/// this is O(population sharing the old/new facts) rather than O(all shapes). -pub(super) fn sync_descriptor_reverse_indices(inner: &mut ShapeTableInner, id: u32) { - let Some(descriptor) = inner.descriptors.get(&id).map(|record| **record) else { - return; - }; - if descriptor.indexed_keys == descriptor.keys { - return; - } - - let old_facts = descriptor_facts_with_keys(descriptor, descriptor.indexed_keys); - let new_facts = descriptor_facts(descriptor); - if descriptor.facts_indexed { - remove_descriptor_id_from_facts_index(inner, old_facts, id); - } - remove_descriptor_id_from_keys_index(inner, descriptor.indexed_keys, id); - if descriptor.facts_indexed { - insert_descriptor_id_sorted(inner.ids_by_facts.entry(new_facts).or_default(), id); - } - insert_descriptor_id_sorted(inner.ids_by_keys.entry(descriptor.keys).or_default(), id); - if let Some(record) = inner.descriptors.get_mut(&id) { - record.indexed_keys = descriptor.keys; - } -} - -pub(super) fn remove_descriptor_and_reverse_indices(inner: &mut ShapeTableInner, id: u32) { - // The record's box is about to be dropped; any cached way naming it must - // stop matching. - invalidate_shape_lookup_cache(); - let Some(descriptor) = inner.descriptors.remove(&id) else { - return; - }; - retire_cached_shape_object_kind(id); - let facts = descriptor_facts_with_keys(*descriptor, descriptor.indexed_keys); - if descriptor.facts_indexed { - remove_descriptor_id_from_facts_index(inner, facts, id); - } - remove_descriptor_id_from_keys_index(inner, descriptor.indexed_keys, id); -} diff --git a/crates/perry-runtime/src/object/shapes_slot_list.rs b/crates/perry-runtime/src/object/shapes_slot_list.rs index 04fe163224..0523e4df43 100644 --- a/crates/perry-runtime/src/object/shapes_slot_list.rs +++ b/crates/perry-runtime/src/object/shapes_slot_list.rs @@ -79,38 +79,7 @@ impl SlotList { } } -use super::{shape_keys_address_is_recycled, ShapeDescriptor}; - -/// Per-descriptor bookkeeping after its keys address has been probed. -/// -/// Lifted out of `scan_shape_table_rekey_mut`'s loop so the memoised path and -/// the probing path cannot drift apart — the probe is what is deduplicated, -/// never the bookkeeping, which still runs once per descriptor. -#[inline] -pub(crate) fn record_shape_scan_outcome( - visitor: &mut crate::gc::RuntimeRootVisitor<'_>, - id: &u32, - descriptor: &mut ShapeDescriptor, - addr: usize, - moved: bool, - dead_descriptor_ids: &mut Vec, - descriptor_rekeys: &mut Vec, -) { - // Validate the POST-visit address. A stale shape key can follow the - // forwarding record of the non-array tenant that recycled its address; - // checking only an unmoved old address misses that case. - if visitor.is_metadata_rewrite_phase() && shape_keys_address_is_recycled(addr) { - dead_descriptor_ids.push(*id); - } else if moved { - descriptor.keys = addr as u64; - } - // A live-object edge can rewrite the boxed `keys` slot before this metadata - // pass. Comparing against the address represented in the reverse maps - // catches both that ordering and a move observed here. - if descriptor.keys != descriptor.indexed_keys { - descriptor_rekeys.push(*id); - } -} +use super::shapes_store::{ShapeRecord, RECORD_FLAG_FACTS_INDEXED}; /// Shift a key index in place after an IN-PLACE delete. /// @@ -233,9 +202,15 @@ pub(crate) unsafe fn retire_owned_shape_history( let keys_addr = keys as u64; let mut inner = crate::state::state().shapes.inner.borrow_mut(); let stale: Vec = inner - .ids_by_keys + .families .get(&keys_addr) - .map(|ids| ids.iter().copied().filter(|&id| id != current).collect()) + .map(|ids| { + ids.as_slice() + .iter() + .copied() + .filter(|&id| id != current) + .collect() + }) .unwrap_or_default(); for id in stale { super::remove_descriptor_and_reverse_indices(&mut inner, id); @@ -253,10 +228,10 @@ pub(crate) unsafe fn retire_owned_shape_history( /// mint-then-stamp path. /// /// A mutable private epoch must not participate in exact-facts interning. -/// Detach it from `ids_by_facts` on entry and leave it in `ids_by_keys`, which -/// keeps GC relocation and squeeze-time retirement exact without hashing six -/// changing facts on every delete and re-add. The boxed descriptor address is -/// stable, so the direct lookup cache observes updated counts immediately. +/// Detach it on entry and leave it in the keys-address family, which keeps GC +/// relocation and squeeze-time retirement exact without re-indexing six +/// changing facts on every delete and re-add. The slab record address is +/// stable, so every later lookup observes the updated counts immediately. pub(crate) unsafe fn try_update_stable_tombstone_shape( obj: *mut crate::object::ObjectHeader, keys: *mut super::ArrayHeader, @@ -278,12 +253,14 @@ pub(crate) unsafe fn try_update_stable_tombstone_shape( return None; } - let mut inner = crate::state::state().shapes.inner.borrow_mut(); - super::sync_descriptor_reverse_indices(&mut inner, id); - let current = **inner.descriptors.get(&id)?; + let table = &crate::state::state().shapes; + let record = table.slab().record_ptr(id)?; + // SAFETY: live slab record, single-threaded agent; read then written + // through the same pointer with nothing else holding a reference. + let current = unsafe { *record }; // A stable id may never silently retarget its collector-owned keys edge. // Array growth that reallocates falls back to a fresh descriptor. - if current.keys != keys as u64 || current.object_kind != super::ShapeObjectKind::Ordinary { + if current.keys != keys as u64 || current.object_kind() != super::ShapeObjectKind::Ordinary { return None; } if current.logical_key_count == logical_key_count @@ -293,21 +270,21 @@ pub(crate) unsafe fn try_update_stable_tombstone_shape( return Some(id); } - if current.facts_indexed { - let old_facts = super::descriptor_facts(current); - super::remove_descriptor_id_from_facts_index(&mut inner, old_facts, id); + // Detach from exact-facts interning, so a mutable private epoch is never + // handed to a second receiver. It stays in the family for GC relocation + // and squeeze retirement. The accelerator was keyed with the address the + // record is indexed under, which is `keys` (the caller proved the edge + // did not move). + if current.has(RECORD_FLAG_FACTS_INDEXED) { + let mut inner = table.inner.borrow_mut(); + inner.facts_remove(current.facts_key_with_keys(keys as u64), id); + } + unsafe { + (*record).logical_key_count = logical_key_count; + (*record).live_inline_slot_count = live_inline_slot_count; + (*record).hole_count = hole_count; + (*record).set(RECORD_FLAG_FACTS_INDEXED, false); } - { - let record = inner - .descriptors - .get_mut(&id) - .expect("stable tombstone descriptor disappeared while borrowed"); - record.logical_key_count = logical_key_count; - record.live_inline_slot_count = live_inline_slot_count; - record.hole_count = hole_count; - record.facts_indexed = false; - } - drop(inner); super::debug_assert_object_shape_parity(obj); Some(id) } @@ -316,10 +293,9 @@ pub(crate) unsafe fn try_update_stable_tombstone_shape( /// record address returned by `shape_descriptor_by_id`. /// /// The first stable mutation must use `try_update_stable_tombstone_shape` to -/// detach exact-facts interning, and a collector-relocated keys edge must use -/// it to repair the reverse index. Between those events the record address is -/// stable, its mutable epoch is deliberately absent from `ids_by_facts`, and -/// no table borrow or hash lookup is needed for a counter-only update. +/// detach exact-facts interning. Between those events the record address is +/// stable, its mutable epoch is deliberately invisible to interning, and no +/// table borrow is needed for a counter-only update. pub(crate) unsafe fn try_update_stable_tombstone_shape_cached( obj: *mut crate::object::ObjectHeader, current: super::ShapeDescriptor, @@ -341,12 +317,17 @@ pub(crate) unsafe fn try_update_stable_tombstone_shape_cached( return None; } - let record = &mut *(current.record as *mut super::ShapeDescriptor); - if record.record != current.record - || record.keys != current.keys - || record.indexed_keys != record.keys - || record.facts_indexed - || record.object_kind != super::ShapeObjectKind::Ordinary + // The caller's copy must still name the live record of THIS id: a + // retired id resolves to nothing, and a record reused under another id + // (never — ids are not recycled) would resolve to a different address. + let live = crate::state::state().shapes.slab().record_ptr(id)?; + if live as usize != current.record { + return None; + } + let record = &mut *live; + if record.keys != current.keys + || record.has(RECORD_FLAG_FACTS_INDEXED) + || record.object_kind() != super::ShapeObjectKind::Ordinary { return None; } @@ -357,11 +338,11 @@ pub(crate) unsafe fn try_update_stable_tombstone_shape_cached( Some(id) } -/// Retire the token of a detached private epoch while reusing its boxed -/// descriptor record. This is the stable-tombstone squeeze counterpart to a -/// full mint: generated caches must observe a new id after slots are -/// compacted, but no exact-facts interning or new descriptor allocation is -/// needed for a record that cannot be shared by another receiver. +/// Retire the token of a detached private epoch while reusing its descriptor +/// record. This is the stable-tombstone squeeze counterpart to a full mint: +/// generated caches must observe a new id after slots are compacted, but no +/// exact-facts interning is needed for a record that cannot be shared by +/// another receiver. pub(crate) unsafe fn rekey_stable_tombstone_shape_after_squeeze( obj: *mut crate::object::ObjectHeader, current: super::ShapeDescriptor, @@ -388,36 +369,41 @@ pub(crate) unsafe fn rekey_stable_tombstone_shape_after_squeeze( super::shape_id_exhausted_abort(); } - let mut inner = crate::state::state().shapes.inner.borrow_mut(); - super::sync_descriptor_reverse_indices(&mut inner, old_id); - let live = **inner.descriptors.get(&old_id)?; - if live.record != current.record - || live.keys != current.keys - || live.indexed_keys != live.keys - || live.facts_indexed - || live.object_kind != super::ShapeObjectKind::Ordinary + let table = &crate::state::state().shapes; + let mut inner = table.inner.borrow_mut(); + let live_ptr = table.slab().record_ptr(old_id)?; + if live_ptr as usize != current.record { + return None; + } + // SAFETY: live slab record, read immediately. + let live = unsafe { *live_ptr }; + if live.keys != current.keys + || live.has(RECORD_FLAG_FACTS_INDEXED) + || live.object_kind() != super::ShapeObjectKind::Ordinary { return None; } - super::invalidate_shape_lookup_cache(); - let mut record = inner.descriptors.remove(&old_id)?; + // Move the record to its new id in place of the old one. The family entry + // is replaced where it stands; a family still keyed under a stale address + // (a rewrite the metadata scan has not yet repaired) simply gains the new + // id under the current one and sheds the old id on that scan. + // SAFETY: no slab reference is held across these two calls. + let mut record = unsafe { table.slab_mut().remove(old_id)? }; record.logical_key_count = logical_key_count; record.live_inline_slot_count = live_inline_slot_count; record.semantic_generation = generation; record.hole_count = hole_count; - if let Some(ids) = inner.ids_by_keys.get_mut(&record.indexed_keys) { - if let Some(pos) = ids.iter().position(|&id| id == old_id) { - ids[pos] = new_id; - ids.sort_unstable(); - } else { - super::insert_descriptor_id_sorted(ids, new_id); - } - } else { - inner.ids_by_keys.insert(record.indexed_keys, vec![new_id]); + super::retire_cached_shape_object_kind(old_id); + unsafe { table.slab_mut().insert(new_id, record) }; + let replaced = inner + .families + .get_mut(&record.keys) + .is_some_and(|ids| ids.replace(old_id, new_id)); + if !replaced { + inner.family_push_back(record.keys, new_id); } inner.indices.remove(&(record.keys as usize)); - inner.descriptors.insert(new_id, record); drop(inner); // #9200: the funnel re-arms the preserved record for a non-nursery @@ -498,9 +484,15 @@ pub(crate) unsafe fn publish_object_shape_holes( // (53.6 s → 25.1 s on the churn benchmark) but ids still accumulated // one per iteration from the append publish. let stale: Vec = inner - .ids_by_keys + .families .get(&(current.keys)) - .map(|ids| ids.iter().copied().filter(|&other| other != id).collect()) + .map(|ids| { + ids.as_slice() + .iter() + .copied() + .filter(|&other| other != id) + .collect() + }) .unwrap_or_default(); for other in stale { super::remove_descriptor_and_reverse_indices(&mut inner, other); @@ -524,42 +516,37 @@ pub(super) fn install_external_shape_id( if !super::is_shape_id(id) || (keys.is_null() && logical_key_count != 0) { return false; } - let descriptor = super::ShapeDescriptor { - keys: keys as usize as u64, - indexed_keys: keys as usize as u64, - facts_indexed: true, - record: 0, - old_carrier: false, - old_carrier_seen: false, - cache_carrier: false, + let keys = keys as usize as u64; + let record = ShapeRecord::new( + keys, logical_key_count, live_inline_slot_count, - semantic_generation: 0, - object_kind: super::ShapeObjectKind::Ordinary, - hole_count: 0, - }; - let facts = super::descriptor_facts(descriptor); - let mut inner = crate::state::state().shapes.inner.borrow_mut(); - if let Some(existing) = inner.descriptors.get(&id) { - return **existing == descriptor; + 0, + super::ShapeObjectKind::Ordinary, + 0, + ); + let table = &crate::state::state().shapes; + let mut inner = table.inner.borrow_mut(); + if let Some(existing) = table.slab().get(id) { + return existing.facts_match( + keys, + logical_key_count, + live_inline_slot_count, + 0, + super::ShapeObjectKind::Ordinary, + 0, + ); } // A worker can have minted an equivalent local descriptor before module // initialization installs the process-global codegen id. Keep both id // descriptors valid for already-published objects and make the external - // id canonical for subsequent births in this agent. - // - // This is the one insert that can REPLACE a live id with a fresh box, so - // the lookup_ways cache has to be invalidated here (the fresh-id insert in - // `intern_shape_descriptor` cannot, and deliberately does not). - super::invalidate_shape_lookup_cache(); - inner - .descriptors - .insert(id, super::box_descriptor(descriptor)); - // An equivalent local descriptor can predate module initialization. Keep - // both reverse-index entries and prefer the external id for subsequent - // births in this agent; already-published local ids remain resolvable. - inner.ids_by_facts.entry(facts).or_default().insert(0, id); - super::insert_descriptor_id_sorted(inner.ids_by_keys.entry(descriptor.keys).or_default(), id); + // id canonical for subsequent births in this agent: it goes to the FRONT + // of its accelerator bucket, which is the order exact-facts interning + // walks. + // SAFETY: no slab reference is held; `slab().get` above returned a copy. + unsafe { table.slab_mut().insert(id, record) }; + inner.facts_push_front(record.facts_key_with_keys(keys), id); + inner.family_push_front(keys, id); true } @@ -573,9 +560,10 @@ pub(super) fn install_external_shape_id( /// descriptor holding the edge, the slot visitor writes the record directly /// and there is nothing left to reconcile. /// -/// The returned address belongs to a BOXED record, so it is stable across -/// descriptor insertion; only `prune_dead_shape_keys` frees one, and that runs -/// at sweep, after every enumeration of the cycle that produced it. +/// The returned address belongs to a slab record, so it is stable across +/// descriptor insertion; a record is only cleared by the table's own +/// retirement paths, and its chunk released at the end of a major +/// collection, after every enumeration of the cycle that produced it. #[cfg(test)] #[inline] pub(crate) fn shape_descriptor_keys_slot(shape_id: u32) -> Option<*mut u64> { @@ -584,11 +572,9 @@ pub(crate) fn shape_descriptor_keys_slot(shape_id: u32) -> Option<*mut u64> { } crate::state::state() .shapes - .inner - .borrow_mut() - .descriptors - .get_mut(&shape_id) - .map(|record| std::ptr::addr_of_mut!(record.keys)) + .slab() + .record_ptr(shape_id) + .map(|record| record as *mut u64) } /// Is `slot` the shared `keys` word of `shape_id`'s descriptor record? @@ -604,15 +590,14 @@ pub(crate) fn shape_id_owns_keys_slot(shape_id: u32, slot: *mut u64) -> bool { if !super::is_shape_id(shape_id) { return false; } - // Immutable borrow on purpose: this runs inside collector walks, and a - // `borrow_mut` here would make the predicate itself a re-entrancy hazard. + // No table borrow at all: this runs inside collector walks, and a + // `RefCell` borrow here would make the predicate itself a re-entrancy + // hazard. The slab is read through a raw pointer. crate::state::state() .shapes - .inner - .borrow() - .descriptors - .get(&shape_id) - .is_some_and(|record| std::ptr::addr_of!(record.keys) as *mut u64 == slot) + .slab() + .record_ptr(shape_id) + .is_some_and(|record| record as *mut u64 == slot) } #[cfg(test)] diff --git a/crates/perry-runtime/src/object/shapes_store.rs b/crates/perry-runtime/src/object/shapes_store.rs new file mode 100644 index 0000000000..b305741f0f --- /dev/null +++ b/crates/perry-runtime/src/object/shapes_store.rs @@ -0,0 +1,776 @@ +//! Storage for the agent-local shape descriptor table (#9706). +//! +//! Two structures, both owned by [`super::ShapeTable`]: +//! +//! * [`ShapeSlab`] — the by-id store. A ShapeId is a process-global monotonic +//! counter (`SHAPE_ID_BASE + n`), so `n` indexes a chunked slab directly: no +//! hash, no per-record heap allocation, and a record address that never +//! moves for the record's lifetime — the property the collector relies on +//! when it enumerates a descriptor's `keys` word as a rewritable slot +//! (#8112) and retains that address across budgeted resumptions. Chunks +//! (32 records) hang off a two-level page directory, are allocated lazily +//! (a worker's ids interleave with the main thread's), and an all-dead chunk +//! is released by [`ShapeSlab::release_empty_chunks`] at the same cadence as +//! the reverse-index shrink (once per major collection). +//! +//! * [`IdList`] — the value of the per-keys-address family index +//! (`ShapeTableInner::families`). One entry per keys array names every +//! descriptor id currently indexed under that address. Exact-facts interning +//! walks the family and compares the remaining facts against the slab +//! record, which is what lets the table drop the second, facts-keyed reverse +//! map it used to carry: a family is small by construction — a SHARED keys +//! array is immutable, so its descriptors differ only in the birth bound or +//! a semantic generation, and an OWNED array retires its growth history +//! eagerly (`retire_owned_shape_siblings`). +//! +//! Measured on the compiled claude-code TUI at idle (`PERRY_GC_CENSUS`), the +//! previous layout — a `PtrHashMap>` beside two +//! `Vec`-valued reverse maps — cost ~330 bytes per live descriptor: +//! a 56-byte record in a 64-byte allocator bin, a 16-byte map entry at 25% +//! load after `shrink_to(2 * len)`, a 57-byte facts-map bucket, and a 33-byte +//! keys-map bucket, plus a 16-byte `Vec` buffer per reverse entry. A packed +//! 32-byte slab record with one 24-byte family bucket per keys array is the +//! same information at a fraction of the bytes. + +use super::{ShapeDescriptor, ShapeObjectKind, SHAPE_ID_BASE}; +use std::cell::UnsafeCell; + +pub(super) const RECORD_FLAG_PRESENT: u8 = 1 << 0; +pub(super) const RECORD_FLAG_FACTS_INDEXED: u8 = 1 << 1; +pub(super) const RECORD_FLAG_OLD_CARRIER: u8 = 1 << 2; +pub(super) const RECORD_FLAG_OLD_CARRIER_SEEN: u8 = 1 << 3; +pub(super) const RECORD_FLAG_CACHE_CARRIER: u8 = 1 << 4; +pub(super) const RECORD_FLAG_KIND_CLASS: u8 = 1 << 5; + +/// The table-owned record of one ShapeId. `keys` is first and 8-aligned: it +/// is the word the collector marks through and rewrites in place. +#[repr(C)] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct ShapeRecord { + /// Raw ArrayHeader address in Perry's fixed-width heap-word ABI (0 for a + /// keyless shape). + pub(super) keys: u64, + pub(super) semantic_generation: u64, + pub(super) logical_key_count: u32, + pub(super) live_inline_slot_count: u32, + pub(super) hole_count: u32, + pub(super) flags: u8, + _pad: [u8; 3], +} + +const _: () = assert!(std::mem::size_of::() == 32); +const _: () = assert!(std::mem::align_of::() == 8); + +impl ShapeRecord { + const EMPTY: ShapeRecord = ShapeRecord { + keys: 0, + semantic_generation: 0, + logical_key_count: 0, + live_inline_slot_count: 0, + hole_count: 0, + flags: 0, + _pad: [0; 3], + }; + + #[inline] + pub(super) fn present(&self) -> bool { + self.flags & RECORD_FLAG_PRESENT != 0 + } + + #[inline] + pub(super) fn has(&self, flag: u8) -> bool { + self.flags & flag != 0 + } + + #[inline] + pub(super) fn set(&mut self, flag: u8, on: bool) { + if on { + self.flags |= flag; + } else { + self.flags &= !flag; + } + } + + #[inline] + pub(super) fn object_kind(&self) -> ShapeObjectKind { + if self.has(RECORD_FLAG_KIND_CLASS) { + ShapeObjectKind::Class + } else { + ShapeObjectKind::Ordinary + } + } + + /// A fresh, facts-indexed record with every liveness bit clear. + pub(super) fn new( + keys: u64, + logical_key_count: u32, + live_inline_slot_count: u32, + semantic_generation: u64, + object_kind: ShapeObjectKind, + hole_count: u32, + ) -> ShapeRecord { + let mut flags = RECORD_FLAG_PRESENT | RECORD_FLAG_FACTS_INDEXED; + if object_kind == ShapeObjectKind::Class { + flags |= RECORD_FLAG_KIND_CLASS; + } + ShapeRecord { + keys, + semantic_generation, + logical_key_count, + live_inline_slot_count, + hole_count, + flags, + _pad: [0; 3], + } + } + + /// Exact-facts identity test (#8067): keys edge, both counts, generation, + /// kind, tombstones. Liveness bits and the facts-indexed bit are storage + /// state, never identity. + #[inline] + pub(super) fn facts_match( + &self, + keys: u64, + logical_key_count: u32, + live_inline_slot_count: u32, + semantic_generation: u64, + object_kind: ShapeObjectKind, + hole_count: u32, + ) -> bool { + self.keys == keys + && self.logical_key_count == logical_key_count + && self.live_inline_slot_count == live_inline_slot_count + && self.semantic_generation == semantic_generation + && self.hole_count == hole_count + && self.object_kind() == object_kind + } + + /// The 64-bit fold of the six identity facts, with `keys` supplied by + /// the caller: the collector rewrites a record's `keys` in place, so the + /// address the record was INDEXED under (its family key) is what the + /// exact-facts accelerator must be probed with until the metadata scan + /// re-indexes it. + #[inline] + pub(super) fn facts_key_with_keys(&self, keys: u64) -> u64 { + facts_key( + keys, + self.logical_key_count, + self.live_inline_slot_count, + self.semantic_generation, + self.object_kind(), + self.hole_count, + ) + } + + /// Copy the record out as the by-value [`ShapeDescriptor`] the rest of the + /// runtime consumes. `record` is the slab address of THIS record, which is + /// what `keys_slot()` and the tombstone fast paths hand back to the table. + #[inline] + pub(super) fn lift(&self, record: *mut ShapeRecord) -> ShapeDescriptor { + ShapeDescriptor { + keys: self.keys, + record: record as usize, + old_carrier: self.has(RECORD_FLAG_OLD_CARRIER), + cache_carrier: self.has(RECORD_FLAG_CACHE_CARRIER), + logical_key_count: self.logical_key_count, + live_inline_slot_count: self.live_inline_slot_count, + semantic_generation: self.semantic_generation, + object_kind: self.object_kind(), + hole_count: self.hole_count, + } + } +} + +/// FNV-1a fold of the six identity facts into the single word the +/// exact-facts accelerator is keyed by. Every field reaches the accumulator +/// (fold, never overwrite — the property `PtrHasher` lacks and the reason the +/// old `ShapeFacts` map could not use it); a 64-bit collision between two +/// live shapes is resolved by the per-hit `facts_match` on the record, so a +/// collision only costs a second record read, never a wrong answer. +#[inline] +pub(super) fn facts_key( + keys: u64, + logical_key_count: u32, + live_inline_slot_count: u32, + semantic_generation: u64, + object_kind: ShapeObjectKind, + hole_count: u32, +) -> u64 { + const FNV_OFFSET_BASIS: u64 = 0xcbf2_9ce4_8422_2325; + const FNV_PRIME: u64 = 0x0000_0100_0000_01b3; + let fold = |acc: u64, word: u64| (acc ^ word).wrapping_mul(FNV_PRIME); + let mut h = fold(FNV_OFFSET_BASIS, keys); + h = fold(h, u64::from(logical_key_count)); + h = fold(h, u64::from(live_inline_slot_count)); + h = fold(h, semantic_generation); + h = fold(h, u64::from(hole_count)); + h = fold(h, u64::from(object_kind == ShapeObjectKind::Class)); + // Final avalanche: FNV keeps most of its entropy in the high bits and + // hashbrown's probe sequence starts from the LOW bits. + h ^ (h >> 32) +} + +/// Records per chunk. Ids are minted far faster than they survive — the +/// compiled claude-code TUI mints ~1.05 M ShapeIds during startup and keeps +/// ~44 k, scattered over the whole range — so a chunk is deliberately SMALL +/// (32 records, 1 KB): an all-dead chunk is released whole, and the smaller +/// the chunk the less of a survivor's neighbourhood it drags along. Measured +/// on that TUI, 256-record chunks held 7.15 MB for those 44 k records and +/// 32-record chunks 4.0 MB. +const CHUNK_SHIFT: usize = 5; +const CHUNK_LEN: usize = 1 << CHUNK_SHIFT; +const CHUNK_MASK: usize = CHUNK_LEN - 1; + +/// Chunk pointers per directory page. The directory is two-level so its +/// size follows the LIVE id range, not the minted one: a long-running server +/// minting a billion ids over its life would otherwise carry a flat +/// `Vec>` of 250 MB at 32 records per chunk. A page is 8 KB and +/// covers 32 K ids; a page whose chunks have all been released is dropped. +const PAGE_SHIFT: usize = 10; +const PAGE_LEN: usize = 1 << PAGE_SHIFT; +const PAGE_MASK: usize = PAGE_LEN - 1; + +/// One lazily allocated run of `CHUNK_LEN` consecutive ids. The cells give +/// the table interior mutability through a shared slab reference: the +/// collector writes liveness bits and the `keys` word through raw record +/// pointers while other code holds only copies (`ShapeDescriptor`). +type Chunk = Box<[UnsafeCell; CHUNK_LEN]>; + +/// One directory page: `PAGE_LEN` chunk slots. +type Page = Box<[Option; PAGE_LEN]>; + +fn new_chunk() -> Chunk { + let mut v: Vec> = Vec::with_capacity(CHUNK_LEN); + v.resize_with(CHUNK_LEN, || UnsafeCell::new(ShapeRecord::EMPTY)); + // Exact length by construction; the conversion moves the allocation. + v.into_boxed_slice() + .try_into() + .unwrap_or_else(|_| unreachable!("chunk vector has CHUNK_LEN cells")) +} + +fn new_page() -> Page { + let mut v: Vec> = Vec::with_capacity(PAGE_LEN); + v.resize_with(PAGE_LEN, || None); + v.into_boxed_slice() + .try_into() + .unwrap_or_else(|_| unreachable!("page vector has PAGE_LEN slots")) +} + +/// The by-id descriptor store. See the module docs. +pub(crate) struct ShapeSlab { + pages: Vec>, + /// Present records. + len: usize, +} + +impl ShapeSlab { + pub(super) fn new() -> Self { + ShapeSlab { + pages: Vec::new(), + len: 0, + } + } + + #[inline] + fn index_of(id: u32) -> Option { + super::is_shape_id(id).then(|| (id - SHAPE_ID_BASE) as usize) + } + + #[inline] + fn id_of(index: usize) -> u32 { + SHAPE_ID_BASE + index as u32 + } + + /// `(page, chunk within page, record within chunk)` of a slab index. + #[inline] + fn split(index: usize) -> (usize, usize, usize) { + ( + index >> (CHUNK_SHIFT + PAGE_SHIFT), + (index >> CHUNK_SHIFT) & PAGE_MASK, + index & CHUNK_MASK, + ) + } + + /// Present records. + #[inline] + pub(super) fn len(&self) -> usize { + self.len + } + + /// The record for `id`, or `None` when the id names no descriptor in this + /// agent. The pointer stays valid until the record is removed; a removal + /// only ever happens through the table's own retirement paths. + #[inline] + pub(super) fn record_ptr(&self, id: u32) -> Option<*mut ShapeRecord> { + let index = Self::index_of(id)?; + let (page, chunk, slot) = Self::split(index); + let chunk = self.pages.get(page)?.as_ref()?[chunk].as_ref()?; + let cell = chunk[slot].get(); + // SAFETY: the cell belongs to a live chunk owned by this slab; reads + // and writes are serialized by the single-threaded agent discipline + // every other shape-table access already relies on. + if unsafe { (*cell).present() } { + Some(cell) + } else { + None + } + } + + /// A copy of the record for `id`. + #[inline] + pub(super) fn get(&self, id: u32) -> Option { + // SAFETY: `record_ptr` only returns a cell of a live chunk. + self.record_ptr(id).map(|p| unsafe { *p }) + } + + /// Lift `id` to the by-value descriptor. + #[inline] + pub(super) fn lift(&self, id: u32) -> Option { + // SAFETY: as in `get`. + self.record_ptr(id).map(|p| unsafe { (*p).lift(p) }) + } + + /// Install `record` under `id`, allocating the page and chunk on first + /// touch. Returns the record it replaced, if the id was already present. + pub(super) fn insert(&mut self, id: u32, mut record: ShapeRecord) -> Option { + let index = Self::index_of(id).expect("ShapeSlab::insert: id outside the ShapeId range"); + record.flags |= RECORD_FLAG_PRESENT; + let (page, chunk, slot) = Self::split(index); + if page >= self.pages.len() { + self.pages.resize_with(page + 1, || None); + } + let page = self.pages[page].get_or_insert_with(new_page); + let chunk = page[chunk].get_or_insert_with(new_chunk); + let cell = chunk[slot].get_mut(); + let previous = cell.present().then_some(*cell); + *cell = record; + if previous.is_none() { + self.len += 1; + } + previous + } + + /// Clear the record under `id`, returning it if it was present. + pub(super) fn remove(&mut self, id: u32) -> Option { + let index = Self::index_of(id)?; + let (page, chunk, slot) = Self::split(index); + let chunk = self.pages.get_mut(page)?.as_mut()?[chunk].as_mut()?; + let cell = chunk[slot].get_mut(); + if !cell.present() { + return None; + } + let previous = *cell; + *cell = ShapeRecord::EMPTY; + self.len -= 1; + Some(previous) + } + + /// Visit every present record in id order. The callback may write + /// through the record pointer; it must not insert or remove. + pub(super) fn for_each(&self, mut f: impl FnMut(u32, *mut ShapeRecord)) { + for (page_index, page) in self.pages.iter().enumerate() { + let Some(page) = page else { + continue; + }; + for (chunk_index, chunk) in page.iter().enumerate() { + let Some(chunk) = chunk else { + continue; + }; + let base = ((page_index << PAGE_SHIFT) | chunk_index) << CHUNK_SHIFT; + for (slot, cell) in chunk.iter().enumerate() { + let p = cell.get(); + // SAFETY: live chunk, single-threaded agent. + if unsafe { (*p).present() } { + f(Self::id_of(base | slot), p); + } + } + } + } + } + + /// Every present id, in id order. + #[cfg(test)] + pub(super) fn ids(&self) -> Vec { + let mut ids = Vec::with_capacity(self.len); + self.for_each(|id, _| ids.push(id)); + ids + } + + /// Free chunks that hold no present record, and pages that hold no + /// chunk. Called once per major collection, after dead-key pruning: + /// retirement is monotonic in id order for the common workload, so the + /// oldest chunks empty first. + pub(super) fn release_empty_chunks(&mut self) { + for page in self.pages.iter_mut() { + let Some(chunks) = page.as_mut() else { + continue; + }; + let mut live_chunks = 0usize; + for chunk in chunks.iter_mut() { + let empty = chunk + .as_ref() + .is_some_and(|c| c.iter().all(|cell| !unsafe { (*cell.get()).present() })); + if empty { + *chunk = None; + } + if chunk.is_some() { + live_chunks += 1; + } + } + if live_chunks == 0 { + *page = None; + } + } + while self.pages.last().is_some_and(Option::is_none) { + self.pages.pop(); + } + self.pages.shrink_to_fit(); + } + + #[cfg(test)] + pub(super) fn clear(&mut self) { + self.pages.clear(); + self.len = 0; + } + + /// Bytes held: the page directory, every allocated page and every + /// allocated chunk. + pub(super) fn estimated_bytes(&self) -> usize { + let mut pages = 0usize; + let mut chunks = 0usize; + for page in self.pages.iter().flatten() { + pages += 1; + chunks += page.iter().filter(|c| c.is_some()).count(); + } + self.pages.capacity() * std::mem::size_of::>() + + pages * PAGE_LEN * std::mem::size_of::>() + + chunks * CHUNK_LEN * std::mem::size_of::() + } + + /// Allocated chunks (diagnostics). + #[cfg(test)] + pub(super) fn chunk_count(&self) -> usize { + self.pages + .iter() + .flatten() + .map(|page| page.iter().filter(|c| c.is_some()).count()) + .sum() + } +} + +/// A compact list of descriptor ids: up to three inline, then a spilled +/// `Vec`. Sized so a family-index bucket is `(u64, IdList)` = 24 bytes. +/// +/// Order is meaningful: [`IdList::push_front`] is how an installed +/// process-global id becomes the canonical answer for exact-facts interning +/// ahead of an equivalent local id (`install_external_shape_id`). +#[derive(Clone, Debug)] +pub(super) enum IdList { + Inline { + len: u8, + ids: [u32; 3], + }, + // The `Box` is the point: an inline `Vec` is 24 bytes and would make every + // bucket 32; the spill is the rare case, so its extra indirection is + // cheaper than eight bytes on every family. + #[allow(clippy::box_collection)] + Spill(Box>), +} + +const _: () = assert!(std::mem::size_of::() == 16); + +impl Default for IdList { + fn default() -> Self { + IdList::Inline { + len: 0, + ids: [0; 3], + } + } +} + +impl IdList { + #[inline] + pub(super) fn as_slice(&self) -> &[u32] { + match self { + IdList::Inline { len, ids } => &ids[..*len as usize], + IdList::Spill(v) => v.as_slice(), + } + } + + #[inline] + pub(super) fn len(&self) -> usize { + self.as_slice().len() + } + + #[inline] + pub(super) fn is_empty(&self) -> bool { + self.len() == 0 + } + + #[inline] + pub(super) fn contains(&self, id: u32) -> bool { + self.as_slice().contains(&id) + } + + fn spill(&mut self) -> &mut Vec { + if let IdList::Inline { len, ids } = self { + let v = ids[..*len as usize].to_vec(); + *self = IdList::Spill(Box::new(v)); + } + match self { + IdList::Spill(v) => v, + IdList::Inline { .. } => unreachable!(), + } + } + + /// Append `id` unless already present. + pub(super) fn push_back(&mut self, id: u32) { + if self.contains(id) { + return; + } + match self { + IdList::Inline { len, ids } if (*len as usize) < ids.len() => { + ids[*len as usize] = id; + *len += 1; + } + _ => self.spill().push(id), + } + } + + /// Prepend `id` unless already present. + pub(super) fn push_front(&mut self, id: u32) { + if self.contains(id) { + return; + } + match self { + IdList::Inline { len, ids } if (*len as usize) < ids.len() => { + ids.copy_within(0..*len as usize, 1); + ids[0] = id; + *len += 1; + } + _ => self.spill().insert(0, id), + } + } + + /// Drop `id` if present; returns whether it was. + pub(super) fn remove(&mut self, id: u32) -> bool { + match self { + IdList::Inline { len, ids } => { + let n = *len as usize; + let Some(pos) = ids[..n].iter().position(|&x| x == id) else { + return false; + }; + ids.copy_within(pos + 1..n, pos); + ids[n - 1] = 0; + *len -= 1; + true + } + IdList::Spill(v) => { + let Some(pos) = v.iter().position(|&x| x == id) else { + return false; + }; + v.remove(pos); + true + } + } + } + + /// Replace `old` with `new` in place (keeps its position); returns + /// whether `old` was present. + pub(super) fn replace(&mut self, old: u32, new: u32) -> bool { + match self { + IdList::Inline { len, ids } => { + let n = *len as usize; + match ids[..n].iter().position(|&x| x == old) { + Some(pos) => { + ids[pos] = new; + true + } + None => false, + } + } + IdList::Spill(v) => match v.iter().position(|&x| x == old) { + Some(pos) => { + v[pos] = new; + true + } + None => false, + }, + } + } + + /// Bytes held outside the containing bucket. + pub(super) fn heap_bytes(&self) -> usize { + match self { + IdList::Inline { .. } => 0, + IdList::Spill(v) => std::mem::size_of::>() + v.capacity() * 4, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn slab_records_are_addressed_by_id_and_keep_their_address() { + let mut slab = ShapeSlab::new(); + let id_a = SHAPE_ID_BASE + 5; + let id_b = SHAPE_ID_BASE + 5 + (CHUNK_LEN * PAGE_LEN) as u32 * 3; + assert_eq!(slab.get(id_a), None); + assert_eq!( + slab.insert( + id_a, + ShapeRecord::new(0x1000, 1, 1, 0, ShapeObjectKind::Ordinary, 0) + ) + .map(|r| r.keys), + None + ); + let a_ptr = slab.record_ptr(id_a).expect("present"); + // A later insert into another chunk must not move the first record. + slab.insert( + id_b, + ShapeRecord::new(0x2000, 2, 2, 7, ShapeObjectKind::Class, 1), + ); + assert_eq!(slab.record_ptr(id_a), Some(a_ptr)); + assert_eq!(slab.len(), 2); + assert_eq!(slab.chunk_count(), 2); + let b = slab.get(id_b).unwrap(); + assert_eq!(b.object_kind(), ShapeObjectKind::Class); + assert_eq!(b.semantic_generation, 7); + assert_eq!(b.hole_count, 1); + assert!(b.facts_match(0x2000, 2, 2, 7, ShapeObjectKind::Class, 1)); + assert!(!b.facts_match(0x2000, 2, 2, 7, ShapeObjectKind::Ordinary, 1)); + // Ids outside the range and never-minted ids resolve to nothing. + assert_eq!(slab.get(0), None); + assert_eq!(slab.get(SHAPE_ID_BASE + 6), None); + assert_eq!(slab.get(super::super::SHAPE_ID_END - 1), None); + assert_eq!(slab.ids(), vec![id_a, id_b]); + // Removal clears the record and, once a chunk is empty, the chunk. + assert_eq!(slab.remove(id_a).map(|r| r.keys), Some(0x1000)); + assert_eq!(slab.remove(id_a), None); + assert_eq!(slab.len(), 1); + slab.release_empty_chunks(); + assert_eq!(slab.chunk_count(), 1); + assert_eq!(slab.get(id_b).map(|r| r.keys), Some(0x2000)); + assert_eq!(slab.remove(id_b).map(|r| r.keys), Some(0x2000)); + slab.release_empty_chunks(); + assert_eq!(slab.chunk_count(), 0); + assert_eq!(slab.estimated_bytes(), 0); + } + + #[test] + fn lifted_descriptor_mirrors_the_record_and_names_its_address() { + let mut slab = ShapeSlab::new(); + let id = SHAPE_ID_BASE + 42; + let mut record = ShapeRecord::new(0x3000, 4, 6, 9, ShapeObjectKind::Ordinary, 2); + record.set(RECORD_FLAG_OLD_CARRIER, true); + record.set(RECORD_FLAG_CACHE_CARRIER, true); + record.set(RECORD_FLAG_FACTS_INDEXED, false); + slab.insert(id, record); + let ptr = slab.record_ptr(id).unwrap(); + let lifted = slab.lift(id).unwrap(); + assert_eq!(lifted.record, ptr as usize); + assert_eq!(lifted.keys, 0x3000); + assert_eq!(lifted.logical_key_count, 4); + assert_eq!(lifted.live_inline_slot_count, 6); + assert_eq!(lifted.semantic_generation, 9); + assert_eq!(lifted.hole_count, 2); + assert!(lifted.old_carrier); + assert!(lifted.cache_carrier); + assert!(!slab.get(id).unwrap().has(RECORD_FLAG_FACTS_INDEXED)); + assert_eq!(lifted.keys_slot(), Some(ptr as *mut u64)); + // Writing through the slot is what an evacuating visitor does. + unsafe { *lifted.keys_slot().unwrap() = 0x4000 }; + assert_eq!(slab.get(id).unwrap().keys, 0x4000); + } + + /// Varying any ONE fact must change the key: a fold that dropped a field + /// would send two different shapes to one bucket for every value of it. + #[test] + fn facts_key_folds_every_field() { + let base = facts_key(0x1111_2222_3333_4444, 7, 3, 9, ShapeObjectKind::Ordinary, 0); + let variants = [ + ( + "keys", + facts_key(0x5555_6666_7777_8888, 7, 3, 9, ShapeObjectKind::Ordinary, 0), + ), + ( + "logical", + facts_key(0x1111_2222_3333_4444, 8, 3, 9, ShapeObjectKind::Ordinary, 0), + ), + ( + "live", + facts_key(0x1111_2222_3333_4444, 7, 4, 9, ShapeObjectKind::Ordinary, 0), + ), + ( + "generation", + facts_key( + 0x1111_2222_3333_4444, + 7, + 3, + 10, + ShapeObjectKind::Ordinary, + 0, + ), + ), + ( + "kind", + facts_key(0x1111_2222_3333_4444, 7, 3, 9, ShapeObjectKind::Class, 0), + ), + ( + "holes", + facts_key(0x1111_2222_3333_4444, 7, 3, 9, ShapeObjectKind::Ordinary, 1), + ), + ]; + for (field, key) in variants { + assert_ne!( + key, base, + "changing `{field}` alone must change the facts key" + ); + } + let record = ShapeRecord::new(0x1111_2222_3333_4444, 7, 3, 9, ShapeObjectKind::Ordinary, 0); + assert_eq!(record.facts_key_with_keys(0x1111_2222_3333_4444), base); + assert_eq!( + record.facts_key_with_keys(0x5555_6666_7777_8888), + variants[0].1 + ); + } + + #[test] + fn id_list_keeps_order_across_the_inline_to_spill_boundary() { + let mut list = IdList::default(); + assert!(list.is_empty()); + list.push_back(2); + list.push_back(3); + list.push_front(1); + list.push_back(2); // duplicate ignored + assert_eq!(list.as_slice(), &[1, 2, 3]); + assert!(matches!(list, IdList::Inline { .. })); + list.push_back(4); + assert!(matches!(list, IdList::Spill(_))); + assert_eq!(list.as_slice(), &[1, 2, 3, 4]); + list.push_front(0); + assert_eq!(list.as_slice(), &[0, 1, 2, 3, 4]); + assert!(list.remove(2)); + assert!(!list.remove(2)); + assert_eq!(list.as_slice(), &[0, 1, 3, 4]); + assert!(list.replace(3, 30)); + assert!(!list.replace(3, 300)); + assert_eq!(list.as_slice(), &[0, 1, 30, 4]); + assert!(list.heap_bytes() >= 4 * 4); + + let mut inline = IdList::default(); + inline.push_back(7); + inline.push_back(8); + inline.push_back(9); + assert!(inline.remove(8)); + assert_eq!(inline.as_slice(), &[7, 9]); + assert!(inline.replace(9, 10)); + assert_eq!(inline.as_slice(), &[7, 10]); + assert!(inline.remove(7)); + assert!(inline.remove(10)); + assert!(inline.is_empty()); + assert_eq!(inline.heap_bytes(), 0); + } +} diff --git a/crates/perry-runtime/src/object/shapes_test_support.rs b/crates/perry-runtime/src/object/shapes_test_support.rs index f012820f5b..311bb982e5 100644 --- a/crates/perry-runtime/src/object/shapes_test_support.rs +++ b/crates/perry-runtime/src/object/shapes_test_support.rs @@ -1,8 +1,8 @@ //! Test-only shape-table helpers, in a sibling file. //! -//! Extracted from `shapes.rs` to keep it under the repo's 2000-line cap; the -//! lookup-way cache pushed it over. A child module, so these keep reaching the -//! parent's private items through `super::`. Moved verbatim. +//! Extracted from `shapes.rs` to keep it under the repo's 2000-line cap. A +//! child module, so these keep reaching the parent's private items through +//! `super::`. use super::*; @@ -73,25 +73,18 @@ pub(crate) fn test_shape_entry_exists(keys_id: usize) -> bool { #[cfg(test)] pub(crate) fn test_shape_descriptor_count() -> usize { - crate::state::state() - .shapes - .inner - .borrow() - .descriptors - .len() + crate::state::state().shapes.slab().len() } #[cfg(test)] pub(crate) fn test_clear_shape_table() { - let mut inner = crate::state::state().shapes.inner.borrow_mut(); - // Every descriptor box is about to be dropped, so every cached way naming - // one has to stop matching. Without this the cache holds dangling - // `Box` addresses and the next hit derefs freed memory. - invalidate_shape_lookup_cache(); + let table = &crate::state::state().shapes; + let mut inner = table.inner.borrow_mut(); inner.indices.clear(); - inner.descriptors.clear(); - inner.ids_by_facts.clear(); - inner.ids_by_keys.clear(); + inner.by_facts.clear(); + inner.families.clear(); + // SAFETY: test-only reset with no slab reference held. + unsafe { table.slab_mut().clear() }; drop(inner); clear_shape_object_kind_cache(); } @@ -99,15 +92,49 @@ pub(crate) fn test_clear_shape_table() { #[cfg(test)] pub(crate) fn test_drop_shape_descriptors(keys_id: usize) { let mut inner = crate::state::state().shapes.inner.borrow_mut(); - let stale = inner - .ids_by_keys - .remove(&(keys_id as u64)) + let stale: Vec = inner + .families + .get(&(keys_id as u64)) + .map(|ids| ids.as_slice().to_vec()) .unwrap_or_default(); for id in stale { remove_descriptor_and_reverse_indices(&mut inner, id); } } +/// Move the family indexed under `old` to `new`, exactly as the metadata +/// scan does after the collector forwarded that keys array. +#[cfg(test)] +pub(crate) fn test_rekey_shape_family(old: usize, new: usize) { + let table = &crate::state::state().shapes; + let mut inner = table.inner.borrow_mut(); + if let Some(ids) = inner.families.remove(&(old as u64)) { + for &id in ids.as_slice() { + let Some(record) = table.slab().get(id) else { + continue; + }; + if record.has(shapes_store::RECORD_FLAG_FACTS_INDEXED) { + inner.facts_remove(record.facts_key_with_keys(old as u64), id); + inner.facts_push_back(record.facts_key_with_keys(new as u64), id); + } + inner.family_push_back(new as u64, id); + } + } +} + +/// The ids currently indexed under `keys_id`, in family order. +#[cfg(test)] +pub(crate) fn test_shape_ids_for_keys(keys_id: usize) -> Vec { + crate::state::state() + .shapes + .inner + .borrow() + .families + .get(&(keys_id as u64)) + .map(|ids| ids.as_slice().to_vec()) + .unwrap_or_default() +} + #[cfg(test)] pub(crate) fn test_seed_shape_entry(keys_id: usize) { crate::state::state() @@ -128,9 +155,5 @@ pub(crate) fn test_seed_shape_entry(keys_id: usize) { #[cfg(test)] pub(crate) fn test_shape_id_for_keys(keys_id: usize) -> Option { - let inner = crate::state::state().shapes.inner.borrow(); - inner - .ids_by_keys - .get(&(keys_id as u64)) - .and_then(|ids| ids.first().copied()) + test_shape_ids_for_keys(keys_id).first().copied() } diff --git a/crates/perry-runtime/src/object/shapes_tests.rs b/crates/perry-runtime/src/object/shapes_tests.rs index ac708f34d1..43475708a2 100644 --- a/crates/perry-runtime/src/object/shapes_tests.rs +++ b/crates/perry-runtime/src/object/shapes_tests.rs @@ -540,7 +540,11 @@ mod descriptor_tests_8067 { external, "the process-global id should be preferred for later births" ); - retain_key_count_versions(keys as u64); + assert_eq!( + test_shape_ids_for_keys(keys), + vec![external, local], + "the external id must lead the family so interning prefers it" + ); assert!(shape_descriptor_by_id(local).is_some()); assert!(shape_descriptor_by_id(external).is_some()); @@ -638,9 +642,9 @@ mod descriptor_tests_8067 { ); assert_eq!(unsafe { *slot }, keys as u64); assert_eq!( - shape_descriptor_by_id(id).unwrap().indexed_keys, - keys as u64, - "newly minted descriptor must record its indexed keys address" + test_shape_ids_for_keys(keys), + vec![id], + "newly minted descriptor must be indexed under its keys address" ); // Writing THROUGH the slot is what an evacuating visitor does. The @@ -649,18 +653,20 @@ mod descriptor_tests_8067 { unsafe { *slot = moved_keys }; assert_eq!(shape_descriptor_by_id(id).unwrap().keys, moved_keys); assert_eq!( - shape_descriptor_by_id(id).unwrap().indexed_keys, - keys as u64, - "an object-edge rewrite must retain the old indexed address until metadata repair" + test_shape_ids_for_keys(keys), + vec![id], + "an object-edge rewrite must leave the family under the old address until metadata repair" + ); + assert!( + test_shape_ids_for_keys(moved_keys as usize).is_empty(), + "the store alone must not re-index the family" ); - // The keys-address reverse index is repaired incrementally by the - // metadata pass, not by the store; force the same one-id repair here. - { - let mut inner = crate::state::state().shapes.inner.borrow_mut(); - sync_descriptor_reverse_indices(&mut inner, id); - } - assert_eq!(shape_descriptor_by_id(id).unwrap().indexed_keys, moved_keys); + // The keys-address family index is repaired by the metadata pass, not + // by the store; force the same one-family repair here. + test_rekey_shape_family(keys, moved_keys as usize); + assert_eq!(test_shape_ids_for_keys(moved_keys as usize), vec![id]); + assert!(test_shape_ids_for_keys(keys).is_empty()); assert_eq!( shape_descriptor_ensure(moved_keys as *const ArrayHeader, 3, 2), Ok(id), @@ -670,7 +676,7 @@ mod descriptor_tests_8067 { .expect("shape range unexpectedly exhausted"); assert_ne!( old_address_id, id, - "incremental repair must remove the stale old-address facts entry" + "incremental repair must remove the stale old-address family entry" ); test_drop_shape_descriptors(moved_keys as usize); assert_eq!( @@ -679,15 +685,17 @@ mod descriptor_tests_8067 { "descriptor rekey did not update the keys-address index" ); test_drop_shape_descriptors(keys); + test_drop_shape_descriptors(keys); } #[test] fn a_boxed_record_keeps_its_keys_slot_across_table_growth() { let _lock = crate::gc::global_side_table_test_lock(); // The prohibition #8067 recorded — "descriptor insertion can reallocate - // the table" — is what BOXING answers. Mint one descriptor, take its - // slot, then mint enough siblings to force several rehashes and assert - // the address never moved. Without the box this fails. + // the table" — is what a stable-address record store answers (a Box + // per record before #9706, a chunked slab since). Mint one descriptor, + // take its slot, then mint enough siblings to grow the store across + // several chunks and assert the address never moved. let keys = 0x8112_0000_0000_1000usize; let id = shape_descriptor_ensure(keys as *const ArrayHeader, 1, 1) .expect("shape range unexpectedly exhausted"); @@ -714,8 +722,11 @@ mod descriptor_tests_8067 { } } + /// #9706: an OWNED keys array's growth history is retired behind the + /// version its single owner now carries, except for a version an + /// optimization cache permanently owns. #[test] - fn key_count_versions_remain_resolvable_until_the_keys_die() { + fn owned_key_count_versions_are_retired_behind_the_current_one() { let _lock = crate::gc::global_side_table_test_lock(); let keys = 0x8067_0000_0000_2100usize; let unrelated_keys = 0x8067_0000_0000_2200usize; @@ -723,29 +734,83 @@ mod descriptor_tests_8067 { .expect("shape range unexpectedly exhausted"); let stale_b = shape_descriptor_ensure(keys as *const ArrayHeader, 1, 2) .expect("shape range unexpectedly exhausted"); + let cached = shape_descriptor_ensure(keys as *const ArrayHeader, 1, 3) + .expect("shape range unexpectedly exhausted"); let current = shape_descriptor_ensure(keys as *const ArrayHeader, 2, 2) .expect("shape range unexpectedly exhausted"); let unrelated = shape_descriptor_ensure(unrelated_keys as *const ArrayHeader, 1, 1) .expect("shape range unexpectedly exhausted"); + // Before retirement every version is resolvable and the family lists + // them in mint order. + assert_eq!( + test_shape_ids_for_keys(keys), + vec![stale_a, stale_b, cached, current] + ); + unsafe { note_cache_carrier(shape_descriptor_by_id(cached)) }; - retain_key_count_versions(keys as u64); + retire_owned_shape_siblings(keys as u64, current); - assert!(shape_descriptor_by_id(stale_a).is_some()); - assert!(shape_descriptor_by_id(stale_b).is_some()); + assert_eq!(shape_descriptor_by_id(stale_a), None); + assert_eq!(shape_descriptor_by_id(stale_b), None); + assert!( + shape_descriptor_by_id(cached).is_some(), + "a cache-carried version must survive same-address retirement" + ); assert!(shape_descriptor_by_id(current).is_some()); assert!(shape_descriptor_by_id(unrelated).is_some()); - let inner = crate::state::state().shapes.inner.borrow(); - let current_ids = inner - .ids_by_keys - .get(&(keys as u64)) - .expect("keys identity disappeared from descriptor index"); - assert_eq!(current_ids.as_slice(), &[stale_a, stale_b, current]); - drop(inner); + assert_eq!(test_shape_ids_for_keys(keys), vec![cached, current]); + // Retired facts re-intern as FRESH ids: nothing can resolve the old ones. + let reminted = shape_descriptor_ensure(keys as *const ArrayHeader, 1, 1) + .expect("shape range unexpectedly exhausted"); + assert_ne!(reminted, stale_a); test_drop_shape_descriptors(keys); test_drop_shape_descriptors(unrelated_keys); } + /// The retirement above is wired to the publish funnel: an in-place + /// append on an OWNED keys array must leave exactly one structural + /// descriptor under that address. + #[test] + fn in_place_owned_append_leaves_one_descriptor_per_keys_address() { + let _lock = crate::gc::global_side_table_test_lock(); + unsafe { + let obj = crate::object::js_object_alloc(0, 0); + let mut keys_before = 0usize; + let mut first_addr_count = 0usize; + for i in 0..96u32 { + let name = format!("owned9706_{i:03}"); + let key = crate::string::js_string_from_bytes(name.as_ptr(), name.len() as u32); + crate::object::js_object_set_field_by_name(obj, key, i as f64); + let keys = crate::object::object_keys_array(obj) as usize; + let stamp = object_shape_stamp(obj); + assert!(is_shape_id(stamp), "receiver must stay stamped"); + let family = test_shape_ids_for_keys(keys); + assert!( + family.contains(&stamp), + "the current stamp must be indexed under the current keys address" + ); + if keys == keys_before { + first_addr_count += 1; + let shared = crate::value::addr_class::try_read_gc_header(keys) + .is_some_and(|h| h.gc_flags & crate::gc::GC_FLAG_SHAPE_SHARED != 0); + if !shared { + assert_eq!( + family.len(), + 1, + "an owned in-place append left growth history alive: {family:?}" + ); + } + } + keys_before = keys; + } + assert!( + first_addr_count > 0, + "fixture premise: some appends must grow the owned array in place" + ); + } + } + #[test] fn shape_drop_does_not_delete_a_potential_siblings_descriptor() { let _lock = crate::gc::global_side_table_test_lock(); @@ -836,23 +901,35 @@ mod descriptor_tests_8067 { } } -/// `ids_by_facts` moved from std's SipHash `RandomState` to `FastKeyHasher`. +/// A multi-field shape key hashed with `FastKeyHasher` must fold every field. /// -/// The hazard that motivated the original "deliberately NOT a `PtrHashMap`" -/// note is real: `PtrHasher`'s `write_*` methods OVERWRITE the accumulator, so -/// a five-field `ShapeFacts` would collapse to its last field and every -/// descriptor sharing that field would collide into one bucket. +/// The shape table's facts-keyed reverse map is gone (#9706 interns through +/// the keys-address family instead), but the hazard this pinned is still +/// live for `gc/layout/typed_shape.rs`'s `RegisteredTypedShapeKey`: +/// `PtrHasher`'s `write_*` methods OVERWRITE the accumulator, so a multi-field +/// key would collapse to its last field and every entry sharing that field +/// would collide into one bucket. /// /// `FastKeyHasher` avoids this by implementing only `write` — the derived /// `Hash`'s `write_u32`/`write_u64` calls all forward there and FOLD with -/// FNV-1a. This test pins that property directly: vary ONE field at a time and -/// require a distinct hash each time. It fails loudly against any hasher that -/// overwrites instead of folding. +/// FNV-1a. This test pins that property directly on the old facts layout: +/// vary ONE field at a time and require a distinct hash each time. It fails +/// loudly against any hasher that overwrites instead of folding. #[test] fn shape_facts_hash_folds_every_field() { use crate::fast_hash::FastKeyHasher; use std::hash::{BuildHasher, Hash, Hasher}; + #[derive(Clone, Copy, Hash)] + struct ShapeFacts { + keys: u64, + logical_key_count: u32, + live_inline_slot_count: u32, + semantic_generation: u64, + object_kind: ShapeObjectKind, + hole_count: u32, + } + fn h(f: &ShapeFacts) -> u64 { let mut hasher = FastKeyHasher.build_hasher(); f.hash(&mut hasher); @@ -928,14 +1005,12 @@ fn shape_facts_hash_folds_every_field() { assert_eq!(h(&base), h(&base.clone()), "hashing must be deterministic"); } -/// The shape lookup cache holds a record's ADDRESS, so it must stop matching -/// the moment that address can change under an id still in use. +/// A removed id must stop resolving at once, and nothing may hand out its +/// record address afterwards. /// -/// A stale way would hand out a pointer to a dropped `Box` — -/// a use-after-free reachable from the hot property path, not a wrong answer. -/// Removal is the funnel that frees a record, so it bumps the epoch; this pins -/// that. Deleting the `invalidate_shape_lookup_cache()` call in -/// `remove_descriptor_and_reverse_indices` fails this test. +/// Before #9706 this pinned the lookup-way cache's invalidation epoch; the +/// slab has no cache in front of it, so the property is asserted directly: +/// removal clears the record and both by-id entry points report `None`. #[test] fn shape_lookup_cache_is_invalidated_when_a_record_is_removed() { let _lock = crate::gc::global_side_table_test_lock(); @@ -945,35 +1020,38 @@ fn shape_lookup_cache_is_invalidated_when_a_record_is_removed() { let id = test_shape_id_for_keys(keys as usize) .expect("a fresh object must have a registered shape"); - // Populate the way. assert!( shape_descriptor_by_id(id).is_some(), "the descriptor must resolve before removal" ); - let epoch_before = crate::state::state().shapes.lookup_epoch.get(); + let record = shape_descriptor_by_id(id).unwrap().record; + assert_ne!(record, 0); - // Drop it through the funnel that frees the box. + // Drop it through the funnel that retires a record. { let mut inner = crate::state::state().shapes.inner.borrow_mut(); remove_descriptor_and_reverse_indices(&mut inner, id); } - assert_ne!( - crate::state::state().shapes.lookup_epoch.get(), - epoch_before, - "removing a record must bump the lookup epoch — a way still naming \ - the freed box would hand out a dangling ShapeDescriptor pointer" - ); assert!( shape_descriptor_by_id(id).is_none(), - "a removed id must not resolve from the cache" + "a removed id must not resolve" + ); + assert_eq!( + shape_live_inline_slot_count_by_id(id), + None, + "the field reader must not read a retired record" + ); + assert_eq!(shape_descriptor_keys_slot(id), None); + assert!( + !shape_id_owns_keys_slot(id, record as *mut u64), + "a retired id must not claim its old record address" ); } } -/// A fresh-id insert must NOT invalidate the cache: it cannot make any existing -/// way wrong, and flushing on every shape creation would defeat the cache in -/// exactly the workloads that build shapes. +/// Minting fresh ids must not move any existing record: the collector may +/// hold a record address across the mint. #[test] fn fresh_shape_creation_does_not_flush_the_lookup_cache() { let _lock = crate::gc::global_side_table_test_lock(); @@ -981,8 +1059,8 @@ fn fresh_shape_creation_does_not_flush_the_lookup_cache() { let a = crate::object::js_object_alloc(0, 0); let keys_a = crate::object::object_keys_array(a); let id_a = test_shape_id_for_keys(keys_a as usize).expect("shape for a"); - assert!(shape_descriptor_by_id(id_a).is_some()); - let epoch = crate::state::state().shapes.lookup_epoch.get(); + let record_a = shape_descriptor_by_id(id_a).expect("resolves").record; + assert_ne!(record_a, 0); // Create more objects — each mints shapes through the fresh-id path. for _ in 0..8 { @@ -991,14 +1069,9 @@ fn fresh_shape_creation_does_not_flush_the_lookup_cache() { } assert_eq!( - crate::state::state().shapes.lookup_epoch.get(), - epoch, - "minting fresh shape ids must not bump the epoch; only removal and \ - the replacing insert may" - ); - assert!( - shape_descriptor_by_id(id_a).is_some(), - "the earlier descriptor must still resolve" + shape_descriptor_by_id(id_a).map(|d| d.record), + Some(record_a), + "minting fresh shape ids must not move an existing record" ); } } diff --git a/scripts/shape_descriptor_census.py b/scripts/shape_descriptor_census.py index a3bcaa6fe2..405ecfddb2 100644 --- a/scripts/shape_descriptor_census.py +++ b/scripts/shape_descriptor_census.py @@ -223,6 +223,7 @@ def assert_authority_surfaces(sources: dict[str, str]) -> None: authority_paths = ( "crates/perry-runtime/src/object/shapes.rs", "crates/perry-runtime/src/object/shapes_slot_list.rs", + "crates/perry-runtime/src/object/shapes_store.rs", "crates/perry-runtime/src/object/mod.rs", "crates/perry-runtime/src/object/live_slots.rs", "crates/perry-codegen/src/lower_call/new_alloc.rs", @@ -248,15 +249,18 @@ def assert_authority_surfaces(sources: dict[str, str]) -> None: ) clean = stripped_sources({path: sources[path] for path in authority_paths}) # `shapes.rs` sits against the repo's 2000-line cap, so helpers keep being - # split into the `shapes_slot_list.rs` sibling as it grows. Read the two as - # ONE logical unit: every `function_body(shapes, ...)` below then finds its - # target wherever it currently lives, instead of silently matching nothing - # the next time a pinned function crosses the split — #8918's exact failure - # mode, where a census inspecting an empty body reports success. + # split into siblings as it grows (`shapes_slot_list.rs`, and since #9706 + # the record store `shapes_store.rs`). Read them as ONE logical unit: every + # `function_body(shapes, ...)` below then finds its target wherever it + # currently lives, instead of silently matching nothing the next time a + # pinned function crosses the split — #8918's exact failure mode, where a + # census inspecting an empty body reports success. shapes = ( clean["crates/perry-runtime/src/object/shapes.rs"] + "\n" + clean["crates/perry-runtime/src/object/shapes_slot_list.rs"] + + "\n" + + clean["crates/perry-runtime/src/object/shapes_store.rs"] ) object_mod = clean["crates/perry-runtime/src/object/mod.rs"] live_slots = clean["crates/perry-runtime/src/object/live_slots.rs"] @@ -301,13 +305,21 @@ def assert_authority_surfaces(sources: dict[str, str]) -> None: raw_write_pics = sources["crates/perry-codegen/src/expr/proxy_reflect.rs"] for pattern, label in ( - # `PtrHashMap` since #8157 (SipHash on a bare u32 was 25% of self time in - # `shapes`). The hasher is free; the BOX is not. Since #8112 the - # collector enumerates `&mut record.keys` as an ordinary GC slot, and a - # budgeted dirty scan can hold that address across mutator resumptions - # that insert descriptors. Un-boxing the value puts the record back in - # the bucket, where a rehash moves it under the collector's feet. - (r"descriptors\s*:\s*(?:[\w:]+::)?(?:Ptr)?HashMap\s*<\s*u32\s*,\s*Box\s*<\s*ShapeDescriptor\s*>", "by-id descriptor table, boxed for a stable keys slot"), + # #9706: the by-id store is a chunked slab indexed by ShapeId. Since + # #8112 the collector enumerates the record's `keys` word as an + # ordinary GC slot, and a budgeted dirty scan can hold that address + # across mutator resumptions that insert descriptors — so a record's + # address must never move for its lifetime. Chunks are individually + # boxed and never reallocated; only the directory of chunk pointers + # grows. Putting records into one flat `Vec` (or back into a rehashing + # bucket) moves them under the collector's feet. + (r"slab\s*:\s*(?:std::cell::)?UnsafeCell\s*<\s*ShapeSlab\s*>", "by-id descriptor slab with stable record addresses"), + (r"type\s+Chunk\s*=\s*Box\s*<\s*\[\s*UnsafeCell\s*<\s*ShapeRecord\s*>\s*;\s*CHUNK_LEN\s*\]\s*>", "slab chunks individually boxed, never reallocated"), + (r"type\s+Page\s*=\s*Box\s*<\s*\[\s*Option\s*<\s*Chunk\s*>\s*;\s*PAGE_LEN\s*\]\s*>", "slab directory pages hold chunk pointers, not records"), + (r"pages\s*:\s*Vec\s*<\s*Option\s*<\s*Page\s*>\s*>", "slab directory is a vector of page pointers"), + # `keys` must stay the FIRST field of the `#[repr(C)]` record: the + # record address IS the rewritable keys slot (`keys_slot`). + (r"#\[repr\(C\)\]\s*(?:#\[[^\]]*\]\s*)*pub\(crate\)\s+struct\s+ShapeRecord\s*\{\s*(?://[^\n]*\n\s*)*pub\(super\)\s+keys\s*:\s*u64", "slab record is repr(C) with the keys word first"), (r"logical_key_count\s*:\s*u32", "exact logical-key fact"), (r"live_inline_slot_count\s*:\s*u32", "exact live-slot fact"), (r"semantic_generation\s*:\s*u64", "semantic transition fact"), @@ -393,8 +405,8 @@ def assert_authority_surfaces(sources: dict[str, str]) -> None: ensure = function_body(shapes, "shape_descriptor_ensure_with_holes") assert_before( ensure, - "inner.descriptors.insert", - "inner.ids_by_facts.entry", + "slab_mut().insert", + "family_push_back", "by-id descriptor before reverse accelerator", ) sync = function_body(shapes, "publish_object_shape_from") @@ -433,18 +445,34 @@ def assert_authority_surfaces(sources: dict[str, str]) -> None: shapes, ): raise CensusError("clear_object_shape_stamp escaped its #[cfg(test)] gate") - retirement = function_body(shapes, "retain_key_count_versions") + # #9706: an OWNED keys array's same-address growth history is retired + # behind the version its single owner now carries. The retirement must be + # scoped to that array's family (never a scan of the whole table), must + # keep the cache-carried versions an optimization cache can reinstall, + # and must run AFTER the successor is stamped and armed (#9200's order). + retirement = function_body(shapes, "retire_owned_shape_siblings") require_code( retirement, - r"ids_by_keys\s*\.\s*remove\s*\(\s*&keys\s*\)", + r"families\s*\.\s*get\s*\(\s*&keys\s*\)", "keys-scoped descriptor lineage index", ) - if re.search(r"descriptors\s*\.\s*(?:iter|values|keys)\s*\(", retirement): - raise CensusError("shape descriptor lineage repair scans the global descriptor table") - if "descriptors.remove" in retirement: - raise CensusError("live-key lineage repair eagerly deletes published descriptors") + if re.search(r"slab\(\)\s*\.\s*for_each\s*\(", retirement): + raise CensusError("owned-history retirement scans the global descriptor table") + require_code( + retirement, + r"RECORD_FLAG_CACHE_CARRIER", + "cache-carried versions survive same-address retirement", + ) + assert_before( + sync, + "stamp_object_shape_id_with_carrier_note", + "retire_owned_shape_siblings", + "successor stamped and armed before the owned history is retired", + ) + # A SHARED array's versions are immutable prefixes other objects may still + # carry; growth and drop of the slot index must not touch them. for name in ("shape_keys_grown", "shape_drop"): - if "descriptors.remove" in function_body(shapes, name): + if "remove_descriptor_and_reverse_indices" in function_body(shapes, name): raise CensusError(f"{name} eagerly deletes a sibling descriptor") require_code( @@ -747,15 +775,26 @@ def run_sabotage_selftests(sources: dict[str, str], baseline: dict[str, object]) ) shapes_path = "crates/perry-runtime/src/object/shapes.rs" - unboxed_table = dict(sources) - unboxed_table[shapes_path] = unboxed_table[shapes_path].replace( - "PtrHashMap>", - "PtrHashMap", + store_path = "crates/perry-runtime/src/object/shapes_store.rs" + flat_slab = dict(sources) + flat_slab[store_path] = flat_slab[store_path].replace( + "type Chunk = Box<[UnsafeCell; CHUNK_LEN]>;", + "type Chunk = Vec>;", 1, ) expect_rejected( - "descriptor record un-boxed back into a rehashing bucket", - lambda: assert_authority_surfaces(unboxed_table), + "slab chunk turned into a reallocating Vec", + lambda: assert_authority_surfaces(flat_slab), + ) + keys_not_first = dict(sources) + keys_not_first[store_path] = keys_not_first[store_path].replace( + " pub(super) keys: u64,\n pub(super) semantic_generation: u64,", + " pub(super) semantic_generation: u64,\n pub(super) keys: u64,", + 1, + ) + expect_rejected( + "keys word moved off the front of the slab record", + lambda: assert_authority_surfaces(keys_not_first), ) ungated_root = dict(sources) @@ -804,11 +843,11 @@ def run_sabotage_selftests(sources: dict[str, str], baseline: dict[str, object]) unscoped_retirement = dict(sources) path = "crates/perry-runtime/src/object/shapes.rs" retirement_body = function_body( - unscoped_retirement[path], "retain_key_count_versions" + unscoped_retirement[path], "retire_owned_shape_siblings" ) unscoped_body, substitutions = re.subn( - r"ids_by_keys\s*\.\s*remove\s*\(\s*&keys\s*\)", - "ids_by_keys.get(&keys).cloned()", + r"families\s*\.\s*get\s*\(\s*&keys\s*\)", + "families.get(&0)", retirement_body, count=1, ) @@ -822,6 +861,19 @@ def run_sabotage_selftests(sources: dict[str, str], baseline: dict[str, object]) lambda: assert_authority_surfaces(unscoped_retirement), ) + early_retirement = dict(sources) + publish_body = function_body(early_retirement[path], "publish_object_shape_from") + early_body = swap_once( + publish_body, + "stamp_object_shape_id_with_carrier_note", + "retire_owned_shape_siblings", + ) + early_retirement[path] = early_retirement[path].replace(publish_body, early_body, 1) + expect_rejected( + "owned history retired before the successor is stamped", + lambda: assert_authority_surfaces(early_retirement), + ) + legacy_ir = dict(sources) path = "crates/perry-codegen/src/expr/property_get/generic_dispatch.rs" legacy_body, substitutions = re.subn( From b87e6c423cf0a9268c3d4d51bf7a4b9c3b500e03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 4 Sep 2026 11:22:13 +0200 Subject: [PATCH 2/5] perf(codegen): reuse receiver validation in counted loops --- crates/perry-codegen/src/collectors/mod.rs | 5 +- .../src/collectors/receiver_regions.rs | 223 +++++++++++++++--- .../src/collectors/receiver_regions_tests.rs | 88 +++++++ crates/perry-codegen/src/expr/index_get.rs | 49 +++- .../src/expr/index_get/guarded_array.rs | 131 +++++++++- crates/perry-codegen/src/stmt/loops.rs | 223 +++++++++++++++--- ...e_9254_receiver_descriptor_counted_loop.rs | 185 +++++++++++++++ ...e_9254_receiver_descriptor_counted_loop.ts | 24 ++ 8 files changed, 855 insertions(+), 73 deletions(-) create mode 100644 crates/perry/tests/issue_9254_receiver_descriptor_counted_loop.rs create mode 100644 test-files/test_issue_9254_receiver_descriptor_counted_loop.ts diff --git a/crates/perry-codegen/src/collectors/mod.rs b/crates/perry-codegen/src/collectors/mod.rs index 83df07fe53..3257750191 100644 --- a/crates/perry-codegen/src/collectors/mod.rs +++ b/crates/perry-codegen/src/collectors/mod.rs @@ -110,7 +110,10 @@ pub(crate) use ptr_numarray::{NumArrayDensity, NumArrayLocal}; pub(crate) use ptr_shape::{ptr_shape_locals_enabled, PtrShapeLocal}; pub(crate) use ptr_shape_callbacks::collect_array_callback_shapes; pub(crate) use ptr_shape_returns::collect_exported_return_shapes; -pub(crate) use receiver_regions::ReceiverDescriptorTable; +pub(crate) use receiver_regions::{ + region_enders_in_stmts_with_trusted_operations, ReceiverArrayValidationKind, + ReceiverDescriptorTable, RegionEnder, +}; pub(crate) use refs::{ collect_let_ids, collect_ref_ids_in_expr, collect_ref_ids_in_stmts, is_clamp_call, }; diff --git a/crates/perry-codegen/src/collectors/receiver_regions.rs b/crates/perry-codegen/src/collectors/receiver_regions.rs index 642028a751..b59d55fd8a 100644 --- a/crates/perry-codegen/src/collectors/receiver_regions.rs +++ b/crates/perry-codegen/src/collectors/receiver_regions.rs @@ -35,8 +35,11 @@ //! the packed/versioned clone's receiver hoist through //! [`ReceiverDescriptorTable`]: the table owns the rooted box, pre-masked base //! handle and poll refresh recipe as one entry, and asks [`boundary_admits`] -//! before carrying that address across a back-edge poll. Other fact tables are -//! migrated one consumer at a time. +//! before carrying that address across a back-edge poll. Phase 3 lets ordinary +//! counted loops attach a conditional plain/numeric-array validation to the +//! same entry, but only after this module proves the loop region contains no +//! ender other than the poll covered by that refresh recipe. Other fact tables +//! are migrated one consumer at a time. //! //! The precedent is `TypeFacts::purity` / `TypeFacts::shape_stability` //! (`collectors/hir_facts.rs`, #854): a subgraph the collector populates and @@ -55,12 +58,10 @@ //! what keeps this file honest: the model is checked against a shipping, //! audited predicate rather than against its own restatement. -// #9254 phase 2: `ReceiverDescriptorTable` and the poll boundary algebra are -// production consumers. Region formation remains lint-only until ordinary -// counted loops migrate in phase 3, so the rest of this module is still dead -// in a non-test build. Keep that incomplete state explicit rather than -// scattering per-item allows; this attribute can go when region formation is -// itself consumed. +// #9254 phases 2/3: `ReceiverDescriptorTable`, the poll boundary algebra and +// region formation are production consumers. Some inventory/lint helpers stay +// test-only while the remaining fact tables await phase 4, so keep that +// incomplete state explicit rather than scattering per-item allows. #![allow(dead_code)] use crate::loop_purity; @@ -261,10 +262,39 @@ pub(crate) struct ReceiverPollRefresh { pub(crate) source_root: String, } +/// Strength of the one-time array validation attached by an ordinary counted +/// loop. Numeric validation includes every plain-array invariant and also +/// proves raw-f64 element representation, so it may serve a plain read too. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ReceiverArrayValidationKind { + Plain, + Numeric, +} + +/// Descriptor data consumed at an ordinary bounded array read. +/// +/// `valid_i1` is loop-invariant. When false the read takes its established +/// guarded fallback and never consumes `base_handle_slot`; when true the +/// region contract guarantees the cached handle remains usable until the next +/// poll refresh. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ReceiverArrayAccess { + pub(crate) valid_i1: String, + pub(crate) base_handle_slot: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ActiveArrayValidation { + contract: ReceiverDescriptor, + kind: ReceiverArrayValidationKind, + valid_i1: String, +} + #[derive(Debug, Clone, PartialEq, Eq)] struct ActiveReceiverDescriptor { contract: ReceiverDescriptor, refresh: ReceiverPollRefresh, + array_validation: Option, } /// Active materialised receiver descriptors for one function lowering. @@ -321,10 +351,71 @@ impl ReceiverDescriptorTable { base_handle_slot, source_root, }, + array_validation: None, }); true } + /// Install phase 3's ordinary-counted-loop descriptor. + /// + /// The caller supplies the exact region enders after replacing only the + /// indexed read that will consume this validation with its trusted form. + /// Both the cached-address and representation contracts must admit every + /// ender before the entry becomes visible to lowering. A duplicate reuses + /// an outer descriptor; callers can query [`Self::array_access`] to learn + /// whether that outer entry is strong enough for their read. + #[allow(clippy::too_many_arguments)] + pub(crate) fn materialize_region_validated_array( + &mut self, + receiver: u32, + rooted_box_slot: String, + base_handle_slot: String, + source_root: String, + kind: ReceiverArrayValidationKind, + valid_i1: String, + enders: &[RegionEnder], + ) -> Result { + if self.contains(receiver) { + return Ok(false); + } + let address_contract = ReceiverDescriptor { + table: "receiver_descriptors", + receiver, + claim: ReceiverClaim::Address, + boundary: FactBoundary::PollRefresh, + // The supplied ender list is the proof: an unwind edge below is + // rejected rather than excused by claiming it was excluded. + excludes_try: false, + }; + for &ender in enders { + boundary_admits(&address_contract, ender)?; + } + let representation_contract = ReceiverDescriptor { + table: "receiver_descriptors[array_validation]", + receiver, + claim: ReceiverClaim::Representation, + boundary: FactBoundary::DynamicExtent, + excludes_try: false, + }; + for &ender in enders { + boundary_admits(&representation_contract, ender)?; + } + self.entries.push(ActiveReceiverDescriptor { + contract: address_contract, + refresh: ReceiverPollRefresh { + rooted_box_slot, + base_handle_slot, + source_root, + }, + array_validation: Some(ActiveArrayValidation { + contract: representation_contract, + kind, + valid_i1, + }), + }); + Ok(true) + } + /// End the dynamic extent of one materialised receiver. pub(crate) fn dematerialize(&mut self, receiver: u32) -> bool { let Some(index) = self @@ -354,6 +445,28 @@ impl ReceiverDescriptorTable { .map(|entry| entry.refresh.base_handle_slot.as_str()) } + /// Conditional validation and refreshed base handle for an ordinary array + /// read. A numeric consumer requires the stronger numeric validation; a + /// plain consumer may reuse either kind. + pub(crate) fn array_access( + &self, + receiver: u32, + require_numeric: bool, + ) -> Option { + let entry = self + .entries + .iter() + .find(|entry| entry.contract.receiver == receiver)?; + let validation = entry.array_validation.as_ref()?; + if require_numeric && validation.kind != ReceiverArrayValidationKind::Numeric { + return None; + } + Some(ReceiverArrayAccess { + valid_i1: validation.valid_i1.clone(), + base_handle_slot: entry.refresh.base_handle_slot.clone(), + }) + } + /// Refresh recipes admitted at a back-edge poll. /// /// Every active entry is checked at the boundary before its recipe is @@ -364,6 +477,9 @@ impl ReceiverDescriptorTable { let mut refreshes = Vec::with_capacity(self.entries.len()); for entry in &self.entries { boundary_admits(&entry.contract, RegionEnder::BackEdgePoll)?; + if let Some(validation) = &entry.array_validation { + boundary_admits(&validation.contract, RegionEnder::BackEdgePoll)?; + } refreshes.push(entry.refresh.clone()); } Ok(refreshes) @@ -390,6 +506,21 @@ pub(crate) fn violations_for( /// Returns the *first* reason found; an expression can qualify several ways /// and the caller only needs to know the region ends. pub(crate) fn expr_region_ender(e: &Expr, is_inert: &dyn Fn(&Expr) -> bool) -> Option { + expr_region_ender_with_trusted_operation(e, is_inert, &|_| false) +} + +fn expr_region_ender_with_trusted_operation( + e: &Expr, + is_inert: &dyn Fn(&Expr) -> bool, + is_trusted_operation: &dyn Fn(&Expr) -> bool, +) -> Option { + // A production consumer may replace one exact operation with a form whose + // guard establishes that it cannot dispatch or allocate. Children still + // run through the ordinary walker before this classification, so trusting + // `arr[i]` never accidentally trusts an effectful `i`. + if is_trusted_operation(e) { + return None; + } match e { // ---- Provably not a relocation point ------------------------------- // Constants, reads of a local/global, and references. No dispatch. @@ -521,37 +652,61 @@ pub(crate) fn region_enders_in_stmts( stmts: &[Stmt], controls: &[&Expr], is_inert: &dyn Fn(&Expr) -> bool, +) -> Vec { + region_enders_in_stmts_with_trusted_operations(stmts, controls, is_inert, &|_| false) +} + +/// Region walk used by a guarded consumer that replaces a precisely matched +/// operation with a non-dispatching form. Only the operation node is trusted; +/// its children retain normal ender classification and execution order. +pub(crate) fn region_enders_in_stmts_with_trusted_operations( + stmts: &[Stmt], + controls: &[&Expr], + is_inert: &dyn Fn(&Expr) -> bool, + is_trusted_operation: &dyn Fn(&Expr) -> bool, ) -> Vec { let mut out = Vec::new(); for s in stmts { - enders_in_stmt(s, is_inert, &mut out); + enders_in_stmt(s, is_inert, is_trusted_operation, &mut out); } for c in controls { - enders_in_expr(c, is_inert, &mut out); + enders_in_expr(c, is_inert, is_trusted_operation, &mut out); } out } -fn enders_in_expr(e: &Expr, is_inert: &dyn Fn(&Expr) -> bool, out: &mut Vec) { +fn enders_in_expr( + e: &Expr, + is_inert: &dyn Fn(&Expr) -> bool, + is_trusted_operation: &dyn Fn(&Expr) -> bool, + out: &mut Vec, +) { // Child expressions execute before the operation represented by their // parent (`f(makeClosure())` allocates the closure before it calls `f`). // Region boundaries are ordered data once a lowering path consumes them, // so a pre-order walk would put the call before the allocation. - perry_hir::walker::walk_expr_children(e, &mut |child| enders_in_expr(child, is_inert, out)); - if let Some(r) = expr_region_ender(e, is_inert) { + perry_hir::walker::walk_expr_children(e, &mut |child| { + enders_in_expr(child, is_inert, is_trusted_operation, out) + }); + if let Some(r) = expr_region_ender_with_trusted_operation(e, is_inert, is_trusted_operation) { out.push(r); } } -fn enders_in_stmt(s: &Stmt, is_inert: &dyn Fn(&Expr) -> bool, out: &mut Vec) { +fn enders_in_stmt( + s: &Stmt, + is_inert: &dyn Fn(&Expr) -> bool, + is_trusted_operation: &dyn Fn(&Expr) -> bool, + out: &mut Vec, +) { match s { // A throw is an unwind edge *and* the helper allocates the Error. Stmt::Throw(e) => { - enders_in_expr(e, is_inert, out); + enders_in_expr(e, is_inert, is_trusted_operation, out); out.push(RegionEnder::UnwindEdge); } Stmt::Let { init: Some(e), .. } | Stmt::Expr(e) | Stmt::Return(Some(e)) => { - enders_in_expr(e, is_inert, out) + enders_in_expr(e, is_inert, is_trusted_operation, out) } Stmt::Let { init: None, .. } | Stmt::Return(None) => {} Stmt::If { @@ -559,13 +714,13 @@ fn enders_in_stmt(s: &Stmt, is_inert: &dyn Fn(&Expr) -> bool, out: &mut Vec { - enders_in_expr(condition, is_inert, out); + enders_in_expr(condition, is_inert, is_trusted_operation, out); for st in then_branch { - enders_in_stmt(st, is_inert, out); + enders_in_stmt(st, is_inert, is_trusted_operation, out); } if let Some(else_branch) = else_branch { for st in else_branch { - enders_in_stmt(st, is_inert, out); + enders_in_stmt(st, is_inert, is_trusted_operation, out); } } } @@ -573,17 +728,17 @@ fn enders_in_stmt(s: &Stmt, is_inert: &dyn Fn(&Expr) -> bool, out: &mut Vec { - enders_in_expr(condition, is_inert, out); + enders_in_expr(condition, is_inert, is_trusted_operation, out); for st in body { - enders_in_stmt(st, is_inert, out); + enders_in_stmt(st, is_inert, is_trusted_operation, out); } out.push(RegionEnder::BackEdgePoll); } Stmt::DoWhile { body, condition } => { for st in body { - enders_in_stmt(st, is_inert, out); + enders_in_stmt(st, is_inert, is_trusted_operation, out); } - enders_in_expr(condition, is_inert, out); + enders_in_expr(condition, is_inert, is_trusted_operation, out); out.push(RegionEnder::BackEdgePoll); } Stmt::For { @@ -593,20 +748,20 @@ fn enders_in_stmt(s: &Stmt, is_inert: &dyn Fn(&Expr) -> bool, out: &mut Vec { if let Some(init) = init { - enders_in_stmt(init, is_inert, out); + enders_in_stmt(init, is_inert, is_trusted_operation, out); } if let Some(condition) = condition { - enders_in_expr(condition, is_inert, out); + enders_in_expr(condition, is_inert, is_trusted_operation, out); } for st in body { - enders_in_stmt(st, is_inert, out); + enders_in_stmt(st, is_inert, is_trusted_operation, out); } if let Some(update) = update { - enders_in_expr(update, is_inert, out); + enders_in_expr(update, is_inert, is_trusted_operation, out); } out.push(RegionEnder::BackEdgePoll); } - Stmt::Labeled { body, .. } => enders_in_stmt(body, is_inert, out), + Stmt::Labeled { body, .. } => enders_in_stmt(body, is_inert, is_trusted_operation, out), // Every statement in a `try` body may divert to the handler. Stmt::Try { body, @@ -614,17 +769,17 @@ fn enders_in_stmt(s: &Stmt, is_inert: &dyn Fn(&Expr) -> bool, out: &mut Vec { for st in body { - enders_in_stmt(st, is_inert, out); + enders_in_stmt(st, is_inert, is_trusted_operation, out); } out.push(RegionEnder::UnwindEdge); if let Some(catch) = catch { for st in &catch.body { - enders_in_stmt(st, is_inert, out); + enders_in_stmt(st, is_inert, is_trusted_operation, out); } } if let Some(finally) = finally { for st in finally { - enders_in_stmt(st, is_inert, out); + enders_in_stmt(st, is_inert, is_trusted_operation, out); } } } @@ -632,13 +787,13 @@ fn enders_in_stmt(s: &Stmt, is_inert: &dyn Fn(&Expr) -> bool, out: &mut Vec { - enders_in_expr(discriminant, is_inert, out); + enders_in_expr(discriminant, is_inert, is_trusted_operation, out); for c in cases { if let Some(t) = &c.test { - enders_in_expr(t, is_inert, out); + enders_in_expr(t, is_inert, is_trusted_operation, out); } for st in &c.body { - enders_in_stmt(st, is_inert, out); + enders_in_stmt(st, is_inert, is_trusted_operation, out); } } } diff --git a/crates/perry-codegen/src/collectors/receiver_regions_tests.rs b/crates/perry-codegen/src/collectors/receiver_regions_tests.rs index f2369daae5..e788ab8708 100644 --- a/crates/perry-codegen/src/collectors/receiver_regions_tests.rs +++ b/crates/perry-codegen/src/collectors/receiver_regions_tests.rs @@ -263,6 +263,94 @@ fn active_descriptor_table_owns_lookup_refresh_and_dynamic_extent() { assert!(!table.dematerialize(OBJ)); } +#[test] +fn ordinary_loop_descriptor_carries_conditional_array_validation() { + let mut table = ReceiverDescriptorTable::default(); + assert!(table + .materialize_region_validated_array( + OBJ, + "%region.box".into(), + "%region.handle".into(), + "%region.source".into(), + ReceiverArrayValidationKind::Numeric, + "%region.valid".into(), + &[RegionEnder::BackEdgePoll], + ) + .expect("the poll is covered by the refresh recipe")); + + let expected = ReceiverArrayAccess { + valid_i1: "%region.valid".into(), + base_handle_slot: "%region.handle".into(), + }; + assert_eq!(table.array_access(OBJ, false), Some(expected.clone())); + assert_eq!(table.array_access(OBJ, true), Some(expected)); + assert_eq!( + table.poll_refreshes().unwrap(), + vec![ReceiverPollRefresh { + rooted_box_slot: "%region.box".into(), + base_handle_slot: "%region.handle".into(), + source_root: "%region.source".into(), + }] + ); +} + +#[test] +fn plain_validation_cannot_serve_a_numeric_read() { + let mut table = ReceiverDescriptorTable::default(); + table + .materialize_region_validated_array( + OBJ, + "%box".into(), + "%handle".into(), + "%source".into(), + ReceiverArrayValidationKind::Plain, + "%valid".into(), + &[], + ) + .unwrap(); + assert!(table.array_access(OBJ, false).is_some()); + assert_eq!(table.array_access(OBJ, true), None); +} + +#[test] +fn ordinary_loop_descriptor_is_not_installed_across_a_call() { + let mut table = ReceiverDescriptorTable::default(); + let violation = table + .materialize_region_validated_array( + OBJ, + "%box".into(), + "%handle".into(), + "%source".into(), + ReceiverArrayValidationKind::Numeric, + "%valid".into(), + &[RegionEnder::CollectingCall], + ) + .expect_err("a call can move and mutate the receiver"); + assert_eq!(violation.ender, RegionEnder::CollectingCall); + assert!(!table.contains(OBJ)); +} + +#[test] +fn trusting_an_operation_does_not_trust_its_effectful_children() { + let read = Expr::IndexGet { + object: Box::new(num(OBJ)), + index: Box::new(call(vec![])), + }; + let ordinary = region_enders_in_stmts(&[Stmt::Expr(read.clone())], &[], &stub_inert); + assert_eq!( + ordinary, + vec![RegionEnder::CollectingCall, RegionEnder::CollectingCall] + ); + + let trusted = region_enders_in_stmts_with_trusted_operations( + &[Stmt::Expr(read)], + &[], + &stub_inert, + &|expr| matches!(expr, Expr::IndexGet { .. }), + ); + assert_eq!(trusted, vec![RegionEnder::CollectingCall]); +} + /// A cached address dies at a call no matter how the table scopes itself — /// scoping is not a substitute for the region being call-free. #[test] diff --git a/crates/perry-codegen/src/expr/index_get.rs b/crates/perry-codegen/src/expr/index_get.rs index 1ed97cda62..baef34326a 100644 --- a/crates/perry-codegen/src/expr/index_get.rs +++ b/crates/perry-codegen/src/expr/index_get.rs @@ -53,7 +53,8 @@ use foreign_counter::{ mod inline_dyn_typed_array; use guarded_array::{ - lower_guarded_array_index_get, lower_packed_f64_loop_index_get, packed_f64_loop_fact, + lower_guarded_array_index_get, lower_packed_f64_loop_index_get, + lower_region_validated_array_index_get, packed_f64_loop_fact, }; use inline_dyn_typed_array::lower_inline_dyn_typed_array_get; @@ -837,8 +838,9 @@ pub(crate) fn lower_numeric_index_get_for_number_context( let repair_slot = receiver_repair_slot(ctx, object); let arr_box = lower_expr(ctx, object)?; let idx_i32 = ctx.block().load(I32, &i32_slot); - return lower_guarded_array_index_get( + return lower_region_validated_array_index_get( ctx, + *arr_id, &arr_box, &idx_i32, "bidx.num", @@ -969,6 +971,44 @@ pub(crate) fn lower_unknown_local_index_get_for_number_context( } fn lower_bounded_array_index_get( + ctx: &mut FnCtx<'_>, + arr_id: u32, + arr_box: &str, + idx_i32: &str, +) -> Result { + let Some(access) = ctx.receiver_descriptors.array_access(arr_id, false) else { + return lower_bounded_array_index_get_checked(ctx, arr_box, idx_i32); + }; + + let fast_idx = ctx.new_block("bidx.receiver_region.fast"); + let fallback_idx = ctx.new_block("bidx.receiver_region.fallback"); + let merge_idx = ctx.new_block("bidx.receiver_region.merge"); + let fast_label = ctx.block_label(fast_idx); + let fallback_label = ctx.block_label(fallback_idx); + let merge_label = ctx.block_label(merge_idx); + ctx.block() + .cond_br(&access.valid_i1, &fast_label, &fallback_label); + + ctx.current_block = fast_idx; + let array_handle = ctx.block().load(I64, &access.base_handle_slot); + let fast_value = + guarded_array::lower_trusted_plain_array_index_get(ctx, &array_handle, idx_i32); + let fast_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); + + ctx.current_block = fallback_idx; + let fallback_value = lower_bounded_array_index_get_checked(ctx, arr_box, idx_i32)?; + let fallback_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); + + ctx.current_block = merge_idx; + Ok(ctx.block().phi( + DOUBLE, + &[(&fast_value, &fast_end), (&fallback_value, &fallback_end)], + )) +} + +fn lower_bounded_array_index_get_checked( ctx: &mut FnCtx<'_>, arr_box: &str, idx_i32: &str, @@ -1646,8 +1686,9 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { let arr_box = lower_expr(ctx, object)?; let idx_i32 = ctx.block().load(I32, &i32_slot); if require_numeric_layout { - return lower_guarded_array_index_get( + return lower_region_validated_array_index_get( ctx, + *arr_id, &arr_box, &idx_i32, "bidx.num", @@ -1656,7 +1697,7 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { repair_slot.as_deref(), ); } - return lower_bounded_array_index_get(ctx, &arr_box, &idx_i32); + return lower_bounded_array_index_get(ctx, *arr_id, &arr_box, &idx_i32); } } } diff --git a/crates/perry-codegen/src/expr/index_get/guarded_array.rs b/crates/perry-codegen/src/expr/index_get/guarded_array.rs index a56d5c0c75..9e235ee48d 100644 --- a/crates/perry-codegen/src/expr/index_get/guarded_array.rs +++ b/crates/perry-codegen/src/expr/index_get/guarded_array.rs @@ -20,7 +20,8 @@ use anyhow::Result; use crate::nanbox::POINTER_MASK_I64; use crate::native_value::{ - BoundsState, BufferAccessMode, LoweredValue, MaterializationReason, NativeRep, SemanticKind, + BoundsProof, BoundsState, BufferAccessMode, LoweredValue, MaterializationReason, NativeRep, + SemanticKind, }; use crate::types::{DOUBLE, I1, I16, I32, I64, I8}; @@ -52,6 +53,134 @@ pub(super) fn lower_trusted_plain_array_index_get( blk.select(I1, &is_hole, DOUBLE, &undefined, &raw) } +fn lower_trusted_numeric_array_index_get( + ctx: &mut FnCtx<'_>, + array_handle: &str, + idx_i32: &str, + coerce_numeric_fallback: bool, +) -> String { + let blk = ctx.block(); + let idx_i64 = blk.zext(I32, idx_i32, I64); + let byte_offset = blk.shl(I64, &idx_i64, "3"); + let with_header = blk.add(I64, &byte_offset, "8"); + let element_addr = blk.add(I64, array_handle, &with_header); + let element_ptr = blk.inttoptr(I64, &element_addr); + let raw = blk.load(DOUBLE, &element_ptr); + if coerce_numeric_fallback { + // A number-context consumer accepts the same raw-f64-or-holes + // contract as the established guarded tier. Phase 3 currently installs + // only the stronger dense numeric descriptor, but retaining the + // canonicalization here keeps this consumer correct if that admission + // widens later. + let is_ordered = blk.fcmp("ord", &raw, &raw); + blk.select(I1, &is_ordered, DOUBLE, &raw, "0x7FF8000000000000") + } else { + raw + } +} + +/// Consume #9254 phase 3's one-time receiver validation at an exact bounded +/// read. `valid_i1` dominates the loop and is invariant; the true arm needs +/// only the refreshed handle load and raw element access, while the false arm +/// is the pre-existing guarded implementation in full. +#[allow(clippy::too_many_arguments)] +pub(super) fn lower_region_validated_array_index_get( + ctx: &mut FnCtx<'_>, + arr_id: u32, + arr_box: &str, + idx_i32: &str, + block_prefix: &str, + require_numeric_layout: bool, + coerce_numeric_fallback: bool, + receiver_slot: Option<&str>, +) -> Result { + let Some(access) = ctx + .receiver_descriptors + .array_access(arr_id, require_numeric_layout) + else { + return lower_guarded_array_index_get( + ctx, + arr_box, + idx_i32, + block_prefix, + require_numeric_layout, + coerce_numeric_fallback, + receiver_slot, + ); + }; + + let fast_idx = ctx.new_block(&format!("{}.receiver_region.fast", block_prefix)); + let fallback_idx = ctx.new_block(&format!("{}.receiver_region.fallback", block_prefix)); + let merge_idx = ctx.new_block(&format!("{}.receiver_region.merge", block_prefix)); + let fast_label = ctx.block_label(fast_idx); + let fallback_label = ctx.block_label(fallback_idx); + let merge_label = ctx.block_label(merge_idx); + ctx.block() + .cond_br(&access.valid_i1, &fast_label, &fallback_label); + + ctx.current_block = fast_idx; + let array_handle = ctx.block().load(I64, &access.base_handle_slot); + let fast_value = if require_numeric_layout { + lower_trusted_numeric_array_index_get(ctx, &array_handle, idx_i32, coerce_numeric_fallback) + } else { + lower_trusted_plain_array_index_get(ctx, &array_handle, idx_i32) + }; + let fast_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); + + if require_numeric_layout { + let lowered = LoweredValue { + semantic: SemanticKind::JsNumber, + rep: NativeRep::F64, + llvm_ty: DOUBLE, + value: fast_value.clone(), + }; + ctx.record_lowered_value_with_access_mode_and_facts( + "NumericArrayIndexGet", + Some(arr_id), + "receiver_descriptor", + &lowered, + Some(BoundsState::Proven { + proof: BoundsProof::LoopGuard, + }), + None, + Some(BufferAccessMode::CheckedNative), + None, + None, + None, + vec![raw_f64_layout_fact( + Some(arr_id), + "consumed", + "receiver_descriptor", + None, + )], + Vec::new(), + false, + false, + vec!["receiver_region=validated_once".to_string()], + ); + } + + ctx.current_block = fallback_idx; + let fallback_value = lower_guarded_array_index_get( + ctx, + arr_box, + idx_i32, + &format!("{}.receiver_region.checked", block_prefix), + require_numeric_layout, + coerce_numeric_fallback, + receiver_slot, + )?; + let fallback_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); + + ctx.current_block = merge_idx; + Ok(ctx.block().phi( + DOUBLE, + &[(&fast_value, &fast_end), (&fallback_value, &fallback_end)], + )) +} + pub(super) fn lower_guarded_array_index_get( ctx: &mut FnCtx<'_>, arr_box: &str, diff --git a/crates/perry-codegen/src/stmt/loops.rs b/crates/perry-codegen/src/stmt/loops.rs index 1bb8d90e51..17891cc498 100644 --- a/crates/perry-codegen/src/stmt/loops.rs +++ b/crates/perry-codegen/src/stmt/loops.rs @@ -3,9 +3,10 @@ use super::*; use crate::expr::{ - array_kind_fact, effect_fact, emit_typed_feedback_register_site, nanbox_pointer_inline, - raw_f64_layout_fact, BoundedIndexPair, PackedF64LoopFact, PackedNumericLoopKind, - TypedFeedbackContract, TypedFeedbackKind, + array_kind_fact, effect_fact, emit_typed_feedback_register_site, + expr_has_numeric_pointer_free_array_layout, nanbox_pointer_inline, raw_f64_layout_fact, + BoundedIndexPair, PackedF64LoopFact, PackedNumericLoopKind, TypedFeedbackContract, + TypedFeedbackKind, }; use crate::loop_purity::body_needs_asm_barrier; use crate::lower_conditional::lower_truthy; @@ -78,6 +79,61 @@ struct LengthHoist { buffer_bounds_width_units: Option, } +/// #9254 phase 3: prove the dynamic extent in which an ordinary +/// `i < arr.length` loop may consume a one-time receiver validation. +/// +/// Only the exact bounded `arr[i]` operation is replaced by the descriptor's +/// non-dispatching load. Its children and every surrounding operation retain +/// the conservative region classification. The outer back-edge poll is added +/// explicitly because the caller passes the loop body rather than a `Stmt::For` +/// node; nested-loop polls are discovered by the statement walker itself. +fn ordinary_counted_array_region_enders( + ctx: &FnCtx<'_>, + hoist: LengthHoist, + update: Option<&perry_hir::Expr>, + body: &[Stmt], +) -> Option> { + use std::cell::Cell; + + if !matches!(hoist.op, perry_hir::CompareOp::Lt) + || hoist.lhs_addend != 0 + || !loop_counter_bounds_are_safe(ctx, hoist.counter_id, update, body) + { + return None; + } + + let saw_bounded_read = Cell::new(false); + let is_trusted_operation = |expr: &perry_hir::Expr| { + let trusted = matches!( + expr, + perry_hir::Expr::IndexGet { object, index } + if matches!(object.as_ref(), perry_hir::Expr::LocalGet(id) if *id == hoist.arr_id) + && matches!(index.as_ref(), perry_hir::Expr::LocalGet(id) if *id == hoist.counter_id) + ); + if trusted { + saw_bounded_read.set(true); + } + trusted + }; + let is_inert = |expr: &perry_hir::Expr| crate::rooting::expr_is_inert_primitive(ctx, expr); + let controls: Vec<&perry_hir::Expr> = update.into_iter().collect(); + let mut enders = crate::collectors::region_enders_in_stmts_with_trusted_operations( + body, + &controls, + &is_inert, + &is_trusted_operation, + ); + if !saw_bounded_read.get() + || enders + .iter() + .any(|ender| !matches!(ender, crate::collectors::RegionEnder::BackEdgePoll)) + { + return None; + } + enders.push(crate::collectors::RegionEnder::BackEdgePoll); + Some(enders) +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum LoopArrayLengthEffect { Preserves, @@ -826,6 +882,113 @@ struct PackedAccumulatorScope { hoisted_receivers: Vec, } +/// Materialize the address half shared by packed clones and phase 3 ordinary +/// counted loops. The returned box is a precise root, while the handle slot is +/// refreshed from it after every fired loop poll. +fn create_poll_refreshed_receiver_cache( + ctx: &mut FnCtx<'_>, + arr_id: u32, +) -> Option<(String, String, String)> { + let source_ref = if let Some(slot) = ctx.locals.get(&arr_id) { + slot.clone() + } else { + format!("@{}", ctx.module_globals.get(&arr_id)?) + }; + let current = ctx.block().load(DOUBLE, &source_ref); + let rooted_box_slot = ctx.func.alloca_entry(DOUBLE); + let base_handle_slot = ctx.func.alloca_entry(I64); + // `root_entry_alloca` hoists the bind into entry setup, so seed the cache + // before that bind can make the collector dereference it. The later store + // publishes the live receiver and the bind makes evacuation rewrite this + // cache itself. + let undefined = crate::nanbox::double_literal(f64::from_bits(crate::nanbox::TAG_UNDEFINED)); + ctx.func + .entry_allocas_push_store(DOUBLE, &undefined, &rooted_box_slot); + ctx.block().store(DOUBLE, ¤t, &rooted_box_slot); + crate::expr::root_entry_alloca(ctx, &rooted_box_slot); + { + let blk = ctx.block(); + let bits = blk.bitcast_double_to_i64(¤t); + let handle = blk.and(I64, &bits, crate::nanbox::POINTER_MASK_I64); + blk.store(I64, &handle, &base_handle_slot); + } + Some((rooted_box_slot, base_handle_slot, source_ref)) +} + +/// Attach a numeric-array validation to an ordinary counted loop. This is +/// intentionally narrower than the descriptor model: phase 3 targets the +/// numeric `arr[i]` shape whose guarded header chain dominates matmul-style +/// kernels. A false one-time guard keeps the established per-read fallback; +/// a true guard makes every exact bounded read a raw load from the refreshed +/// handle slot. +fn materialize_ordinary_counted_array_descriptor( + ctx: &mut FnCtx<'_>, + hoist: LengthHoist, + update: Option<&perry_hir::Expr>, + body: &[Stmt], +) -> bool { + if ctx.receiver_descriptors.contains(hoist.arr_id) + || !expr_has_numeric_pointer_free_array_layout( + ctx, + &perry_hir::Expr::LocalGet(hoist.arr_id), + ) + { + return false; + } + let Some(enders) = ordinary_counted_array_region_enders(ctx, hoist, update, body) else { + return false; + }; + let Some((rooted_box_slot, base_handle_slot, source_root)) = + create_poll_refreshed_receiver_cache(ctx, hoist.arr_id) + else { + return false; + }; + + let feedback_site_id = emit_typed_feedback_register_site( + ctx, + TypedFeedbackKind::ArrayElement, + "array[index].receiver_region", + TypedFeedbackContract::numeric_array_get_index(), + ); + let receiver = ctx.block().load(DOUBLE, &rooted_box_slot); + let guard_i32 = ctx.block().call( + I32, + "js_typed_feedback_numeric_array_index_get_guard", + &[ + (I64, &feedback_site_id), + (DOUBLE, &receiver), + // Receiver-only validation: the surrounding strict length bound + // supplies the per-use index proof. + (I32, "0"), + (I32, "0"), + ], + ); + let valid_i1 = ctx.block().icmp_ne(I32, &guard_i32, "0"); + + // The numeric guard is non-collecting, but it may verify/rewrite boxed + // numeric slots into raw-f64 representation. Derive the cached handle + // after that operation so the ordering is explicit in the IR contract. + { + let blk = ctx.block(); + let fresh = blk.load(DOUBLE, &rooted_box_slot); + let bits = blk.bitcast_double_to_i64(&fresh); + let handle = blk.and(I64, &bits, crate::nanbox::POINTER_MASK_I64); + blk.store(I64, &handle, &base_handle_slot); + } + + ctx.receiver_descriptors + .materialize_region_validated_array( + hoist.arr_id, + rooted_box_slot, + base_handle_slot, + source_root, + crate::collectors::ReceiverArrayValidationKind::Numeric, + valid_i1, + &enders, + ) + .expect("region analysis admitted only poll-refreshable boundaries") +} + impl PackedAccumulatorScope { fn empty() -> Self { Self { @@ -933,40 +1096,15 @@ impl PackedAccumulatorScope { if ctx.receiver_descriptors.contains(*arr_id) { continue; } - let source_ref = if let Some(slot) = ctx.locals.get(arr_id) { - slot.clone() - } else if let Some(global_name) = ctx.module_globals.get(arr_id) { - format!("@{}", global_name) - } else { + let Some((rooted_box_slot, base_handle_slot, source_ref)) = + create_poll_refreshed_receiver_cache(ctx, *arr_id) + else { continue; }; - let current = ctx.block().load(DOUBLE, &source_ref); - let alloca = ctx.func.alloca_entry(DOUBLE); - let handle_alloca = ctx.func.alloca_entry(I64); - // `root_entry_alloca` hoists the bind into entry setup, so seed - // the cache before that bind can make the collector dereference - // it. The later store publishes the live receiver and the bind - // makes evacuation rewrite this cache itself. Under native roots - // the bind becomes an addrspace(1) value that mem2reg can still - // promote, retaining the receiver-cache fast path while making - // its liveness across a strided poll explicit to the checker. - let undef = crate::nanbox::double_literal(f64::from_bits(crate::nanbox::TAG_UNDEFINED)); - ctx.func.entry_allocas_push_store(DOUBLE, &undef, &alloca); - { - let blk = ctx.block(); - blk.store(DOUBLE, ¤t, &alloca); - } - crate::expr::root_entry_alloca(ctx, &alloca); - { - let blk = ctx.block(); - let bits = blk.bitcast_double_to_i64(¤t); - let handle = blk.and(I64, &bits, crate::nanbox::POINTER_MASK_I64); - blk.store(I64, &handle, &handle_alloca); - } let installed = ctx.receiver_descriptors.materialize_poll_refreshed_address( *arr_id, - alloca, - handle_alloca, + rooted_box_slot, + base_handle_slot, source_ref, // Packed loop admission rejects Stmt::Try. A throw may // leave the clone, but no descriptor is live in the @@ -6890,6 +7028,19 @@ pub(super) fn lower_for_after_init_with_i32_bound( None }; + // #9254 phase 3: once both the strict loop bound and its i32 storage are + // concrete, validate a numeric receiver once for the dynamic extent of + // this ordinary loop. Specialized clones keep precedence and own their + // existing descriptors; this path is for the generic counted-loop tier. + let ordinary_receiver_descriptor_installed = + if !in_call_free_clone && hoisted_length_slot.is_some() && i32_length_slot.is_some() { + hoist_classification.is_some_and(|hoist| { + materialize_ordinary_counted_array_descriptor(ctx, hoist, update, body) + }) + } else { + false + }; + // Issue #168: when the `i < arr.length` peephole didn't fire, also // detect the simpler `i < n` shape where `n` is a statically proven // loop-invariant i32 local. Emitting `fptosi(n)` once at the loop head @@ -7242,6 +7393,12 @@ pub(super) fn lower_for_after_init_with_i32_bound( ctx.loop_targets.pop(); + if ordinary_receiver_descriptor_installed { + let arr_id = hoisted_length_arr_id.expect("installed descriptor has a length receiver"); + let removed = ctx.receiver_descriptors.dematerialize(arr_id); + debug_assert!(removed, "loop owns the receiver descriptor it installed"); + } + // Pop the hoisted-length entry so nested loops or sibling loops // don't see a stale slot. Repsel Phase 1: only when THIS site inserted // it — a canonical-i32 counter's slot is its ONLY storage and must diff --git a/crates/perry/tests/issue_9254_receiver_descriptor_counted_loop.rs b/crates/perry/tests/issue_9254_receiver_descriptor_counted_loop.rs new file mode 100644 index 0000000000..a985bc0a74 --- /dev/null +++ b/crates/perry/tests/issue_9254_receiver_descriptor_counted_loop.rs @@ -0,0 +1,185 @@ +//! Regression coverage for #9254 phase 3: ordinary `i < arr.length` loops can +//! validate a numeric receiver once, then carry its refreshed address through +//! loop polls instead of repeating the full structural guard at every read. + +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +fn perry_bin() -> PathBuf { + PathBuf::from(env!("CARGO_BIN_EXE_perry")) +} + +fn compile(dir: &Path, source: &str) -> (PathBuf, String) { + let entry = dir.join("main.ts"); + let output = dir.join("main_bin"); + std::fs::write(&entry, source).expect("write entry"); + + let compile = Command::new(perry_bin()) + .current_dir(dir) + .arg("compile") + .arg(&entry) + .arg("-o") + .arg(&output) + .env("PERRY_NO_CACHE", "1") + .env("PERRY_NO_AUTO_OPTIMIZE", "1") + .env("PERRY_LLVM_KEEP_IR", "1") + .env_remove("PERRY_GC_MOVING_LOOP_POLLS") + .output() + .expect("run perry compile"); + assert!( + compile.status.success(), + "perry compile failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&compile.stdout), + String::from_utf8_lossy(&compile.stderr) + ); + ( + output, + String::from_utf8_lossy(&compile.stderr).into_owned(), + ) +} + +fn kept_ir(stderr: &str) -> String { + let path = stderr + .lines() + .find_map(|line| line.split("kept LLVM IR: ").nth(1)) + .map(str::trim) + .map(PathBuf::from) + .unwrap_or_else(|| panic!("PERRY_LLVM_KEEP_IR did not report an IR path\n{stderr}")); + std::fs::read_to_string(path).expect("read kept LLVM IR") +} + +fn run(bin: &Path, dir: &Path, scheduled_gc: bool) -> Output { + let mut command = Command::new(bin); + command.current_dir(dir); + for key in [ + "PERRY_GC_SCHEDULE_SEED", + "PERRY_GC_SCHEDULE_RATE", + "PERRY_GC_SCHEDULE_ALLOC_KB", + "PERRY_GC_FORCE_EVACUATE", + "PERRY_GC_VERIFY_EVACUATION", + "PERRY_GC_PROTECT_FROMSPACE", + "PERRY_GC_PROTECT_FROMSPACE_DEPTH", + ] { + command.env_remove(key); + } + if scheduled_gc { + command + .env("PERRY_GC_SCHEDULE_SEED", "9254") + .env("PERRY_GC_SCHEDULE_RATE", "1") + .env("PERRY_GC_SCHEDULE_ALLOC_KB", "0") + .env("PERRY_GC_FORCE_EVACUATE", "1") + .env("PERRY_GC_VERIFY_EVACUATION", "1") + .env("PERRY_GC_PROTECT_FROMSPACE", "1") + .env("PERRY_GC_PROTECT_FROMSPACE_DEPTH", "64"); + } + command.output().expect("run compiled binary") +} + +fn verdict_field(stderr: &str, field: &str) -> u64 { + let verdict = stderr + .lines() + .rev() + .find(|line| line.contains("[gc-schedule] forced_collections=")) + .unwrap_or_else(|| panic!("scheduled run emitted no exercise verdict\n{stderr}")); + verdict + .split_ascii_whitespace() + .find_map(|part| part.strip_prefix(&format!("{field}="))) + .and_then(|value| value.parse().ok()) + .unwrap_or_else(|| panic!("scheduled verdict has no numeric {field}\n{verdict}")) +} + +const VALUES: &str = "[0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5, 9.5, 10.5, 11.5, 12.5, \ + 13.5, 14.5, 15.5, 16.5, 17.5, 18.5, 19.5, 20.5, 21.5, 22.5, 23.5, 24.5, \ + 25.5, 26.5, 27.5, 28.5, 29.5, 30.5, 31.5]"; + +fn admitted_source() -> &'static str { + include_str!("../../../test-files/test_issue_9254_receiver_descriptor_counted_loop.ts") +} + +#[test] +fn ordinary_counted_loop_consumes_the_descriptor_and_refreshes_it_at_real_polls() { + let dir = tempfile::tempdir().expect("tempdir"); + let (bin, compile_stderr) = compile(dir.path(), admitted_source()); + let ir = kept_ir(&compile_stderr); + + let fast_start = ir + .find("\nbidx.num.receiver_region.fast") + .unwrap_or_else(|| panic!("the ordinary bounded read did not consume a descriptor\n{ir}")); + let fast_tail = &ir[fast_start + 1..]; + let fast_end = fast_tail + .find("\nbidx.num.receiver_region.fallback") + .expect("receiver-region fallback block follows its fast block"); + let fast_block = &fast_tail[..fast_end]; + assert!( + fast_block.contains("load i64") && fast_block.contains("load double"), + "descriptor fast block must load the refreshed handle and raw element\n{fast_block}" + ); + assert!( + !fast_block.contains("call "), + "the descriptor fast block repeated a runtime guard/call\n{fast_block}" + ); + assert!( + ir.contains("call i32 @js_typed_feedback_numeric_array_index_get_guard") + && ir.contains("i32 0, i32 0"), + "receiver-only numeric validation was not emitted in the preheader\n{ir}" + ); + assert!( + ir.contains("call void @js_gc_loop_safepoint"), + "the subject has no real poll and cannot prove refresh safety\n{ir}" + ); + + let plain = run(&bin, dir.path(), false); + assert!( + plain.status.success(), + "plain run failed\nstderr:\n{}", + String::from_utf8_lossy(&plain.stderr) + ); + assert_eq!(String::from_utf8_lossy(&plain.stdout), "32\n0\n"); + + let scheduled = run(&bin, dir.path(), true); + assert!( + scheduled.status.success(), + "scheduled moving-GC run failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&scheduled.stdout), + String::from_utf8_lossy(&scheduled.stderr) + ); + assert_eq!(scheduled.stdout, plain.stdout); + let scheduled_stderr = String::from_utf8_lossy(&scheduled.stderr); + for field in ["copying_minors", "moved_objects", "loop_polls"] { + assert!( + verdict_field(&scheduled_stderr, field) > 0, + "scheduled run did not exercise {field}\n{scheduled_stderr}" + ); + } +} + +#[test] +fn allocating_region_keeps_the_per_read_guard() { + let source = format!( + r#" +function sum(a: number[]): number {{ + let total = 0; + for (let i = 0; i < a.length; i++) {{ + const scratch = {{ value: i }}; + if (scratch.value < -1) total = total + 1000000; + total = total + a[i]; + }} + return total; +}} +const values: number[] = {VALUES}; +console.log(sum(values)); +"# + ); + let dir = tempfile::tempdir().expect("tempdir"); + let (_, compile_stderr) = compile(dir.path(), &source); + let ir = kept_ir(&compile_stderr); + assert!( + !ir.contains("receiver_region.fast"), + "an allocation/user-code-capable region incorrectly carried the descriptor\n{ir}" + ); + assert!( + ir.contains("bidx.num.guard.deref") + || ir.contains("call i32 @js_typed_feedback_numeric_array_index_get_guard"), + "negative control did not retain the established per-read guard\n{ir}" + ); +} diff --git a/test-files/test_issue_9254_receiver_descriptor_counted_loop.ts b/test-files/test_issue_9254_receiver_descriptor_counted_loop.ts new file mode 100644 index 0000000000..20a4042916 --- /dev/null +++ b/test-files/test_issue_9254_receiver_descriptor_counted_loop.ts @@ -0,0 +1,24 @@ +function sum(a: number[]): number { + let matched = 0; + for (let i = 0; i < a.length; i++) { + // Keep this on the ordinary counted-loop lowering: the switch makes the + // specialized packed-loop tiers decline without adding a collection point. + switch (i & 0) { + case 1: + matched = -1000000; + break; + } + if (a[i] === i + 0.5) matched++; + } + return matched; +} + +const values: number[] = [ + 0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, + 8.5, 9.5, 10.5, 11.5, 12.5, 13.5, 14.5, 15.5, + 16.5, 17.5, 18.5, 19.5, 20.5, 21.5, 22.5, 23.5, + 24.5, 25.5, 26.5, 27.5, 28.5, 29.5, 30.5, 31.5, +]; +console.log(sum(values)); +// A failed one-time receiver validation must retain the guarded fallback. +console.log(sum(["x", "y"] as any)); From 77902c1077e789d6190838c78ffa32e4ce74bdea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 4 Sep 2026 11:27:08 +0200 Subject: [PATCH 3/5] docs: add PR 9712 changelog fragment --- .../9712-receiver-descriptor-counted-loops.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 changelog.d/9712-receiver-descriptor-counted-loops.md diff --git a/changelog.d/9712-receiver-descriptor-counted-loops.md b/changelog.d/9712-receiver-descriptor-counted-loops.md new file mode 100644 index 0000000000..925731d1d9 --- /dev/null +++ b/changelog.d/9712-receiver-descriptor-counted-loops.md @@ -0,0 +1,15 @@ +**Ordinary counted loops now consume the shared receiver descriptor model** +(#9254 phase 3). For a strict, call-free `i < array.length` numeric-array loop, +codegen validates the receiver and element layout once in the preheader, keeps +the boxed receiver precisely rooted, and carries a cached base handle through +the loop. Exact bounded `array[i]` reads select an invariant raw-load arm instead +of repeating the receiver tag, header, integrity, length, capacity, and layout +checks at every use; a failed one-time validation retains the existing guarded +fallback semantics. + +Admission is intentionally conservative: the shared region analysis rejects +calls, allocations, coercions, suspensions, unwind edges, and every unmodelled +operation, while fired back-edge GC polls refresh both the rooted receiver and +its derived handle. Regression coverage pins the call-free fast block, the +allocating-region rejection, the guard-miss fallback, and a real rate-1 moving +collection with evacuated from-space protected. From c9ee9ea83c30537a7710428a71a4e5abde29488a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 4 Sep 2026 12:50:04 +0200 Subject: [PATCH 4/5] refactor(codegen): retire receiver fact tables --- crates/perry-codegen/src/codegen/closure.rs | 5 - crates/perry-codegen/src/codegen/entry.rs | 10 - crates/perry-codegen/src/codegen/function.rs | 9 +- crates/perry-codegen/src/codegen/method.rs | 10 - .../src/collectors/ptr_numarray.rs | 2 +- .../src/collectors/receiver_regions.rs | 448 +++++++++++++++--- .../src/collectors/receiver_regions_tests.rs | 235 +++++++-- crates/perry-codegen/src/expr/binary.rs | 2 +- .../perry-codegen/src/expr/buffer_access.rs | 14 +- crates/perry-codegen/src/expr/buffer_views.rs | 39 +- .../perry-codegen/src/expr/i32_fast_path.rs | 12 +- crates/perry-codegen/src/expr/index_get.rs | 26 +- .../src/expr/index_get/foreign_counter.rs | 8 +- .../src/expr/index_get/guarded_array.rs | 4 +- crates/perry-codegen/src/expr/index_set.rs | 18 +- .../perry-codegen/src/expr/literals_vars.rs | 2 +- .../perry-codegen/src/expr/masked_window.rs | 4 +- crates/perry-codegen/src/expr/mod.rs | 69 +-- .../perry-codegen/src/expr/native_memory.rs | 4 +- crates/perry-codegen/src/expr/property_get.rs | 12 +- .../src/expr/proven_view_access.rs | 60 +-- .../src/expr/ptr_numarray_access.rs | 11 +- crates/perry-codegen/src/expr/range_facts.rs | 30 +- .../src/expr/ta_param_f64_read.rs | 2 +- .../perry-codegen/src/expr/typed_array_rmw.rs | 4 +- .../perry-codegen/src/expr/u8_buffer_read.rs | 2 +- .../src/stmt/let_buffer_views.rs | 2 +- crates/perry-codegen/src/stmt/let_stmt.rs | 2 +- crates/perry-codegen/src/stmt/loops.rs | 171 +++---- .../src/stmt/masked_window_region.rs | 90 ++-- .../src/stmt/stable_packed_loop.rs | 4 +- .../src/stmt/stable_packed_typed_array.rs | 10 +- .../src/type_analysis/numeric.rs | 12 +- crates/perry-codegen/src/type_analysis/pod.rs | 4 +- 34 files changed, 862 insertions(+), 475 deletions(-) diff --git a/crates/perry-codegen/src/codegen/closure.rs b/crates/perry-codegen/src/codegen/closure.rs index 2d8196e645..57f15b9af3 100644 --- a/crates/perry-codegen/src/codegen/closure.rs +++ b/crates/perry-codegen/src/codegen/closure.rs @@ -1190,11 +1190,7 @@ pub(super) fn compile_closure( class_keys_slots: HashMap::new(), class_shape_slots: HashMap::new(), class_header_images: HashMap::new(), - cached_lengths: HashMap::new(), array_length_snapshots: HashMap::new(), - bounded_index_pairs: Vec::new(), - packed_f64_loop_facts: Vec::new(), - masked_window_array_facts: Vec::new(), string_window_array_facts: Vec::new(), masked_region_scalar_locals: std::collections::HashSet::new(), suppressed_cleared_shadow_slots: std::collections::HashSet::new(), @@ -1298,7 +1294,6 @@ pub(super) fn compile_closure( property_get_ic_override: None, typed_parse_rodata: Vec::new(), buffer_data_slots: HashMap::new(), - buffer_view_slots: HashMap::new(), native_arena_owner_aliases: HashMap::new(), native_arena_ambiguous_owner_aliases: HashSet::new(), disable_buffer_fast_path: cross_module.disable_buffer_fast_path, diff --git a/crates/perry-codegen/src/codegen/entry.rs b/crates/perry-codegen/src/codegen/entry.rs index 9b4f036cfb..b063835dc1 100644 --- a/crates/perry-codegen/src/codegen/entry.rs +++ b/crates/perry-codegen/src/codegen/entry.rs @@ -968,11 +968,7 @@ pub(super) fn compile_module_entry( class_keys_slots: HashMap::new(), class_shape_slots: HashMap::new(), class_header_images: HashMap::new(), - cached_lengths: HashMap::new(), array_length_snapshots: HashMap::new(), - bounded_index_pairs: Vec::new(), - packed_f64_loop_facts: Vec::new(), - masked_window_array_facts: Vec::new(), string_window_array_facts: Vec::new(), masked_region_scalar_locals: std::collections::HashSet::new(), suppressed_cleared_shadow_slots: std::collections::HashSet::new(), @@ -1085,7 +1081,6 @@ pub(super) fn compile_module_entry( property_get_ic_override: None, typed_parse_rodata: Vec::new(), buffer_data_slots: HashMap::new(), - buffer_view_slots: HashMap::new(), native_arena_owner_aliases: HashMap::new(), native_arena_ambiguous_owner_aliases: HashSet::new(), disable_buffer_fast_path: cross_module.disable_buffer_fast_path, @@ -1767,11 +1762,7 @@ pub(super) fn compile_module_entry( class_keys_slots: HashMap::new(), class_shape_slots: HashMap::new(), class_header_images: HashMap::new(), - cached_lengths: HashMap::new(), array_length_snapshots: HashMap::new(), - bounded_index_pairs: Vec::new(), - packed_f64_loop_facts: Vec::new(), - masked_window_array_facts: Vec::new(), string_window_array_facts: Vec::new(), masked_region_scalar_locals: std::collections::HashSet::new(), suppressed_cleared_shadow_slots: std::collections::HashSet::new(), @@ -1884,7 +1875,6 @@ pub(super) fn compile_module_entry( property_get_ic_override: None, typed_parse_rodata: Vec::new(), buffer_data_slots: HashMap::new(), - buffer_view_slots: HashMap::new(), native_arena_owner_aliases: HashMap::new(), native_arena_ambiguous_owner_aliases: HashSet::new(), disable_buffer_fast_path: cross_module.disable_buffer_fast_path, diff --git a/crates/perry-codegen/src/codegen/function.rs b/crates/perry-codegen/src/codegen/function.rs index 5a2a9f7073..8e58e0805b 100644 --- a/crates/perry-codegen/src/codegen/function.rs +++ b/crates/perry-codegen/src/codegen/function.rs @@ -1137,11 +1137,7 @@ pub(super) fn compile_function( class_keys_slots: HashMap::new(), class_shape_slots: HashMap::new(), class_header_images: HashMap::new(), - cached_lengths: HashMap::new(), array_length_snapshots: HashMap::new(), - bounded_index_pairs: Vec::new(), - packed_f64_loop_facts: Vec::new(), - masked_window_array_facts: Vec::new(), string_window_array_facts: Vec::new(), masked_region_scalar_locals: std::collections::HashSet::new(), suppressed_cleared_shadow_slots: std::collections::HashSet::new(), @@ -1250,7 +1246,6 @@ pub(super) fn compile_function( property_get_ic_override: None, typed_parse_rodata: Vec::new(), buffer_data_slots: HashMap::new(), - buffer_view_slots: HashMap::new(), native_arena_owner_aliases: HashMap::new(), native_arena_ambiguous_owner_aliases: HashSet::new(), disable_buffer_fast_path: cross_module.disable_buffer_fast_path, @@ -1319,7 +1314,7 @@ pub(super) fn compile_function( let scope_idx = ctx.buffer_alias_base + ctx.buffer_data_slots.len() as u32; ctx.buffer_data_slots .insert(p.id, (buf_slot.clone(), scope_idx)); - ctx.buffer_view_slots.insert( + ctx.receiver_descriptors.materialize_buffer_view( p.id, BufferViewSlot { data_slot: buf_slot, @@ -1383,7 +1378,7 @@ pub(super) fn compile_function( let scope_idx = ctx.buffer_alias_base + ctx.buffer_data_slots.len() as u32; ctx.buffer_data_slots .insert(p.id, (data_slot.clone(), scope_idx)); - ctx.buffer_view_slots.insert( + ctx.receiver_descriptors.materialize_buffer_view( p.id, BufferViewSlot { data_slot, diff --git a/crates/perry-codegen/src/codegen/method.rs b/crates/perry-codegen/src/codegen/method.rs index 75e5b8706d..e4464d2d8e 100644 --- a/crates/perry-codegen/src/codegen/method.rs +++ b/crates/perry-codegen/src/codegen/method.rs @@ -543,11 +543,7 @@ pub(super) fn compile_method( class_keys_slots: HashMap::new(), class_shape_slots: HashMap::new(), class_header_images: HashMap::new(), - cached_lengths: HashMap::new(), array_length_snapshots: HashMap::new(), - bounded_index_pairs: Vec::new(), - packed_f64_loop_facts: Vec::new(), - masked_window_array_facts: Vec::new(), string_window_array_facts: Vec::new(), masked_region_scalar_locals: std::collections::HashSet::new(), suppressed_cleared_shadow_slots: std::collections::HashSet::new(), @@ -662,7 +658,6 @@ pub(super) fn compile_method( property_get_ic_override: None, typed_parse_rodata: Vec::new(), buffer_data_slots: HashMap::new(), - buffer_view_slots: HashMap::new(), native_arena_owner_aliases: HashMap::new(), native_arena_ambiguous_owner_aliases: HashSet::new(), disable_buffer_fast_path: cross_module.disable_buffer_fast_path, @@ -1712,11 +1707,7 @@ pub(super) fn compile_static_method( class_keys_slots: HashMap::new(), class_shape_slots: HashMap::new(), class_header_images: HashMap::new(), - cached_lengths: HashMap::new(), array_length_snapshots: HashMap::new(), - bounded_index_pairs: Vec::new(), - packed_f64_loop_facts: Vec::new(), - masked_window_array_facts: Vec::new(), string_window_array_facts: Vec::new(), masked_region_scalar_locals: std::collections::HashSet::new(), suppressed_cleared_shadow_slots: std::collections::HashSet::new(), @@ -1820,7 +1811,6 @@ pub(super) fn compile_static_method( property_get_ic_override: None, typed_parse_rodata: Vec::new(), buffer_data_slots: HashMap::new(), - buffer_view_slots: HashMap::new(), native_arena_owner_aliases: HashMap::new(), native_arena_ambiguous_owner_aliases: HashSet::new(), disable_buffer_fast_path: cross_module.disable_buffer_fast_path, diff --git a/crates/perry-codegen/src/collectors/ptr_numarray.rs b/crates/perry-codegen/src/collectors/ptr_numarray.rs index 33ab9b8eb9..d00e795a5f 100644 --- a/crates/perry-codegen/src/collectors/ptr_numarray.rs +++ b/crates/perry-codegen/src/collectors/ptr_numarray.rs @@ -73,7 +73,7 @@ //! has `int_range` proof `[0, max]` with `max < proven_initial_length` //! (length can only grow — `pop`-class shrinkers are disqualified — so //! `idx < initial_length <= current length` holds forever), or the site is -//! a `bounded_index_pairs` loop read. Everything unproven falls back to the +//! a bounded-index receiver descriptor read. Everything unproven falls back to the //! Phase 4a.1/4a.2 guarded tiers, which maintain the same invariants. //! * **Hole observability (density gating)**: guard-free READS are emitted //! only in ToNumber contexts (the Phase 4a number-context reader), where a diff --git a/crates/perry-codegen/src/collectors/receiver_regions.rs b/crates/perry-codegen/src/collectors/receiver_regions.rs index b59d55fd8a..68477914f4 100644 --- a/crates/perry-codegen/src/collectors/receiver_regions.rs +++ b/crates/perry-codegen/src/collectors/receiver_regions.rs @@ -3,22 +3,26 @@ //! //! # Why this exists //! -//! Phase 1 found sixteen separate receiver-keyed fact mechanisms on `FnCtx` -//! (`cached_lengths`, `bounded_index_pairs`, `packed_f64_loop_facts`, -//! `masked_window_array_facts`, `buffer_view_slots`, `int_range_facts`, +//! Phase 1 found sixteen separate receiver-keyed fact mechanisms on `FnCtx`. +//! The original issue singled out six (`cached_lengths`, +//! `bounded_index_pairs`, `packed_f64_loop_facts`, +//! `masked_window_array_facts`, `buffer_view_slots`, and the +//! `packed_receiver_*` trio); Phase 4 has now moved all six into this table. +//! The expanded audit also records `int_range_facts`, +//! `bounded_buffer_index_pairs`, `guarded_buffer_index_pairs`, //! `element_shape_loop_facts`, `class_field_loop_facts`, //! `versioned_indexed_loop_facts`, `stable_packed_loop_facts`, -//! `string_window_array_facts`, `buffer_data_slots`, the two class-shape slot -//! maps and the `packed_receiver_*` trio). Each answers the same two questions +//! `string_window_array_facts`, `buffer_data_slots`, and `class_keys_slots`. +//! Historically, each answered the same two questions //! — *what do we know about this receiver* and *how long may we believe it* — //! and each answers the second question in a different, hand-rolled way: //! //! | mechanism | tables | //! |---|---| -//! | `retain(|f| f.scope_id != id)` at scope exit | `bounded_index_pairs`, `packed_f64_loop_facts`, `masked_window_array_facts` | -//! | insert/remove pair with no id | `cached_lengths`, `packed_receiver_*` | -//! | mutable field downgraded in place, never removed | `buffer_view_slots` | -//! | reloaded at the safepoint instead of invalidated | `packed_receiver_*` | +//! | `retain(|f| f.scope_id != id)` at scope exit | `receiver_descriptors[bounded_index]` / `[packed_f64_loop]` / `[masked_window_array]` (formerly three independent tables) | +//! | insert/remove pair with no id | `receiver_descriptors[cached_length]` and the base address payload (formerly `cached_lengths` and `packed_receiver_*`) | +//! | mutable field downgraded in place, never removed | `receiver_descriptors[buffer_view]` (formerly `buffer_view_slots`) | +//! | reloaded at the safepoint instead of invalidated | base address payload (formerly `packed_receiver_*`) | //! //! and a fifth boundary — the **unwind edge** — is expressed by none of them. //! It is honoured today only indirectly: the packed matcher rejects @@ -38,8 +42,10 @@ //! before carrying that address across a back-edge poll. Phase 3 lets ordinary //! counted loops attach a conditional plain/numeric-array validation to the //! same entry, but only after this module proves the loop region contains no -//! ender other than the poll covered by that refresh recipe. Other fact tables -//! are migrated one consumer at a time. +//! ender other than the poll covered by that refresh recipe. Phase 4 folds the +//! five remaining mechanisms named by the proposal into this table: cached +//! lengths, bounded indices, packed and masked representations, and +//! non-moving buffer views. //! //! The precedent is `TypeFacts::purity` / `TypeFacts::shape_stability` //! (`collectors/hir_facts.rs`, #854): a subgraph the collector populates and @@ -58,13 +64,14 @@ //! what keeps this file honest: the model is checked against a shipping, //! audited predicate rather than against its own restatement. -// #9254 phases 2/3: `ReceiverDescriptorTable`, the poll boundary algebra and -// region formation are production consumers. Some inventory/lint helpers stay -// test-only while the remaining fact tables await phase 4, so keep that -// incomplete state explicit rather than scattering per-item allows. +// `ReceiverDescriptorTable`, the poll boundary algebra and region formation +// are production consumers. Inventory/lint helpers remain test-only, so keep +// their allowance central rather than scattering per-item attributes. #![allow(dead_code)] +use crate::expr::{MaskedWindowArrayFact, PackedF64LoopFact}; use crate::loop_purity; +use crate::native_value::BufferViewSlot; use perry_hir::{CompareOp, Expr, Stmt, UnaryOp}; /// Why a no-relocation region ends. @@ -160,6 +167,11 @@ pub(crate) enum ReceiverClaim { /// outright; this is the only claim for which a region boundary is /// load-bearing rather than incidental. Address, + /// A raw address into storage whose allocation is explicitly non-moving. + /// Collection and unwind do not stale it; receiver reassignment, backing + /// replacement, disposal and alias escape are handled by descriptor-table + /// degradation APIs instead. + NonMovingAddress, } /// One table's claim about one receiver, in the shared vocabulary. @@ -172,8 +184,8 @@ pub(crate) struct ReceiverDescriptor { pub(crate) claim: ReceiverClaim, pub(crate) boundary: FactBoundary, /// Whether the tier that owns this table structurally excludes `Stmt::Try` - /// from the region it forms (the packed matcher does; `buffer_view_slots` - /// has no region at all). + /// from the region it forms (the packed matcher does; a buffer-view + /// descriptor has no region at all). pub(crate) excludes_try: bool, } @@ -207,7 +219,7 @@ pub(crate) fn boundary_admits( match desc.claim { // A length or an index range is a value. Relocation does not touch it. // It dies at mutation, which no ender in this enum implies on its own. - ReceiverClaim::ScalarRelation => Ok(()), + ReceiverClaim::ScalarRelation | ReceiverClaim::NonMovingAddress => Ok(()), // A representation claim survives relocation but not arbitrary user // code, which can convert the receiver's storage out from under it. @@ -290,11 +302,28 @@ struct ActiveArrayValidation { valid_i1: String, } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone)] struct ActiveReceiverDescriptor { contract: ReceiverDescriptor, - refresh: ReceiverPollRefresh, - array_validation: Option, + data: ActiveReceiverData, +} + +#[derive(Debug, Clone)] +enum ActiveReceiverData { + Address { + refresh: ReceiverPollRefresh, + array_validation: Option, + }, + CachedLength { + slot: String, + }, + BoundedIndex { + index_local_id: u32, + scope_id: u32, + }, + PackedF64Loop(PackedF64LoopFact), + MaskedWindowArray(MaskedWindowArrayFact), + BufferView(BufferViewSlot), } /// Active materialised receiver descriptors for one function lowering. @@ -310,9 +339,10 @@ pub(crate) struct ReceiverDescriptorTable { impl ReceiverDescriptorTable { /// Whether an active scope has already materialised `receiver`. pub(crate) fn contains(&self, receiver: u32) -> bool { - self.entries - .iter() - .any(|entry| entry.contract.receiver == receiver) + self.entries.iter().any(|entry| { + entry.contract.receiver == receiver + && matches!(&entry.data, ActiveReceiverData::Address { .. }) + }) } /// Install the first production descriptor consumer (#9254 phase 2): a @@ -346,12 +376,14 @@ impl ReceiverDescriptorTable { .expect("a poll-refreshed receiver descriptor must survive its poll boundary"); self.entries.push(ActiveReceiverDescriptor { contract, - refresh: ReceiverPollRefresh { - rooted_box_slot, - base_handle_slot, - source_root, + data: ActiveReceiverData::Address { + refresh: ReceiverPollRefresh { + rooted_box_slot, + base_handle_slot, + source_root, + }, + array_validation: None, }, - array_validation: None, }); true } @@ -402,27 +434,307 @@ impl ReceiverDescriptorTable { } self.entries.push(ActiveReceiverDescriptor { contract: address_contract, - refresh: ReceiverPollRefresh { - rooted_box_slot, - base_handle_slot, - source_root, + data: ActiveReceiverData::Address { + refresh: ReceiverPollRefresh { + rooted_box_slot, + base_handle_slot, + source_root, + }, + array_validation: Some(ActiveArrayValidation { + contract: representation_contract, + kind, + valid_i1, + }), }, - array_validation: Some(ActiveArrayValidation { - contract: representation_contract, - kind, - valid_i1, - }), }); Ok(true) } + /// Install a loop-invariant `receiver.length` value for one dynamic + /// extent. Unlike an address, this scalar survives every relocation + /// boundary; ownership still belongs in the descriptor table so the + /// extent cannot drift from the receiver fact it serves. + pub(crate) fn materialize_cached_length(&mut self, receiver: u32, slot: String) -> bool { + let contract = ReceiverDescriptor { + table: "receiver_descriptors[cached_length]", + receiver, + claim: ReceiverClaim::ScalarRelation, + boundary: FactBoundary::DynamicExtent, + excludes_try: false, + }; + self.entries.push(ActiveReceiverDescriptor { + contract, + data: ActiveReceiverData::CachedLength { slot }, + }); + true + } + + /// End the dynamic extent of a cached length without disturbing another + /// active descriptor payload for the same receiver. + pub(crate) fn dematerialize_cached_length(&mut self, receiver: u32) -> bool { + let Some(index) = self.entries.iter().rposition(|entry| { + entry.contract.receiver == receiver + && matches!(&entry.data, ActiveReceiverData::CachedLength { .. }) + }) else { + return false; + }; + self.entries.remove(index); + true + } + + /// The loop-invariant boxed-double length slot for `receiver`. + pub(crate) fn cached_length_slot(&self, receiver: u32) -> Option<&str> { + self.entries.iter().rev().find_map(|entry| { + if entry.contract.receiver != receiver { + return None; + } + match &entry.data { + ActiveReceiverData::CachedLength { slot } => Some(slot.as_str()), + ActiveReceiverData::Address { .. } + | ActiveReceiverData::BoundedIndex { .. } + | ActiveReceiverData::PackedF64Loop(_) + | ActiveReceiverData::MaskedWindowArray(_) + | ActiveReceiverData::BufferView(_) => None, + } + }) + } + + /// Record that `index_local_id` is in bounds for `receiver` throughout a + /// lexical loop-proof scope. Scalar relations survive safepoints, while + /// reassignment and scope exit invalidate them through the table APIs + /// below. + pub(crate) fn materialize_bounded_index( + &mut self, + receiver: u32, + index_local_id: u32, + scope_id: u32, + ) { + let contract = ReceiverDescriptor { + table: "receiver_descriptors[bounded_index]", + receiver, + claim: ReceiverClaim::ScalarRelation, + boundary: FactBoundary::ScopeId, + excludes_try: false, + }; + self.entries.push(ActiveReceiverDescriptor { + contract, + data: ActiveReceiverData::BoundedIndex { + index_local_id, + scope_id, + }, + }); + } + + /// Whether the current descriptor scope proves this exact receiver/index + /// pair in bounds. + pub(crate) fn has_bounded_index(&self, receiver: u32, index_local_id: u32) -> bool { + self.entries.iter().any(|entry| { + entry.contract.receiver == receiver + && matches!( + &entry.data, + ActiveReceiverData::BoundedIndex { + index_local_id: active_index, + .. + } if *active_index == index_local_id + ) + }) + } + + /// Invalidate bounded-index relations whose receiver or index binding was + /// reassigned. + pub(crate) fn invalidate_bounded_indices_for_local(&mut self, local_id: u32) { + self.entries.retain(|entry| { + !(entry.contract.receiver == local_id + && matches!(&entry.data, ActiveReceiverData::BoundedIndex { .. })) + && !matches!( + &entry.data, + ActiveReceiverData::BoundedIndex { index_local_id, .. } + if *index_local_id == local_id + ) + }); + } + + /// Install a packed numeric-array representation fact for one guarded + /// clone. The producing matcher excludes calls and `try`; the remaining + /// back-edge poll is admitted here by the shared boundary algebra. + pub(crate) fn materialize_packed_f64_loop(&mut self, fact: PackedF64LoopFact) { + let contract = ReceiverDescriptor { + table: "receiver_descriptors[packed_f64_loop]", + receiver: fact.array_local_id, + claim: ReceiverClaim::Representation, + boundary: FactBoundary::ScopeId, + excludes_try: true, + }; + boundary_admits(&contract, RegionEnder::BackEdgePoll) + .expect("a packed representation descriptor must survive its loop poll"); + self.entries.push(ActiveReceiverDescriptor { + contract, + data: ActiveReceiverData::PackedF64Loop(fact), + }); + } + + /// Packed representation facts in installation order. Nested consumers + /// may reverse this iterator to prefer their innermost scope. + pub(crate) fn packed_f64_loop_facts( + &self, + ) -> impl DoubleEndedIterator { + self.entries.iter().filter_map(|entry| match &entry.data { + ActiveReceiverData::PackedF64Loop(fact) => Some(fact), + _ => None, + }) + } + + pub(crate) fn has_packed_f64_loop_facts(&self) -> bool { + self.packed_f64_loop_facts().next().is_some() + } + + /// Install a statically bounded masked-window representation fact for one + /// guarded clone. Its producer admits only a call-free scalar region and + /// excludes `try`, leaving the loop poll as the sole region boundary. + pub(crate) fn materialize_masked_window_array(&mut self, fact: MaskedWindowArrayFact) { + let contract = ReceiverDescriptor { + table: "receiver_descriptors[masked_window_array]", + receiver: fact.array_local_id, + claim: ReceiverClaim::Representation, + boundary: FactBoundary::ScopeId, + excludes_try: true, + }; + boundary_admits(&contract, RegionEnder::BackEdgePoll) + .expect("a masked-window representation descriptor must survive its loop poll"); + self.entries.push(ActiveReceiverDescriptor { + contract, + data: ActiveReceiverData::MaskedWindowArray(fact), + }); + } + + pub(crate) fn masked_window_array_facts( + &self, + ) -> impl DoubleEndedIterator { + self.entries.iter().filter_map(|entry| match &entry.data { + ActiveReceiverData::MaskedWindowArray(fact) => Some(fact), + _ => None, + }) + } + + /// Install or replace the function-lifetime native storage descriptor for + /// a Buffer/TypedArray receiver. The pointed-to allocation is non-moving; + /// all semantic invalidation is expressed by mutating or removing this + /// payload through the APIs below. + pub(crate) fn materialize_buffer_view( + &mut self, + receiver: u32, + view: BufferViewSlot, + ) -> Option { + let previous = self.dematerialize_buffer_view(receiver); + let contract = ReceiverDescriptor { + table: "receiver_descriptors[buffer_view]", + receiver, + claim: ReceiverClaim::NonMovingAddress, + boundary: FactBoundary::InPlaceDegradation, + excludes_try: false, + }; + for ender in [RegionEnder::BackEdgePoll, RegionEnder::UnwindEdge] { + boundary_admits(&contract, ender) + .expect("a non-moving buffer-view address survives relocation boundaries"); + } + self.entries.push(ActiveReceiverDescriptor { + contract, + data: ActiveReceiverData::BufferView(view), + }); + previous + } + + pub(crate) fn dematerialize_buffer_view( + &mut self, + receiver: impl std::borrow::Borrow, + ) -> Option { + let receiver = *receiver.borrow(); + let index = self.entries.iter().position(|entry| { + entry.contract.receiver == receiver + && matches!(&entry.data, ActiveReceiverData::BufferView(_)) + })?; + let entry = self.entries.remove(index); + let ActiveReceiverData::BufferView(view) = entry.data else { + unreachable!("buffer-view lookup selected another descriptor payload") + }; + Some(view) + } + + pub(crate) fn contains_buffer_view(&self, receiver: impl std::borrow::Borrow) -> bool { + self.buffer_view(receiver).is_some() + } + + pub(crate) fn buffer_view( + &self, + receiver: impl std::borrow::Borrow, + ) -> Option<&BufferViewSlot> { + let receiver = *receiver.borrow(); + self.entries.iter().find_map(|entry| { + if entry.contract.receiver != receiver { + return None; + } + match &entry.data { + ActiveReceiverData::BufferView(view) => Some(view), + _ => None, + } + }) + } + + pub(crate) fn buffer_view_mut( + &mut self, + receiver: impl std::borrow::Borrow, + ) -> Option<&mut BufferViewSlot> { + let receiver = *receiver.borrow(); + self.entries.iter_mut().find_map(|entry| { + if entry.contract.receiver != receiver { + return None; + } + match &mut entry.data { + ActiveReceiverData::BufferView(view) => Some(view), + _ => None, + } + }) + } + + pub(crate) fn buffer_views(&self) -> impl Iterator { + self.entries.iter().filter_map(|entry| match &entry.data { + ActiveReceiverData::BufferView(view) => Some((entry.contract.receiver, view)), + _ => None, + }) + } + + pub(crate) fn buffer_views_mut(&mut self) -> impl Iterator { + self.entries + .iter_mut() + .filter_map(|entry| match &mut entry.data { + ActiveReceiverData::BufferView(view) => Some((entry.contract.receiver, view)), + _ => None, + }) + } + + /// End every descriptor fact owned by a lexical proof scope. Each Phase 4 + /// migration adds its scoped payload here, replacing a separate + /// `retain(scope_id)` discipline at the lowering site. + pub(crate) fn dematerialize_scope(&mut self, scope_id: u32) -> usize { + let before = self.entries.len(); + self.entries.retain(|entry| { + let active_scope = match &entry.data { + ActiveReceiverData::BoundedIndex { scope_id, .. } => Some(*scope_id), + ActiveReceiverData::PackedF64Loop(fact) => Some(fact.scope_id), + ActiveReceiverData::MaskedWindowArray(fact) => Some(fact.scope_id), + _ => None, + }; + active_scope != Some(scope_id) + }); + before - self.entries.len() + } + /// End the dynamic extent of one materialised receiver. pub(crate) fn dematerialize(&mut self, receiver: u32) -> bool { - let Some(index) = self - .entries - .iter() - .position(|entry| entry.contract.receiver == receiver) - else { + let Some(index) = self.entries.iter().position(|entry| { + entry.contract.receiver == receiver + && matches!(&entry.data, ActiveReceiverData::Address { .. }) + }) else { return false; }; self.entries.remove(index); @@ -431,18 +743,22 @@ impl ReceiverDescriptorTable { /// The promotable, precise-root box slot consumed by `LocalGet`. pub(crate) fn rooted_box_slot(&self, receiver: u32) -> Option<&str> { - self.entries - .iter() - .find(|entry| entry.contract.receiver == receiver) - .map(|entry| entry.refresh.rooted_box_slot.as_str()) + self.entries.iter().find_map(|entry| match &entry.data { + ActiveReceiverData::Address { refresh, .. } if entry.contract.receiver == receiver => { + Some(refresh.rooted_box_slot.as_str()) + } + _ => None, + }) } /// The pre-masked base-handle slot consumed by packed address math. pub(crate) fn base_handle_slot(&self, receiver: u32) -> Option<&str> { - self.entries - .iter() - .find(|entry| entry.contract.receiver == receiver) - .map(|entry| entry.refresh.base_handle_slot.as_str()) + self.entries.iter().find_map(|entry| match &entry.data { + ActiveReceiverData::Address { refresh, .. } if entry.contract.receiver == receiver => { + Some(refresh.base_handle_slot.as_str()) + } + _ => None, + }) } /// Conditional validation and refreshed base handle for an ordinary array @@ -453,17 +769,24 @@ impl ReceiverDescriptorTable { receiver: u32, require_numeric: bool, ) -> Option { - let entry = self - .entries - .iter() - .find(|entry| entry.contract.receiver == receiver)?; - let validation = entry.array_validation.as_ref()?; + let entry = self.entries.iter().find(|entry| { + entry.contract.receiver == receiver + && matches!(&entry.data, ActiveReceiverData::Address { .. }) + })?; + let ActiveReceiverData::Address { + refresh, + array_validation, + } = &entry.data + else { + unreachable!("address lookup selected a non-address descriptor") + }; + let validation = array_validation.as_ref()?; if require_numeric && validation.kind != ReceiverArrayValidationKind::Numeric { return None; } Some(ReceiverArrayAccess { valid_i1: validation.valid_i1.clone(), - base_handle_slot: entry.refresh.base_handle_slot.clone(), + base_handle_slot: refresh.base_handle_slot.clone(), }) } @@ -476,11 +799,18 @@ impl ReceiverDescriptorTable { pub(crate) fn poll_refreshes(&self) -> Result, BoundaryViolation> { let mut refreshes = Vec::with_capacity(self.entries.len()); for entry in &self.entries { + let ActiveReceiverData::Address { + refresh, + array_validation, + } = &entry.data + else { + continue; + }; boundary_admits(&entry.contract, RegionEnder::BackEdgePoll)?; - if let Some(validation) = &entry.array_validation { + if let Some(validation) = array_validation { boundary_admits(&validation.contract, RegionEnder::BackEdgePoll)?; } - refreshes.push(entry.refresh.clone()); + refreshes.push(refresh.clone()); } Ok(refreshes) } diff --git a/crates/perry-codegen/src/collectors/receiver_regions_tests.rs b/crates/perry-codegen/src/collectors/receiver_regions_tests.rs index e788ab8708..ee1a5913ef 100644 --- a/crates/perry-codegen/src/collectors/receiver_regions_tests.rs +++ b/crates/perry-codegen/src/collectors/receiver_regions_tests.rs @@ -110,10 +110,11 @@ fn desc( } } -/// `cached_lengths` / `bounded_index_pairs`: a value, not an address. +/// Cached-length and bounded-index descriptor payloads are values, not +/// addresses. fn cached_length_desc() -> ReceiverDescriptor { desc( - "cached_lengths", + "receiver_descriptors[cached_length]", ReceiverClaim::ScalarRelation, FactBoundary::DynamicExtent, false, @@ -131,22 +132,22 @@ fn poll_refreshed_receiver_desc() -> ReceiverDescriptor { ) } -/// `buffer_view_slots`: an address, function-lifetime, degraded in place, and -/// with no region that excludes `Stmt::Try`. +/// A buffer-view descriptor names non-moving storage, is function-lifetime, +/// degrades in place, and has no region that excludes `Stmt::Try`. fn buffer_view_desc() -> ReceiverDescriptor { desc( - "buffer_view_slots", - ReceiverClaim::Address, + "receiver_descriptors[buffer_view]", + ReceiverClaim::NonMovingAddress, FactBoundary::InPlaceDegradation, false, ) } -/// `packed_f64_loop_facts`: a representation claim, scope-id bounded, inside a -/// matcher that rejects `Stmt::Try`. +/// The packed-f64 descriptor payload is a representation claim, scope-id +/// bounded, inside a matcher that rejects `Stmt::Try`. fn packed_f64_desc() -> ReceiverDescriptor { desc( - "packed_f64_loop_facts", + "receiver_descriptors[packed_f64_loop]", ReceiverClaim::Representation, FactBoundary::ScopeId, true, @@ -167,8 +168,8 @@ const ALL_ENDERS: [RegionEnder; 6] = [ // --------------------------------------------------------------------------- /// A moving collection changes an object's ADDRESS, never its length. This is -/// the reason `cached_lengths` needs no safepoint logic at all, and it has to -/// fall out of the model rather than be special-cased per table. +/// the reason a cached-length descriptor needs no safepoint logic at all, and +/// it has to fall out of the model rather than be special-cased per table. #[test] fn a_scalar_relation_survives_every_relocation_point() { let d = cached_length_desc(); @@ -263,6 +264,136 @@ fn active_descriptor_table_owns_lookup_refresh_and_dynamic_extent() { assert!(!table.dematerialize(OBJ)); } +#[test] +fn cached_length_is_a_coexisting_descriptor_payload_with_its_own_extent() { + let mut table = ReceiverDescriptorTable::default(); + assert!(table.materialize_cached_length(OBJ, "%length".into())); + assert_eq!(table.cached_length_slot(OBJ), Some("%length")); + assert!(table.materialize_cached_length(OBJ, "%nested".into())); + assert_eq!(table.cached_length_slot(OBJ), Some("%nested")); + assert!(table.dematerialize_cached_length(OBJ)); + assert_eq!(table.cached_length_slot(OBJ), Some("%length")); + + // A scalar fact and a poll-refreshed address for one receiver coexist; + // ending either extent must leave the other one intact. + assert!(table.materialize_poll_refreshed_address( + OBJ, + "%box.root".into(), + "%base.handle".into(), + "%source.root".into(), + true, + )); + assert!(table.dematerialize_cached_length(OBJ)); + assert_eq!(table.cached_length_slot(OBJ), None); + assert_eq!(table.base_handle_slot(OBJ), Some("%base.handle")); + assert!(!table.dematerialize_cached_length(OBJ)); + assert!(table.dematerialize(OBJ)); +} + +#[test] +fn bounded_indices_share_scope_and_reassignment_invalidation() { + let mut table = ReceiverDescriptorTable::default(); + table.materialize_bounded_index(OBJ, NUM, 7); + table.materialize_bounded_index(OBJ, NUM, 8); + table.materialize_bounded_index(OBJ + 1, NUM2, 8); + assert!(table.has_bounded_index(OBJ, NUM)); + + assert_eq!(table.dematerialize_scope(7), 1); + assert!(table.has_bounded_index(OBJ, NUM)); + table.invalidate_bounded_indices_for_local(NUM); + assert!(!table.has_bounded_index(OBJ, NUM)); + assert!(table.has_bounded_index(OBJ + 1, NUM2)); + table.invalidate_bounded_indices_for_local(OBJ + 1); + assert!(!table.has_bounded_index(OBJ + 1, NUM2)); +} + +#[test] +fn representation_payloads_use_the_common_scope_boundary() { + let mut table = ReceiverDescriptorTable::default(); + table.materialize_packed_f64_loop(crate::expr::PackedF64LoopFact { + index_local_id: NUM, + array_local_id: OBJ, + scope_id: 17, + guard_id: "%guard".into(), + store_side_exit_label: "slow".into(), + array_kind: crate::expr::PackedNumericLoopKind::F64, + allow_holes: false, + numeric_accumulators: vec![NUM2], + window_validated: true, + affine_indices: false, + }); + let fact = table + .packed_f64_loop_facts() + .next() + .expect("packed descriptor must be queryable"); + assert_eq!(fact.array_local_id, OBJ); + assert_eq!(fact.numeric_accumulators, vec![NUM2]); + assert!(table.has_packed_f64_loop_facts()); + table.materialize_masked_window_array(crate::expr::MaskedWindowArrayFact { + array_local_id: OBJ + 1, + scope_id: 17, + guard_id: "%window.guard".into(), + min_idx: 0, + max_idx_exclusive: 16, + values_i32: false, + numeric_accumulators: Vec::new(), + elem: crate::expr::MaskedWindowElem::PlainF64, + allows_stores: false, + }); + assert_eq!( + table + .masked_window_array_facts() + .next() + .expect("masked descriptor must be queryable") + .max_idx_exclusive, + 16 + ); + assert_eq!(table.dematerialize_scope(17), 2); + assert!(!table.has_packed_f64_loop_facts()); + assert!(table.masked_window_array_facts().next().is_none()); +} + +fn test_buffer_view(data_slot: &str) -> crate::native_value::BufferViewSlot { + crate::native_value::BufferViewSlot { + data_slot: data_slot.into(), + length_slot: None, + scope_idx: Some(3), + elem: crate::native_value::BufferElem::U8, + element_width_bytes: 1, + index_unit: crate::native_value::BufferIndexUnit::Byte, + view_byte_offset: Some(0), + length_offset_from_data: -8, + alias: crate::native_value::AliasState::NoAliasProven, + length_source: Some(crate::native_value::LengthSource::Constant(32)), + native_owned: None, + pointer_state: crate::native_value::BufferViewPointerState::Stable, + storage_inline_proven: true, + } +} + +#[test] +fn non_moving_buffer_view_uses_descriptor_lookup_and_in_place_degradation() { + let mut table = ReceiverDescriptorTable::default(); + assert!(table + .materialize_buffer_view(OBJ, test_buffer_view("%data")) + .is_none()); + assert!(table.contains_buffer_view(OBJ)); + table.buffer_view_mut(OBJ).unwrap().alias = crate::native_value::AliasState::MayAlias; + let (view_id, view) = table.buffer_views().next().unwrap(); + assert_eq!(view_id, OBJ); + assert_eq!(view.data_slot, "%data"); + let old = table + .materialize_buffer_view(OBJ, test_buffer_view("%replacement")) + .expect("replacement returns the prior payload"); + assert_eq!(old.data_slot, "%data"); + assert_eq!( + table.buffer_view(OBJ).map(|view| view.data_slot.as_str()), + Some("%replacement") + ); + assert!(table.dematerialize_buffer_view(OBJ).is_some()); + assert!(!table.contains_buffer_view(OBJ)); +} + #[test] fn ordinary_loop_descriptor_carries_conditional_array_validation() { let mut table = ReceiverDescriptorTable::default(); @@ -370,27 +501,16 @@ fn a_cached_address_dies_at_a_collecting_call_under_every_boundary() { } } -/// THE phase-1 finding. `buffer_view_slots` caches a raw data pointer, is -/// function-lifetime, and is never removed — only downgraded in place. Nothing -/// structurally stops an entry registered before a `try` from being consulted -/// inside the `catch` handler, and `lower_try` clears no fact table. -/// -/// It is sound in the shipped compiler for a reason outside the model (typed -/// and buffer storage is marked non-movable and never relocates). The model -/// flags it anyway, and that is correct behaviour for phase 1: the tier is -/// relying on a property of the storage kind that its own boundary mechanism -/// does not state. When phase 2 gives descriptors a non-movable-storage -/// attribute this becomes a clean pass; until then a flag is the honest answer. +/// Phase 4 closes the phase-1 finding for buffer views: their descriptor now +/// states that the pointed-to allocation is non-moving. Safepoints and unwind +/// cannot stale that address; assignment, backing replacement, disposal and +/// alias escape still degrade or remove the payload through table APIs. #[test] -fn an_address_claim_with_no_try_exclusion_is_flagged_on_the_unwind_edge() { - let v = boundary_admits(&buffer_view_desc(), RegionEnder::UnwindEdge) - .expect_err("function-lifetime address claim reaches the catch handler"); - assert_eq!(v.table, "buffer_view_slots"); - assert_eq!(v.ender, RegionEnder::UnwindEdge); - - // A tier that DOES exclude `Try` from its region is not flagged: the - // packed matcher rejects `Stmt::Try` outright, so no handler can observe - // its cache. +fn a_non_moving_address_survives_every_relocation_boundary() { + for ender in ALL_ENDERS { + assert!(boundary_admits(&buffer_view_desc(), ender).is_ok()); + } + assert!(boundary_admits(&poll_refreshed_receiver_desc(), RegionEnder::UnwindEdge).is_ok()); } @@ -813,14 +933,14 @@ struct TableRow { fn inventory() -> Vec { vec![ TableRow { - table: "cached_lengths", + table: "receiver_descriptors[cached_length]", claim: ReceiverClaim::ScalarRelation, boundary: FactBoundary::DynamicExtent, excludes_try: false, unwind_safe_by: "a length is a value; relocation moves the object, not the number", }, TableRow { - table: "bounded_index_pairs", + table: "receiver_descriptors[bounded_index]", claim: ReceiverClaim::ScalarRelation, boundary: FactBoundary::ScopeId, excludes_try: false, @@ -850,7 +970,7 @@ fn inventory() -> Vec { lexical-order invalidation precedes catch lowering", }, TableRow { - table: "packed_f64_loop_facts", + table: "receiver_descriptors[packed_f64_loop]", claim: ReceiverClaim::Representation, boundary: FactBoundary::ScopeId, excludes_try: true, @@ -858,7 +978,7 @@ fn inventory() -> Vec { only with the #9185 accumulator flush at the throw site", }, TableRow { - table: "masked_window_array_facts", + table: "receiver_descriptors[masked_window_array]", claim: ReceiverClaim::Representation, boundary: FactBoundary::ScopeId, excludes_try: true, @@ -914,8 +1034,8 @@ fn inventory() -> Vec { before every call/invoke (capture/nested modes)", }, TableRow { - table: "buffer_view_slots", - claim: ReceiverClaim::Address, + table: "receiver_descriptors[buffer_view]", + claim: ReceiverClaim::NonMovingAddress, boundary: FactBoundary::InPlaceDegradation, excludes_try: false, unwind_safe_by: "storage kind is non-movable (GC_TYPE_TYPED_ARRAY/BUFFER); the fact \ @@ -940,13 +1060,40 @@ fn inventory() -> Vec { ] } +/// The six mechanisms named by #9254 must no longer own independent `FnCtx` +/// fields or lifecycle rules. Their inventory rows now identify payloads of +/// the one descriptor table (the base address/refresh payload keeps the table's +/// unqualified name). +#[test] +fn the_original_six_mechanisms_are_receiver_descriptor_payloads() { + let migrated = inventory() + .into_iter() + .filter(|row| { + row.table == "receiver_descriptors" || row.table.starts_with("receiver_descriptors[") + }) + .map(|row| row.table) + .collect::>(); + assert_eq!( + migrated, + std::collections::BTreeSet::from([ + "receiver_descriptors", + "receiver_descriptors[bounded_index]", + "receiver_descriptors[buffer_view]", + "receiver_descriptors[cached_length]", + "receiver_descriptors[masked_window_array]", + "receiver_descriptors[packed_f64_loop]", + ]) + ); +} + /// The inventory, run through the model. /// /// A flag here is NOT a bug report. It says: *this table's stated boundary /// does not by itself license its claim across an unwind edge* — the safety /// comes from somewhere the boundary mechanism cannot express, recorded in -/// `unwind_safe_by`. That gap is the thing #9254 proposes to close, and -/// pinning the exact set is how phase 2 proves it closed one. +/// `unwind_safe_by`. Phase 4 removes the original proposal's buffer-view row +/// from this set; pinning the remaining expanded-audit rows keeps follow-up +/// migrations honest. #[test] fn the_inventory_flags_exactly_the_tables_whose_unwind_safety_is_external() { let flagged: Vec<&str> = inventory() @@ -978,12 +1125,9 @@ fn the_inventory_flags_exactly_the_tables_whose_unwind_safety_is_external() { // unconstrained. Safety rests on a post-hoc call-free scan in one // mode and a before-call dirty bit in the others. "stable_packed_loop_facts", - // Immutable-fact tables: a cached pointer into storage the GC - // marks non-movable, or a root the collector rewrites in place. - // Sound, but for a reason the boundary vocabulary cannot state — - // which is exactly why phase 2 needs a non-movable-storage - // attribute on the descriptor. - "buffer_view_slots", + // The remaining immutable-fact tables still rely on a non-moving + // allocation or a root rewritten in place without stating that + // property in their boundary contract. "buffer_data_slots", "class_keys_slots", ], @@ -1049,6 +1193,7 @@ fn the_inventory_covers_every_claim_kind_and_every_boundary_mechanism() { ReceiverClaim::ScalarRelation, ReceiverClaim::Representation, ReceiverClaim::Address, + ReceiverClaim::NonMovingAddress, ] { assert!( rows.iter().any(|r| r.claim == claim), diff --git a/crates/perry-codegen/src/expr/binary.rs b/crates/perry-codegen/src/expr/binary.rs index 4910f3ad0c..04be6b3799 100644 --- a/crates/perry-codegen/src/expr/binary.rs +++ b/crates/perry-codegen/src/expr/binary.rs @@ -568,7 +568,7 @@ fn lower_arithmetic_operand(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result<(String, return Ok((value, true)); } // #7494: the guarded tier above declines outright for a receiver - // tracked in `ctx.buffer_view_slots` (its own "don't shadow" comment) + // tracked by a buffer-view descriptor (its own "don't shadow" comment) // because that tracked view owns a STRONGER-bounds native path // (`lower_typed_array_load`). Nothing routed a number-context read to // it, so a proven in-bounds buffer-view typed-array read still fell diff --git a/crates/perry-codegen/src/expr/buffer_access.rs b/crates/perry-codegen/src/expr/buffer_access.rs index 34c86c65c4..21b67cf133 100644 --- a/crates/perry-codegen/src/expr/buffer_access.rs +++ b/crates/perry-codegen/src/expr/buffer_access.rs @@ -253,7 +253,7 @@ pub(crate) fn can_lower_buffer_access_without_calls( } let (buffer_local_id, view) = match buffer_expr { - Expr::LocalGet(id) => match ctx.buffer_view_slots.get(id) { + Expr::LocalGet(id) => match ctx.receiver_descriptors.buffer_view(id) { Some(view) => (*id, view), None => return false, }, @@ -293,7 +293,7 @@ pub(crate) fn lower_buffer_access_proof( } let (buffer_local_id, view) = match buffer_expr { - Expr::LocalGet(id) => match ctx.buffer_view_slots.get(id).cloned() { + Expr::LocalGet(id) => match ctx.receiver_descriptors.buffer_view(id).cloned() { Some(view) => (*id, view), None => return Ok(None), }, @@ -526,8 +526,8 @@ pub(crate) fn lower_buffer_store( // view storage must use the dynamic path when the RHS can collect. let value_crosses_cached_view = match buffer_expr { Expr::LocalGet(id) => ctx - .buffer_view_slots - .get(id) + .receiver_descriptors + .buffer_view(id) .is_some_and(|view| !view.storage_inline_proven), _ => false, } && rooting::operand_may_collect(ctx, value_expr); @@ -629,7 +629,7 @@ pub(crate) fn lower_typed_array_load( index_expr: &Expr, ) -> Result> { let view = match array_expr { - Expr::LocalGet(id) => match ctx.buffer_view_slots.get(id).cloned() { + Expr::LocalGet(id) => match ctx.receiver_descriptors.buffer_view(id).cloned() { Some(view) if view.index_unit == BufferIndexUnit::Element => view, _ => return Ok(None), }, @@ -704,7 +704,7 @@ pub(crate) fn lower_typed_array_load( /// Numeric-context sink for a buffer-view-tracked typed-array element read. /// /// `ta_param_f64_read::checked_typed_array_f64_kind` declines outright for a -/// receiver tracked in `ctx.buffer_view_slots` — its own doc comment says the +/// receiver tracked by a buffer-view descriptor — its own doc comment says the /// tracked view "owns this receiver via its own (stronger-bounds) native /// path", i.e. [`lower_typed_array_load`] above. That path is real (the /// general, non-arithmetic `IndexGet` dispatch already calls it), but until @@ -747,7 +747,7 @@ pub(crate) fn lower_typed_array_store( value_expr: &Expr, ) -> Result> { let view = match array_expr { - Expr::LocalGet(id) => match ctx.buffer_view_slots.get(id).cloned() { + Expr::LocalGet(id) => match ctx.receiver_descriptors.buffer_view(id).cloned() { Some(view) if view.index_unit == BufferIndexUnit::Element => view, _ => return Ok(None), }, diff --git a/crates/perry-codegen/src/expr/buffer_views.rs b/crates/perry-codegen/src/expr/buffer_views.rs index d44c47a670..b08d7e14cb 100644 --- a/crates/perry-codegen/src/expr/buffer_views.rs +++ b/crates/perry-codegen/src/expr/buffer_views.rs @@ -90,7 +90,7 @@ pub(crate) fn buffer_view_lowered_value( pub(crate) fn downgrade_buffer_alias(ctx: &mut FnCtx<'_>, id: u32, reason: MaterializationReason) { let mut effective_reason = reason.clone(); - if let Some(view) = ctx.buffer_view_slots.get_mut(&id) { + if let Some(view) = ctx.receiver_descriptors.buffer_view_mut(id) { if view.native_owned.is_some() && matches!(reason, MaterializationReason::UnknownCallEscape) { effective_reason = MaterializationReason::EscapingUnownedPointer; @@ -120,19 +120,19 @@ pub(crate) fn invalidate_buffer_view_pointer( reason: MaterializationReason, ) { let affected_ids = if let Some(data_slot) = ctx - .buffer_view_slots - .get(&id) + .receiver_descriptors + .buffer_view(id) .map(|view| view.data_slot.clone()) { - ctx.buffer_view_slots - .iter() - .filter_map(|(view_id, view)| (view.data_slot == data_slot).then_some(*view_id)) + ctx.receiver_descriptors + .buffer_views() + .filter_map(|(view_id, view)| (view.data_slot == data_slot).then_some(view_id)) .collect::>() } else { vec![id] }; for affected_id in affected_ids { - if let Some(view) = ctx.buffer_view_slots.get_mut(&affected_id) { + if let Some(view) = ctx.receiver_descriptors.buffer_view_mut(affected_id) { view.pointer_state = BufferViewPointerState::Invalidated { reason: reason.clone(), }; @@ -172,7 +172,7 @@ pub(crate) fn invalidate_native_owned_views_for_owner( reason: MaterializationReason, ) { let mut invalidated = Vec::new(); - for (view_id, view) in ctx.buffer_view_slots.iter_mut() { + for (view_id, view) in ctx.receiver_descriptors.buffer_views_mut() { let Some(native) = view.native_owned.as_ref() else { continue; }; @@ -180,7 +180,7 @@ pub(crate) fn invalidate_native_owned_views_for_owner( continue; } invalidate_native_owned_view(view, &reason); - invalidated.push(*view_id); + invalidated.push(view_id); } for view_id in invalidated { ctx.buffer_hazard_reasons.insert(view_id, reason.clone()); @@ -200,12 +200,12 @@ pub(crate) fn invalidate_native_owned_views_for_dispose(ctx: &mut FnCtx<'_>, own fn invalidate_all_native_owned_views(ctx: &mut FnCtx<'_>, reason: MaterializationReason) { let mut invalidated = Vec::new(); - for (view_id, view) in ctx.buffer_view_slots.iter_mut() { + for (view_id, view) in ctx.receiver_descriptors.buffer_views_mut() { if view.native_owned.is_none() { continue; } invalidate_native_owned_view(view, &reason); - invalidated.push(*view_id); + invalidated.push(view_id); } for view_id in invalidated { ctx.buffer_hazard_reasons.insert(view_id, reason.clone()); @@ -238,7 +238,7 @@ pub(crate) fn alias_buffer_view_slot( source_id: u32, reason: MaterializationReason, ) { - let Some(mut view) = ctx.buffer_view_slots.get(&source_id).cloned() else { + let Some(mut view) = ctx.receiver_descriptors.buffer_view(source_id).cloned() else { return; }; let reason = if view.native_owned.is_some() { @@ -249,7 +249,8 @@ pub(crate) fn alias_buffer_view_slot( downgrade_buffer_alias(ctx, source_id, reason.clone()); view.alias = AliasState::MayAlias; view.scope_idx = None; - ctx.buffer_view_slots.insert(alias_id, view); + ctx.receiver_descriptors + .materialize_buffer_view(alias_id, view); ctx.buffer_hazard_reasons.insert(alias_id, reason); } @@ -275,8 +276,8 @@ pub(crate) fn attach_buffer_view_pointer_state_for_expr(ctx: &mut FnCtx<'_>, exp return; }; let Some(state) = ctx - .buffer_view_slots - .get(id) + .receiver_descriptors + .buffer_view(id) .map(|view| view.pointer_state.clone()) else { return; @@ -311,12 +312,12 @@ pub(crate) fn update_buffer_view_for_assignment( let handle_ptr = blk.inttoptr(I64, &handle); let data_ptr = blk.gep(I8, &handle_ptr, &[(I32, "8")]); let data_slot = ctx - .buffer_view_slots - .get(&id) + .receiver_descriptors + .buffer_view(id) .map(|view| view.data_slot.clone()) .unwrap_or_else(|| ctx.func.alloca_entry(PTR)); ctx.block().store(PTR, &data_ptr, &data_slot); - ctx.buffer_view_slots.insert( + ctx.receiver_descriptors.materialize_buffer_view( id, BufferViewSlot { data_slot, @@ -338,7 +339,7 @@ pub(crate) fn update_buffer_view_for_assignment( }, ); } else { - ctx.buffer_view_slots.remove(&id); + ctx.receiver_descriptors.dematerialize_buffer_view(id); } ctx.buffer_hazard_reasons .insert(id, MaterializationReason::Reassignment); diff --git a/crates/perry-codegen/src/expr/i32_fast_path.rs b/crates/perry-codegen/src/expr/i32_fast_path.rs index 17229c14dc..ac66adbab5 100644 --- a/crates/perry-codegen/src/expr/i32_fast_path.rs +++ b/crates/perry-codegen/src/expr/i32_fast_path.rs @@ -516,7 +516,7 @@ fn ta_int_elem_load_is_i32_provable(ctx: &FnCtx<'_>, object: &Expr, index: &Expr let Expr::LocalGet(id) = object else { return false; }; - let Some(view) = ctx.buffer_view_slots.get(id) else { + let Some(view) = ctx.receiver_descriptors.buffer_view(id) else { return false; }; if !view.pointer_state.is_stable() @@ -612,7 +612,7 @@ fn checked_typed_array_i32_kind( }; // A tracked buffer view (proven-bounds unchecked path, or a Buffer param) // owns this receiver; don't shadow it. - if ctx.buffer_view_slots.contains_key(id) { + if ctx.receiver_descriptors.contains_buffer_view(id) { return None; } // Class proof: function-local first, then — for a MODULE-GLOBAL typed array @@ -793,8 +793,8 @@ fn packed_i32_loop_index_get_fact(ctx: &FnCtx<'_>, e: &Expr) -> Option, e: &Expr) -> Option, object: &Expr) -> bool { - if matches!(object, Expr::LocalGet(id) if ctx.buffer_view_slots.contains_key(id)) { + if matches!(object, Expr::LocalGet(id) if ctx.receiver_descriptors.contains_buffer_view(id)) { return true; } // This predicate selects only runtime-validated typed-array helpers (or a - // `buffer_view_slots` proof that invalidates on assignment), as documented + // buffer-view descriptor proof that invalidates on assignment), as documented // above. Preserve the declared kind as a hint for that dynamic fallback; // the general `static_type_of` deliberately drops reassigned bindings. let ty = match object { @@ -236,11 +236,7 @@ fn numeric_index_has_loop_array_index_proof(ctx: &FnCtx<'_>, object: &Expr, inde if packed_f64_loop_offset_read(ctx, *arr_id, index).is_some() { return true; } - offset == 0 - && ctx - .bounded_index_pairs - .iter() - .any(|fact| fact.array_local_id == *arr_id && fact.index_local_id == idx_id) + offset == 0 && ctx.receiver_descriptors.has_bounded_index(*arr_id, idx_id) } fn numeric_index_needs_runtime_key(ctx: &FnCtx<'_>, object: &Expr, index: &Expr) -> bool { @@ -829,11 +825,7 @@ pub(crate) fn lower_numeric_index_get_for_number_context( } } if let (Expr::LocalGet(arr_id), Expr::LocalGet(idx_id)) = (object.as_ref(), index.as_ref()) { - if ctx - .bounded_index_pairs - .iter() - .any(|fact| fact.index_local_id == *idx_id && fact.array_local_id == *arr_id) - { + if ctx.receiver_descriptors.has_bounded_index(*arr_id, *idx_id) { if let Some(i32_slot) = ctx.i32_counter_slots.get(idx_id).cloned() { let repair_slot = receiver_repair_slot(ctx, object); let arr_box = lower_expr(ctx, object)?; @@ -1203,7 +1195,7 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { if typed_array_index_needs_runtime_key(ctx, object.as_ref(), index.as_ref()) { if runtime_key_may_expose_typed_array_backing_buffer(index) { if let Expr::LocalGet(id) = object.as_ref() { - if ctx.buffer_view_slots.contains_key(id) { + if ctx.receiver_descriptors.contains_buffer_view(id) { invalidate_buffer_view_pointer( ctx, *id, @@ -1678,9 +1670,7 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { if let (Expr::LocalGet(arr_id), Expr::LocalGet(idx_id)) = (object.as_ref(), index.as_ref()) { - if ctx.bounded_index_pairs.iter().any(|fact| { - fact.index_local_id == *idx_id && fact.array_local_id == *arr_id - }) { + if ctx.receiver_descriptors.has_bounded_index(*arr_id, *idx_id) { if let Some(i32_slot) = ctx.i32_counter_slots.get(idx_id).cloned() { let repair_slot = receiver_repair_slot(ctx, object); let arr_box = lower_expr(ctx, object)?; diff --git a/crates/perry-codegen/src/expr/index_get/foreign_counter.rs b/crates/perry-codegen/src/expr/index_get/foreign_counter.rs index af52b7ae39..5f0f1bdda3 100644 --- a/crates/perry-codegen/src/expr/index_get/foreign_counter.rs +++ b/crates/perry-codegen/src/expr/index_get/foreign_counter.rs @@ -24,8 +24,8 @@ pub(crate) fn foreign_packed_loop_read( return None; } let fact = ctx - .packed_f64_loop_facts - .iter() + .receiver_descriptors + .packed_f64_loop_facts() .rev() .find(|fact| { fact.array_local_id == arr_id @@ -169,8 +169,8 @@ pub(crate) fn affine_packed_loop_read( return None; } let fact = ctx - .packed_f64_loop_facts - .iter() + .receiver_descriptors + .packed_f64_loop_facts() .rev() .find(|fact| fact.array_local_id == arr_id && fact.affine_indices)? .clone(); diff --git a/crates/perry-codegen/src/expr/index_get/guarded_array.rs b/crates/perry-codegen/src/expr/index_get/guarded_array.rs index 9e235ee48d..a7fdc91289 100644 --- a/crates/perry-codegen/src/expr/index_get/guarded_array.rs +++ b/crates/perry-codegen/src/expr/index_get/guarded_array.rs @@ -646,8 +646,8 @@ pub(super) fn packed_f64_loop_fact( arr_id: u32, idx_id: u32, ) -> Option { - ctx.packed_f64_loop_facts - .iter() + ctx.receiver_descriptors + .packed_f64_loop_facts() .find(|fact| fact.array_local_id == arr_id && fact.index_local_id == idx_id) .cloned() } diff --git a/crates/perry-codegen/src/expr/index_set.rs b/crates/perry-codegen/src/expr/index_set.rs index 9be4d7eb80..67317672ca 100644 --- a/crates/perry-codegen/src/expr/index_set.rs +++ b/crates/perry-codegen/src/expr/index_set.rs @@ -117,7 +117,7 @@ fn lower_value_for_dynamic_index_set( /// #7494: `static_type_of`, not `receiver_class_name` — see the sibling in /// `index_get.rs` for the full rationale. In short: every tier this predicate -/// gates is either `ctx.buffer_view_slots`-tracked (which reassignment +/// gates is either tracked by a buffer-view descriptor (which reassignment /// already invalidates on its own) or a genuinely dynamic runtime call that /// re-validates the object's actual GC kind, so `receiver_class_name`'s /// blanket "reassigned local → unknown" answer only broke the dynamic- @@ -128,7 +128,7 @@ fn lower_value_for_dynamic_index_set( /// typed-array object (data at byte 16): a type-confused write, not a missed /// optimization. fn is_width_tracked_typed_array_receiver(ctx: &FnCtx<'_>, object: &Expr) -> bool { - if matches!(object, Expr::LocalGet(id) if ctx.buffer_view_slots.contains_key(id)) { + if matches!(object, Expr::LocalGet(id) if ctx.receiver_descriptors.contains_buffer_view(id)) { return true; } let ty = match object { @@ -203,8 +203,8 @@ pub(super) fn packed_f64_loop_fact( arr_id: u32, idx_id: u32, ) -> Option { - ctx.packed_f64_loop_facts - .iter() + ctx.receiver_descriptors + .packed_f64_loop_facts() .find(|fact| fact.array_local_id == arr_id && fact.index_local_id == idx_id) .cloned() } @@ -247,11 +247,7 @@ fn numeric_index_has_loop_array_index_proof(ctx: &FnCtx<'_>, object: &Expr, inde if packed_f64_loop_fact_for_index(ctx, *arr_id, index).is_some() { return true; } - offset == 0 - && ctx - .bounded_index_pairs - .iter() - .any(|fact| fact.array_local_id == *arr_id && fact.index_local_id == idx_id) + offset == 0 && ctx.receiver_descriptors.has_bounded_index(*arr_id, idx_id) } fn numeric_index_needs_runtime_key(ctx: &FnCtx<'_>, object: &Expr, index: &Expr) -> bool { @@ -954,9 +950,7 @@ pub(crate) fn lower( if let (Expr::LocalGet(arr_id), Expr::LocalGet(idx_id)) = (object.as_ref(), index.as_ref()) { - if ctx.bounded_index_pairs.iter().any(|fact| { - fact.index_local_id == *idx_id && fact.array_local_id == *arr_id - }) { + if ctx.receiver_descriptors.has_bounded_index(*arr_id, *idx_id) { let Some(i32_slot) = ctx.i32_counter_slots.get(idx_id).cloned() else { return lower_array_index_set_via_runtime_key( ctx, diff --git a/crates/perry-codegen/src/expr/literals_vars.rs b/crates/perry-codegen/src/expr/literals_vars.rs index 6bf582dfc0..eea5d38544 100644 --- a/crates/perry-codegen/src/expr/literals_vars.rs +++ b/crates/perry-codegen/src/expr/literals_vars.rs @@ -863,7 +863,7 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { emit_root_nanbox_store_on_block(ctx.block(), &v, &g_ref); } super::record_native_arena_owner_assignment(ctx, *id, value.as_ref()); - if ctx.buffer_view_slots.contains_key(id) + if ctx.receiver_descriptors.contains_buffer_view(id) || matches!( value.as_ref(), Expr::BufferAlloc { .. } | Expr::BufferAllocUnsafe(_) | Expr::Uint8ArrayNew(_) diff --git a/crates/perry-codegen/src/expr/masked_window.rs b/crates/perry-codegen/src/expr/masked_window.rs index 77690ac7dc..0513bf122d 100644 --- a/crates/perry-codegen/src/expr/masked_window.rs +++ b/crates/perry-codegen/src/expr/masked_window.rs @@ -33,8 +33,8 @@ pub(crate) fn masked_window_fact_for_index( index: &Expr, ) -> Option { let (lo, hi) = crate::collectors::static_index_window(index)?; - ctx.masked_window_array_facts - .iter() + ctx.receiver_descriptors + .masked_window_array_facts() .rev() .find(|fact| { fact.array_local_id == arr_id && lo >= fact.min_idx && hi < fact.max_idx_exclusive diff --git a/crates/perry-codegen/src/expr/mod.rs b/crates/perry-codegen/src/expr/mod.rs index 065cf16678..dc6f0e332d 100644 --- a/crates/perry-codegen/src/expr/mod.rs +++ b/crates/perry-codegen/src/expr/mod.rs @@ -18,9 +18,9 @@ use crate::collectors::NativeRegionFactGraph; use crate::function::LlFunction; use crate::nanbox::double_literal; use crate::native_value::{ - AliasState, BoundedBufferIndex, BoundsProof, BoundsState, BufferAccessMode, BufferViewSlot, - ExpectedNativeRep, GuardedBufferIndex, LoweredValue, MaterializationReason, NativeFactUse, - NativeRep, NativeRepRecord, + AliasState, BoundedBufferIndex, BoundsProof, BoundsState, BufferAccessMode, ExpectedNativeRep, + GuardedBufferIndex, LoweredValue, MaterializationReason, NativeFactUse, NativeRep, + NativeRepRecord, }; use crate::strings::StringPool; use crate::type_analysis::{is_bigint_expr, is_bool_expr, is_numeric_expr}; @@ -971,49 +971,12 @@ pub(crate) struct FnCtx<'a> { /// as the fallback, an SSA value composed in the entry region. pub class_header_images: std::collections::HashMap<(String, u64), HeaderImageSource>, - /// Per-arr-local cached `arr.length` slots — populated by - /// `lower_for` when it spots the well-known shape - /// `for (...; i < arr.length; ...) { body }` and proves via - /// `stmt_preserves_array_length` that the body doesn't change - /// `arr.length`. The `PropertyGet { object: LocalGet(arr_id), - /// property: "length" }` lowering checks this map and, if found, - /// emits a `load double, ptr ` instead of unboxing the - /// array and doing a fresh `load i32` of the length field. - /// - /// Saves the per-iteration length reload (which LLVM's LICM - /// declines to do because the IndexSet slow path is an external - /// call that LLVM can't prove won't modify the length). - pub cached_lengths: std::collections::HashMap, - /// Immutable locals initialized from an exact `receiver.length` read, /// keyed by the snapshot local. The read itself retains ordinary property /// semantics; a later counted-loop guard may use the association only /// after proving the receiver is a packed Array/Array-subclass. pub array_length_snapshots: std::collections::HashMap, - /// `(counter_local_id, array_local_id)` pairs that are guaranteed - /// inbounds inside the current loop nest — populated by - /// `lower_for` when it detects the same `for (...; i < arr.length; - /// ...)` shape that drives `cached_lengths`. The IndexSet codegen - /// (`lower_index_set_fast`) checks this set: if `arr[i] = expr` - /// where `(i, arr)` is in the set, the IndexSet skips its - /// runtime bound check + cap check + realloc fallback entirely - /// and emits a single inline-store sequence. - /// - /// The for-loop guarantees `i < arr.length` is true at the cond - /// check, and `stmt_preserves_array_length` already proved the - /// body can't change `arr.length` or reassign `i`, so the - /// IndexSet site can rely on `i < arr.length` without rechecking. - pub bounded_index_pairs: Vec, - - /// Scoped loop-versioning facts for `for (...; i < arr.length; i++)` - /// clones guarded by `js_typed_feedback_packed_f64_array_loop_guard`. - /// Inside the fast clone, `arr[i]` and `arr[i] = numeric_expr` can lower - /// directly to raw `double` load/store because the loop-entry guard proves - /// the array is a live packed raw-f64 plain Array and the loop proof keeps - /// `i` in bounds. - pub packed_f64_loop_facts: Vec, - pub masked_window_array_facts: Vec, /// Scoped facts established by the string-array masked-window loop /// versioner. The entry guard proves every slot in the window is an /// in-bounds SSO-or-heap string, so reads may bypass ordinary array @@ -1061,7 +1024,7 @@ pub(crate) struct FnCtx<'a> { /// Parallel i32 counter slots for integer loop counters that are /// used as bounded array indices. When a for-loop counter is in - /// `integer_locals` AND appears in `bounded_index_pairs`, `lower_for` + /// `integer_locals` AND has a bounded-index descriptor, `lower_for` /// allocates a parallel i32 alloca tracked here. The `Expr::Update` /// lowering increments the i32 slot alongside the normal double slot, /// and the IndexGet/IndexSet bounded fast-path loads the i32 directly @@ -1083,13 +1046,11 @@ pub(crate) struct FnCtx<'a> { /// root slot during the clone is harmless to a GC scan. pub numeric_accumulator_f64_slots: std::collections::HashMap, pub transition_cache_base_slot: Option, - /// #9254 phase 2: active materialised receiver descriptors. The first - /// consumer is the packed fast clone's #9111 cache: one entry owns the - /// frame-rooted receiver box, its pre-masked base-handle slot and the - /// source-root refresh recipe. `LocalGet`, packed address math and the - /// armed poll all query this table instead of coordinating three parallel - /// maps. The table checks its address claim against the shared region - /// boundary algebra before carrying it across a fired poll. + /// #9254 active materialised receiver descriptors. Entries carry + /// poll-refreshed receiver addresses, cached scalar lengths and bounds, + /// scoped packed/masked representation proofs, or non-moving native + /// buffer views. Lookup, nested-scope teardown and boundary validation + /// live in that table instead of independent `FnCtx` maps. pub receiver_descriptors: crate::collectors::ReceiverDescriptorTable, /// When set, `emit_armed_gc_loop_safepoint` gates its VOLATILE armed /// load on `(counter & 63) == 0`, so the poll's serialization cost (and @@ -1598,11 +1559,6 @@ pub(crate) struct FnCtx<'a> { /// different buffers don't alias (fixes the vectorizer's "unsafe /// dependent memory operations" remark). pub buffer_data_slots: std::collections::HashMap, - /// Codegen-level native buffer views keyed by LocalId. This is the - /// representation model behind `buffer_data_slots`: raw pointer access can - /// exist with `AliasState::Unknown`, while noalias metadata requires a - /// proven/guarded alias state at the consumer. - pub buffer_view_slots: std::collections::HashMap, /// Local owner-handle aliases for native arenas. Values are canonical /// owner local ids used by native-owned typed-array view proof state. pub native_arena_owner_aliases: std::collections::HashMap, @@ -1706,13 +1662,6 @@ pub struct I18nLowerCtx { pub currencies: Vec<(String, String)>, } -#[derive(Debug, Clone)] -pub(crate) struct BoundedIndexPair { - pub index_local_id: u32, - pub array_local_id: u32, - pub scope_id: u32, -} - #[derive(Clone, Debug)] pub(crate) struct VersionedIndexedArrayFact { pub local_id: u32, diff --git a/crates/perry-codegen/src/expr/native_memory.rs b/crates/perry-codegen/src/expr/native_memory.rs index 0e9462ac14..8cea1b7a76 100644 --- a/crates/perry-codegen/src/expr/native_memory.rs +++ b/crates/perry-codegen/src/expr/native_memory.rs @@ -176,12 +176,12 @@ fn proven_view( let Expr::LocalGet(local_id) = expr else { return None; }; - // `buffer_view_slots` below is the representation proof and every write + // The buffer-view descriptor below is the representation proof and every write // invalidates its pointer state. The type is only an early dispatch hint. if !is_native_memory_typed_view(ctx.local_type_hint(local_id)) { return None; } - let slot = ctx.buffer_view_slots.get(local_id)?.clone(); + let slot = ctx.receiver_descriptors.buffer_view(local_id)?.clone(); if !slot.pointer_state.is_stable() { return None; } diff --git a/crates/perry-codegen/src/expr/property_get.rs b/crates/perry-codegen/src/expr/property_get.rs index bb7ea9445c..64e00dc857 100644 --- a/crates/perry-codegen/src/expr/property_get.rs +++ b/crates/perry-codegen/src/expr/property_get.rs @@ -133,7 +133,7 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { } if property == "buffer" { if let Expr::LocalGet(id) = object.as_ref() { - if ctx.buffer_view_slots.contains_key(id) { + if ctx.receiver_descriptors.contains_buffer_view(id) { super::invalidate_buffer_view_pointer( ctx, *id, @@ -299,9 +299,9 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // data_ptr 16 bytes past the header, so the hardcoded `-8` here read // the packed `kind|elem_size` bytes (Int32→0x404=1028, // Float64→0x807=2055) instead of `.length`. Prefer the co-registered - // `buffer_view_slots` entry, which carries the correct + // buffer-view descriptor, which carries the correct // `length_offset_from_data` (and a `length_slot` for native views). - let view = ctx.buffer_view_slots.get(&arr_id).cloned(); + let view = ctx.receiver_descriptors.buffer_view(arr_id).cloned(); let length_slot = view.as_ref().and_then(|v| v.length_slot.clone()); let length_offset = view .as_ref() @@ -419,7 +419,11 @@ pub(crate) fn lower(ctx: &mut FnCtx<'_>, expr: &Expr) -> Result { // iteration — LLVM's LICM declines to hoist it because the // IndexSet's slow path is an opaque external call. if let Expr::LocalGet(arr_id) = object.as_ref() { - if let Some(slot) = ctx.cached_lengths.get(arr_id).cloned() { + if let Some(slot) = ctx + .receiver_descriptors + .cached_length_slot(*arr_id) + .map(str::to_owned) + { return Ok(ctx.block().load(DOUBLE, &slot)); } } diff --git a/crates/perry-codegen/src/expr/proven_view_access.rs b/crates/perry-codegen/src/expr/proven_view_access.rs index e4d88911d8..cc51334c9b 100644 --- a/crates/perry-codegen/src/expr/proven_view_access.rs +++ b/crates/perry-codegen/src/expr/proven_view_access.rs @@ -118,23 +118,25 @@ pub(crate) fn local_is_proven_int_store_view(ctx: &FnCtx<'_>, id: u32) -> bool { if ctx.closure_captures.contains_key(&id) { return false; } - ctx.buffer_view_slots.get(&id).is_some_and(|view| { - view.pointer_state.is_stable() - && view.storage_inline_proven - && view.native_owned.is_none() - && view.index_unit == BufferIndexUnit::Element - && view.alias.allows_noalias() - && view.scope_idx.is_some() - && matches!( - view.elem, - BufferElem::I8 - | BufferElem::U8 - | BufferElem::I16 - | BufferElem::U16 - | BufferElem::I32 - | BufferElem::U32 - ) - }) + ctx.receiver_descriptors + .buffer_view(id) + .is_some_and(|view| { + view.pointer_state.is_stable() + && view.storage_inline_proven + && view.native_owned.is_none() + && view.index_unit == BufferIndexUnit::Element + && view.alias.allows_noalias() + && view.scope_idx.is_some() + && matches!( + view.elem, + BufferElem::I8 + | BufferElem::U8 + | BufferElem::I16 + | BufferElem::U16 + | BufferElem::I32 + | BufferElem::U32 + ) + }) } /// The proven view for `object`, when every gate for the checked tier holds. @@ -149,7 +151,7 @@ fn proven_view_for( let Expr::LocalGet(id) = object else { return None; }; - let view = ctx.buffer_view_slots.get(id)?.clone(); + let view = ctx.receiver_descriptors.buffer_view(id)?.clone(); if !view.pointer_state.is_stable() || !view.storage_inline_proven || view.native_owned.is_some() @@ -205,16 +207,18 @@ pub(crate) fn is_proven_u32_view_read(ctx: &FnCtx<'_>, value: &Expr) -> bool { let Expr::LocalGet(id) = object.as_ref() else { return false; }; - ctx.buffer_view_slots.get(id).is_some_and(|view| { - view.pointer_state.is_stable() - && view.storage_inline_proven - && view.native_owned.is_none() - && view.index_unit == BufferIndexUnit::Element - && view.alias.allows_noalias() - && view.scope_idx.is_some() - && matches!(view.elem, BufferElem::U32) - && crate::stmt::stable_packed_loop::has_u32_index_fact(ctx, index) - }) + ctx.receiver_descriptors + .buffer_view(id) + .is_some_and(|view| { + view.pointer_state.is_stable() + && view.storage_inline_proven + && view.native_owned.is_none() + && view.index_unit == BufferIndexUnit::Element + && view.alias.allows_noalias() + && view.scope_idx.is_some() + && matches!(view.elem, BufferElem::U32) + && crate::stmt::stable_packed_loop::has_u32_index_fact(ctx, index) + }) } fn proven_u32_view_value(ctx: &FnCtx<'_>, value: &Expr) -> bool { diff --git a/crates/perry-codegen/src/expr/ptr_numarray_access.rs b/crates/perry-codegen/src/expr/ptr_numarray_access.rs index cca729ee7c..03b4527beb 100644 --- a/crates/perry-codegen/src/expr/ptr_numarray_access.rs +++ b/crates/perry-codegen/src/expr/ptr_numarray_access.rs @@ -138,11 +138,7 @@ pub(crate) fn try_lower_num_array_guard_free_get( ); } if let Expr::LocalGet(idx_id) = index { - if ctx - .bounded_index_pairs - .iter() - .any(|f| f.index_local_id == *idx_id && f.array_local_id == *arr_id) - { + if ctx.receiver_descriptors.has_bounded_index(*arr_id, *idx_id) { if let Some(i32_slot) = ctx.i32_counter_slots.get(idx_id).cloned() { let arr_box = lower_expr(ctx, &Expr::LocalGet(*arr_id))?; let idx_i32 = ctx.block().load(I32, &i32_slot); @@ -189,10 +185,7 @@ pub(crate) fn try_lower_num_array_guard_free_set( } else { match index { Expr::LocalGet(idx_id) - if ctx - .bounded_index_pairs - .iter() - .any(|f| f.index_local_id == *idx_id && f.array_local_id == *arr_id) => + if ctx.receiver_descriptors.has_bounded_index(*arr_id, *idx_id) => { ctx.i32_counter_slots.get(idx_id).cloned() } diff --git a/crates/perry-codegen/src/expr/range_facts.rs b/crates/perry-codegen/src/expr/range_facts.rs index 4d0614611e..49dc12798f 100644 --- a/crates/perry-codegen/src/expr/range_facts.rs +++ b/crates/perry-codegen/src/expr/range_facts.rs @@ -224,7 +224,7 @@ fn int_typed_array_load_range( let Expr::LocalGet(id) = object else { return None; }; - let view = ctx.buffer_view_slots.get(id)?; + let view = ctx.receiver_descriptors.buffer_view(id)?; if view.index_unit != BufferIndexUnit::Element || !view.alias.allows_noalias() || view.scope_idx.is_none() @@ -548,18 +548,18 @@ pub(crate) fn invalidate_local_write_facts(ctx: &mut FnCtx<'_>, id: u32) { .retain(|fact| fact.index_local_id != id && fact.buffer_local_id != id); ctx.guarded_buffer_index_pairs .retain(|fact| fact.index_local_id != id && fact.buffer_local_id != id); - ctx.bounded_index_pairs - .retain(|fact| fact.index_local_id != id && fact.array_local_id != id); + ctx.receiver_descriptors + .invalidate_bounded_indices_for_local(id); let mut stale_length_views = Vec::new(); let mut owner_reassignment_views = Vec::new(); - for (view_id, view) in ctx.buffer_view_slots.iter_mut() { + for (view_id, view) in ctx.receiver_descriptors.buffer_views_mut() { if matches!( view.length_source.as_ref(), Some(LengthSource::Local { id: source_id, .. }) if *source_id == id ) { view.length_source = Some(LengthSource::Unknown); - stale_length_views.push(*view_id); + stale_length_views.push(view_id); } if view .native_owned @@ -571,7 +571,7 @@ pub(crate) fn invalidate_local_write_facts(ctx: &mut FnCtx<'_>, id: u32) { if let Some(native) = view.native_owned.as_mut() { native.owner_rooted = false; } - owner_reassignment_views.push(*view_id); + owner_reassignment_views.push(view_id); } } for view_id in stale_length_views { @@ -746,7 +746,7 @@ pub(crate) fn bounds_for_buffer_access_width( // Deliberately narrow: a non-constant length (a parameter, a grown array) // still declines, because then there is nothing to compare the interval // against. - if let Some(view) = ctx.buffer_view_slots.get(&buffer_local_id) { + if let Some(view) = ctx.receiver_descriptors.buffer_view(buffer_local_id) { if let Some(length) = view .length_source .as_ref() @@ -768,7 +768,7 @@ pub(crate) fn bounds_for_buffer_access_width( } } if let Some(index_value) = constant_i64_expr(ctx, index) { - let Some(view) = ctx.buffer_view_slots.get(&buffer_local_id) else { + let Some(view) = ctx.receiver_descriptors.buffer_view(buffer_local_id) else { return BoundsState::Unknown; }; let length = view @@ -798,7 +798,7 @@ fn range_bounds_for_buffer_access( index: &Expr, bounds_width_units: u32, ) -> BoundsState { - let Some(view) = ctx.buffer_view_slots.get(&buffer_local_id) else { + let Some(view) = ctx.receiver_descriptors.buffer_view(buffer_local_id) else { return BoundsState::Unknown; }; // Ctx-aware range facts first; fall back to the ctx-free syntactic window @@ -935,7 +935,10 @@ fn guarded_buffer_index( if width < 1 || width > u32::MAX as i64 { return None; } - if !ctx.buffer_view_slots.contains_key(&buffer_local_id) { + if !ctx + .receiver_descriptors + .contains_buffer_view(buffer_local_id) + { return None; } let nonnegative = ctx.nonnegative_integer_locals.contains(&index_local_id) @@ -1022,8 +1025,9 @@ pub(crate) fn effective_alias_state_for_access( }; } let noalias_candidate_count = ctx - .buffer_view_slots - .values() + .receiver_descriptors + .buffer_views() + .map(|(_, view)| view) .filter(|slot| slot.scope_idx.is_some() && slot.alias.allows_noalias()) .count(); if noalias_candidate_count >= 2 { @@ -1038,7 +1042,7 @@ fn native_owned_view_has_overlapping_alias(ctx: &FnCtx<'_>, view: &BufferViewSlo return false; }; let scope_idx = view.scope_idx; - ctx.buffer_view_slots.values().any(|other| { + ctx.receiver_descriptors.buffer_views().any(|(_, other)| { if other.scope_idx == scope_idx { return false; } diff --git a/crates/perry-codegen/src/expr/ta_param_f64_read.rs b/crates/perry-codegen/src/expr/ta_param_f64_read.rs index 7f3a03e558..1fb01ac6ac 100644 --- a/crates/perry-codegen/src/expr/ta_param_f64_read.rs +++ b/crates/perry-codegen/src/expr/ta_param_f64_read.rs @@ -97,7 +97,7 @@ fn checked_typed_array_f64_kind( }; // A tracked buffer view owns this receiver via its own (stronger-bounds) // native path; don't shadow it. - if ctx.buffer_view_slots.contains_key(id) { + if ctx.receiver_descriptors.contains_buffer_view(id) { return None; } // Class proof: first the function-local proof, then — for a MODULE-GLOBAL diff --git a/crates/perry-codegen/src/expr/typed_array_rmw.rs b/crates/perry-codegen/src/expr/typed_array_rmw.rs index 230903fed6..8768d2cadd 100644 --- a/crates/perry-codegen/src/expr/typed_array_rmw.rs +++ b/crates/perry-codegen/src/expr/typed_array_rmw.rs @@ -76,8 +76,8 @@ fn exact_alias_root(ctx: &FnCtx<'_>, mut id: u32) -> u32 { fn receiver_is_uint32_candidate(ctx: &FnCtx<'_>, id: u32) -> Option { let root = exact_alias_root(ctx, id); if ctx - .buffer_view_slots - .get(&root) + .receiver_descriptors + .buffer_view(root) .is_some_and(|view| matches!(view.elem, BufferElem::U32)) { return Some(root); diff --git a/crates/perry-codegen/src/expr/u8_buffer_read.rs b/crates/perry-codegen/src/expr/u8_buffer_read.rs index b70e0864d1..8bf941158a 100644 --- a/crates/perry-codegen/src/expr/u8_buffer_read.rs +++ b/crates/perry-codegen/src/expr/u8_buffer_read.rs @@ -62,7 +62,7 @@ fn u8_buffer_receiver_eligible(ctx: &FnCtx<'_>, object: &Expr) -> bool { let Expr::LocalGet(id) = object else { return false; }; - if ctx.buffer_view_slots.contains_key(id) { + if ctx.receiver_descriptors.contains_buffer_view(id) { return false; } let class = crate::type_analysis::receiver_class_name(ctx, object) diff --git a/crates/perry-codegen/src/stmt/let_buffer_views.rs b/crates/perry-codegen/src/stmt/let_buffer_views.rs index dbad595f20..6e6169dcd9 100644 --- a/crates/perry-codegen/src/stmt/let_buffer_views.rs +++ b/crates/perry-codegen/src/stmt/let_buffer_views.rs @@ -94,7 +94,7 @@ pub(super) fn register_noalias_buffer_view( } None => None, }; - ctx.buffer_view_slots.insert( + ctx.receiver_descriptors.materialize_buffer_view( id, BufferViewSlot { data_slot, diff --git a/crates/perry-codegen/src/stmt/let_stmt.rs b/crates/perry-codegen/src/stmt/let_stmt.rs index ffdbaf540b..fabc412735 100644 --- a/crates/perry-codegen/src/stmt/let_stmt.rs +++ b/crates/perry-codegen/src/stmt/let_stmt.rs @@ -350,7 +350,7 @@ pub(crate) fn lower_let( let auto_captures = crate::type_analysis::compute_auto_captures(ctx, params, body, captures); for cap_id in auto_captures { - if ctx.buffer_view_slots.contains_key(&cap_id) + if ctx.receiver_descriptors.contains_buffer_view(cap_id) || ctx.known_noalias_buffer_locals.contains(&cap_id) { crate::expr::downgrade_buffer_alias( diff --git a/crates/perry-codegen/src/stmt/loops.rs b/crates/perry-codegen/src/stmt/loops.rs index 17891cc498..52afd86ef2 100644 --- a/crates/perry-codegen/src/stmt/loops.rs +++ b/crates/perry-codegen/src/stmt/loops.rs @@ -5,8 +5,7 @@ use super::*; use crate::expr::{ array_kind_fact, effect_fact, emit_typed_feedback_register_site, expr_has_numeric_pointer_free_array_layout, nanbox_pointer_inline, raw_f64_layout_fact, - BoundedIndexPair, PackedF64LoopFact, PackedNumericLoopKind, TypedFeedbackContract, - TypedFeedbackKind, + PackedF64LoopFact, PackedNumericLoopKind, TypedFeedbackContract, TypedFeedbackKind, }; use crate::loop_purity::body_needs_asm_barrier; use crate::lower_conditional::lower_truthy; @@ -1463,18 +1462,19 @@ fn lower_packed_f64_versioned_for( false, ); acc_scope.hoist_receivers(ctx, &[matched.array_id]); - ctx.packed_f64_loop_facts.push(PackedF64LoopFact { - index_local_id: matched.counter_id, - array_local_id: matched.array_id, - scope_id: packed_scope_id, - guard_id: guard_id.to_string(), - store_side_exit_label: acc_scope.fact_side_exit(&slow_pre_label), - array_kind: matched.array_kind, - allow_holes: false, - window_validated: false, - affine_indices: false, - numeric_accumulators: acc_scope.accumulators.clone(), - }); + ctx.receiver_descriptors + .materialize_packed_f64_loop(PackedF64LoopFact { + index_local_id: matched.counter_id, + array_local_id: matched.array_id, + scope_id: packed_scope_id, + guard_id: guard_id.to_string(), + store_side_exit_label: acc_scope.fact_side_exit(&slow_pre_label), + array_kind: matched.array_kind, + allow_holes: false, + window_validated: false, + affine_indices: false, + numeric_accumulators: acc_scope.accumulators.clone(), + }); // The guard just proved a live, non-forwarded plain array, and the // matched body cannot change its length (in-bounds stores only, no // calls/closures/awaits) — so hoist the length ONCE as the fast clone's @@ -1503,8 +1503,8 @@ fn lower_packed_f64_versioned_for( Some((matched.counter_id, hoisted_len_i32)), )?; ctx.poll_stride_counter_slot = saved_stride; - ctx.packed_f64_loop_facts - .retain(|fact| fact.scope_id != packed_scope_id); + ctx.receiver_descriptors + .dematerialize_scope(packed_scope_id); acc_scope.finish(ctx); if !ctx.block().is_terminated() { ctx.block().br(&merge_label); @@ -2978,45 +2978,47 @@ fn push_packed_f64_range_facts( ) { for access in &matched.arrays { if access.counter.is_some() { - ctx.packed_f64_loop_facts.push(PackedF64LoopFact { - index_local_id: matched.counter_id, - array_local_id: access.array_id, - scope_id, - guard_id: guard_id.to_string(), - store_side_exit_label: slow_pre_label.to_string(), - array_kind: PackedNumericLoopKind::F64, - // Dense mode proved the window hole-free — loads need no - // hole check / side exit. Classic range mode stays - // hole-tolerant. - allow_holes: !matched.dense, - window_validated: true, - affine_indices: false, - numeric_accumulators: numeric_accumulators.to_vec(), - }); + ctx.receiver_descriptors + .materialize_packed_f64_loop(PackedF64LoopFact { + index_local_id: matched.counter_id, + array_local_id: access.array_id, + scope_id, + guard_id: guard_id.to_string(), + store_side_exit_label: slow_pre_label.to_string(), + array_kind: PackedNumericLoopKind::F64, + // Dense mode proved the window hole-free — loads need no + // hole check / side exit. Classic range mode stays + // hole-tolerant. + allow_holes: !matched.dense, + window_validated: true, + affine_indices: false, + numeric_accumulators: numeric_accumulators.to_vec(), + }); } // #9253: an affine access publishes a receiver-only fact. No window // was validated, so reads bounds-check per access; holes are excluded // because the receiver guard proves fully-packed raw f64. if access.affine { - ctx.packed_f64_loop_facts.push(PackedF64LoopFact { - index_local_id: matched.counter_id, - array_local_id: access.array_id, - scope_id, - guard_id: guard_id.to_string(), - store_side_exit_label: slow_pre_label.to_string(), - array_kind: PackedNumericLoopKind::F64, - allow_holes: false, - // True when the entry guard proved this array's whole affine - // window at the loop's endpoints — the reads then skip both - // the range clamp and the per-read bounds check. - window_validated: affine_window_proven.contains(&access.array_id), - affine_indices: true, - numeric_accumulators: numeric_accumulators.to_vec(), - }); + ctx.receiver_descriptors + .materialize_packed_f64_loop(PackedF64LoopFact { + index_local_id: matched.counter_id, + array_local_id: access.array_id, + scope_id, + guard_id: guard_id.to_string(), + store_side_exit_label: slow_pre_label.to_string(), + array_kind: PackedNumericLoopKind::F64, + allow_holes: false, + // True when the entry guard proved this array's whole affine + // window at the loop's endpoints — the reads then skip both + // the range clamp and the per-read bounds check. + window_validated: affine_window_proven.contains(&access.array_id), + affine_indices: true, + numeric_accumulators: numeric_accumulators.to_vec(), + }); } if let Some((lo, hi)) = access.stat { - ctx.masked_window_array_facts - .push(crate::expr::MaskedWindowArrayFact { + ctx.receiver_descriptors.materialize_masked_window_array( + crate::expr::MaskedWindowArrayFact { array_local_id: access.array_id, scope_id, guard_id: guard_id.to_string(), @@ -3026,7 +3028,8 @@ fn push_packed_f64_range_facts( elem: crate::expr::MaskedWindowElem::PlainF64, allows_stores: allow_masked_stores, numeric_accumulators: numeric_accumulators.to_vec(), - }); + }, + ); } } } @@ -3118,8 +3121,8 @@ fn lower_masked_window_ta_tier( let (lo, hi) = access .stat .expect("TA tiers require static-window accesses"); - ctx.masked_window_array_facts - .push(crate::expr::MaskedWindowArrayFact { + ctx.receiver_descriptors.materialize_masked_window_array( + crate::expr::MaskedWindowArrayFact { array_local_id: arr_id, scope_id, guard_id: guard_id.to_string(), @@ -3129,7 +3132,8 @@ fn lower_masked_window_ta_tier( elem, allows_stores: false, numeric_accumulators: Vec::new(), - }); + }, + ); } lower_for_after_init_with_i32_bound( ctx, @@ -3140,8 +3144,7 @@ fn lower_masked_window_ta_tier( loop_label, Some((matched.counter_id, bound_i32.to_string())), )?; - ctx.masked_window_array_facts - .retain(|fact| fact.scope_id != scope_id); + ctx.receiver_descriptors.dematerialize_scope(scope_id); if !ctx.block().is_terminated() { ctx.block().br(merge_label); } @@ -3458,10 +3461,7 @@ fn lower_packed_f64_range_versioned_for( Some((matched.counter_id, bound_i32.clone())), )?; ctx.poll_stride_counter_slot = saved_stride; - ctx.packed_f64_loop_facts - .retain(|fact| fact.scope_id != scope_i32); - ctx.masked_window_array_facts - .retain(|fact| fact.scope_id != scope_i32); + ctx.receiver_descriptors.dematerialize_scope(scope_i32); acc_scope.finish(ctx); if !ctx.block().is_terminated() { ctx.block().br(&merge_label); @@ -3507,10 +3507,7 @@ fn lower_packed_f64_range_versioned_for( Some((matched.counter_id, bound_i32.clone())), )?; ctx.poll_stride_counter_slot = saved_stride; - ctx.packed_f64_loop_facts - .retain(|fact| fact.scope_id != scope_f64); - ctx.masked_window_array_facts - .retain(|fact| fact.scope_id != scope_f64); + ctx.receiver_descriptors.dematerialize_scope(scope_f64); acc_scope.finish(ctx); if !ctx.block().is_terminated() { ctx.block().br(&merge_label); @@ -3566,10 +3563,8 @@ fn lower_packed_f64_range_versioned_for( Some((matched.counter_id, bound_i32.clone())), )?; ctx.poll_stride_counter_slot = saved_stride; - ctx.packed_f64_loop_facts - .retain(|fact| fact.scope_id != packed_scope_id); - ctx.masked_window_array_facts - .retain(|fact| fact.scope_id != packed_scope_id); + ctx.receiver_descriptors + .dematerialize_scope(packed_scope_id); acc_scope.finish(ctx); if !ctx.block().is_terminated() { ctx.block().br(&merge_label); @@ -6908,7 +6903,7 @@ pub(super) fn lower_for_after_init_with_i32_bound( }); let hoisted_buffer_bounds_width = hoist_classification.and_then(|hoist| { hoist.buffer_bounds_width_units.filter(|_| { - ctx.buffer_view_slots.contains_key(&hoist.arr_id) + ctx.receiver_descriptors.contains_buffer_view(hoist.arr_id) && loop_counter_bounds_are_safe(ctx, hoist.counter_id, update, body) }) }); @@ -6949,18 +6944,21 @@ pub(super) fn lower_for_after_init_with_i32_bound( )?; let slot = ctx.func.alloca_entry(DOUBLE); ctx.block().store(DOUBLE, &arr_box_loaded, &slot); - ctx.cached_lengths.insert(hoist.arr_id, slot.clone()); + let installed = ctx + .receiver_descriptors + .materialize_cached_length(hoist.arr_id, slot.clone()); + debug_assert!(installed, "loop must own its cached-length descriptor"); Some(slot) }; // Also tell `lower_index_set_fast` (and similar sites) that // `arr[counter_id]` is statically inbounds for this body, so // it can skip the runtime length-load + bound check. if hoisted_index_bounds_are_safe { - ctx.bounded_index_pairs.push(BoundedIndexPair { - index_local_id: hoist.counter_id, - array_local_id: hoist.arr_id, - scope_id: loop_proof_scope_id, - }); + ctx.receiver_descriptors.materialize_bounded_index( + hoist.arr_id, + hoist.counter_id, + loop_proof_scope_id, + ); } if let Some(bounds_width_units) = hoisted_buffer_bounds_width { ctx.bounded_buffer_index_pairs.push(BoundedBufferIndex { @@ -7130,7 +7128,10 @@ pub(super) fn lower_for_after_init_with_i32_bound( if local_bound_index_bounds_are_safe { if let Some(buffer_ids) = ctx.min_length_bounds.get(&bound_id).cloned() { for buffer_local_id in buffer_ids { - if ctx.buffer_view_slots.contains_key(&buffer_local_id) { + if ctx + .receiver_descriptors + .contains_buffer_view(buffer_local_id) + { ctx.bounded_buffer_index_pairs.push(BoundedBufferIndex { index_local_id: counter_id, buffer_local_id, @@ -7144,11 +7145,11 @@ pub(super) fn lower_for_after_init_with_i32_bound( } } let alloc_bound_ids: Vec = ctx - .buffer_view_slots - .iter() + .receiver_descriptors + .buffer_views() .filter_map(|(buffer_local_id, view)| match &view.length_source { Some(LengthSource::Local { id, addend }) if *id == bound_id && *addend >= 0 => { - Some(*buffer_local_id) + Some(buffer_local_id) } _ => None, }) @@ -7409,8 +7410,10 @@ pub(super) fn lower_for_after_init_with_i32_bound( ctx.i32_counter_slots.remove(&hoist.counter_id); } } - if let Some(arr_id) = hoisted_length_arr_id { - ctx.cached_lengths.remove(&arr_id); + if hoisted_length_slot.is_some() { + let arr_id = hoisted_length_arr_id.expect("a cached length has a receiver"); + let removed = ctx.receiver_descriptors.dematerialize_cached_length(arr_id); + debug_assert!(removed, "loop must retire its cached-length descriptor"); } let _ = hoisted_length_slot; // Pop the i32 counter slot we inserted for the `i < n` number-bound @@ -7426,8 +7429,8 @@ pub(super) fn lower_for_after_init_with_i32_bound( // the counter's existing (Let-site) i32 slot or keeps its own private one // out of `ctx.i32_counter_slots` entirely (#6072). let _ = dynamic_i32_bound; - ctx.bounded_index_pairs - .retain(|fact| fact.scope_id != loop_proof_scope_id); + ctx.receiver_descriptors + .dematerialize_scope(loop_proof_scope_id); ctx.bounded_buffer_index_pairs .retain(|fact| fact.scope_id != loop_proof_scope_id); ctx.guarded_buffer_index_pairs @@ -7588,7 +7591,7 @@ pub(crate) fn emit_gc_loop_safepoint( // be re-derived per element — which is why striding it 1-in-64 (#9316) // did not recover the loss and removing it does. Measured on // `bench_numeric_array_numeric`: 45 -> 38 ms against node's 38. - || !ctx.packed_f64_loop_facts.is_empty() + || ctx.receiver_descriptors.has_packed_f64_loop_facts() || ctx.versioned_indexed_loop_facts.last().is_some_and(|fact| { matches!( fact.guard_mode, @@ -8295,8 +8298,8 @@ fn min_length_bound_can_use_static_i32(ctx: &crate::expr::FnCtx<'_>, bound_id: u }; !buffer_ids.is_empty() && buffer_ids.iter().all(|buffer_id| { - ctx.buffer_view_slots - .get(buffer_id) + ctx.receiver_descriptors + .buffer_view(buffer_id) .and_then(|view| view.length_source.as_ref()) .is_some_and(|source| length_source_can_use_static_i32(ctx, source)) }) diff --git a/crates/perry-codegen/src/stmt/masked_window_region.rs b/crates/perry-codegen/src/stmt/masked_window_region.rs index 15da4ed148..26fb3740c4 100644 --- a/crates/perry-codegen/src/stmt/masked_window_region.rs +++ b/crates/perry-codegen/src/stmt/masked_window_region.rs @@ -515,8 +515,8 @@ pub(super) fn try_match_masked_window_region( // range-loop fast copy) — its reads inline through that fact; a // second, nested versioning would only add per-iteration probes. if ctx - .masked_window_array_facts - .iter() + .receiver_descriptors + .masked_window_array_facts() .any(|fact| fact.array_local_id == access.array_id) { continue; @@ -530,8 +530,8 @@ pub(super) fn try_match_masked_window_region( // escape, aliasing) clear those markers when the tracked data slot can // no longer be trusted, and this tier emits NO runtime re-validation. let proven_ta_i32 = ctx - .buffer_view_slots - .get(&access.array_id) + .receiver_descriptors + .buffer_view(access.array_id) .is_some_and(|view| { view.pointer_state.is_stable() && view.storage_inline_proven @@ -872,23 +872,24 @@ pub(super) fn lower_masked_window_region( let ta_scope_id = ctx.next_loop_proof_scope_id(); for array in ®ion.arrays { let view = ctx - .buffer_view_slots - .get(&array.array_id) + .receiver_descriptors + .buffer_view(array.array_id) .cloned() .expect("proven region array must have a view slot"); let data_ptr_val = ctx.block().load(crate::types::PTR, &view.data_slot); let data_i64 = ctx.block().ptrtoint(&data_ptr_val, I64); - ctx.masked_window_array_facts.push(MaskedWindowArrayFact { - array_local_id: array.array_id, - scope_id: ta_scope_id, - guard_id: "masked_region_ta_i32_proven".to_string(), - min_idx: array.lo, - max_idx_exclusive: array.hi + 1, - values_i32: true, - allows_stores: false, - elem: MaskedWindowElem::TaI32 { data_ptr: data_i64 }, - numeric_accumulators: Vec::new(), - }); + ctx.receiver_descriptors + .materialize_masked_window_array(MaskedWindowArrayFact { + array_local_id: array.array_id, + scope_id: ta_scope_id, + guard_id: "masked_region_ta_i32_proven".to_string(), + min_idx: array.lo, + max_idx_exclusive: array.hi + 1, + values_i32: true, + allows_stores: false, + elem: MaskedWindowElem::TaI32 { data_ptr: data_i64 }, + numeric_accumulators: Vec::new(), + }); } let privatize = ctx.try_depth == 0; let result = lower_region_copy( @@ -900,8 +901,7 @@ pub(super) fn lower_masked_window_region( privatize, true, ); - ctx.masked_window_array_facts - .retain(|fact| fact.scope_id != ta_scope_id); + ctx.receiver_descriptors.dematerialize_scope(ta_scope_id); return result; } @@ -995,17 +995,18 @@ pub(super) fn lower_masked_window_region( } let ta_scope_id = ctx.next_loop_proof_scope_id(); for (array, (arr_id, data_ptr)) in region.arrays.iter().zip(hoisted) { - ctx.masked_window_array_facts.push(MaskedWindowArrayFact { - array_local_id: arr_id, - scope_id: ta_scope_id, - guard_id: "masked_region_ta_i32".to_string(), - min_idx: array.lo, - max_idx_exclusive: array.hi + 1, - values_i32: true, - allows_stores: false, - elem: MaskedWindowElem::TaI32 { data_ptr }, - numeric_accumulators: Vec::new(), - }); + ctx.receiver_descriptors + .materialize_masked_window_array(MaskedWindowArrayFact { + array_local_id: arr_id, + scope_id: ta_scope_id, + guard_id: "masked_region_ta_i32".to_string(), + min_idx: array.lo, + max_idx_exclusive: array.hi + 1, + values_i32: true, + allows_stores: false, + elem: MaskedWindowElem::TaI32 { data_ptr }, + numeric_accumulators: Vec::new(), + }); } let privatize = ctx.try_depth == 0; lower_region_copy( @@ -1020,8 +1021,7 @@ pub(super) fn lower_masked_window_region( // towers (#6794 follow-up (a)). true, )?; - ctx.masked_window_array_facts - .retain(|fact| fact.scope_id != ta_scope_id); + ctx.receiver_descriptors.dematerialize_scope(ta_scope_id); if !ctx.block().is_terminated() { ctx.block().br(&merge_label); } @@ -1030,17 +1030,18 @@ pub(super) fn lower_masked_window_region( ctx.current_block = plain_pre_idx; let plain_scope_id = ctx.next_loop_proof_scope_id(); for array in ®ion.arrays { - ctx.masked_window_array_facts.push(MaskedWindowArrayFact { - array_local_id: array.array_id, - scope_id: plain_scope_id, - guard_id: "masked_region_plain_f64".to_string(), - min_idx: array.lo, - max_idx_exclusive: array.hi + 1, - values_i32: false, - allows_stores: false, - elem: MaskedWindowElem::PlainF64, - numeric_accumulators: Vec::new(), - }); + ctx.receiver_descriptors + .materialize_masked_window_array(MaskedWindowArrayFact { + array_local_id: array.array_id, + scope_id: plain_scope_id, + guard_id: "masked_region_plain_f64".to_string(), + min_idx: array.lo, + max_idx_exclusive: array.hi + 1, + values_i32: false, + allows_stores: false, + elem: MaskedWindowElem::PlainF64, + numeric_accumulators: Vec::new(), + }); } lower_region_copy( ctx, @@ -1053,8 +1054,7 @@ pub(super) fn lower_masked_window_region( // maintained by no write — keep the ordinary Number lowering here. false, )?; - ctx.masked_window_array_facts - .retain(|fact| fact.scope_id != plain_scope_id); + ctx.receiver_descriptors.dematerialize_scope(plain_scope_id); if !ctx.block().is_terminated() { ctx.block().br(&merge_label); } diff --git a/crates/perry-codegen/src/stmt/stable_packed_loop.rs b/crates/perry-codegen/src/stmt/stable_packed_loop.rs index eb4ba96279..e2479aec36 100644 --- a/crates/perry-codegen/src/stmt/stable_packed_loop.rs +++ b/crates/perry-codegen/src/stmt/stable_packed_loop.rs @@ -1384,8 +1384,8 @@ pub(crate) fn has_numeric_index_fact(ctx: &FnCtx<'_>, expr: &Expr) -> bool { // to the slow clone before the value is consumed. I32/U32 kinds // materialize via sitofp/uitofp — numeric by construction. if ctx - .packed_f64_loop_facts - .iter() + .receiver_descriptors + .packed_f64_loop_facts() .any(|fact| fact.array_local_id == *array_id && fact.index_local_id == *counter_id) { return true; diff --git a/crates/perry-codegen/src/stmt/stable_packed_typed_array.rs b/crates/perry-codegen/src/stmt/stable_packed_typed_array.rs index 15d171fe07..1d1f53c18e 100644 --- a/crates/perry-codegen/src/stmt/stable_packed_typed_array.rs +++ b/crates/perry-codegen/src/stmt/stable_packed_typed_array.rs @@ -252,7 +252,7 @@ pub(super) fn find_candidate( || ctx.boxed_vars.contains(&id) || ctx.closure_captures.contains_key(&id) || ctx.reassigned_locals.contains(&id) - || ctx.buffer_view_slots.contains_key(&id) + || ctx.receiver_descriptors.contains_buffer_view(id) || !matches!( crate::type_analysis::static_type_of(ctx, &Expr::LocalGet(id)), None | Some(perry_hir::types::Type::Any) @@ -332,7 +332,7 @@ fn reserve_alias_scope(ctx: &mut FnCtx<'_>, data_slot: &str) -> u32 { while ctx.locals.contains_key(&reservation) || ctx.module_globals.contains_key(&reservation) || ctx.buffer_data_slots.contains_key(&reservation) - || ctx.buffer_view_slots.contains_key(&reservation) + || ctx.receiver_descriptors.contains_buffer_view(reservation) { reservation = reservation.wrapping_sub(1); } @@ -362,7 +362,7 @@ pub(super) fn install_views(ctx: &mut FnCtx<'_>, admission: &Admission) -> Insta length.clone() }); let scope_idx = reserve_alias_scope(ctx, &data_slot); - let old = ctx.buffer_view_slots.insert( + let old = ctx.receiver_descriptors.materialize_buffer_view( *id, BufferViewSlot { data_slot, @@ -393,9 +393,9 @@ pub(super) fn install_views(ctx: &mut FnCtx<'_>, admission: &Admission) -> Insta pub(super) fn restore_views(ctx: &mut FnCtx<'_>, installed: InstalledViews) { for (id, old) in installed.previous { if let Some(old) = old { - ctx.buffer_view_slots.insert(id, old); + ctx.receiver_descriptors.materialize_buffer_view(id, old); } else { - ctx.buffer_view_slots.remove(&id); + ctx.receiver_descriptors.dematerialize_buffer_view(id); } } } diff --git a/crates/perry-codegen/src/type_analysis/numeric.rs b/crates/perry-codegen/src/type_analysis/numeric.rs index 850f324cfc..03dd401291 100644 --- a/crates/perry-codegen/src/type_analysis/numeric.rs +++ b/crates/perry-codegen/src/type_analysis/numeric.rs @@ -181,8 +181,8 @@ pub(crate) fn is_numeric_expr(ctx: &FnCtx<'_>, e: &Expr) -> bool { // in-clone write is numeric-preserving by the accumulator // walk. || ctx - .packed_f64_loop_facts - .iter() + .receiver_descriptors + .packed_f64_loop_facts() .rev() .any(|fact| fact.numeric_accumulators.contains(id)) // #9160: the string-window clone admits the accumulator only @@ -192,8 +192,8 @@ pub(crate) fn is_numeric_expr(ctx: &FnCtx<'_>, e: &Expr) -> bool { // The dense masked-window clone's twin: same entry tag // check, same numeric-preserving write proof. || ctx - .masked_window_array_facts - .iter() + .receiver_descriptors + .masked_window_array_facts() .rev() .any(|fact| fact.numeric_accumulators.contains(id)) || ctx @@ -500,8 +500,8 @@ pub(crate) fn is_numeric_expr(ctx: &FnCtx<'_>, e: &Expr) -> bool { // and every representable element kind is numeric. This runtime // fact is stronger than the erasable declaration consulted below. if ctx - .buffer_view_slots - .get(arr_id) + .receiver_descriptors + .buffer_view(arr_id) .is_some_and(|view| view.elem.is_number_valued()) { return true; diff --git a/crates/perry-codegen/src/type_analysis/pod.rs b/crates/perry-codegen/src/type_analysis/pod.rs index a59e55f63d..c225dc4e48 100644 --- a/crates/perry-codegen/src/type_analysis/pod.rs +++ b/crates/perry-codegen/src/type_analysis/pod.rs @@ -586,8 +586,8 @@ pub(crate) fn numeric_proof_is_declared_only(ctx: &FnCtx<'_>, expr: &Expr) -> bo // Do not discard that stronger fact and reclassify the read // from its erasable source annotation. if ctx - .buffer_view_slots - .get(arr_id) + .receiver_descriptors + .buffer_view(arr_id) .is_some_and(|view| view.elem.is_number_valued()) { return false; From 02e16d718e178524882f828a352fff101fbfa4d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Fri, 4 Sep 2026 12:52:32 +0200 Subject: [PATCH 5/5] docs: add PR 9716 changelog fragment --- .../9716-receiver-descriptor-table-retirement.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 changelog.d/9716-receiver-descriptor-table-retirement.md diff --git a/changelog.d/9716-receiver-descriptor-table-retirement.md b/changelog.d/9716-receiver-descriptor-table-retirement.md new file mode 100644 index 0000000000..f9d03c977c --- /dev/null +++ b/changelog.d/9716-receiver-descriptor-table-retirement.md @@ -0,0 +1,12 @@ +**Receiver proof state now has one lifecycle owner** (#9254 phase 4). Codegen's +cached array lengths, bounded indices, packed-f64 loop facts, masked-window +facts, and native buffer views now live as typed payloads in the shared receiver +descriptor table. Together with the poll-refreshed receiver addresses migrated +in phases 2 and 3, this retires all six independent mechanisms named by the +receiver-region proposal. + +The table now owns nested dynamic extents, lexical-scope teardown, reassignment +invalidation, and non-moving buffer-view address contracts. Existing fast-path +producers and consumers keep their established fallback behavior, while tests +pin nested-loop restoration, shared scope cleanup, temporary view replacement, +and moving-GC refresh behavior.