From 023ddb100c25a3aedba03b03b9ba64b7829456cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Wed, 23 Sep 2026 17:45:00 +0000 Subject: [PATCH 1/3] perf(runtime): answer a latched-megamorphic read from the receiver's shape key list A site whose way state is latched megamorphic re-ran the whole generic miss machinery on every read: ~730 instructions on a 40-shape `o.kind` site (node: ~60), dominated by the receiver re-classification, the inherited-read-cache probe, the re-priming of a site that will not prime, and a by-name scan. Before that, the slow entry now asks the receiver's own shape record: an Ordinary, generation-0, hole-free shape stores an own inline key at its position in its canonical key list, so the key's position below the key count and the live inline slot count IS its inline slot, and the value is loaded from there. The site's compact word keeps an unmatchable low half and carries the last answered slot in its high half as the next guess, which the receiver's key list confirms before any scan. Dictionary, generation > 0, tombstones, spill, inherited keys, descriptors and `length` fall through unchanged. --- .../object/field_get_set/ic_miss/ic_slow.rs | 193 ++++++++++++++++++ crates/perry-runtime/src/object/shapes.rs | 102 +++++++++ 2 files changed, 295 insertions(+) diff --git a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs index 26f50a93c3..01abc64495 100644 --- a/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs +++ b/crates/perry-runtime/src/object/field_get_set/ic_miss/ic_slow.rs @@ -267,6 +267,63 @@ pub extern "C" fn js_object_get_field_ic_slow( } } } + // --- 2b. a MEGAMORPHIC site: the receiver's shape answers --- + // + // A site whose way state is latched negative will not be primed + // again, so the miss handler below would re-derive the receiver + // class, probe the inherited-read cache, try to prime and scan + // by name — ~700 instructions per read, measured on a 40-shape + // `o.kind` site (node: ~51). The receiver's own shape already + // knows the answer: an ordinary own data key's inline slot is its + // position in the shape's canonical key list. Anything the shape + // cannot answer by position (dictionary, generation > 0, + // tombstones, spill, inherited, descriptors) falls through + // unchanged. + // `length` is excluded (UTF-16 length word first, so the byte + // compare runs only for 6-unit keys): an Array-subclass receiver serves it + // from its elements store, not from a key position. + if plain && !key.is_null() && !key_is_length(key) { + let cache = crate::object::pic_slot_peek(cache_slot); + if !cache.is_null() + && (*cache)[crate::object::field_get_set::ic_miss::PIC_WAY_STATE] < 0 + { + if let Some(rec) = + crate::object::shapes::shape_record_by_id((*obj).parent_class_id) + { + // The site's last primed slot is the guess (the + // compact word's high half; a spill entry's + // flipped id carries no inline slot to guess). + let word = if packed.is_null() { + u64::MAX + } else { + (*packed).load(Ordering::Relaxed) + }; + let hint = (word >> 32) as usize; + if let Some(slot) = rec.inline_slot_of_key(key, hint) { + #[cfg(test)] + crate::object::shapes::SHAPE_ANSWERED_READS + .fetch_add(1, Ordering::Relaxed); + // Keep the answer as the site's next slot GUESS + // (owner-approved form: a guess the receiver's + // shape confirms). Only while the word's low + // half is unmatchable (`PACKED_GET_EMPTY`'s + // 0xFFFF_FFFF): the inline ShapeId compare can + // never equal it, and `packed_get_decode` reads + // it as no entry. + if slot != hint && !packed.is_null() && word as u32 == u32::MAX { + (*packed).store( + ((slot as u64) << 32) | u64::from(u32::MAX), + Ordering::Relaxed, + ); + } + let field = (obj as *const u8) + .add(std::mem::size_of::() + slot * 8) + as *const f64; + return *field; + } + } + } + } // (There is no third arm. An object-backed Array subclass used // to be served here by a class-wide "named-prefix" token held // in cache word 2 and matched against the receiver's @@ -285,6 +342,13 @@ pub extern "C" fn js_object_get_field_ic_slow( super::ic_miss::get_field_ic_miss_impl(obj, key, cache_slot, packed) } +/// `key` spells `length` — six bytes, compared directly (no UTF-8 validation). +#[inline] +unsafe fn key_is_length(key: *const crate::StringHeader) -> bool { + (*key).byte_len == 6 + && std::slice::from_raw_parts(crate::string::string_data(key), 6) == b"length" +} + #[cfg(test)] mod tests { use super::*; @@ -301,6 +365,135 @@ mod tests { (obj as u64 & 0x0000_FFFF_FFFF_FFFF) as i64 } + /// Build one receiver per distinct shape: every object gets `pos`, `end`, + /// `kind` (so `kind` sits at slot 2 in all of them) and then ONE distinct + /// extra key, which forks the shape. Returns (receivers, kind key). + fn megamorphic_receivers<'s>( + scope: &'s crate::gc::RuntimeHandleScope, + n: usize, + ) -> Vec> { + let mut out = Vec::new(); + for i in 0..n { + let obj = scope.root_raw_mut_ptr(crate::object::js_object_alloc(0, 8)); + for (k, v) in [ + (&b"pos"[..], 1.0), + (&b"end"[..], 2.0), + (&b"kind"[..], 100.0 + i as f64), + ] { + let key = scope.root_string_ptr(key_of(k)); + obj.with_mut_ptr(|o| { + key.with_const_ptr(|kp| crate::object::js_object_set_field_by_name(o, kp, v)) + }); + } + let extra = format!("x{i}"); + let key = scope.root_string_ptr(key_of(extra.as_bytes())); + obj.with_mut_ptr(|o| { + key.with_const_ptr(|kp| crate::object::js_object_set_field_by_name(o, kp, 7.0)) + }); + out.push(obj); + } + out + } + + /// S3: once a site has latched megamorphic, a read is answered by the + /// RECEIVER'S SHAPE (its key list), for every one of 48 shapes — and the + /// answer is the receiver's own value, not the value of whichever shape + /// last primed the site. + #[test] + fn a_latched_megamorphic_site_is_answered_by_the_receivers_shape() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let objs = megamorphic_receivers(&scope, 48); + let kind = scope.root_string_ptr(key_of(b"kind")); + let mut cache: PicCache = [0; PIC_CACHE_WORDS]; + let mut slot: PicCacheSlot = &mut cache; + let packed = AtomicU64::new(0); + let read = |o: &crate::gc::RuntimeHandle<'_>, slot: &mut PicCacheSlot| { + o.with_mut_ptr(|p: *mut ObjectHeader| { + kind.with_const_ptr(|k| js_object_get_field_ic_slow(handle(p), k, slot, &packed)) + }) + }; + // Drive the site until it latches. + for round in 0..4 { + for (i, o) in objs.iter().enumerate() { + assert_eq!( + read(o, &mut slot), + 100.0 + i as f64, + "round {round} receiver {i}" + ); + } + } + assert!( + cache[crate::object::field_get_set::ic_miss::PIC_WAY_STATE] < 0, + "48 shapes must latch the site megamorphic: state {}", + cache[crate::object::field_get_set::ic_miss::PIC_WAY_STATE] + ); + let before = + crate::object::shapes::SHAPE_ANSWERED_READS.load(std::sync::atomic::Ordering::Relaxed); + for (i, o) in objs.iter().enumerate() { + assert_eq!( + read(o, &mut slot), + 100.0 + i as f64, + "latched read, receiver {i}" + ); + } + let answered = crate::object::shapes::SHAPE_ANSWERED_READS + .load(std::sync::atomic::Ordering::Relaxed) + - before; + // The one receiver whose shape the compact word still names is served + // by the word itself (inline, in emitted code; step 2 here). Every + // other latched read is answered by its receiver's shape. + assert!( + answered >= 47, + "every latched read the word cannot serve must be answered by the shape: {answered}" + ); + // A WRONG slot guess (the compact word's high half) must not change the + // answer: the shape confirms or refutes the guess. + packed.store(5u64 << 32, std::sync::atomic::Ordering::Relaxed); + for (i, o) in objs.iter().enumerate() { + assert_eq!( + read(o, &mut slot), + 100.0 + i as f64, + "wrong guess, receiver {i}" + ); + } + } + + /// S3 declines what a key POSITION cannot answer: a key the shape does not + /// have (inherited/absent) still reaches the full miss handler. + #[test] + fn a_latched_site_still_answers_an_absent_key_through_the_miss_handler() { + let _lock = crate::gc::global_side_table_test_lock(); + let scope = crate::gc::RuntimeHandleScope::new(); + let objs = megamorphic_receivers(&scope, 48); + let absent = scope.root_string_ptr(key_of(b"notthere")); + let mut cache: PicCache = [0; PIC_CACHE_WORDS]; + let mut slot: PicCacheSlot = &mut cache; + let packed = AtomicU64::new(0); + let kind = scope.root_string_ptr(key_of(b"kind")); + for _ in 0..4 { + for o in &objs { + o.with_mut_ptr(|p: *mut ObjectHeader| { + kind.with_const_ptr(|k| { + js_object_get_field_ic_slow(handle(p), k, &mut slot, &packed) + }) + }); + } + } + for o in &objs { + let v = o.with_mut_ptr(|p: *mut ObjectHeader| { + absent.with_const_ptr(|k| { + js_object_get_field_ic_slow(handle(p), k, &mut slot, &packed) + }) + }); + assert_eq!( + v.to_bits(), + crate::value::TAG_UNDEFINED, + "an absent key reads undefined" + ); + } + } + /// A plain own data read that has never primed: the entry must fall all the /// way through to the miss handler, answer the field, and leave the site /// primed exactly as the old `js_object_get_field_ic_miss_packed` edge did. diff --git a/crates/perry-runtime/src/object/shapes.rs b/crates/perry-runtime/src/object/shapes.rs index 0312fb05f5..6e433fe4c4 100644 --- a/crates/perry-runtime/src/object/shapes.rs +++ b/crates/perry-runtime/src/object/shapes.rs @@ -221,6 +221,108 @@ impl ShapeRecordRef { } } +/// Test instrument: reads the receiver's shape answered at a latched +/// megamorphic site (compiled into test builds only). +#[cfg(test)] +pub(crate) static SHAPE_ANSWERED_READS: std::sync::atomic::AtomicU64 = + std::sync::atomic::AtomicU64::new(0); + +impl ShapeRecordRef { + /// The INLINE slot at which this shape stores `key`, answered from the + /// shape's own canonical key list — or `None` when the shape cannot answer + /// by position alone. + /// + /// The position of a key in the keys list is its slot exactly when the + /// shape is `Ordinary`, generation 0 (no descriptor/prototype mutation + /// minted it) and hole-free; a position below `live_inline_slot_count` is + /// an inline slot of every receiver carrying the shape (the invariant the + /// read cache's prime already relies on). The list is bounded by the + /// SHAPE's key count, never the backing's length (#10969: one backing per + /// growth chain). + /// + /// A stored key matches the site's key by identity, or else by (byte + /// length, bytes): a canonical list holds the string its first grower + /// passed, which is usually NOT the read site's pooled literal. The + /// site's slot guess is tried first. Allocation-free, never calls user + /// code. + #[inline] + pub(crate) unsafe fn inline_slot_of_key( + self, + key: *const crate::StringHeader, + hint: usize, + ) -> Option { + let r = &*self.0.as_ptr(); + if r.object_kind() != ShapeObjectKind::Ordinary + || r.semantic_generation != 0 + || r.hole_count != 0 + || r.keys == 0 + { + return None; + } + let (slots, len) = + super::keys_array_dense_slots_resolved(r.keys as usize as *const ArrayHeader); + if slots.is_null() { + return None; + } + let bound = len + .min(r.logical_key_count as usize) + .min(r.live_inline_slot_count as usize); + // A canonical list holds heap strings of its own (NOT the site's pooled + // key — measured: every stored key of a literal-born shape is a + // distinct heap string) or SSO immediates, so a stored key matches by + // identity, by SSO identity, or by (byte length, bytes). + let klen = (*key).byte_len as usize; + let kdata = crate::string::string_data(key); + let heap_bits = crate::JSValue::string_ptr(key as *mut crate::StringHeader).bits(); + let matches = |bits: u64| -> bool { + if bits == heap_bits { + return true; + } + match bits >> 48 { + 0x7FFF => { + let sp = (bits & 0x0000_FFFF_FFFF_FFFF) as *const crate::StringHeader; + !sp.is_null() + && (*sp).byte_len as usize == klen + && bytes_eq(crate::string::string_data(sp), kdata, klen) + } + // An SSO immediate in the list: rare; compare its bytes. + 0x7FF9 => { + let mut buf = [0u8; crate::value::SHORT_STRING_MAX_LEN]; + crate::string::js_string_key_bytes(crate::JSValue::from_bits(bits), &mut buf) + == Some(std::slice::from_raw_parts(kdata, klen)) + } + _ => false, + } + }; + // The site's slot guess first: the receiver's shape confirms it. + if hint < bound && matches((*slots.add(hint)).to_bits()) { + return Some(hint); + } + (0..bound).find(|&i| i != hint && matches((*slots.add(i)).to_bits())) + } +} + +/// Byte equality without a libc call for the short keys property names are. +#[inline] +unsafe fn bytes_eq(a: *const u8, b: *const u8, n: usize) -> bool { + let mut i = 0; + while i + 8 <= n { + if std::ptr::read_unaligned(a.add(i) as *const u64) + != std::ptr::read_unaligned(b.add(i) as *const u64) + { + return false; + } + i += 8; + } + while i < n { + if *a.add(i) != *b.add(i) { + return false; + } + i += 1; + } + true +} + impl PartialEq for ShapeDescriptor { fn eq(&self, other: &Self) -> bool { self.keys == other.keys From acb5ea66b99b5105da627299e1a65a0757982565 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 26 Sep 2026 13:12:30 +0000 Subject: [PATCH 2/3] docs(changelog): megamorphic reads answered by the receiver's shape key list --- changelog.d/megamorphic-read-shape-key-list.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelog.d/megamorphic-read-shape-key-list.md diff --git a/changelog.d/megamorphic-read-shape-key-list.md b/changelog.d/megamorphic-read-shape-key-list.md new file mode 100644 index 0000000000..4b98d97b62 --- /dev/null +++ b/changelog.d/megamorphic-read-shape-key-list.md @@ -0,0 +1,4 @@ +Megamorphic property reads (a site that has seen more shapes than its inline +cache holds, such as `node.kind` across a compiler AST) are answered from the +receiver's own shape key list instead of the generic miss handler. A 40-shape +`o.kind` read drops from ~730 to ~315 instructions. From 0cae43da168af9e39916caf935cd328b0a9cd491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ralph=20K=C3=BCpper?= Date: Sat, 26 Sep 2026 22:23:15 +0000 Subject: [PATCH 3/3] changelog: name the fragment after PR #11438 --- ...shape-key-list.md => 11438-megamorphic-read-shape-key-list.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changelog.d/{megamorphic-read-shape-key-list.md => 11438-megamorphic-read-shape-key-list.md} (100%) diff --git a/changelog.d/megamorphic-read-shape-key-list.md b/changelog.d/11438-megamorphic-read-shape-key-list.md similarity index 100% rename from changelog.d/megamorphic-read-shape-key-list.md rename to changelog.d/11438-megamorphic-read-shape-key-list.md