diff --git a/CLAUDE.md b/CLAUDE.md index 2774741392..4dcfeb40b2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co Perry is a native TypeScript compiler written in Rust that compiles TypeScript source code directly to native executables. It uses SWC for TypeScript parsing and LLVM for code generation. -**Current Version:** 0.5.1559 +**Current Version:** 0.5.1560 ## TypeScript Parity Status diff --git a/Cargo.lock b/Cargo.lock index 6a4abfd6d8..e7074f405b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5690,7 +5690,7 @@ checksum = "1542e48011813fbdf3c075da4a4ed53ee93c816eef62e36eb5064a6fd2be10a5" [[package]] name = "perry" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "anyhow", "base64 0.22.1", @@ -5754,7 +5754,7 @@ dependencies = [ [[package]] name = "perry-api-manifest" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "perry-dispatch", "serde", @@ -5762,7 +5762,7 @@ dependencies = [ [[package]] name = "perry-audio-miniaudio" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "cc", "libc", @@ -5771,7 +5771,7 @@ dependencies = [ [[package]] name = "perry-codegen" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "aho-corasick", "anyhow", @@ -5789,7 +5789,7 @@ dependencies = [ [[package]] name = "perry-codegen-arkts" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "anyhow", "perry-hir", @@ -5797,7 +5797,7 @@ dependencies = [ [[package]] name = "perry-codegen-glance" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "anyhow", "perry-hir", @@ -5805,7 +5805,7 @@ dependencies = [ [[package]] name = "perry-codegen-js" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "anyhow", "perry-dispatch", @@ -5814,7 +5814,7 @@ dependencies = [ [[package]] name = "perry-codegen-swiftui" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "anyhow", "perry-hir", @@ -5822,7 +5822,7 @@ dependencies = [ [[package]] name = "perry-codegen-wasm" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "anyhow", "base64 0.22.1", @@ -5834,7 +5834,7 @@ dependencies = [ [[package]] name = "perry-codegen-wear-tiles" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "anyhow", "perry-hir", @@ -5842,7 +5842,7 @@ dependencies = [ [[package]] name = "perry-container-compose" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "anyhow", "async-trait", @@ -5870,14 +5870,14 @@ dependencies = [ [[package]] name = "perry-container-e2e" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "anyhow", ] [[package]] name = "perry-diagnostics" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "serde", "serde_json", @@ -5885,7 +5885,7 @@ dependencies = [ [[package]] name = "perry-dispatch" -version = "0.5.1559" +version = "0.5.1560" [[package]] name = "perry-doc-fixture-my-bindings" @@ -5896,7 +5896,7 @@ dependencies = [ [[package]] name = "perry-doc-tests" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "anyhow", "clap", @@ -5911,7 +5911,7 @@ dependencies = [ [[package]] name = "perry-ext-ads" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "block2", "objc2", @@ -5921,7 +5921,7 @@ dependencies = [ [[package]] name = "perry-ext-argon2" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "argon2", "perry-ffi", @@ -5930,7 +5930,7 @@ dependencies = [ [[package]] name = "perry-ext-axios" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "perry-ffi", "reqwest", @@ -5939,7 +5939,7 @@ dependencies = [ [[package]] name = "perry-ext-bcrypt" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "bcrypt", "perry-ffi", @@ -5947,7 +5947,7 @@ dependencies = [ [[package]] name = "perry-ext-better-sqlite3" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "perry-ffi", "rusqlite", @@ -5955,7 +5955,7 @@ dependencies = [ [[package]] name = "perry-ext-cheerio" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "perry-ffi", "scraper", @@ -5963,7 +5963,7 @@ dependencies = [ [[package]] name = "perry-ext-commander" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "perry-ffi", "perry-runtime", @@ -5971,7 +5971,7 @@ dependencies = [ [[package]] name = "perry-ext-cron" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "chrono", "cron", @@ -5981,7 +5981,7 @@ dependencies = [ [[package]] name = "perry-ext-dayjs" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "chrono", "perry-ffi", @@ -5989,7 +5989,7 @@ dependencies = [ [[package]] name = "perry-ext-decimal" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "perry-ffi", "rust_decimal", @@ -5997,7 +5997,7 @@ dependencies = [ [[package]] name = "perry-ext-dotenv" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "perry-ffi", "serde_json", @@ -6005,7 +6005,7 @@ dependencies = [ [[package]] name = "perry-ext-ethers" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "perry-ffi", "rand 0.10.2", @@ -6013,7 +6013,7 @@ dependencies = [ [[package]] name = "perry-ext-events" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "perry-ffi", "perry-runtime", @@ -6021,14 +6021,14 @@ dependencies = [ [[package]] name = "perry-ext-exponential-backoff" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "perry-ffi", ] [[package]] name = "perry-ext-fastify" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "bytes", "http-body-util", @@ -6046,7 +6046,7 @@ dependencies = [ [[package]] name = "perry-ext-fetch" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "bytes", "lazy_static", @@ -6059,7 +6059,7 @@ dependencies = [ [[package]] name = "perry-ext-http" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "base64 0.22.1", "bytes", @@ -6091,7 +6091,7 @@ dependencies = [ [[package]] name = "perry-ext-ioredis" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "lazy_static", "perry-ffi", @@ -6101,7 +6101,7 @@ dependencies = [ [[package]] name = "perry-ext-jsonwebtoken" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "base64 0.22.1", "jsonwebtoken", @@ -6112,7 +6112,7 @@ dependencies = [ [[package]] name = "perry-ext-lru-cache" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "lru", "perry-ffi", @@ -6121,7 +6121,7 @@ dependencies = [ [[package]] name = "perry-ext-moment" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "chrono", "perry-ffi", @@ -6129,7 +6129,7 @@ dependencies = [ [[package]] name = "perry-ext-mongodb" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "bson", "futures-util", @@ -6141,7 +6141,7 @@ dependencies = [ [[package]] name = "perry-ext-mysql2" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "chrono", "perry-ffi", @@ -6153,7 +6153,7 @@ dependencies = [ [[package]] name = "perry-ext-nanoid" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "nanoid", "perry-ffi", @@ -6162,7 +6162,7 @@ dependencies = [ [[package]] name = "perry-ext-net" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "bytes", "perry-ffi", @@ -6177,7 +6177,7 @@ dependencies = [ [[package]] name = "perry-ext-node-forge" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "const-oid 0.10.2", "der 0.8.1", @@ -6196,7 +6196,7 @@ dependencies = [ [[package]] name = "perry-ext-nodemailer" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "lettre", "perry-ffi", @@ -6206,7 +6206,7 @@ dependencies = [ [[package]] name = "perry-ext-parcel-watcher" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "notify", "perry-ffi", @@ -6218,7 +6218,7 @@ dependencies = [ [[package]] name = "perry-ext-pdf" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "perry-ffi", "printpdf", @@ -6226,7 +6226,7 @@ dependencies = [ [[package]] name = "perry-ext-pg" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "perry-ffi", "sqlx", @@ -6235,7 +6235,7 @@ dependencies = [ [[package]] name = "perry-ext-qs" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "perry-ffi", "perry-runtime", @@ -6244,7 +6244,7 @@ dependencies = [ [[package]] name = "perry-ext-ratelimit" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "governor", "perry-ffi", @@ -6252,7 +6252,7 @@ dependencies = [ [[package]] name = "perry-ext-sharp" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "fast_image_resize", "image", @@ -6263,7 +6263,7 @@ dependencies = [ [[package]] name = "perry-ext-streams" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "lazy_static", "perry-ffi", @@ -6272,7 +6272,7 @@ dependencies = [ [[package]] name = "perry-ext-typescript" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "anyhow", "perry-ffi", @@ -6292,7 +6292,7 @@ dependencies = [ [[package]] name = "perry-ext-undici" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "perry-ffi", "perry-runtime", @@ -6301,7 +6301,7 @@ dependencies = [ [[package]] name = "perry-ext-uuid" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "perry-ffi", "uuid", @@ -6309,7 +6309,7 @@ dependencies = [ [[package]] name = "perry-ext-validator" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "perry-ffi", "perry-validation", @@ -6318,7 +6318,7 @@ dependencies = [ [[package]] name = "perry-ext-ws" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "futures-util", "lazy_static", @@ -6331,7 +6331,7 @@ dependencies = [ [[package]] name = "perry-ext-zlib" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "brotli", "flate2", @@ -6341,7 +6341,7 @@ dependencies = [ [[package]] name = "perry-ffi" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "dashmap 6.2.1", "once_cell", @@ -6351,7 +6351,7 @@ dependencies = [ [[package]] name = "perry-hir" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "anyhow", "perry-api-manifest", @@ -6372,11 +6372,11 @@ dependencies = [ [[package]] name = "perry-native-registration" -version = "0.5.1559" +version = "0.5.1560" [[package]] name = "perry-parser" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "anyhow", "perry-diagnostics", @@ -6390,7 +6390,7 @@ dependencies = [ [[package]] name = "perry-perex" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "perex", "regex", @@ -6398,7 +6398,7 @@ dependencies = [ [[package]] name = "perry-runtime" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "ahash", "anyhow", @@ -6458,14 +6458,14 @@ dependencies = [ [[package]] name = "perry-runtime-static" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "perry-runtime", ] [[package]] name = "perry-stdlib" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "aes 0.8.4", "aes 0.9.1", @@ -6560,14 +6560,14 @@ dependencies = [ [[package]] name = "perry-stdlib-static" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "perry-stdlib", ] [[package]] name = "perry-transform" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "anyhow", "perry-hir", @@ -6576,7 +6576,7 @@ dependencies = [ [[package]] name = "perry-ui" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "perry-ffi", "perry-ui-model", @@ -6584,7 +6584,7 @@ dependencies = [ [[package]] name = "perry-ui-android" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "base64 0.22.1", "itoa", @@ -6602,7 +6602,7 @@ dependencies = [ [[package]] name = "perry-ui-geisterhand" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "rand 0.10.2", "serde", @@ -6612,7 +6612,7 @@ dependencies = [ [[package]] name = "perry-ui-gtk4" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "base64 0.22.1", "cairo-rs 0.22.9", @@ -6635,7 +6635,7 @@ dependencies = [ [[package]] name = "perry-ui-ios" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "base64 0.22.1", "block2", @@ -6652,7 +6652,7 @@ dependencies = [ [[package]] name = "perry-ui-macos" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "base64 0.22.1", "block2", @@ -6669,7 +6669,7 @@ dependencies = [ [[package]] name = "perry-ui-model" -version = "0.5.1559" +version = "0.5.1560" [[package]] name = "perry-ui-test" @@ -6680,11 +6680,11 @@ dependencies = [ [[package]] name = "perry-ui-testkit" -version = "0.5.1559" +version = "0.5.1560" [[package]] name = "perry-ui-tvos" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "base64 0.22.1", "block2", @@ -6701,7 +6701,7 @@ dependencies = [ [[package]] name = "perry-ui-visionos" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "base64 0.22.1", "block2", @@ -6718,7 +6718,7 @@ dependencies = [ [[package]] name = "perry-ui-watchos" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "block2", "libc", @@ -6732,7 +6732,7 @@ dependencies = [ [[package]] name = "perry-ui-windows" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "base64 0.22.1", "libc", @@ -6751,7 +6751,7 @@ dependencies = [ [[package]] name = "perry-ui-windows-winui" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "base64 0.22.1", "libc", @@ -6764,7 +6764,7 @@ dependencies = [ [[package]] name = "perry-updater" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "anyhow", "base64 0.22.1", @@ -6780,7 +6780,7 @@ dependencies = [ [[package]] name = "perry-validation" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "idna", "regex", @@ -6790,7 +6790,7 @@ dependencies = [ [[package]] name = "perry-wasm-host" -version = "0.5.1559" +version = "0.5.1560" dependencies = [ "wasmi", ] diff --git a/Cargo.toml b/Cargo.toml index 3dd88af770..ad66fe02de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -338,7 +338,7 @@ codegen-units = 1 codegen-units = 1 [workspace.package] -version = "0.5.1559" +version = "0.5.1560" edition = "2021" license = "MIT" repository = "https://github.com/PerryTS/perry" diff --git a/changelog.d/10218-dynamic-index-get-one-exit.md b/changelog.d/10218-dynamic-index-get-one-exit.md new file mode 100644 index 0000000000..24515662fc --- /dev/null +++ b/changelog.d/10218-dynamic-index-get-one-exit.md @@ -0,0 +1,14 @@ +Collapse the inline guarded element read for a dynamically typed receiver +(`a[i]` on an erased or union type) from ~51 basic blocks and two runtime call +sites per site to 21 blocks and one call. The dense-Array arm and the +elements-backed Array-subclass probe stay inline and byte-identical; the +shape-carried subclass IC tower (which never fired in the shipped +configuration), the lazy-JSON probe and the spill paths move behind the +existing `js_packed_arraylike_index_get` dispatcher; the typed-array ladder +collapses from eight element kinds onto four element widths, with the brand +test decided on the tag alone. Per site 345 → 173 IR instructions. On +prettier/plugins/flow.mjs `.text` shrinks a further 15.8 % on top of the +budget and property-get changes, ordinary workloads keep their instruction +counts and RSS, and dynamic-receiver typed-array and plain-array reads run +10–32 % fewer instructions. The dispatcher now checks `GC_FLAG_FORWARDED` +before probing a lazy JSON array, which #10098 made movable. diff --git a/crates/perry-codegen/src/block.rs b/crates/perry-codegen/src/block.rs index e54ad61124..dbb5a7ad51 100644 --- a/crates/perry-codegen/src/block.rs +++ b/crates/perry-codegen/src/block.rs @@ -870,6 +870,24 @@ impl LlBlock { r } + /// Reinterpret a 32-bit integer lane as an IEEE-754 single. + /// + /// The dynamic `obj[i]` typed-array arm loads a 4-byte lane ONCE and + /// resolves `Int32Array` / `Uint32Array` / `Float32Array` from it with + /// `select`s instead of three loads behind three branches, so the + /// `Float32Array` form needs this reinterpretation of the same register. + pub fn bitcast_i32_to_float(&mut self, val: &str) -> String { + let r = self.reg(); + self.push_inst(crate::inst::LlInst::Cast { + dst: r.clone(), + op: "bitcast", + from: "i32", + v: val.to_string(), + to: "float", + }); + r + } + pub fn sitofp(&mut self, from_ty: LlvmType, val: &str, to_ty: LlvmType) -> String { let r = self.reg(); self.push_inst(crate::inst::LlInst::Cast { diff --git a/crates/perry-codegen/src/codegen/index_method_clone_tests.rs b/crates/perry-codegen/src/codegen/index_method_clone_tests.rs index 27d2964158..cff9a0d6cb 100644 --- a/crates/perry-codegen/src/codegen/index_method_clone_tests.rs +++ b/crates/perry-codegen/src/codegen/index_method_clone_tests.rs @@ -804,15 +804,16 @@ fn u31_bitset_clone_uses_one_guarded_uint32_load_and_keeps_dynamic_miss() { && clone.contains("load i32") && clone.contains("call double @js_dyn_index_get(") && clone.contains("call double @js_dynamic_bitand(") - && !clone.contains("tav.k.i8") - && !clone.contains("tav.k.f64"), + && !clone.contains("arrlike.ic.header") + && !clone.contains("js_packed_arraylike_index_get"), "the u31 clone must use the monomorphic Uint32 bitset tier and a canonical miss:\n{clone}" ); assert!( - generic.contains("tav.k.i8") - && generic.contains("tav.k.f64") + generic.contains("arrlike.ic.header") + && generic.contains("call double @js_packed_arraylike_index_get(") && generic.contains("call double @js_dynamic_bitand("), - "the unproven body must retain the full dynamic typed-array and BigInt behavior:\n{generic}" + "the unproven body must retain the complete dynamic element read (inline hit \ + plus its one out-of-line exit) and the BigInt behavior:\n{generic}" ); } @@ -843,7 +844,7 @@ fn u31_transition_clone_returns_a_proved_cached_array_hit_without_second_get() { .and_then(|tail| tail.split("\ncached_field_index.normal.").next()) .is_some_and(|fast_return| fast_return.contains("ret double")) && clone.contains("cached_field_index.normal") - && clone.contains("tav.k.i8") + && clone.contains("arrlike.ic.header") && clone.contains("if.then"), "the proved truthy hit must return directly while the complete original body remains as fallback:\n{clone}" ); diff --git a/crates/perry-codegen/src/expr/index_get/inline_dyn_typed_array.rs b/crates/perry-codegen/src/expr/index_get/inline_dyn_typed_array.rs index 4f3fd5c8db..c313c54b93 100644 --- a/crates/perry-codegen/src/expr/index_get/inline_dyn_typed_array.rs +++ b/crates/perry-codegen/src/expr/index_get/inline_dyn_typed_array.rs @@ -1,9 +1,48 @@ -//! Inline guarded typed-array element read for a dynamically-typed receiver. +//! The inline HIT of the dynamically-typed `obj[i]` element read: four guarded +//! arms and one out-of-line exit. //! -//! Split out of `index_get.rs` to keep that file under the 2000-line cap. -//! Pure mechanical move — the items below are verbatim copies (only the -//! visibility of the entry point is widened to `pub(super)` so the trunk's -//! call sites keep compiling). +//! Split out of `index_get.rs` to keep that file under the 2000-line cap; the +//! entry point stays `pub(super)` so the trunk's call sites keep compiling. +//! +//! # What is inline and what is not (#T2, "inline hit, one exit") +//! +//! The site keeps four guarded arms and ONE out-of-line call. Each arm is here +//! because removing it was measured and cost runtime; each absent arm is +//! absent because keeping it was measured and cost only bytes. +//! +//! 1. receiver NaN-box tag / heap-band and canonical-index checks, then the +//! managed `GcHeader` kind (+ forwarding) load; +//! 2. the packed ordinary `GC_TYPE_ARRAY` arm (bounds check, element load, +//! hole -> `undefined`); +//! 3. the typed-array arm, collapsed from eight per-kind load blocks behind +//! a seven-block kind dispatch onto the four ELEMENT WIDTHS the header +//! already stores (`tav.w1/w2/w4/w8`), each resolving its own +//! signedness/float form with `select`s — behind #10118's tag-only brand +//! test, so a non-typed-array receiver still pays one `icmp` to leave; +//! 4. the elements-backed Array-subclass probe (`ObjectMeta.elements`), +//! which is the DEFAULT representation of `class X extends Array`. +//! +//! Everything else reaches `js_packed_arraylike_index_get`, the site's single +//! non-feedback out-of-line call and the same call the old `arrlike.ic.miss` +//! block made: BigInt/Float16 lanes, a live typed-array view, the +//! lazy-JSON-array tier (which moved INTO that helper) and the whole +//! shape-carried Array-subclass IC tower. +//! +//! The tower is the large removal — fifteen blocks per site +//! (`arrlike.ic.{shape,identity,exact,family_meta,family_token,bounds, +//! length_inline,length_spill_meta,length_spill_ptr,length_spill_load,range, +//! inline,spill,spill_ptr,spill_load,spill_or_miss}`) — and it is removable +//! because it CANNOT HIT in the shipped configuration: its hit needs a primed +//! layout cache, and the only writer of that cache reaches +//! `build_dense_layout` only when `elements_of(obj)` is null, which the +//! default elements store makes false for every Array subclass. See the +//! `arrlike.elem.*` section below for the full argument. +//! +//! This used to emit ~50 basic blocks and ~393 pre-RS4GC instructions per +//! `a[i]`; on `prettier/plugins/flow.mjs` — a program that never constructs a +//! typed array and never subclasses `Array` — that tower was 55% of all +//! emitted IR across 10,778 sites, and each of its ~4 runtime calls is a +//! statepoint whose `.perry_gcmap` scales with the live GC values at the site. //! //! # Rooting (Layer 1, slice 4) //! @@ -19,37 +58,25 @@ use crate::types::{DOUBLE, F32, I1, I16, I32, I64, I8, PTR}; use super::FnCtx; -/// #5525 follow-up: emit a guarded **inline** typed-array element read for an -/// `obj[i]` whose receiver static type is erased (`any`/unknown) but is, at -/// runtime, commonly an owning numeric typed array reached through an untyped -/// param — exactly bcryptjs's `S[i]`/`P[i]` Blowfish boxes (~600M reads for one -/// cost-12 `compareSync`). Instead of an out-of-line `js_dyn_index_get` call + -/// `lookup_typed_array_kind` + `js_number_coerce` per element, this inlines: -/// 1. receiver-is-pointer NaN-box guard, -/// 2. a read of the process-global `PERRY_TA_VIEW_GUARD` (must be 0 → every -/// live typed array uses inline storage, so `data_ptr == header + 16`), -/// 3. a `GC_TYPE_TYPED_ARRAY` brand read from the receiver's managed header -/// (`obj_type == 11`) plus the element kind read from the -/// `TypedArrayHeader` (must be a non-BigInt kind ≤ `KIND_UINT8_CLAMPED`), -/// 4. an index validity + bounds check against the header `length`, -/// 5. a direct per-kind element load + int↔f64 widen, -/// and falls back to the existing `js_dyn_index_get` slow path on ANY guard -/// miss (non-pointer, cache miss, view live, BigInt/Float16 kind, OOB / -/// fractional / negative index, runtime-string or symbol key). Because every -/// rejected case defers to the unchanged runtime helper, semantics are -/// identical; only the hot monomorphic numeric-typed-array case is short-cut. +/// Emit the guarded inline element read for an `obj[i]` whose receiver static +/// type is erased (`any`/unknown). +/// /// `obj_box` / `idx_d` are the already-lowered receiver and index (DOUBLE). /// /// `coerce_slow_to_number`: when the read is used in a context that will /// `ToNumber` the result regardless (a non-`+` arithmetic / bitwise operand — -/// `^`, `-`, `*`, `<<`, …, all of which `ToNumber` their operands; see -/// [`lower_unknown_local_index_get_for_number_context`]), the cold slow branch's -/// `js_dyn_index_get` result is wrapped in `js_number_coerce` here so the merged -/// value is *always* a Number. The hot per-kind fast branches already produce a -/// Number, so the caller can skip the per-element site `js_number_coerce` it -/// would otherwise emit — moving that coercion off bcrypt's ~600M-read hot path -/// and onto the cache-miss path only. `false` leaves the slow result boxed -/// (the general `obj[i]` read, whose result may legitimately be a non-Number). +/// `^`, `-`, `*`, `<<`, …; see [`lower_unknown_local_index_get_for_number_ +/// context`]), EVERY arm wraps its result in `js_number_coerce` so the merged +/// value is always a Number and the caller can skip the per-element site +/// coerce it would otherwise emit. `false` leaves the result boxed (the +/// general `obj[i]` read, whose result may legitimately be a non-Number). +/// +/// The typed-array arms never coerce: their value is a Number by +/// construction, which is the #5525 property that made inlining them worth it +/// (bcryptjs's `S[i]`/`P[i]` Blowfish boxes, ~600M reads for one cost-12 +/// `compareSync`). +/// +/// [`lower_unknown_local_index_get_for_number_context`]: super::lower_unknown_local_index_get_for_number_context pub(super) fn lower_inline_dyn_typed_array_get( ctx: &mut FnCtx<'_>, obj_box: &str, @@ -61,364 +88,65 @@ pub(super) fn lower_inline_dyn_typed_array_get( let pointer_tag = crate::nanbox::POINTER_TAG_I64; let pointer_mask = crate::nanbox::POINTER_MASK_I64; - let fast_idx = ctx.new_block("tav.get.fast"); - let load_idx = ctx.new_block("tav.get.load"); - let slow_idx = ctx.new_block("tav.get.slow"); - let merge_idx = ctx.new_block("tav.get.merge"); - let fast_label = ctx.block_label(fast_idx); - let load_label = ctx.block_label(load_idx); - let slow_label = ctx.block_label(slow_idx); - let merge_label = ctx.block_label(merge_idx); - - // ---- entry: combined cache/kind/range guard -> fast | slow ---- - let entry_guard = { - let blk = ctx.block(); - let obj_bits = blk.bitcast_double_to_i64(obj_box); - let raw = blk.and(I64, &obj_bits, pointer_mask); - // is_pointer: (bits & TAG_MASK) == POINTER_TAG - let tagged = blk.and(I64, &obj_bits, &tag_mask); - let is_ptr = blk.icmp_eq(I64, &tagged, pointer_tag); - // view guard must be 0 (all typed arrays inline-storage) - let vg = blk.load(I64, "@PERRY_TA_VIEW_GUARD"); - let vg_zero = blk.icmp_eq(I64, &vg, "0"); - // Heap-band magnitude before any dereference: the same floor and - // ceiling the guarded Array tiers apply (`is_plausible_heap_addr`). - let above_handle_band = blk.icmp_ugt(I64, &raw, "1048575"); - let below_heap_limit = blk.icmp_ult(I64, &raw, "140737488355328"); - let heap_candidate = blk.and(I1, &above_handle_band, &below_heap_limit); - let g0 = blk.and(I1, &is_ptr, &vg_zero); - blk.and(I1, &g0, &heap_candidate) - }; - let brand_idx = ctx.new_block("tav.get.brand"); - let brand_label = ctx.block_label(brand_idx); - ctx.block().cond_br(&entry_guard, &brand_label, &slow_label); - - // ---- brand: managed-header tag + header kind -> fast | slow ---- - // - // Every typed array carries a real `GC_TYPE_TYPED_ARRAY` GcHeader (the - // 2026-07-09 audit) whose payload starts with `TypedArrayHeader` - // {length u32, capacity u32, kind u8, ...}. Reading the brand and the kind - // from the object itself replaces the 64-slot direct-mapped - // `PERRY_TA_KIND_CACHE` probe, which every ordinary-array registry miss - // also writes NEGATIVE entries into: a hot typed array whose slot kept - // being evicted (the wolf-ecs archetype `mask` reads) missed this tier on - // every access and paid the complete dynamic read. The header tag is - // ABA-proof for a value held by live code: the arena rewrites `obj_type` - // before it hands the address out again, and a live reference keeps the - // typed array alive. - // The brand test is the FIRST thing every indexed read on an unknown - // receiver executes, and until #10118 it computed the whole guard set -- - // element kind, both index range checks, three ANDs -- before finding out - // the receiver was not a typed array at all. A `JSON.parse` array, and any - // ordinary Array behind an erased receiver, paid that on every element - // read forever. Decide on the tag alone and leave; the rest of the guard - // set is only meaningful once the tag says typed array. - let kind_guard_idx = ctx.new_block("tav.get.kind_guard"); - let kind_guard_label = ctx.block_label(kind_guard_idx); - ctx.current_block = brand_idx; - let is_typed_array = { - let blk = ctx.block(); - let obj_bits = blk.bitcast_double_to_i64(obj_box); - let raw = blk.and(I64, &obj_bits, pointer_mask); - let gc_type_addr = blk.sub(I64, &raw, "8"); - let gc_type_ptr = blk.inttoptr(I64, &gc_type_addr); - let gc_type = blk.load(I8, &gc_type_ptr); - blk.icmp_eq(I8, &gc_type, "11") // GC_TYPE_TYPED_ARRAY - }; - ctx.block() - .cond_br(&is_typed_array, &kind_guard_label, &slow_label); - - ctx.current_block = kind_guard_idx; - let entry_guard = { - let blk = ctx.block(); - let obj_bits = blk.bitcast_double_to_i64(obj_box); - let raw = blk.and(I64, &obj_bits, pointer_mask); - let kind_addr = blk.add(I64, &raw, "8"); - let kind_ptr = blk.inttoptr(I64, &kind_addr); - let kind_i8 = blk.load(I8, &kind_ptr); - let kind = blk.zext(I8, &kind_i8, I64); - // loadable numeric kind = kind <= 8 (KIND_INT8=0 .. KIND_UINT8_CLAMPED=8; - // rejects BigInt 9/10 and Float16 11). - let kind_ok = blk.icmp_ule(I64, &kind, "8"); - // index float-range pre-checks (well-defined on NaN → false): the - // fptosi in the load block is only reached when these hold, so its - // result is never poison there. - let idx_ge0 = blk.fcmp("oge", idx_d, "0.0"); - let idx_lt = blk.fcmp("olt", idx_d, "4294967296.0"); - let g = blk.and(I1, &kind_ok, &idx_ge0); - blk.and(I1, &g, &idx_lt) - }; - ctx.block().cond_br(&entry_guard, &fast_label, &slow_label); - - // ---- fast: validate integer index + bounds -> load | slow ---- - ctx.current_block = fast_idx; - let (raw, idx_i64, kind) = { - let blk = ctx.block(); - let obj_bits = blk.bitcast_double_to_i64(obj_box); - let raw = blk.and(I64, &obj_bits, pointer_mask); - // kind re-read from the header (cheap; keeps the fast block - // self-contained). - let kind_addr = blk.add(I64, &raw, "8"); - let kind_ptr = blk.inttoptr(I64, &kind_addr); - let kind_i8 = blk.load(I8, &kind_ptr); - let kind = blk.zext(I8, &kind_i8, I64); - // idx is in [0, 2^32) (entry guard) so fptosi i64 is well-defined. - let idx_i64 = blk.fptosi(DOUBLE, idx_d, I64); - (raw, idx_i64, kind) - }; - let fast_ok = { - let blk = ctx.block(); - // reject fractional indices: sitofp(idx_i64) == idx_d - let idx_back = blk.sitofp(I64, &idx_i64, DOUBLE); - let is_int = blk.fcmp("oeq", &idx_back, idx_d); - // bounds: idx < header.length (u32 at offset 0) - let hdr_ptr = blk.inttoptr(I64, &raw); - let len = blk.load(I32, &hdr_ptr); - let len_i64 = blk.zext(I32, &len, I64); - let in_bounds = blk.icmp_ult(I64, &idx_i64, &len_i64); - blk.and(I1, &is_int, &in_bounds) - }; - ctx.block().cond_br(&fast_ok, &load_label, &slow_label); - - // ---- load: per-kind direct element load (data = header + 16) ---- - ctx.current_block = load_idx; - // (value, end_label) for each per-kind load block, collected for the merge. - let mut kind_incoming: Vec<(String, String)>; - { - // Per-kind load blocks. Each computes the element address from - // `data = raw + 16` and `off = idx * elem_size`, loads the native - // slot, and widens to f64. We branch on `kind` via a cond_br chain. - // kinds: 0 I8, 1 U8, 2 I16, 3 U16, 4 I32, 5 U32, 6 F32, 7 F64, - // 8 U8Clamped (== U8 load). All others were excluded by the entry - // guard (kind <= 8). - let data_base = { - let blk = ctx.block(); - blk.add(I64, &raw, "16") - }; - // Helper closure-like inline: build a block that loads with a given - // element byte-width shift + LLVM elem type + widen, then brs to merge. - // We emit explicit blocks since closures can't borrow ctx mutably here. - - // Create the per-kind blocks up front. - let b_i8 = ctx.new_block("tav.k.i8"); - let b_u8 = ctx.new_block("tav.k.u8"); - let b_i16 = ctx.new_block("tav.k.i16"); - let b_u16 = ctx.new_block("tav.k.u16"); - let b_i32 = ctx.new_block("tav.k.i32"); - let b_u32 = ctx.new_block("tav.k.u32"); - let b_f32 = ctx.new_block("tav.k.f32"); - let b_f64 = ctx.new_block("tav.k.f64"); - let l_i8 = ctx.block_label(b_i8); - let l_u8 = ctx.block_label(b_u8); - let l_i16 = ctx.block_label(b_i16); - let l_u16 = ctx.block_label(b_u16); - let l_i32 = ctx.block_label(b_i32); - let l_u32 = ctx.block_label(b_u32); - let l_f32 = ctx.block_label(b_f32); - let l_f64 = ctx.block_label(b_f64); - - // Dispatch chain on `kind` (in the load block). - let chk = |ctx: &mut FnCtx<'_>, k: &str, hit: &str, next_idx: usize| { - let next_label = ctx.block_label(next_idx); - let cond = ctx.block().icmp_eq(I64, &kind, k); - ctx.block().cond_br(&cond, hit, &next_label); - }; - // 0..7 explicit; kind 8 (U8Clamped) shares the U8 load as the final - // else (no further branch needed — entry guard already proved kind<=8). - let c1 = ctx.new_block("tav.kd1"); - let c2 = ctx.new_block("tav.kd2"); - let c3 = ctx.new_block("tav.kd3"); - let c4 = ctx.new_block("tav.kd4"); - let c5 = ctx.new_block("tav.kd5"); - let c6 = ctx.new_block("tav.kd6"); - let c7 = ctx.new_block("tav.kd7"); - chk(ctx, "0", &l_i8, c1); - ctx.current_block = c1; - chk(ctx, "1", &l_u8, c2); - ctx.current_block = c2; - chk(ctx, "2", &l_i16, c3); - ctx.current_block = c3; - chk(ctx, "3", &l_u16, c4); - ctx.current_block = c4; - chk(ctx, "4", &l_i32, c5); - ctx.current_block = c5; - chk(ctx, "5", &l_u32, c6); - ctx.current_block = c6; - chk(ctx, "6", &l_f32, c7); - ctx.current_block = c7; - // remaining: kind 7 → f64, else (8) → u8. - let is_f64 = ctx.block().icmp_eq(I64, &kind, "7"); - ctx.block().cond_br(&is_f64, &l_f64, &l_u8); - - // Each per-kind block: compute elem addr, load, widen, br merge. - // off = idx << shift; addr = data_base + off. - let mut incoming: Vec<(String, String)> = Vec::new(); - // I8 (sext), U8 (zext), I16 (sext), U16 (zext) via the small-int helper. - incoming.push(emit_inline_ta_int_load( - ctx, - b_i8, - &idx_i64, - &data_base, - &merge_label, - "0", - I8, - true, - )); - incoming.push(emit_inline_ta_int_load( - ctx, - b_u8, - &idx_i64, - &data_base, - &merge_label, - "0", - I8, - false, - )); - incoming.push(emit_inline_ta_int_load( - ctx, - b_i16, - &idx_i64, - &data_base, - &merge_label, - "1", - I16, - true, - )); - incoming.push(emit_inline_ta_int_load( - ctx, - b_u16, - &idx_i64, - &data_base, - &merge_label, - "1", - I16, - false, - )); - // I32: load i32, sitofp directly (sext to i32 is a no-op). - { - ctx.current_block = b_i32; - let blk = ctx.block(); - let off = blk.shl(I64, &idx_i64, "2"); - let addr = blk.add(I64, &data_base, &off); - let ptr = blk.inttoptr(I64, &addr); - let raw_elem = blk.load(I32, &ptr); - let val = blk.sitofp(I32, &raw_elem, DOUBLE); - let end_label = blk.label.clone(); - blk.br(&merge_label); - incoming.push((val, end_label)); - } - // U32: load i32, treat as unsigned → uitofp. - { - ctx.current_block = b_u32; - let blk = ctx.block(); - let off = blk.shl(I64, &idx_i64, "2"); - let addr = blk.add(I64, &data_base, &off); - let ptr = blk.inttoptr(I64, &addr); - let raw_elem = blk.load(I32, &ptr); - let val = blk.uitofp(I32, &raw_elem, DOUBLE); - let end_label = blk.label.clone(); - blk.br(&merge_label); - incoming.push((val, end_label)); - } - // F32: load float, fpext. - { - ctx.current_block = b_f32; - let blk = ctx.block(); - let off = blk.shl(I64, &idx_i64, "2"); - let addr = blk.add(I64, &data_base, &off); - let ptr = blk.inttoptr(I64, &addr); - let raw_elem = blk.load(F32, &ptr); - let val = blk.fpext(F32, &raw_elem, DOUBLE); - let end_label = blk.label.clone(); - blk.br(&merge_label); - incoming.push((val, end_label)); - } - // F64: load double raw. - { - ctx.current_block = b_f64; - let blk = ctx.block(); - let off = blk.shl(I64, &idx_i64, "3"); - let addr = blk.add(I64, &data_base, &off); - let ptr = blk.inttoptr(I64, &addr); - let val = blk.load(DOUBLE, &ptr); - let end_label = blk.label.clone(); - blk.br(&merge_label); - incoming.push((val, end_label)); - } - - // Hand the collected per-kind (value,label) pairs to the final merge. - kind_incoming = incoming; - } - - // ---- typed-array miss: Array-subclass shape IC, then dispatcher ---- - ctx.current_block = slow_idx; + // #9708: the site owns an 8-byte pointer SLOT (`__bss`, null until the + // runtime primes it), not the cache words. The site no longer reads the + // cache at all — the tier that did (the shape-carried Array-subclass IC) + // lives behind the exit now — but it still owns the slot, because the exit + // is handed the slot's ADDRESS, which is a link-time constant needing no + // load. let site_id = ctx.ic_site_counter; ctx.ic_site_counter += 1; let cache_name = super::super::inline_cache_global_name(ctx, site_id); ctx.ic_globals.push(cache_name.clone()); - // #9708: the cache sits behind a pointer slot the runtime fills on the - // first shape-carried prime. `arrlike.ic.shape` reads word 0 inside a - // flat predicate, so it reads through `key_cache`: the real cache when - // present, else the slot itself — 8 bytes of null, i.e. a zero identity, - // which fails `key_nonzero` exactly as the all-zero global did. Every - // later word is read only past that edge, through the real pointer. - let ic_slot = crate::expr::emit_inline_cache_slot(ctx, &cache_name); - let cache_ref = ic_slot.cache.clone(); - let key_cache = ctx - .block() - .select(I1, &ic_slot.present, PTR, &cache_ref, &ic_slot.slot_ref); + let slot_ref = format!("@{cache_name}"); let object_header_idx = ctx.new_block("arrlike.ic.header"); let object_brand_idx = ctx.new_block("arrlike.ic.brand"); let object_array_guard_idx = ctx.new_block("arrlike.ic.array_guard"); let object_array_load_idx = ctx.new_block("arrlike.ic.array_load"); - let object_shape_idx = ctx.new_block("arrlike.ic.shape"); - let object_identity_idx = ctx.new_block("arrlike.ic.identity"); - let object_exact_idx = ctx.new_block("arrlike.ic.exact"); - let object_family_meta_idx = ctx.new_block("arrlike.ic.family_meta"); - let object_family_token_idx = ctx.new_block("arrlike.ic.family_token"); - let object_bounds_idx = ctx.new_block("arrlike.ic.bounds"); - let object_length_inline_idx = ctx.new_block("arrlike.ic.length_inline"); - let object_length_spill_meta_idx = ctx.new_block("arrlike.ic.length_spill_meta"); - let object_length_spill_ptr_idx = ctx.new_block("arrlike.ic.length_spill_ptr"); - let object_length_spill_load_idx = ctx.new_block("arrlike.ic.length_spill_load"); - let object_range_idx = ctx.new_block("arrlike.ic.range"); - let object_inline_idx = ctx.new_block("arrlike.ic.inline"); - let object_spill_idx = ctx.new_block("arrlike.ic.spill"); - let object_spill_ptr_idx = ctx.new_block("arrlike.ic.spill_ptr"); - let object_spill_load_idx = ctx.new_block("arrlike.ic.spill_load"); + let ta_brand_idx = ctx.new_block("tav.brand"); + let ta_kind_guard_idx = ctx.new_block("tav.kind_guard"); + let ta_width_idx = ctx.new_block("tav.width"); + let ta_width4_idx = ctx.new_block("tav.width4"); + let ta_width2_idx = ctx.new_block("tav.width2"); + let ta_w8_idx = ctx.new_block("tav.w8"); + let ta_w4_idx = ctx.new_block("tav.w4"); + let ta_w2_idx = ctx.new_block("tav.w2"); + let ta_w1_idx = ctx.new_block("tav.w1"); + let elem_kind_idx = ctx.new_block("arrlike.elem.kind"); + let elem_meta_idx = ctx.new_block("arrlike.elem.meta"); + let elem_store_idx = ctx.new_block("arrlike.elem.store"); + let elem_bounds_idx = ctx.new_block("arrlike.elem.bounds"); + let elem_load_idx = ctx.new_block("arrlike.elem.load"); + let elem_value_idx = ctx.new_block("arrlike.elem.value"); let object_miss_idx = ctx.new_block("arrlike.ic.miss"); + let merge_idx = ctx.new_block("arrlike.ic.merge"); let object_header_label = ctx.block_label(object_header_idx); let object_brand_label = ctx.block_label(object_brand_idx); let object_array_guard_label = ctx.block_label(object_array_guard_idx); let object_array_load_label = ctx.block_label(object_array_load_idx); - let object_shape_label = ctx.block_label(object_shape_idx); - let object_identity_label = ctx.block_label(object_identity_idx); - let object_exact_label = ctx.block_label(object_exact_idx); - let object_family_meta_label = ctx.block_label(object_family_meta_idx); - let object_family_token_label = ctx.block_label(object_family_token_idx); - let object_bounds_label = ctx.block_label(object_bounds_idx); - let object_length_inline_label = ctx.block_label(object_length_inline_idx); - let object_length_spill_meta_label = ctx.block_label(object_length_spill_meta_idx); - let object_length_spill_ptr_label = ctx.block_label(object_length_spill_ptr_idx); - let object_length_spill_load_label = ctx.block_label(object_length_spill_load_idx); - let object_range_label = ctx.block_label(object_range_idx); - let object_inline_label = ctx.block_label(object_inline_idx); - let object_spill_label = ctx.block_label(object_spill_idx); - let object_spill_ptr_label = ctx.block_label(object_spill_ptr_idx); - let object_spill_load_label = ctx.block_label(object_spill_load_idx); + let ta_brand_label = ctx.block_label(ta_brand_idx); + let ta_kind_guard_label = ctx.block_label(ta_kind_guard_idx); + let ta_width_label = ctx.block_label(ta_width_idx); + let ta_width4_label = ctx.block_label(ta_width4_idx); + let ta_width2_label = ctx.block_label(ta_width2_idx); + let ta_w8_label = ctx.block_label(ta_w8_idx); + let ta_w4_label = ctx.block_label(ta_w4_idx); + let ta_w2_label = ctx.block_label(ta_w2_idx); + let ta_w1_label = ctx.block_label(ta_w1_idx); + let elem_kind_label = ctx.block_label(elem_kind_idx); + let elem_meta_label = ctx.block_label(elem_meta_idx); + let elem_store_label = ctx.block_label(elem_store_idx); + let elem_bounds_label = ctx.block_label(elem_bounds_idx); + let elem_load_label = ctx.block_label(elem_load_idx); + let elem_value_label = ctx.block_label(elem_value_idx); let object_miss_label = ctx.block_label(object_miss_idx); - let meta_ptr_size: u64 = if crate::target_layout::target_is_ilp32(ctx.target_triple) { - 4 - } else { - 8 - }; - let meta_offset = - crate::target_layout::object_meta_slot_offset_bytes(ctx.target_triple).to_string(); + let merge_label = ctx.block_label(merge_idx); // Reject every non-pointer / handle-band / noncanonical-index case before - // touching a managed header. The miss helper retains full ToPropertyKey, - // Proxy, string, descriptor, hole and prototype-chain semantics. + // touching a managed header. The slow exit retains full ToPropertyKey, + // typed-array, Proxy, string, descriptor, hole and prototype-chain + // semantics. let heap_floor = crate::target_layout::heap_addr_lower_bound_inclusive(ctx.target_triple).to_string(); let heap_ceiling = @@ -442,10 +170,7 @@ pub(super) fn lower_inline_dyn_typed_array_get( .cond_br(&object_entry_ok, &object_header_label, &object_miss_label); // One validated managed header feeds two tiers: a direct ordinary-Array - // load and the Array-subclass shape/family IC. The old miss path handled - // only the latter, so every unknown-receiver plain Array read immediately - // called the full polymorphic dispatcher despite having all guard inputs - // available here. + // load, the typed-array arm and the elements-backed Array-subclass probe. ctx.current_block = object_header_idx; let object_idx_i64 = ctx.block().fptosi(DOUBLE, idx_d, I64); let object_idx_back = ctx.block().sitofp(I64, &object_idx_i64, DOUBLE); @@ -463,157 +188,15 @@ pub(super) fn lower_inline_dyn_typed_array_get( ctx.block() .cond_br(&header_ok, &object_brand_label, &object_miss_label); - // An elements-backed Array-subclass instance (`ObjectMeta.elements`, - // perry-runtime `array/subclass_elements.rs`): its indexed elements live - // in a real Array hanging off the meta record, so the read is the plain - // Array read on that inner array — no shape IC, no family token. A miss - // of this probe (no meta, no store) is the shape-carried form and keeps - // the IC below; an out-of-bounds index or a hole goes to the complete - // dispatcher (prototype chain). - let elem_kind_idx = ctx.new_block("arrlike.elem.kind"); - let elem_meta_idx = ctx.new_block("arrlike.elem.meta"); - let elem_store_idx = ctx.new_block("arrlike.elem.store"); - let elem_bounds_idx = ctx.new_block("arrlike.elem.bounds"); - let elem_load_idx = ctx.new_block("arrlike.elem.load"); - let elem_value_idx = ctx.new_block("arrlike.elem.value"); - let elem_kind_label = ctx.block_label(elem_kind_idx); - let elem_meta_label = ctx.block_label(elem_meta_idx); - let elem_store_label = ctx.block_label(elem_store_idx); - let elem_bounds_label = ctx.block_label(elem_bounds_idx); - let elem_load_label = ctx.block_label(elem_load_idx); - let elem_value_label = ctx.block_label(elem_value_idx); - // A `JSON.parse` result is `GC_TYPE_LAZY_ARRAY`, not `GC_TYPE_ARRAY`, so - // every one of its indexed reads used to fall straight through to - // `arrlike.ic.miss` and re-classify the receiver three more times - // (`js_packed_arraylike_index_get` -> `js_array_get_f64` -> - // `json_tape::cached_read::lazy_get`). Once a scan or a random-access flip - // has installed the ordinary array, that whole chain resolves one word; - // serve it here instead, on exactly the proof `lazy_get` already uses. - // The blocks are declared here; they are reached from `arrlike.elem.kind` - // below, after the ordinary-Array and elements-subclass probes both miss. - let lazy_kind_idx = ctx.new_block("arrlike.lazy.kind"); - let lazy_call_idx = ctx.new_block("arrlike.lazy.call"); - let lazy_value_idx = ctx.new_block("arrlike.lazy.value"); - let lazy_kind_label = ctx.block_label(lazy_kind_idx); - let lazy_call_label = ctx.block_label(lazy_call_idx); - let lazy_value_label = ctx.block_label(lazy_value_idx); - + // `GC_TYPE_ARRAY` takes the direct guarded load. Everything else is offered + // to the typed-array arm, then to the elements-backed Array-subclass + // probe; `GC_TYPE_LAZY_ARRAY`, native Buffers and every exotic cell fail + // both brand tests and are classified by the slow exit. Both `tav.brand` + // and `arrlike.elem.kind` re-test the brand they need before they read a + // header word, so nothing else can reach those loads. ctx.current_block = object_brand_idx; ctx.block() - .cond_br(&is_array, &object_array_guard_label, &elem_kind_label); - - ctx.current_block = elem_kind_idx; - let elem_is_object = ctx.block().icmp_eq(I8, &gc_type, "2"); - ctx.block() - .cond_br(&elem_is_object, &elem_meta_label, &lazy_kind_label); - ctx.current_block = elem_meta_idx; - let elem_meta_addr = ctx.block().add(I64, &object_raw, &meta_offset); - let elem_meta_slot_ptr = ctx.block().inttoptr(I64, &elem_meta_addr); - let elem_meta_loaded = ctx.block().load( - if meta_ptr_size == 4 { I32 } else { I64 }, - &elem_meta_slot_ptr, - ); - let elem_meta_i64 = if meta_ptr_size == 4 { - ctx.block().zext(I32, &elem_meta_loaded, I64) - } else { - elem_meta_loaded - }; - let elem_has_meta = ctx.block().icmp_ne(I64, &elem_meta_i64, "0"); - ctx.block() - .cond_br(&elem_has_meta, &elem_store_label, &object_shape_label); - ctx.current_block = elem_store_idx; - let elem_meta_ptr = ctx.block().inttoptr(I64, &elem_meta_i64); - // `ObjectMeta.elements` is word 12 (offset 96; pinned by a const assert - // in perry-runtime `object/mod.rs`). - let elem_store_slot_ptr = ctx.block().gep(I64, &elem_meta_ptr, &[(I64, "12")]); - let elem_store_i64 = ctx.block().load(I64, &elem_store_slot_ptr); - let elem_has_store = ctx.block().icmp_ne(I64, &elem_store_i64, "0"); - ctx.block() - .cond_br(&elem_has_store, &elem_bounds_label, &object_shape_label); - ctx.current_block = elem_bounds_idx; - let elem_type_addr = ctx.block().sub(I64, &elem_store_i64, "8"); - let elem_type_ptr = ctx.block().inttoptr(I64, &elem_type_addr); - let elem_type = ctx.block().load(I8, &elem_type_ptr); - let elem_is_array = ctx.block().icmp_eq(I8, &elem_type, "1"); - let elem_flags_addr = ctx.block().sub(I64, &elem_store_i64, "7"); - let elem_flags_ptr = ctx.block().inttoptr(I64, &elem_flags_addr); - let elem_flags = ctx.block().load(I8, &elem_flags_ptr); - let elem_fwd = ctx.block().and(I8, &elem_flags, "128"); - let elem_not_fwd = ctx.block().icmp_eq(I8, &elem_fwd, "0"); - let elem_store_ptr = ctx.block().inttoptr(I64, &elem_store_i64); - let elem_length = ctx.block().load(I32, &elem_store_ptr); - let elem_length_i64 = ctx.block().zext(I32, &elem_length, I64); - let elem_in_bounds = ctx.block().icmp_ult(I64, &object_idx_i64, &elem_length_i64); - let elem_ok = ctx.block().and(I1, &elem_is_array, &elem_not_fwd); - let elem_ok = ctx.block().and(I1, &elem_ok, &elem_in_bounds); - ctx.block() - .cond_br(&elem_ok, &elem_load_label, &object_miss_label); - ctx.current_block = elem_load_idx; - let elem_bytes = ctx.block().shl(I64, &object_idx_i64, "3"); - let elem_elements_addr = ctx.block().array_elements_addr(&elem_store_i64); - let elem_addr = ctx.block().add(I64, &elem_elements_addr, &elem_bytes); - let elem_ptr = ctx.block().inttoptr(I64, &elem_addr); - let elem_raw = ctx.block().load(DOUBLE, &elem_ptr); - let elem_bits = ctx.block().bitcast_double_to_i64(&elem_raw); - let elem_is_hole = ctx - .block() - .icmp_eq(I64, &elem_bits, crate::nanbox::TAG_HOLE_I64); - ctx.block() - .cond_br(&elem_is_hole, &object_miss_label, &elem_value_label); - ctx.current_block = elem_value_idx; - let elem_value = if coerce_slow_to_number { - ctx.block() - .call(DOUBLE, "js_number_coerce", &[(DOUBLE, &elem_raw)]) - } else { - elem_raw - }; - let elem_end_label = ctx.block().label.clone(); - ctx.block().br(&merge_label); - kind_incoming.push((elem_value, elem_end_label)); - - // `GC_TYPE_LAZY_ARRAY` (perry-runtime `gc/types.rs`). This tier hangs off - // the Array-subclass probe's miss edge, after the ordinary-Array and - // elements-subclass probes have both declined the receiver. - ctx.current_block = lazy_kind_idx; - let lazy_is_lazy = ctx.block().icmp_eq(I8, &gc_type, "9"); - ctx.block() - .cond_br(&lazy_is_lazy, &lazy_call_label, &object_miss_label); - - // One call into `json_tape::cached_read::js_lazy_array_index_probe`, which - // is `lazy_get`'s two non-allocating branches and nothing else. That skips - // the dispatcher chain (`js_packed_arraylike_index_get` -> - // `js_array_get_f64` -> `lazy_get`) without inlining the whole proof at - // every indexed read site: the inline form grew this function ~10% and cost - // rows it never executes on up to 5% to code layout alone. - // - // `TAG_HOLE` means "this read needs the rooted accessor" -- unambiguous, - // because a hole is never a value a read yields, and holes already route to - // the miss helper. Cold elements, descriptors, out-of-bounds, growth stubs - // and a stale length mirror all come back as that. The probe cannot - // allocate, run user code or collect, so no extra rooting is required here. - ctx.current_block = lazy_call_idx; - let lazy_raw_i64 = object_raw.clone(); - let lazy_probe = ctx.block().call( - DOUBLE, - "js_lazy_array_index_probe", - &[(I64, &lazy_raw_i64), (I64, &object_idx_i64)], - ); - let lazy_probe_bits = ctx.block().bitcast_double_to_i64(&lazy_probe); - let lazy_declined = ctx - .block() - .icmp_eq(I64, &lazy_probe_bits, crate::nanbox::TAG_HOLE_I64); - ctx.block() - .cond_br(&lazy_declined, &object_miss_label, &lazy_value_label); - ctx.current_block = lazy_value_idx; - let lazy_value = if coerce_slow_to_number { - ctx.block() - .call(DOUBLE, "js_number_coerce", &[(DOUBLE, &lazy_probe)]) - } else { - lazy_probe - }; - let lazy_end_label = ctx.block().label.clone(); - ctx.block().br(&merge_label); - kind_incoming.push((lazy_value, lazy_end_label)); + .cond_br(&is_array, &object_array_guard_label, &ta_brand_label); // Ordinary Array: the receiver tag and forwarding state were checked in // the predecessor. Reject descriptors or any process-wide prototype @@ -682,256 +265,301 @@ pub(super) fn lower_inline_dyn_typed_array_get( }; let array_end_label = ctx.block().label.clone(); ctx.block().br(&merge_label); - kind_incoming.push((array_value, array_end_label)); - // The runtime publishes either an exact `(class, ShapeId)` identity or a - // high-bit Array-subclass dense-tail family token. The latter lives in - // ObjectMeta and survives only the exact learned numeric push/pop edges; - // every generic structural or descriptor mutation retires it before the - // mutation is observable. This lets lifecycle-heavy subclasses traverse - // a thousand historical tail shapes without thrashing a monomorphic IC. - ctx.current_block = object_shape_idx; - let is_object = ctx.block().icmp_eq(I8, &gc_type, "2"); - let object_ptr = ctx.block().inttoptr(I64, &object_raw); - let class_id = ctx.block().load(I32, &object_ptr); - let shape_addr = ctx.block().add(I64, &object_raw, "4"); - let shape_ptr = ctx.block().inttoptr(I64, &shape_addr); - let shape_id = ctx.block().load(I32, &shape_ptr); - let class64 = ctx.block().zext(I32, &class_id, I64); - let shape64 = ctx.block().zext(I32, &shape_id, I64); - let class_high = ctx.block().shl(I64, &class64, "32"); - let live_key = ctx.block().or(I64, &class_high, &shape64); - let cached_key_ptr = ctx.block().gep(I64, &key_cache, &[(I64, "0")]); - let cached_key = ctx.block().load(I64, &cached_key_ptr); - let key_nonzero = ctx.block().icmp_ne(I64, &cached_key, "0"); - let object_ok = ctx.block().and(I1, &is_object, &key_nonzero); - ctx.block() - .cond_br(&object_ok, &object_identity_label, &object_miss_label); - - ctx.current_block = object_identity_idx; - let family_token_bit = crate::nanbox::i64_literal(1u64 << 63); - let family_bits = ctx.block().and(I64, &cached_key, &family_token_bit); - let is_family = ctx.block().icmp_ne(I64, &family_bits, "0"); + // ---- typed-array arm: ONE guarded read, four element widths ---- + // + // #5525's ladder was eight per-kind load blocks behind a seven-block kind + // dispatch, and measuring it out of line (this exit + a Rust fast path) + // cost a dynamically-typed `Float64Array` sum **+122.9% walltime and + // +206.3% instructions** — a call and its statepoint per element. It is + // back inline, collapsed onto the ELEMENT WIDTH the header already stores + // (`TypedArrayHeader::elem_size`, byte 9): four load blocks instead of + // fifteen, each of which resolves its own signedness/float form with + // `select`s rather than more branches. + // + // (The literal "cache the last-seen kind, inline one guarded load" form is + // not expressible in static codegen: the LOAD TYPE is what varies per + // kind, so a runtime-cached kind still cannot pick it. Width is the + // coarsest split that keeps every load in bounds — reading 8 bytes from a + // `Uint8Array`'s last element is not ours to take.) + // + // `PERRY_TA_VIEW_GUARD == 0` is the licence to compute the data pointer as + // `header + 16` without consulting the view registries; a raised guard, + // like a BigInt lane or an out-of-range index, leaves through + // `arrlike.ic.miss` — the same exit the old `tav.get.slow` edge reached. + // + // This arm sits AHEAD of the object probe, which was measured both ways: + // putting it on the probe's decline edge instead saves an object receiver + // one brand test it always fails, and that was worth nothing on any + // workload (the row it was meant to fix, `bench_histogram_numarray`, read + // +0.15% retired instructions either way and +0.00% once rebuilt), while + // costing every typed-array element read an `icmp`+branch — `dyn_ta_f64` + // went from -9.98% to -6.65%. + // + // #10118: the brand test DECIDES ON THE TAG ALONE. Everything else in the + // guard set — the view guard, the element kind and its range test, the + // bounds check — is meaningful only once the tag says typed array, so it + // sits behind the tag in `tav.kind_guard`. An Array-subclass instance or a + // `JSON.parse` array reaching this arm pays one `icmp` and leaves, instead + // of three loads and three ANDs to reach a branch it was always going to + // take. The typed-array path reaches the same guard set by the same + // AND-reduction and is unchanged. + ctx.current_block = ta_brand_idx; + let is_typed_array = ctx.block().icmp_eq(I8, &gc_type, "11"); // GC_TYPE_TYPED_ARRAY ctx.block() - .cond_br(&is_family, &object_family_meta_label, &object_exact_label); - - ctx.current_block = object_exact_idx; - let key_matches = ctx.block().icmp_eq(I64, &live_key, &cached_key); + .cond_br(&is_typed_array, &ta_kind_guard_label, &elem_kind_label); + + // Past the tag, a receiver this guard rejects (a raised view guard, a + // BigInt/Float16 lane, an out-of-range index) cannot be an ordinary object + // either, so it leaves straight through the exit rather than re-testing + // `GC_TYPE_OBJECT` it is guaranteed to fail. + ctx.current_block = ta_kind_guard_idx; + let (ta_kind, ta_ok) = { + let blk = ctx.block(); + let view_guard = blk.load(I64, "@PERRY_TA_VIEW_GUARD"); + let inline_storage = blk.icmp_eq(I64, &view_guard, "0"); + let kind_addr = blk.add(I64, &object_raw, "8"); + let kind_ptr = blk.inttoptr(I64, &kind_addr); + let kind_i8 = blk.load(I8, &kind_ptr); + let kind = blk.zext(I8, &kind_i8, I64); + // kinds 0..=8 (Int8 .. Uint8Clamped); rejects BigInt 9/10 and Float16 + // 11, whose lanes are not plain Numbers. + let kind_ok = blk.icmp_ule(I64, &kind, "8"); + // `length` is `TypedArrayHeader` word 0. + let len_ptr = blk.inttoptr(I64, &object_raw); + let len = blk.load(I32, &len_ptr); + let len_i64 = blk.zext(I32, &len, I64); + let in_bounds = blk.icmp_ult(I64, &object_idx_i64, &len_i64); + let ok = blk.and(I1, &inline_storage, &kind_ok); + (kind, blk.and(I1, &ok, &in_bounds)) + }; ctx.block() - .cond_br(&key_matches, &object_bounds_label, &object_miss_label); - - ctx.current_block = object_family_meta_idx; - let family_meta_addr = ctx.block().add(I64, &object_raw, &meta_offset); - let family_meta_slot_ptr = ctx.block().inttoptr(I64, &family_meta_addr); - let family_meta_loaded = ctx.block().load( - if meta_ptr_size == 4 { I32 } else { I64 }, - &family_meta_slot_ptr, - ); - let family_meta_i64 = if meta_ptr_size == 4 { - ctx.block().zext(I32, &family_meta_loaded, I64) - } else { - family_meta_loaded + .cond_br(&ta_ok, &ta_width_label, &object_miss_label); + + // `elem_size` (byte 9) is written from `kind` by `typed_array_alloc`, so + // the brand guard's `kind <= KIND_UINT8_CLAMPED` already bounds it to + // {1,2,4,8} — the same pairing the runtime's own `load_at` trusts for its + // offset and its load type. `tav.w1` is the final else, not a fourth test. + ctx.current_block = ta_width_idx; + let (ta_elem_size, ta_addr) = { + let blk = ctx.block(); + let size_addr = blk.add(I64, &object_raw, "9"); + let size_ptr = blk.inttoptr(I64, &size_addr); + let size_i8 = blk.load(I8, &size_ptr); + let elem_size = blk.zext(I8, &size_i8, I64); + let offset = blk.mul(I64, &object_idx_i64, &elem_size); + // `data = header + size_of::()`, proven by the + // cleared view guard above. + let data_base = blk.add(I64, &object_raw, "16"); + (elem_size, blk.add(I64, &data_base, &offset)) }; - let family_has_meta = ctx.block().icmp_ne(I64, &family_meta_i64, "0"); - ctx.block().cond_br( - &family_has_meta, - &object_family_token_label, - &object_miss_label, - ); + let is_width8 = ctx.block().icmp_eq(I64, &ta_elem_size, "8"); + ctx.block() + .cond_br(&is_width8, &ta_w8_label, &ta_width4_label); - ctx.current_block = object_family_token_idx; - let family_meta_ptr = ctx.block().inttoptr(I64, &family_meta_i64); - // repr(C) ObjectMeta word 6 is the move-stable Array-subclass named-prefix - // token. The dense-tail miss helper only publishes it after proving that - // the canonical numeric suffix immediately follows that prefix. - let family_token_ptr = ctx.block().gep(I64, &family_meta_ptr, &[(I64, "6")]); - let live_family_token = ctx.block().load(I64, &family_token_ptr); - let family_matches = ctx.block().icmp_eq(I64, &live_family_token, &cached_key); + ctx.current_block = ta_width4_idx; + let is_width4 = ctx.block().icmp_eq(I64, &ta_elem_size, "4"); ctx.block() - .cond_br(&family_matches, &object_bounds_label, &object_miss_label); + .cond_br(&is_width4, &ta_w4_label, &ta_width2_label); - // The exact shape or family token proves the cached slots. `length` may - // itself be in ObjectMeta::spill (wolf-ecs Archetype has four declared - // fields before Array-subclass init installs it), so split its load just - // like the element load below. Check the live value against the admitted - // dense-prefix high-water mark on every hit; a generic length-only grow - // therefore cannot expose holes through this tier. - ctx.current_block = object_bounds_idx; - let length_slot_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "1")]); - let length_slot = ctx.block().load(I64, &length_slot_ptr); - let element_base_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "2")]); - let element_base = ctx.block().load(I64, &element_base_ptr); - let dense_prefix_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "3")]); - let dense_prefix = ctx.block().load(I64, &dense_prefix_ptr); - let inline_bound_ptr = ctx.block().gep(I64, &cache_ref, &[(I64, "4")]); - let inline_bound = ctx.block().load(I64, &inline_bound_ptr); - let object_header_size = - crate::target_layout::object_header_size_bytes(ctx.target_triple).to_string(); - let length_is_inline = ctx.block().icmp_ult(I64, &length_slot, &inline_bound); - ctx.block().cond_br( - &length_is_inline, - &object_length_inline_label, - &object_length_spill_meta_label, - ); + ctx.current_block = ta_width2_idx; + let is_width2 = ctx.block().icmp_eq(I64, &ta_elem_size, "2"); + ctx.block().cond_br(&is_width2, &ta_w2_label, &ta_w1_label); - ctx.current_block = object_length_inline_idx; - let length_bytes = ctx.block().shl(I64, &length_slot, "3"); - let length_offset = ctx.block().add(I64, &length_bytes, &object_header_size); - let length_addr = ctx.block().add(I64, &object_raw, &length_offset); - let length_ptr = ctx.block().inttoptr(I64, &length_addr); - let inline_length = ctx.block().load(DOUBLE, &length_ptr); - let inline_length_end = ctx.block().label.clone(); - ctx.block().br(&object_range_label); + // Width 8: `Float64Array` is the only non-BigInt kind of this width, so + // the stored lane IS the value. + ctx.current_block = ta_w8_idx; + let ta_w8_value = { + let blk = ctx.block(); + let ptr = blk.inttoptr(I64, &ta_addr); + blk.load(DOUBLE, &ptr) + }; + let ta_w8_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); - ctx.current_block = object_length_spill_meta_idx; - let length_meta_addr = ctx.block().add(I64, &object_raw, &meta_offset); - let length_meta_slot_ptr = ctx.block().inttoptr(I64, &length_meta_addr); - let length_meta_loaded = ctx.block().load( - if meta_ptr_size == 4 { I32 } else { I64 }, - &length_meta_slot_ptr, - ); - let length_meta_i64 = if meta_ptr_size == 4 { - ctx.block().zext(I32, &length_meta_loaded, I64) - } else { - length_meta_loaded + // Width 4: `Int32Array` (4), `Uint32Array` (5), `Float32Array` (6). Both + // integer forms come from ONE load: the zero-extended lane is the unsigned + // value, `shl`+`ashr` is the signed one, and `sitofp i64` is exact for + // both because a u32 fits a signed i64. The float form reinterprets the + // same lane; neither computation can trap, so a `select` replaces the + // branch. + ctx.current_block = ta_w4_idx; + let ta_w4_value = { + let blk = ctx.block(); + let ptr = blk.inttoptr(I64, &ta_addr); + let lane = blk.load(I32, &ptr); + let unsigned = blk.zext(I32, &lane, I64); + let widened = blk.shl(I64, &unsigned, "32"); + let signed = blk.ashr(I64, &widened, "32"); + let is_signed = blk.icmp_eq(I64, &ta_kind, "4"); + let integral = blk.select(I1, &is_signed, I64, &signed, &unsigned); + let as_number = blk.sitofp(I64, &integral, DOUBLE); + let as_f32 = blk.bitcast_i32_to_float(&lane); + let widened_f32 = blk.fpext(F32, &as_f32, DOUBLE); + let is_f32 = blk.icmp_eq(I64, &ta_kind, "6"); + blk.select(I1, &is_f32, DOUBLE, &widened_f32, &as_number) }; - let length_has_meta = ctx.block().icmp_ne(I64, &length_meta_i64, "0"); - ctx.block().cond_br( - &length_has_meta, - &object_length_spill_ptr_label, - &object_miss_label, - ); + let ta_w4_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); - ctx.current_block = object_length_spill_ptr_idx; - let length_meta_ptr = ctx.block().inttoptr(I64, &length_meta_i64); - let length_spill_slot_ptr = ctx.block().gep(I64, &length_meta_ptr, &[(I64, "4")]); - let length_spill_i64 = ctx.block().load(I64, &length_spill_slot_ptr); - let length_has_spill = ctx.block().icmp_ne(I64, &length_spill_i64, "0"); - let safe_length_spill_i64 = ctx.block().select( - I1, - &length_has_spill, - I64, - &length_spill_i64, - &length_meta_i64, - ); - let length_spill_ptr = ctx.block().inttoptr(I64, &safe_length_spill_i64); - let length_spill_len = ctx.block().load(I32, &length_spill_ptr); - let length_spill_len_i64 = ctx.block().zext(I32, &length_spill_len, I64); - let length_in_spill = ctx - .block() - .icmp_ult(I64, &length_slot, &length_spill_len_i64); - let length_spill_ok = ctx.block().and(I1, &length_has_spill, &length_in_spill); - ctx.block().cond_br( - &length_spill_ok, - &object_length_spill_load_label, - &object_miss_label, - ); + // Width 2: `Int16Array` (2) and `Uint16Array` (3). + ctx.current_block = ta_w2_idx; + let ta_w2_value = { + let blk = ctx.block(); + let ptr = blk.inttoptr(I64, &ta_addr); + let lane = blk.load(I16, &ptr); + let unsigned = blk.zext(I16, &lane, I64); + let widened = blk.shl(I64, &unsigned, "48"); + let signed = blk.ashr(I64, &widened, "48"); + let is_signed = blk.icmp_eq(I64, &ta_kind, "2"); + let integral = blk.select(I1, &is_signed, I64, &signed, &unsigned); + blk.sitofp(I64, &integral, DOUBLE) + }; + let ta_w2_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); - ctx.current_block = object_length_spill_load_idx; - let length_element_word = ctx.block().add(I64, &length_slot, "1"); - let length_element_ptr = - ctx.block() - .gep_inbounds(I64, &length_spill_ptr, &[(I64, &length_element_word)]); - let spilled_length = ctx.block().load(DOUBLE, &length_element_ptr); - let spilled_length_end = ctx.block().label.clone(); - ctx.block().br(&object_range_label); + // Width 1: `Int8Array` (0), `Uint8Array` (1) and `Uint8ClampedArray` (8) — + // the clamped kind stores plain bytes, so it shares the unsigned form. + ctx.current_block = ta_w1_idx; + let ta_w1_value = { + let blk = ctx.block(); + let ptr = blk.inttoptr(I64, &ta_addr); + let lane = blk.load(I8, &ptr); + let unsigned = blk.zext(I8, &lane, I64); + let widened = blk.shl(I64, &unsigned, "56"); + let signed = blk.ashr(I64, &widened, "56"); + let is_signed = blk.icmp_eq(I64, &ta_kind, "0"); + let integral = blk.select(I1, &is_signed, I64, &signed, &unsigned); + blk.sitofp(I64, &integral, DOUBLE) + }; + let ta_w1_end = ctx.block().label.clone(); + ctx.block().br(&merge_label); - ctx.current_block = object_range_idx; - let live_length = ctx.block().phi( - DOUBLE, - &[ - (&inline_length, &inline_length_end), - (&spilled_length, &spilled_length_end), - ], - ); - let below_length = ctx.block().fcmp("olt", idx_d, &live_length); - let below_prefix = ctx.block().icmp_ult(I64, &object_idx_i64, &dense_prefix); - let in_dense_range = ctx.block().and(I1, &below_length, &below_prefix); - let object_slot = ctx.block().add(I64, &element_base, &object_idx_i64); - let slot_is_inline = ctx.block().icmp_ult(I64, &object_slot, &inline_bound); - let inline_ok = ctx.block().and(I1, &in_dense_range, &slot_is_inline); - let slot_is_spilled = ctx.block().xor(I1, &slot_is_inline, "true"); - let range_but_spilled = ctx.block().and(I1, &in_dense_range, &slot_is_spilled); - let spill_or_miss_idx = ctx.new_block("arrlike.ic.spill_or_miss"); - let spill_or_miss_label = ctx.block_label(spill_or_miss_idx); - ctx.block() - .cond_br(&inline_ok, &object_inline_label, &spill_or_miss_label); - ctx.current_block = spill_or_miss_idx; + // ---- elements-backed Array-subclass probe ---- + // + // `class X extends Array` instances own a real `GC_TYPE_ARRAY` in + // `ObjectMeta.elements` (`perry-runtime/src/array/subclass_elements.rs`), + // so the read is the plain Array read on that inner array: meta word -> + // `elements` (word 12) -> bounds -> slot. A miss (no meta, no store) and + // every hole or out-of-range index go to the exit, which keeps the + // complete prototype-chain semantics. + // + // This tier is INLINE because the elements store is the DEFAULT + // representation. Moving it out of line cost an `Array`-subclass read loop + // **+71.2% instructions and +60.0% walltime** (`dyn_arraylike_object`). + // + // What is NOT here any more is the shape-carried IC tower — + // `arrlike.ic.{shape,identity,exact,family_meta,family_token,bounds, + // length_inline,length_spill_meta,length_spill_ptr,length_spill_load, + // range,inline,spill,spill_ptr,spill_load,spill_or_miss}`, fifteen blocks + // at every site. Its hit requires a primed layout cache, and the only + // writer of that cache (`js_packed_arraylike_index_get` -> + // `dense_layout_for_validated_object` -> `build_dense_layout`) is reached + // only when `elements_of(obj)` is NULL and the receiver passes + // `is_array_subclass_class_id`. With the elements store on — the shipped + // default, whose kill switch `PERRY_ARRAY_SUBCLASS_ELEMENTS=0` its own + // doc calls "a bisecting kill switch, not a supported mode" — that cache + // is never primed, so `cached_key` is always zero and the whole tower + // exits on its first test. It was fifteen unreachable blocks per site, + // inlined at 10,778 sites in `prettier/plugins/flow.mjs` alone. Under the + // kill switch those receivers now take one call per read instead. + ctx.current_block = elem_kind_idx; + let elem_is_object = ctx.block().icmp_eq(I8, &gc_type, "2"); ctx.block() - .cond_br(&range_but_spilled, &object_spill_label, &object_miss_label); + .cond_br(&elem_is_object, &elem_meta_label, &object_miss_label); - ctx.current_block = object_inline_idx; - let inline_bytes = ctx.block().shl(I64, &object_slot, "3"); - let inline_offset = ctx.block().add(I64, &inline_bytes, &object_header_size); - let inline_addr = ctx.block().add(I64, &object_raw, &inline_offset); - let inline_ptr = ctx.block().inttoptr(I64, &inline_addr); - let inline_raw = ctx.block().load(DOUBLE, &inline_ptr); - let inline_value = if coerce_slow_to_number { - ctx.block() - .call(DOUBLE, "js_number_coerce", &[(DOUBLE, &inline_raw)]) + ctx.current_block = elem_meta_idx; + let meta_ptr_size: u64 = if crate::target_layout::target_is_ilp32(ctx.target_triple) { + 4 } else { - inline_raw + 8 }; - let inline_end_label = ctx.block().label.clone(); - ctx.block().br(&merge_label); - - // Wide subclass instances store absolute field slots in the object-owned - // spill Array. Reload both moving pointers from the live receiver; the IC - // itself contains only scalar offsets. - ctx.current_block = object_spill_idx; - let meta_addr = ctx.block().add(I64, &object_raw, &meta_offset); - let meta_slot_ptr = ctx.block().inttoptr(I64, &meta_addr); - let meta_loaded = ctx - .block() - .load(if meta_ptr_size == 4 { I32 } else { I64 }, &meta_slot_ptr); - let meta_i64 = if meta_ptr_size == 4 { - ctx.block().zext(I32, &meta_loaded, I64) + let meta_offset = + crate::target_layout::object_meta_slot_offset_bytes(ctx.target_triple).to_string(); + let elem_meta_addr = ctx.block().add(I64, &object_raw, &meta_offset); + let elem_meta_slot_ptr = ctx.block().inttoptr(I64, &elem_meta_addr); + let elem_meta_loaded = ctx.block().load( + if meta_ptr_size == 4 { I32 } else { I64 }, + &elem_meta_slot_ptr, + ); + let elem_meta_i64 = if meta_ptr_size == 4 { + ctx.block().zext(I32, &elem_meta_loaded, I64) } else { - meta_loaded + elem_meta_loaded }; - let has_meta = ctx.block().icmp_ne(I64, &meta_i64, "0"); + let elem_has_meta = ctx.block().icmp_ne(I64, &elem_meta_i64, "0"); + ctx.block() + .cond_br(&elem_has_meta, &elem_store_label, &object_miss_label); + + ctx.current_block = elem_store_idx; + let elem_meta_ptr = ctx.block().inttoptr(I64, &elem_meta_i64); + // `ObjectMeta.elements` is word 12 (offset 96; pinned by a const assert + // in perry-runtime `object/mod.rs`). + let elem_store_slot_ptr = ctx.block().gep(I64, &elem_meta_ptr, &[(I64, "12")]); + let elem_store_i64 = ctx.block().load(I64, &elem_store_slot_ptr); + let elem_has_store = ctx.block().icmp_ne(I64, &elem_store_i64, "0"); ctx.block() - .cond_br(&has_meta, &object_spill_ptr_label, &object_miss_label); + .cond_br(&elem_has_store, &elem_bounds_label, &object_miss_label); - ctx.current_block = object_spill_ptr_idx; - let meta_ptr = ctx.block().inttoptr(I64, &meta_i64); - let spill_slot_ptr = ctx.block().gep(I64, &meta_ptr, &[(I64, "4")]); - let spill_i64 = ctx.block().load(I64, &spill_slot_ptr); - let has_spill = ctx.block().icmp_ne(I64, &spill_i64, "0"); - // Keep the hot path to one bounds branch without speculatively loading - // through a null spill pointer: ObjectMeta is live here and is a safe - // address for the ignored length load when `spill_i64 == 0`. - let safe_spill_i64 = ctx + ctx.current_block = elem_bounds_idx; + let elem_type_addr = ctx.block().sub(I64, &elem_store_i64, "8"); + let elem_type_ptr = ctx.block().inttoptr(I64, &elem_type_addr); + let elem_type = ctx.block().load(I8, &elem_type_ptr); + let elem_is_array = ctx.block().icmp_eq(I8, &elem_type, "1"); + let elem_flags_addr = ctx.block().sub(I64, &elem_store_i64, "7"); + let elem_flags_ptr = ctx.block().inttoptr(I64, &elem_flags_addr); + let elem_flags = ctx.block().load(I8, &elem_flags_ptr); + let elem_fwd = ctx.block().and(I8, &elem_flags, "128"); + let elem_not_fwd = ctx.block().icmp_eq(I8, &elem_fwd, "0"); + let elem_store_ptr = ctx.block().inttoptr(I64, &elem_store_i64); + let elem_length = ctx.block().load(I32, &elem_store_ptr); + let elem_length_i64 = ctx.block().zext(I32, &elem_length, I64); + let elem_in_bounds = ctx.block().icmp_ult(I64, &object_idx_i64, &elem_length_i64); + let elem_ok = ctx.block().and(I1, &elem_is_array, &elem_not_fwd); + let elem_ok = ctx.block().and(I1, &elem_ok, &elem_in_bounds); + ctx.block() + .cond_br(&elem_ok, &elem_load_label, &object_miss_label); + + ctx.current_block = elem_load_idx; + let elem_bytes = ctx.block().shl(I64, &object_idx_i64, "3"); + let elem_elements_addr = ctx.block().array_elements_addr(&elem_store_i64); + let elem_addr = ctx.block().add(I64, &elem_elements_addr, &elem_bytes); + let elem_ptr = ctx.block().inttoptr(I64, &elem_addr); + let elem_raw = ctx.block().load(DOUBLE, &elem_ptr); + let elem_bits = ctx.block().bitcast_double_to_i64(&elem_raw); + let elem_is_hole = ctx .block() - .select(I1, &has_spill, I64, &spill_i64, &meta_i64); - let spill_ptr = ctx.block().inttoptr(I64, &safe_spill_i64); - let spill_len = ctx.block().load(I32, &spill_ptr); - let spill_len_i64 = ctx.block().zext(I32, &spill_len, I64); - let spill_in_bounds = ctx.block().icmp_ult(I64, &object_slot, &spill_len_i64); - let spill_ok = ctx.block().and(I1, &has_spill, &spill_in_bounds); + .icmp_eq(I64, &elem_bits, crate::nanbox::TAG_HOLE_I64); ctx.block() - .cond_br(&spill_ok, &object_spill_load_label, &object_miss_label); + .cond_br(&elem_is_hole, &object_miss_label, &elem_value_label); - ctx.current_block = object_spill_load_idx; - let spill_element_word = ctx.block().add(I64, &object_slot, "1"); - let spill_element_ptr = - ctx.block() - .gep_inbounds(I64, &spill_ptr, &[(I64, &spill_element_word)]); - let spill_raw = ctx.block().load(DOUBLE, &spill_element_ptr); - let spill_value = if coerce_slow_to_number { + ctx.current_block = elem_value_idx; + let elem_value = if coerce_slow_to_number { ctx.block() - .call(DOUBLE, "js_number_coerce", &[(DOUBLE, &spill_raw)]) + .call(DOUBLE, "js_number_coerce", &[(DOUBLE, &elem_raw)]) } else { - spill_raw + elem_raw }; - let spill_end_label = ctx.block().label.clone(); + let elem_end_label = ctx.block().label.clone(); ctx.block().br(&merge_label); + // The site's ONE out-of-line edge, and it is the SAME call the old + // `arrlike.ic.miss` block made: every arm this change stopped inlining was + // an acceleration of a decision `js_packed_arraylike_index_get` already + // makes, and it is handed this site's own cache slot, so neither the + // answer nor the primed words moved. + // + // Deliberately NOT a new four-argument entry point with the cold + // `ToNumber` folded into a flag: the extra argument and its test cost + // +0.43% retired instructions on `object_deep_clone` and +0.18% on + // `json_parse_1mb`, paid by every receiver that reaches this exit, to + // spare a `js_number_coerce` from an arm that no TypeScript fixture can + // reach (see `lower_unknown_local_index_get_for_number_context`). ctx.current_block = object_miss_idx; let slow_raw = ctx.block().call( DOUBLE, "js_packed_arraylike_index_get", - &[(DOUBLE, obj_box), (DOUBLE, idx_d), (PTR, &ic_slot.slot_ref)], + &[(DOUBLE, obj_box), (DOUBLE, idx_d), (PTR, &slot_ref)], ); // In a number context, coerce the (possibly boxed) slow result here so the // merge phi is uniformly a Number and the arithmetic caller skips its own @@ -947,49 +575,18 @@ pub(super) fn lower_inline_dyn_typed_array_get( let slow_end_label = ctx.block().label.clone(); ctx.block().br(&merge_label); - kind_incoming.push((inline_value, inline_end_label)); - kind_incoming.push((spill_value, spill_end_label)); - - // ---- final merge: one phi over every per-kind fast end + the slow end ---- + // ---- final merge: the two inline hits and the single exit ---- ctx.current_block = merge_idx; - let mut incoming_refs: Vec<(&str, &str)> = kind_incoming - .iter() - .map(|(v, l)| (v.as_str(), l.as_str())) - .collect(); - incoming_refs.push((slow_val.as_str(), slow_end_label.as_str())); - ctx.block().phi(DOUBLE, &incoming_refs) -} - -/// Emit one per-kind small-integer (1/2-byte) typed-array element load block for -/// [`lower_inline_dyn_typed_array_get`]: switches to `blk_idx`, computes the -/// element address (`data_base + (idx << shift)`), loads `elem_ty`, sign-/zero- -/// extends to i32, converts to f64, and branches to `merge_label`. Returns the -/// `(value, end_label)` pair for the merge phi. -#[allow(clippy::too_many_arguments)] -fn emit_inline_ta_int_load( - ctx: &mut FnCtx<'_>, - blk_idx: usize, - idx_i64: &str, - data_base: &str, - merge_label: &str, - shift: &str, - elem_ty: crate::types::LlvmType, - signed: bool, -) -> (String, String) { - ctx.current_block = blk_idx; - let blk = ctx.block(); - let off = blk.shl(I64, idx_i64, shift); - let addr = blk.add(I64, data_base, &off); - let ptr = blk.inttoptr(I64, &addr); - let raw_elem = blk.load(elem_ty, &ptr); - let val = if signed { - let i32v = blk.sext(elem_ty, &raw_elem, I32); - blk.sitofp(I32, &i32v, DOUBLE) - } else { - let i32v = blk.zext(elem_ty, &raw_elem, I32); - blk.uitofp(I32, &i32v, DOUBLE) - }; - let end_label = blk.label.clone(); - blk.br(merge_label); - (val, end_label) + ctx.block().phi( + DOUBLE, + &[ + (ta_w8_value.as_str(), ta_w8_end.as_str()), + (ta_w4_value.as_str(), ta_w4_end.as_str()), + (ta_w2_value.as_str(), ta_w2_end.as_str()), + (ta_w1_value.as_str(), ta_w1_end.as_str()), + (array_value.as_str(), array_end_label.as_str()), + (elem_value.as_str(), elem_end_label.as_str()), + (slow_val.as_str(), slow_end_label.as_str()), + ], + ) } diff --git a/crates/perry-codegen/src/expr/index_get_claim_tests.rs b/crates/perry-codegen/src/expr/index_get_claim_tests.rs index c5da000d3f..c3dc09104e 100644 --- a/crates/perry-codegen/src/expr/index_get_claim_tests.rs +++ b/crates/perry-codegen/src/expr/index_get_claim_tests.rs @@ -126,8 +126,39 @@ fn numeric_layout_oob_array_read_returns_undefined_inline() { ); } +/// The ordered block labels of ONE dynamic element-read site, with the +/// per-site numeric suffix stripped. +fn dynamic_index_site_blocks(ir: &str) -> Vec { + ir.lines() + .filter(|line| !line.starts_with(char::is_whitespace)) + .filter_map(|line| line.trim_end().strip_suffix(':')) + .filter(|label| label.starts_with("arrlike.") || label.starts_with("tav.")) + .map(|label| { + label + .rsplit_once('.') + .filter(|(_, suffix)| suffix.chars().all(|c| c.is_ascii_digit())) + .map_or(label.to_string(), |(head, _)| head.to_string()) + }) + .collect() +} + +/// #T2 ("inline hit, one exit"): the emitted `obj[i]` for an erased receiver +/// keeps exactly two inline hits — the packed ordinary-Array arm and the +/// object-backed MRU cache hit — and routes everything else through ONE +/// runtime call. +/// +/// This replaces `unknown_numeric_read_guards_dense_subclass_families_and_ +/// spilled_length`, which pinned the tower those arms used to be inlined +/// into (eight typed-array kind arms behind a seven-block kind dispatch, the +/// dense-tail family-token tier, the spilled-`length` and spilled-element +/// tiers, the elements-backed subclass probe and the lazy-JSON-array probe — +/// ~50 blocks and ~316 pre-RS4GC instructions per site, 55% of all IR on +/// `prettier/plugins/flow.mjs`). Each of those was an acceleration of a +/// decision `js_packed_arraylike_index_get` already makes, and the exit still +/// calls it with this site's own cache slot, so neither the answer nor the +/// primed cache words moved. #[test] -fn unknown_numeric_read_guards_dense_subclass_families_and_spilled_length() { +fn unknown_numeric_read_is_one_inline_hit_and_one_out_of_line_exit() { let ir = ir_for( "unknown_dense_subclass_read", vec![ @@ -159,30 +190,213 @@ fn unknown_numeric_read_guards_dense_subclass_families_and_spilled_length() { }, ], ); - assert!( - ir.contains("arrlike.ic.family_token"), - "the generated IC must compare the move-stable dense-tail family token:\n{ir}" - ); - assert!( - ir.contains("arrlike.ic.array_guard") && ir.contains("arrlike.ic.array_load"), - "an ordinary Array behind the erased receiver must retain a direct guarded load:\n{ir}" - ); - assert!( - ir.contains("arrlike.ic.length_spill_load"), - "an Array-subclass whose length slot spilled must retain an inline IC tier:\n{ir}" - ); - assert!( - ir.contains("arrlike.ic.range") && ir.contains("arrlike.ic.miss"), - "the live length and cached dense-prefix bound must retain a semantic side exit:\n{ir}" - ); - assert!( - ir.contains("arrlike.lazy.kind") && ir.contains("js_lazy_array_index_probe"), - "a lazy JSON array must reach its probe from the cache, not the dispatcher:\n{ir}" + // The complete emitted shape of one site, in order. A change here is a + // change to the per-site code-size contract and must be measured, not + // waved through. + assert_eq!( + dynamic_index_site_blocks(&ir), + vec![ + "arrlike.ic.header", + "arrlike.ic.brand", + "arrlike.ic.array_guard", + "arrlike.ic.array_load", + "tav.brand", + "tav.kind_guard", + "tav.width", + "tav.width4", + "tav.width2", + "tav.w8", + "tav.w4", + "tav.w2", + "tav.w1", + "arrlike.elem.kind", + "arrlike.elem.meta", + "arrlike.elem.store", + "arrlike.elem.bounds", + "arrlike.elem.load", + "arrlike.elem.value", + "arrlike.ic.miss", + "arrlike.ic.merge", + ], + "the dynamic element read must emit exactly the inline hit plus one exit:\n{ir}" + ); + // Exactly one runtime call for the whole site, and it is the exit. + assert_eq!( + ir.matches("call double @js_packed_arraylike_index_get(") + .count(), + 1, + "the site must have exactly one out-of-line edge:\n{ir}" + ); + for absent in [ + // the per-kind typed-array ladder, collapsed onto element width + "tav.get.brand", + "tav.k.i8", + "tav.k.f64", + "tav.kd1", + // the whole shape-carried Array-subclass IC tower, which cannot hit + // while the elements store is the default representation + "arrlike.ic.shape", + "arrlike.ic.identity", + "arrlike.ic.exact", + "arrlike.ic.family_meta", + "arrlike.ic.family_token", + "arrlike.ic.bounds", + "arrlike.ic.length_inline", + "arrlike.ic.length_spill_meta", + "arrlike.ic.length_spill_ptr", + "arrlike.ic.length_spill_load", + "arrlike.ic.range", + "arrlike.ic.inline", + "arrlike.ic.spill_or_miss", + "arrlike.ic.spill_ptr", + "arrlike.ic.spill_load", + // the lazy-JSON-array tier + "arrlike.lazy.kind", + "arrlike.lazy.call", + // and the runtime entries only those arms called + "js_lazy_array_index_probe", + "js_dyn_index_get", + "js_number_coerce", + ] { + assert!( + !ir.contains(absent), + "`{absent}` must no longer be emitted at a dynamic element-read site:\n{ir}" + ); + } + // The inline hits themselves: a guarded ordinary-Array element load, and + // the elements-backed Array-subclass probe's own load. + let array_load = super::class_field_barrier_tests::block_body(&ir, "arrlike.ic.array_load.") + .expect("the ordinary-Array load block exists"); + assert!( + array_load.contains("load double") && array_load.contains("select i1"), + "the packed Array hit must stay a direct load with an inline hole->undefined:\n{array_load}" ); + let store = super::class_field_barrier_tests::block_body(&ir, "arrlike.elem.store.") + .expect("the elements-store probe block exists"); assert!( - !ir.contains("arrlike.lazy.sparse") && !ir.contains("arrlike.lazy.guard"), - "the lazy proof belongs in the probe, not inlined at every read site:\n{ir}" + store.contains("getelementptr i64, ptr %") && store.contains(", i64 12"), + "the probe must load ObjectMeta.elements at word 12:\n{store}" ); + let elem_load = super::class_field_barrier_tests::block_body(&ir, "arrlike.elem.load.") + .expect("the elements-store load block exists"); + assert!( + elem_load.contains("load double") && !elem_load.contains("call "), + "the elements-backed hit must load the element with no runtime call:\n{elem_load}" + ); + // Nothing inline reads the site's cache any more; only the exit mentions + // it, and only as an ADDRESS — a link-time constant needing no load. (The + // tier that used to load word 0 through `emit_inline_cache_slot` is the + // shape-carried tower, now behind the exit.) + for block in dynamic_index_site_blocks(&ir) { + let body = super::class_field_barrier_tests::block_body(&ir, &format!("{block}.")) + .unwrap_or_else(|| panic!("{block} block exists")); + if block == "arrlike.ic.miss" { + assert!( + body.contains("ptr @perry_ic_") && !body.contains("load ptr, ptr @perry_ic_"), + "the exit must take the slot's address, not its contents:\n{body}" + ); + } else { + assert!( + !body.contains("@perry_ic_"), + "no inline arm may touch the site's cache slot ({block}):\n{body}" + ); + } + } +} + +/// In a number context every arm's value must be a Number, or the merge phi +/// is not uniformly one. The coercion is therefore COUPLED: each inline hit +/// and the exit either all wrap their result in `js_number_coerce` or none +/// does. +/// +/// The exit deliberately does NOT fold that coercion into a flag argument of +/// its own (a fourth parameter on `js_packed_arraylike_index_get`): the extra +/// argument and its test are paid by every receiver that reaches the exit, +/// and measured +0.43% retired instructions on `object_deep_clone`, +0.18% on +/// `json_parse_1mb` and +0.13% on `batch` — to spare a `js_number_coerce` +/// from an arm that no TypeScript fixture can reach. +/// +/// `coerce_slow_to_number = true` is currently **unreachable from +/// TypeScript**, on `main` as well as here: `lower_binary` routes `-`/`*`/`/` +/// with an unproven operand to `lower_guarded_numeric_arith` and the bitwise +/// ops to the `ToInt32` lowering, so `lower_arithmetic_operand` — the only +/// caller of `lower_unknown_local_index_get_for_number_context` — is not +/// reached by an erased-receiver element read. Measured: no +/// `js_number_coerce` is emitted inside the site blocks of any of eleven +/// probe shapes (`a[i] * 2`, `a[i] ^ 0`, `a[i] - 1`, `a[i] | 0`, `2 - a[i]`, +/// `a[i] << 1`, `a[i] * a[i]`, a loop accumulator, and the declared-array +/// claim forms `a[b[i]] - 1`, `a[i] * 3`, `a[k] - 1`) under the base +/// toolchain either. This test therefore pins the COUPLING rather than a +/// literal expectation: it fails the moment a site coerces on one arm and not +/// another. +#[test] +fn the_number_context_coercion_is_coupled_across_every_arm() { + for name in [ + "unknown_dense_subclass_read", + "unknown_dense_subclass_read_number_context", + ] { + let ir = ir_for( + name, + vec![ + Stmt::Let { + id: ITEMS, + name: "items".to_string(), + ty: Type::Any, + mutable: false, + init: Some(Expr::PropertyGet { + object: Box::new(Expr::Object(vec![( + "value".to_string(), + Expr::Array(vec![Expr::Number(7.0)]), + )])), + property: "value".to_string(), + byte_offset: 0, + }), + }, + Stmt::Let { + id: RESULT, + name: "result".to_string(), + ty: Type::Number, + mutable: false, + init: Some(Expr::Binary { + op: BinaryOp::Sub, + left: Box::new(Expr::IndexGet { + object: Box::new(Expr::LocalGet(ITEMS)), + index: Box::new(Expr::Integer(0)), + }), + right: Box::new(Expr::Number(1.0)), + }), + }, + ], + ); + assert_eq!( + dynamic_index_site_blocks(&ir).len(), + 21, + "{name}: a number context must not change the emitted block shape:\n{ir}" + ); + let miss = super::class_field_barrier_tests::block_body(&ir, "arrlike.ic.miss.") + .unwrap_or_else(|| panic!("{name}: the exit block exists")); + assert!( + miss.contains("call double @js_packed_arraylike_index_get("), + "{name}: the exit must be the dispatcher call:\n{miss}" + ); + let coerces = miss.contains("call double @js_number_coerce("); + assert_eq!( + miss.matches("call ").count(), + if coerces { 2 } else { 1 }, + "{name}: the exit is one receiver classification, plus a ToNumber only \ + in a number context:\n{miss}" + ); + for hit in ["arrlike.ic.array_load.", "arrlike.elem.value."] { + let body = super::class_field_barrier_tests::block_body(&ir, hit) + .unwrap_or_else(|| panic!("{name}: {hit} block exists")); + assert_eq!( + body.contains("call double @js_number_coerce("), + coerces, + "{name}: {hit} must coerce exactly when the exit does, or the merge \ + phi is not uniformly a Number:\n{body}" + ); + } + } } fn dynamic_symbol_access_ir(symbol_init: Expr, field: Option<&str>) -> String { @@ -289,14 +503,17 @@ fn erased_symbol_annotation_does_not_bypass_runtime_validation() { ); } -/// The inline dynamic typed-array read brands the receiver off its managed -/// `GC_TYPE_TYPED_ARRAY` header and reads the element kind from the -/// `TypedArrayHeader` itself, instead of probing the 64-slot direct-mapped -/// `PERRY_TA_KIND_CACHE` that every ordinary-array registry miss also writes -/// negative entries into (a hot typed array kept getting evicted and missed -/// the tier on every access). +/// #T2: a typed-array receiver behind an erased type leaves through the +/// site's single exit instead of the eight inline element-kind arms. +/// +/// This replaces `unknown_numeric_read_brands_typed_arrays_off_the_header_ +/// not_the_kind_cache`, which pinned that inlined ladder. The brand is still +/// read off the managed `GcHeader` and never from the 64-slot direct-mapped +/// `PERRY_TA_KIND_CACHE` — the runtime exit reads the `TypedArrayHeader` +/// itself, exactly as the inline arms did — so the #5525 property that made +/// them worth inlining is intact; only their per-site code is gone. #[test] -fn unknown_numeric_read_brands_typed_arrays_off_the_header_not_the_kind_cache() { +fn unknown_numeric_read_routes_typed_arrays_through_the_single_exit() { let ir = ir_for( "unknown_typed_array_read_brand", vec![ @@ -326,23 +543,80 @@ fn unknown_numeric_read_brands_typed_arrays_off_the_header_not_the_kind_cache() }, ], ); - assert!( - ir.contains("tav.get.brand"), - "the inline typed-array tier must brand the receiver off its header:\n{ir}" - ); - let brand = super::class_field_barrier_tests::block_body(&ir, "tav.get.brand.") - .expect("brand block exists"); - assert!( - brand.contains("icmp eq i8") && brand.contains(", 11"), - "the brand block must test GC_TYPE_TYPED_ARRAY (11):\n{brand}" - ); - assert!( - brand.contains("load i8"), - "the element kind must be read from the TypedArrayHeader:\n{brand}" + // The site reads ONE managed-header brand byte, and it selects the + // ordinary-Array arm; every other `obj_type` — typed arrays included — + // continues to the object arm's own `GC_TYPE_OBJECT` test and, failing + // that, to the exit. + let header = super::class_field_barrier_tests::block_body(&ir, "arrlike.ic.header.") + .expect("the managed-header block exists"); + assert!( + header.contains("load i8") && header.contains(", 1\n"), + "the site must brand the receiver off its GcHeader:\n{header}" + ); + // The typed-array arm is back inline (measured: out of line it cost a + // dynamically-typed `Float64Array` sum +122.9% walltime / +206.3% + // instructions), but collapsed onto the ELEMENT WIDTH the header stores + // rather than the element KIND: four load blocks, not eight behind a + // seven-block dispatch. + for width in ["tav.w1", "tav.w2", "tav.w4", "tav.w8"] { + assert!( + ir.contains(width), + "the width-collapsed typed-array arm must emit `{width}`:\n{ir}" + ); + } + for gone in ["tav.get.", "tav.k.", "tav.kd"] { + assert!( + !ir.contains(gone), + "the per-kind ladder `{gone}` must not come back:\n{ir}" + ); + } + // #10118: the brand test decides on the TAG ALONE. Everything an + // Array-subclass or `JSON.parse` receiver would otherwise compute before + // failing it — the view guard, the element kind, the bounds check — sits + // behind the tag in `tav.kind_guard`. + let ta_brand = super::class_field_barrier_tests::block_body(&ir, "tav.brand.") + .expect("the typed-array brand block exists"); + assert!( + ta_brand.contains(", 11") && ta_brand.contains("arrlike.elem.kind"), + "the arm must test GC_TYPE_TYPED_ARRAY and decline to the object arm:\n{ta_brand}" + ); + assert_eq!( + ( + ta_brand.matches("icmp ").count(), + ta_brand.matches("load ").count(), + ta_brand.matches("and i1").count() + ), + (1, 0, 0), + "the brand test must be ONE compare on the already-loaded tag — no kind \ + load, no view-guard load, no AND-reduction:\n{ta_brand}" + ); + let ta_kind_guard = super::class_field_barrier_tests::block_body(&ir, "tav.kind_guard.") + .expect("the typed-array kind/bounds guard exists"); + assert!( + ta_kind_guard.contains("@PERRY_TA_VIEW_GUARD") && ta_kind_guard.contains("arrlike.ic.miss"), + "inline storage, the element kind and the bounds check belong behind the \ + tag, and their miss leaves through the single exit:\n{ta_kind_guard}" + ); + let w4 = super::class_field_barrier_tests::block_body(&ir, "tav.w4.") + .expect("the 4-byte width block exists"); + assert_eq!( + w4.matches("load ").count(), + 1, + "Int32Array/Uint32Array/Float32Array must resolve from ONE load:\n{w4}" + ); + assert_eq!( + w4.matches("select ").count(), + 2, + "signedness and the float form must be `select`s, not branches:\n{w4}" + ); + assert!( + ir.contains("call double @js_packed_arraylike_index_get("), + "a BigInt/Float16 lane, a live view or an out-of-bounds typed-array read \ + must still reach the single exit:\n{ir}" ); assert!( !ir.contains("@PERRY_TA_KIND_CACHE"), - "the inline read must no longer depend on the kind cache:\n{ir}" + "neither the site nor its exit may depend on the kind cache:\n{ir}" ); } @@ -414,52 +688,30 @@ fn any_typed_dynamic_key_takes_the_numeric_tiers_when_it_is_an_array_index() { "the integer test must be the fptosi/sitofp round trip:\n{exact}" ); assert!( - ir.contains("tav.get.brand") && ir.contains("arrlike.ic.family_token"), - "an integer key must reach the inline typed-array and dense-subclass tiers:\n{ir}" + ir.contains("arrlike.ic.header") && ir.contains("arrlike.elem.store"), + "an integer key must reach the inline element-read hit:\n{ir}" ); - // Only an ordinary ObjectHeader has the `meta` slot used by the - // elements-backed Array-subclass probe. Native Buffers and other exotic - // managed cells must leave through the complete dispatcher before that - // load; interpreting their header word at offset 8 as ObjectMeta crashes. + // Only an ordinary `ObjectHeader` has the words the MRU cache hit reads, + // so the object arm must re-test `GC_TYPE_OBJECT` itself: the brand block + // only proves "not GC_TYPE_ARRAY". Native Buffers, typed arrays, lazy + // JSON arrays and every other exotic managed cell must leave through the + // exit BEFORE that load — reading their header word at offset 0/4 as a + // `(class_id, ShapeId)` identity would compare garbage. let kind = super::class_field_barrier_tests::block_body(&ir, "arrlike.elem.kind.") - .expect("the elements-store object-kind guard exists"); - assert!( - kind.contains("icmp eq i8") && kind.contains(", 2") && kind.contains("arrlike.elem.meta"), - "only GC_TYPE_OBJECT may reach the ObjectMeta.elements load:\n{kind}" - ); - // #10114 put the lazy-JSON-array tier on this guard's miss edge, so the - // exit is one block further out than it used to be. Pin BOTH hops rather - // than the old block adjacency: a non-object must fall to the lazy kind - // test, and anything that is not GC_TYPE_LAZY_ARRAY (9) must still leave - // through the complete dispatcher at `arrlike.ic.miss`. Native Buffers and - // other exotic managed cells reach that exit unchanged; what must never - // happen is either tier reading their header word at offset 8 as - // ObjectMeta. - assert!( - kind.contains("arrlike.lazy.kind"), - "a non-object must fall through to the lazy tier's own kind test:\n{kind}" - ); - let lazy_kind = super::class_field_barrier_tests::block_body(&ir, "arrlike.lazy.kind.") - .expect("the lazy-array kind guard exists"); - assert!( - lazy_kind.contains("icmp eq i8") - && lazy_kind.contains(", 9") - && lazy_kind.contains("arrlike.lazy.call") - && lazy_kind.contains("arrlike.ic.miss"), - "only GC_TYPE_LAZY_ARRAY may reach the lazy probe; everything else must \ - still exit through the complete dispatcher:\n{lazy_kind}" - ); - // The elements-backed subclass probe sits ahead of the shape IC: meta - // word → `ObjectMeta.elements` (word 12) → inner-array bounds → slot. - let store = super::class_field_barrier_tests::block_body(&ir, "arrlike.elem.store.") - .expect("the elements-store probe block exists"); + .expect("the object-kind guard exists"); assert!( - store.contains("getelementptr i64, ptr %") && store.contains(", i64 12"), - "the probe must load ObjectMeta.elements at word 12:\n{store}" + kind.contains("icmp eq i8") && kind.contains(", 2") && kind.contains("arrlike.ic.miss"), + "only GC_TYPE_OBJECT may reach the ObjectMeta.elements load; everything \ + else must leave through the single exit:\n{kind}" ); + // The elements-backed subclass probe, the lazy-JSON-array probe and the + // dense-tail family token now live behind that exit rather than at every + // read site. + let miss = super::class_field_barrier_tests::block_body(&ir, "arrlike.ic.miss.") + .expect("the exit block exists"); assert!( - ir.contains("arrlike.elem.bounds") && ir.contains("arrlike.elem.load"), - "the probe must bounds-check and load from the inner array:\n{ir}" + miss.contains("call double @js_packed_arraylike_index_get("), + "the exit must be one call:\n{miss}" ); assert!( ir.contains("call double @js_array_get_index_or_string("), @@ -539,9 +791,9 @@ fn claimed_array_receiver_brands_before_committing_a_canonical_key_to_the_plain_ ); assert!( ir.contains("aidx.claimed.other") - && ir.matches("arrlike.ic.family_token").count() >= 2 - && ir.matches("tav.get.brand").count() >= 2, - "every other heap receiver must reach the inline typed-array and dense-subclass tiers from BOTH the canonical and the runtime-key arm:\n{ir}" + && ir.matches("arrlike.elem.store").count() >= 2 + && ir.matches("call double @js_packed_arraylike_index_get(").count() >= 2, + "every other heap receiver must reach the inline element-read hit and its single exit from BOTH the canonical and the runtime-key arm:\n{ir}" ); } diff --git a/crates/perry-codegen/src/runtime_decls/strings.rs b/crates/perry-codegen/src/runtime_decls/strings.rs index 0da4b188c1..de13335899 100644 --- a/crates/perry-codegen/src/runtime_decls/strings.rs +++ b/crates/perry-codegen/src/runtime_decls/strings.rs @@ -727,7 +727,13 @@ pub fn declare_phase_b_strings(module: &mut LlModule) { ); module.declare_function("js_dyn_index_get", DOUBLE, &[DOUBLE, DOUBLE]); // #8655: guarded packed-array / dense Array-subclass read before the - // fully generic dynamic dispatcher. Used by unknown-receiver loop reads. + // fully generic dynamic dispatcher. Used by unknown-receiver loop reads, + // and (#T2) the single out-of-line exit of the emitted dynamic `obj[i]` + // site (`expr/index_get/inline_dyn_typed_array.rs`): it absorbs the + // per-kind typed-array ladder's rejected cases, the whole shape-carried + // Array-subclass IC tower and the lazy-JSON-array probe, each of which was + // an acceleration of a decision this helper already makes — and it is + // handed the SAME site cache slot, so the primed words are unchanged. module.declare_function( "js_packed_arraylike_index_get", DOUBLE, diff --git a/crates/perry-runtime/src/array/index_get_exit_tests.rs b/crates/perry-runtime/src/array/index_get_exit_tests.rs new file mode 100644 index 0000000000..738d3ecb47 --- /dev/null +++ b/crates/perry-runtime/src/array/index_get_exit_tests.rs @@ -0,0 +1,379 @@ +//! #T2 ("inline hit, one exit"): the arms the emitted dynamic `obj[i]` read +//! stopped inlining must still be served, in the same order, with the same +//! cache effects, by its single out-of-line exit +//! [`js_packed_arraylike_index_get`]. +//! +//! Every test here is a **differential** against +//! [`crate::value::js_dyn_index_get`], the complete generic accessor that the +//! exit's fast tiers exist to shortcut, because that is the exact contract: +//! every tier — the ordinary-Array arm, the elements-backed Array-subclass +//! probe, the shape-carried layout cache and (new here) the lazy-JSON-array +//! probe — must produce the value the generic path would have produced. A +//! test that only asserted "did not panic" would pass for a stub; these assert +//! the element VALUE and, where a cache is involved, the primed words. +//! +//! The typed-array cases matter even though the emitted site now serves an +//! in-bounds, inline-storage, non-BigInt read from its own `tav.w1/w2/w4/w8` +//! arm: every case that arm's guard rejects — a raised `PERRY_TA_VIEW_GUARD`, +//! a BigInt or Float16 lane, an out-of-range or fractional index — routes the +//! same read to this exit, so the two must produce the same element. + +use super::subclass::js_packed_arraylike_index_get; +use super::{ArrayHeader, ArrayLikePicCache, ArrayLikePicCacheSlot, ARRAYLIKE_PIC_WORDS}; +use crate::array::{js_array_alloc, js_array_push_f64}; +use crate::object::ObjectHeader; +use crate::typedarray::{ + js_typed_array_get, js_typed_array_set, typed_array_alloc, TypedArrayHeader, KIND_BIGINT64, + KIND_BIGUINT64, KIND_FLOAT32, KIND_FLOAT64, KIND_INT16, KIND_INT32, KIND_INT8, KIND_UINT16, + KIND_UINT32, KIND_UINT8, KIND_UINT8_CLAMPED, +}; + +/// The reserved parent class id `class X extends Array` records. +const CLASS_ID_ARRAY: u32 = 0xFFFF_0024; + +/// Every element kind a dynamic `obj[i]` can read as a Number, in the order +/// the old per-kind `tav.kd1..7` chain tested them. The site's arms are now +/// keyed on element WIDTH, so these nine kinds map onto four load blocks — +/// which is exactly why each one is read here at three distinct indices. +const INLINE_KINDS: [u8; 9] = [ + KIND_INT8, + KIND_UINT8, + KIND_INT16, + KIND_UINT16, + KIND_INT32, + KIND_UINT32, + KIND_FLOAT32, + KIND_FLOAT64, + KIND_UINT8_CLAMPED, +]; + +fn nanbox(p: *const u8) -> f64 { + crate::value::js_nanbox_pointer(p as i64) +} + +fn typed(kind: u8, values: &[f64]) -> *mut TypedArrayHeader { + let ta = typed_array_alloc(kind, values.len() as u32); + for (i, v) in values.iter().enumerate() { + js_typed_array_set(ta, i as i32, *v); + } + ta +} + +fn plain(values: &[f64]) -> *mut ArrayHeader { + let mut arr = js_array_alloc(values.len() as u32); + for v in values { + arr = js_array_push_f64(arr, *v); + } + arr +} + +/// A `class X extends Array` instance with `count` pushed numeric elements and +/// `declared` named inline slots ahead of them — the shape that gives the site +/// an object-backed array-like receiver. +/// The shape-carried representation guard the object-backed tiers need: with +/// the default elements store installed, `ObjectMeta.elements` answers every +/// read and NO layout cache is ever primed — which would make the cache +/// assertions below vacuous. +fn shape_carried() -> crate::array::subclass_elements::ArraySubclassRepresentationGuard { + crate::array::subclass_elements::ArraySubclassRepresentationGuard::shape_carried() +} + +fn array_subclass( + class_id: u32, + declared: u32, + packed_keys: &[u8], + count: u32, +) -> *mut ObjectHeader { + crate::object::js_register_class_parent(class_id, CLASS_ID_ARRAY); + let keys = crate::object::js_build_class_keys_array( + class_id, + declared, + packed_keys.as_ptr(), + packed_keys.len() as u32, + ); + let obj = + crate::object::js_object_alloc_class_inline_keys(class_id, CLASS_ID_ARRAY, declared, keys); + crate::node_stream::js_array_subclass_init(nanbox(obj as *const u8), 0.0); + for i in 0..count { + js_array_push_f64(obj as *mut ArrayHeader, f64::from(i) + 11.0); + } + obj +} + +/// The contract in one place: every tier of the site's exit must answer what +/// the complete generic accessor answers. +/// +/// The generic side runs FIRST and on its own, so the comparison cannot be +/// satisfied by the exit priming state the generic path then reads. +#[track_caller] +fn assert_matches_generic(receiver: f64, index: f64, what: &str) { + let expected = crate::value::js_dyn_index_get(receiver, index); + let mut cache: ArrayLikePicCache = [0; ARRAYLIKE_PIC_WORDS]; + let mut slot: ArrayLikePicCacheSlot = &mut cache; + let actual = js_packed_arraylike_index_get(receiver, index, &mut slot); + assert_eq!( + actual.to_bits(), + expected.to_bits(), + "{what}: the site exit must answer exactly what the generic accessor \ + answers (exit {actual}, generic {expected})" + ); +} + +#[test] +fn the_exit_reads_every_element_kind_the_inline_arm_can_serve() { + let _serialized = crate::array::test_serialize(); + // A value that only survives correct per-kind truncation/sign handling, so + // a stub that returned 0.0 — or that picked the wrong width — cannot pass. + for kind in INLINE_KINDS { + let ta = typed(kind, &[0.0, 7.0, 0.0]); + let receiver = nanbox(ta as *const u8); + for index in 0..3u32 { + let expected = js_typed_array_get(ta, index as i32); + let actual = + js_packed_arraylike_index_get(receiver, f64::from(index), std::ptr::null_mut()); + assert_eq!( + actual.to_bits(), + expected.to_bits(), + "kind {kind} index {index}: the exit must read the same element the inline arm would" + ); + } + assert_eq!( + js_packed_arraylike_index_get(receiver, 1.0, std::ptr::null_mut()), + 7.0, + "kind {kind}: the fixture value must survive the per-kind load" + ); + } +} + +#[test] +fn each_kind_reads_its_own_lane_width_not_a_neighbours() { + let _serialized = crate::array::test_serialize(); + // Distinct per-lane values: a load of the wrong width at index 1 reads + // element 0's or element 2's bytes and cannot produce 2.0. + for kind in INLINE_KINDS { + let ta = typed(kind, &[1.0, 2.0, 3.0]); + let receiver = nanbox(ta as *const u8); + for (index, expect) in [(0u32, 1.0), (1, 2.0), (2, 3.0)] { + assert_eq!( + js_packed_arraylike_index_get(receiver, f64::from(index), std::ptr::null_mut()), + expect, + "kind {kind}: element {index} must be read at its own lane offset" + ); + } + } +} + +#[test] +fn out_of_range_fractional_and_negative_indices_match_the_generic_accessor() { + let _serialized = crate::array::test_serialize(); + let ta = typed(KIND_INT32, &[5.0, 6.0]); + let receiver = nanbox(ta as *const u8); + for (index, what) in [ + (2.0, "one past the end"), + (1e9, "far out of range"), + (-1.0, "negative"), + (0.5, "fractional"), + (f64::NAN, "NaN"), + (4_294_967_296.0, "above the array-index range"), + ] { + assert_matches_generic(receiver, index, what); + } +} + +#[test] +fn bigint_and_float16_kinds_are_not_served_as_numbers() { + let _serialized = crate::array::test_serialize(); + // The inline ladder guarded `kind <= KIND_UINT8_CLAMPED` precisely because + // a BigInt lane is a NaN-boxed pointer, not a Number. The exit keeps that + // bound and defers, so `ta[i]` still round-trips as a `bigint`. + for kind in [KIND_BIGINT64, KIND_BIGUINT64] { + let ta = typed_array_alloc(kind, 2); + let receiver = nanbox(ta as *const u8); + // Deliberately NOT an exact-bits differential: each call allocates a + // fresh BigInt, so two reads of the same lane are two distinct + // pointers. What must hold is the TAG — the site's arm keeps the + // `kind <= KIND_UINT8_CLAMPED` bound precisely because a BigInt lane + // is not a Number, and the exit it defers to must produce the BigInt. + let through_exit = js_packed_arraylike_index_get(receiver, 0.0, std::ptr::null_mut()); + let through_dispatcher = js_packed_arraylike_index_get(receiver, 0.0, std::ptr::null_mut()); + for (value, via) in [(through_exit, "exit"), (through_dispatcher, "dispatcher")] { + assert!( + crate::value::JSValue::from_bits(value.to_bits()).is_bigint(), + "kind {kind} via the {via}: a BigInt lane must come back NaN-boxed as a BigInt, \ + not read as a Number" + ); + } + } +} + +#[test] +fn a_live_view_changes_the_route_and_not_the_element() { + let _serialized = crate::array::test_serialize(); + let ta = typed(KIND_FLOAT64, &[1.5, 2.5]); + let receiver = nanbox(ta as *const u8); + let direct = js_packed_arraylike_index_get(receiver, 1.0, std::ptr::null_mut()); + assert_eq!(direct, 2.5, "the exit must serve this receiver"); + + // The site's inline arm computes the data pointer as `header + 16`, and + // its whole licence to do that is the cleared process-wide view guard. A + // raised guard sends the read here instead, and it must still answer 2.5 — + // through the dispatcher, whose `data_ptr` consults the view registries. + crate::typedarray::PERRY_TA_VIEW_GUARD.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let guarded = js_packed_arraylike_index_get(receiver, 1.0, std::ptr::null_mut()); + crate::typedarray::PERRY_TA_VIEW_GUARD.fetch_sub(1, std::sync::atomic::Ordering::Relaxed); + assert_eq!( + guarded, 2.5, + "a raised view guard must change the ROUTE, never the element" + ); +} + +#[test] +fn ordinary_arrays_holes_and_lazy_json_arrays_match_the_generic_accessor() { + let _serialized = crate::array::test_serialize(); + let arr = plain(&[3.0, 4.0, 5.0]); + let receiver = nanbox(arr as *const u8); + for index in [0.0, 2.0, 3.0] { + assert_matches_generic(receiver, index, "an ordinary Array"); + } + + // `JSON.parse` yields GC_TYPE_LAZY_ARRAY, the receiver the emitted + // `arrlike.lazy.*` tier used to serve inline through + // `js_lazy_array_index_probe`. + let json = b"[10,20,30]"; + let text = crate::string::js_string_from_bytes(json.as_ptr(), json.len() as u32); + let parsed = f64::from_bits(unsafe { crate::json::js_json_parse(text) }.bits()); + for index in [0.0, 1.0, 2.0, 3.0] { + assert_matches_generic(parsed, index, "a lazy JSON array"); + } + assert_eq!( + js_packed_arraylike_index_get(parsed, 1.0, std::ptr::null_mut()), + 20.0, + "the lazy tier must still produce the element, not undefined" + ); +} + +#[test] +fn an_array_subclass_read_primes_the_site_cache_and_matches_the_generic_accessor() { + let _representation = shape_carried(); + let _global = crate::gc::global_side_table_test_lock(); + crate::object::array_tail_transition::test_clear(); + let obj = array_subclass(0x0074_86A1, 2, b"sset\0mask\0", 3); + let receiver = nanbox(obj as *const u8); + + // Two reads through two SEPARATE fresh caches: the words the exit + // publishes are a function of the receiver, not of what the site already + // held, which is what makes "cache decisions unchanged" checkable. + let mut first_cache: ArrayLikePicCache = [0; ARRAYLIKE_PIC_WORDS]; + let mut first_slot: ArrayLikePicCacheSlot = &mut first_cache; + let mut second_cache: ArrayLikePicCache = [0; ARRAYLIKE_PIC_WORDS]; + let mut second_slot: ArrayLikePicCacheSlot = &mut second_cache; + assert_eq!( + js_packed_arraylike_index_get(receiver, 0.0, &mut first_slot), + 11.0 + ); + assert_eq!( + js_packed_arraylike_index_get(receiver, 0.0, &mut second_slot), + 11.0 + ); + assert_ne!( + first_cache[0], 0, + "the fixture must actually prime a cache, or this test is vacuous" + ); + assert_eq!( + first_cache, second_cache, + "miss + prime must publish the identical layout words every time" + ); + for index in [1.0, 2.0, 3.0] { + assert_matches_generic(receiver, index, "a dense Array subclass"); + } +} + +#[test] +fn a_spilled_length_subclass_and_its_family_token_match_the_generic_accessor() { + let _representation = shape_carried(); + let _global = crate::gc::global_side_table_test_lock(); + crate::object::array_tail_transition::test_clear(); + // Four declared named fields push the Array-subclass `length` and the + // element slots past the inline region: the `arrlike.ic.length_spill_*` + // and `arrlike.ic.spill*` tiers, plus the high-bit family token that + // `arrlike.ic.family_meta`/`family_token` used to compare inline. + let obj = array_subclass(0x0074_86A2, 4, b"a\0b\0c\0d\0", 4); + let receiver = nanbox(obj as *const u8); + + let mut cache: ArrayLikePicCache = [0; ARRAYLIKE_PIC_WORDS]; + let mut slot: ArrayLikePicCacheSlot = &mut cache; + assert_eq!( + js_packed_arraylike_index_get(receiver, 0.0, &mut slot), + 11.0 + ); + assert_ne!( + cache[0], 0, + "the fixture must actually prime a cache, or this test is vacuous" + ); + assert!( + cache[1] >= cache[4], + "this fixture must actually spill the length slot, or the spill arms are untested \ + (length slot {}, inline bound {})", + cache[1], + cache[4] + ); + for index in [0.0, 1.0, 3.0, 4.0] { + assert_matches_generic(receiver, index, "a spilled Array subclass"); + } +} + +#[test] +fn a_plain_object_and_a_non_pointer_receiver_match_the_generic_accessor() { + let _global = crate::gc::global_side_table_test_lock(); + let obj = crate::object::js_object_alloc(0x0074_86A3, 2); + let key = crate::string::js_string_from_bytes(b"0".as_ptr(), 1); + crate::object::js_object_set_field_by_name(obj, key, 17.0); + assert_matches_generic(nanbox(obj as *const u8), 0.0, "a plain object"); + assert_matches_generic(nanbox(obj as *const u8), 1.0, "a plain object, absent key"); + // A non-pointer receiver: `(42)[0]` is `undefined`, not a throw, so it is + // safe to compare here. `undefined[0]` / `null[0]` deliberately are NOT — + // they raise a TypeError, and the emitted site routes them to this exit + // precisely so the dispatcher can raise it. + assert_matches_generic(42.0, 0.0, "a non-pointer receiver"); +} + +/// The exit returns a BOXED element, which is why a number context still +/// wraps it in `js_number_coerce` at the site (`arrlike.ic.miss`) instead of +/// folding the coercion into the call — folding it would have cost every +/// receiver that reaches this exit an extra argument and its test. +#[test] +fn the_exit_returns_a_boxed_element_that_a_number_context_must_coerce() { + let _serialized = crate::array::test_serialize(); + // A string element is the case that distinguishes "coerced" from "not": + // `ToNumber("12")` is 12, and the raw read is a NaN-boxed string. + let mut arr = js_array_alloc(1); + let text = crate::string::js_string_from_bytes(b"12".as_ptr(), 2); + arr = js_array_push_f64(arr, crate::value::js_nanbox_string(text as i64)); + let receiver = nanbox(arr as *const u8); + + let raw = js_packed_arraylike_index_get(receiver, 0.0, std::ptr::null_mut()); + assert!( + crate::value::JSValue::from_bits(raw.to_bits()).is_any_string(), + "the exit must leave the element boxed" + ); + assert_eq!( + crate::builtins::js_number_coerce(js_packed_arraylike_index_get( + receiver, + 0.0, + std::ptr::null_mut(), + )), + 12.0, + "the site's number-context `js_number_coerce` must apply ToNumber" + ); + // `undefined` is the OOB answer, and `ToNumber(undefined)` is NaN — the + // property the emitted merge phi depends on in a number context. + assert!( + crate::builtins::js_number_coerce(js_packed_arraylike_index_get( + receiver, + 9.0, + std::ptr::null_mut(), + )) + .is_nan(), + "a coerced out-of-bounds read must be NaN, not the undefined box" + ); +} diff --git a/crates/perry-runtime/src/array/mod.rs b/crates/perry-runtime/src/array/mod.rs index e98d3efd54..fc2c981e47 100644 --- a/crates/perry-runtime/src/array/mod.rs +++ b/crates/perry-runtime/src/array/mod.rs @@ -58,6 +58,8 @@ mod dense_move_tests; #[cfg(test)] mod forwarding_tests; #[cfg(test)] +mod index_get_exit_tests; +#[cfg(test)] mod push_pop_tests; #[cfg(test)] mod spread_dense_tests; diff --git a/crates/perry-runtime/src/array/subclass_packed_index.rs b/crates/perry-runtime/src/array/subclass_packed_index.rs index 2d50e3eef1..6872ec1f6d 100644 --- a/crates/perry-runtime/src/array/subclass_packed_index.rs +++ b/crates/perry-runtime/src/array/subclass_packed_index.rs @@ -44,6 +44,45 @@ pub extern "C" fn js_packed_arraylike_index_get( if let Some(header) = unsafe { crate::value::addr_class::try_read_gc_header(raw as usize) } { + // #10114's lazy-JSON-array tier. It used to be three emitted + // blocks and a direct `js_lazy_array_index_probe` call at + // EVERY indexed read site; it lives here now, so a site pays + // nothing for it and a `JSON.parse` result still skips the + // `js_array_get_f64` -> `lazy_get` chain (~227 retired + // instructions for `rows[7].id`). `TAG_HOLE` is the probe's + // "this read needs the rooted accessor" signal — unambiguous, + // because a hole is never a value a read yields — and covers + // cold elements, descriptors, out-of-bounds, growth-forwarding + // stubs and a stale `cached_length` mirror, all of which fall + // through to the unchanged accessor below. + // + // The `GC_FLAG_FORWARDED` test is load-bearing, not defensive: + // the probe's contract is "a live, UNFORWARDED + // `GC_TYPE_LAZY_ARRAY` pointer -- the caller proves that from + // the GC header", and since #10098 a lazy array is movable and + // nursery-resident, so a forwarded one is reachable here. When + // this tier was three emitted blocks that proof came from + // `arrlike.ic.header`, which dominated them; folding the tier + // into this helper moved the obligation here with it. A + // forwarded receiver falls through to `js_array_get_f64`, + // whose `clean_arr_ptr` resolves the forwarding pointer. + // + // Nothing between the header read and the probe can collect + // (the probe is `CannotCollect` and `try_read_gc_header` only + // reads), so `raw` cannot go stale across this window. + if header.obj_type == crate::gc::GC_TYPE_LAZY_ARRAY + && header.gc_flags & crate::gc::GC_FLAG_FORWARDED == 0 + { + let probed = unsafe { + crate::json_tape::js_lazy_array_index_probe( + raw as i64, + i64::from(index_u32), + ) + }; + if probed.to_bits() != crate::value::TAG_HOLE { + return probed; + } + } if matches!( header.obj_type, crate::gc::GC_TYPE_ARRAY | crate::gc::GC_TYPE_LAZY_ARRAY diff --git a/crates/perry-runtime/src/json_tape.rs b/crates/perry-runtime/src/json_tape.rs index e380a91eff..ae0c47bd00 100644 --- a/crates/perry-runtime/src/json_tape.rs +++ b/crates/perry-runtime/src/json_tape.rs @@ -16,7 +16,7 @@ use crate::value::JSValue; use std::cell::Cell; mod cached_read; -pub use cached_read::lazy_get; +pub use cached_read::{js_lazy_array_index_probe, lazy_get}; mod iterative; pub(crate) use iterative::materialize_iterative; mod mutation; diff --git a/scripts/shape_descriptor_census_baseline.json b/scripts/shape_descriptor_census_baseline.json index 0082782542..0f821ab3b5 100644 --- a/scripts/shape_descriptor_census_baseline.json +++ b/scripts/shape_descriptor_census_baseline.json @@ -2,7 +2,6 @@ "codegen_object_header_size_callsite_multiset": { "crates/perry-codegen/src/codegen/artifacts.rs|crate::target_layout::object_header_size_bytes(target_triple),": 1, "crates/perry-codegen/src/expr/element_shape_guard.rs|let header_skip = crate::target_layout::object_header_size_bytes(ctx.target_triple).to_string();": 1, - "crates/perry-codegen/src/expr/index_get/inline_dyn_typed_array.rs|crate::target_layout::object_header_size_bytes(ctx.target_triple).to_string();": 1, "crates/perry-codegen/src/expr/member_update.rs|let header_skip = crate::target_layout::object_header_size_bytes(": 1, "crates/perry-codegen/src/expr/property_get.rs|crate::target_layout::object_header_size_bytes(ctx.target_triple)": 3, "crates/perry-codegen/src/expr/property_get/composed_ics.rs|crate::target_layout::object_header_size_bytes(ctx.target_triple).to_string();": 1, @@ -63,7 +62,7 @@ "crates/perry-runtime/src/object/test_root_accessors.rs|keys_array|access|let inline = unsafe { (*st.object_hot.shape_inline_cache.get())[slot].keys_array as usize };": 1 }, "summary": { - "codegen_object_header_size_sites": 43, + "codegen_object_header_size_sites": 42, "raw_member_files": 12, "raw_member_sites": { "keys_array": 35