From 57cdc835117ca038ed816af920546d3ac9f88dbd Mon Sep 17 00:00:00 2001 From: dianqk Date: Fri, 22 May 2026 18:03:10 +0800 Subject: [PATCH] Avoid cloning CFG-related analysis data. --- .../src/diagnostics/conflict_errors.rs | 7 +- compiler/rustc_middle/src/mir/basic_blocks.rs | 82 ++++++++++++------- compiler/rustc_middle/src/mir/mod.rs | 2 +- compiler/rustc_mir_transform/src/gvn.rs | 11 ++- compiler/rustc_mir_transform/src/prettify.rs | 3 +- .../rustc_mir_transform/src/ssa_range_prop.rs | 38 ++++----- 6 files changed, 82 insertions(+), 61 deletions(-) diff --git a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs index 9a64a063fcf5c..befd530e9dccd 100644 --- a/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs +++ b/compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs @@ -3708,8 +3708,11 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> { location: Location, ) -> impl Iterator { if location.statement_index == 0 { - let predecessors = body.basic_blocks.predecessors()[location.block].to_vec(); - Either::Left(predecessors.into_iter().map(move |bb| body.terminator_loc(bb))) + Either::Left( + body.basic_blocks.predecessors()[location.block] + .iter() + .map(move |&bb| body.terminator_loc(bb)), + ) } else { Either::Right(std::iter::once(Location { statement_index: location.statement_index - 1, diff --git a/compiler/rustc_middle/src/mir/basic_blocks.rs b/compiler/rustc_middle/src/mir/basic_blocks.rs index 41e21749f3920..bef9fd82c7b96 100644 --- a/compiler/rustc_middle/src/mir/basic_blocks.rs +++ b/compiler/rustc_middle/src/mir/basic_blocks.rs @@ -15,44 +15,72 @@ use crate::mir::{BasicBlock, BasicBlockData, START_BLOCK}; pub struct BasicBlocks<'tcx> { basic_blocks: IndexVec>, /// Use an `Arc` so we can share the cache when we clone the MIR body, as borrowck does. - cache: Arc, + cache: Arc, } // Typically 95%+ of basic blocks have 4 or fewer predecessors. -type Predecessors = IndexVec>; +pub type Predecessors = IndexVec>; #[derive(Clone, Default, Debug)] -struct Cache { +pub struct CfgCache { predecessors: OnceLock, - reverse_postorder: OnceLock>, + reverse_postorder: OnceLock>, dominators: OnceLock>, } +impl CfgCache { + #[inline] + pub fn dominators(&self, bbs: &'_ BasicBlocks<'_>) -> &Dominators { + self.dominators.get_or_init(|| dominators(bbs)) + } + + #[inline] + pub fn reverse_postorder( + &self, + bbs: &'_ BasicBlocks<'_>, + ) -> &IndexSlice { + self.reverse_postorder.get_or_init(|| { + let mut rpo: IndexVec = + Postorder::new(&bbs.basic_blocks, START_BLOCK, None).collect(); + rpo.raw.reverse(); + rpo + }) + } + + #[inline] + pub fn predecessors(&self, bbs: &'_ BasicBlocks<'_>) -> &Predecessors { + self.predecessors.get_or_init(|| { + let mut preds = IndexVec::from_elem(SmallVec::new(), &bbs.basic_blocks); + for (bb, data) in bbs.basic_blocks.iter_enumerated() { + for succ in data.terminator().successors() { + preds[succ].push(bb); + } + } + preds + }) + } +} + impl<'tcx> BasicBlocks<'tcx> { #[inline] pub fn new(basic_blocks: IndexVec>) -> Self { - BasicBlocks { basic_blocks, cache: Arc::new(Cache::default()) } + BasicBlocks { basic_blocks, cache: Arc::new(CfgCache::default()) } + } + + #[inline] + pub fn cfg_cache(&self) -> Arc { + self.cache.clone() } #[inline] pub fn dominators(&self) -> &Dominators { - self.cache.dominators.get_or_init(|| dominators(self)) + self.cache.dominators(self) } /// Returns predecessors for each basic block. #[inline] pub fn predecessors(&self) -> &Predecessors { - self.cache.predecessors.get_or_init(|| { - let mut preds = IndexVec::from_elem(SmallVec::new(), &self.basic_blocks); - for (bb, data) in self.basic_blocks.iter_enumerated() { - if let Some(term) = &data.terminator { - for succ in term.successors() { - preds[succ].push(bb); - } - } - } - preds - }) + self.cache.predecessors(self) } /// Returns basic blocks in a reverse postorder. @@ -61,12 +89,8 @@ impl<'tcx> BasicBlocks<'tcx> { /// /// [`traversal::reverse_postorder`]: crate::mir::traversal::reverse_postorder #[inline] - pub fn reverse_postorder(&self) -> &[BasicBlock] { - self.cache.reverse_postorder.get_or_init(|| { - let mut rpo: Vec<_> = Postorder::new(&self.basic_blocks, START_BLOCK, None).collect(); - rpo.reverse(); - rpo - }) + pub fn reverse_postorder(&self) -> &IndexSlice { + self.cache.reverse_postorder(self) } /// Returns mutable reference to basic blocks. Invalidates CFG cache. @@ -100,11 +124,11 @@ impl<'tcx> BasicBlocks<'tcx> { pub fn invalidate_cfg_cache(&mut self) { if let Some(cache) = Arc::get_mut(&mut self.cache) { // If we only have a single reference to this cache, clear it. - *cache = Cache::default(); + *cache = CfgCache::default(); } else { // If we have several references to this cache, overwrite the pointer itself so other // users can continue to use their (valid) cache. - self.cache = Arc::new(Cache::default()); + self.cache = Arc::new(CfgCache::default()); } } } @@ -149,21 +173,21 @@ impl<'tcx> graph::Predecessors for BasicBlocks<'tcx> { } // Done here instead of in `structural_impls.rs` because `Cache` is private, as is `basic_blocks`. -TrivialTypeTraversalImpls! { Cache } +TrivialTypeTraversalImpls! { CfgCache } -impl Encodable for Cache { +impl Encodable for CfgCache { #[inline] fn encode(&self, _s: &mut S) {} } -impl Decodable for Cache { +impl Decodable for CfgCache { #[inline] fn decode(_: &mut D) -> Self { Default::default() } } -impl StableHash for Cache { +impl StableHash for CfgCache { #[inline] fn stable_hash(&self, _: &mut Hcx, _: &mut StableHasher) {} } diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs index 57c2883ef42e4..40c9315d5379a 100644 --- a/compiler/rustc_middle/src/mir/mod.rs +++ b/compiler/rustc_middle/src/mir/mod.rs @@ -36,7 +36,7 @@ use crate::ty::{ TypingEnv, UserTypeAnnotationIndex, }; -mod basic_blocks; +pub mod basic_blocks; mod consts; pub mod coverage; mod generic_graph; diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs index d185c11452dbe..3252408d3bfee 100644 --- a/compiler/rustc_mir_transform/src/gvn.rs +++ b/compiler/rustc_mir_transform/src/gvn.rs @@ -137,8 +137,8 @@ impl<'tcx> crate::MirPass<'tcx> for GVN { let typing_env = body.typing_env(tcx); let ssa = SsaLocals::new(tcx, body, typing_env); - // Clone dominators because we need them while mutating the body. - let dominators = body.basic_blocks.dominators().clone(); + let cfg_cache = body.basic_blocks.cfg_cache(); + let dominators = cfg_cache.dominators(&body.basic_blocks); let arena = DroplessArena::default(); let mut state = @@ -149,8 +149,7 @@ impl<'tcx> crate::MirPass<'tcx> for GVN { state.assign(local, opaque); } - let reverse_postorder = body.basic_blocks.reverse_postorder().to_vec(); - for bb in reverse_postorder { + for &bb in cfg_cache.reverse_postorder(&body.basic_blocks) { let data = &mut body.basic_blocks.as_mut_preserves_cfg()[bb]; state.visit_basic_block_data(bb, data); } @@ -399,7 +398,7 @@ struct VnState<'body, 'a, 'tcx> { /// - `Some(Some(op))` are successful computations. evaluated: IndexVec>>>, ssa: &'body SsaLocals, - dominators: Dominators, + dominators: &'a Dominators, reused_locals: DenseBitSet, arena: &'a DroplessArena, } @@ -410,7 +409,7 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> { body: &Body<'tcx>, typing_env: ty::TypingEnv<'tcx>, ssa: &'body SsaLocals, - dominators: Dominators, + dominators: &'a Dominators, local_decls: &'body LocalDecls<'tcx>, arena: &'a DroplessArena, ) -> Self { diff --git a/compiler/rustc_mir_transform/src/prettify.rs b/compiler/rustc_mir_transform/src/prettify.rs index 8217feff24eca..8f832433739bf 100644 --- a/compiler/rustc_mir_transform/src/prettify.rs +++ b/compiler/rustc_mir_transform/src/prettify.rs @@ -23,8 +23,7 @@ impl<'tcx> crate::MirPass<'tcx> for ReorderBasicBlocks { } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let rpo: IndexVec = - body.basic_blocks.reverse_postorder().iter().copied().collect(); + let rpo = body.basic_blocks.reverse_postorder(); if rpo.iter().is_sorted() { return; } diff --git a/compiler/rustc_mir_transform/src/ssa_range_prop.rs b/compiler/rustc_mir_transform/src/ssa_range_prop.rs index 348c3bc2c9119..c6e23b7d13922 100644 --- a/compiler/rustc_mir_transform/src/ssa_range_prop.rs +++ b/compiler/rustc_mir_transform/src/ssa_range_prop.rs @@ -13,7 +13,7 @@ use rustc_abi::WrappingRange; use rustc_const_eval::interpret::Scalar; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::graph::dominators::Dominators; -use rustc_index::bit_set::DenseBitSet; +use rustc_middle::mir::basic_blocks::Predecessors; use rustc_middle::mir::visit::MutVisitor; use rustc_middle::mir::{BasicBlock, Body, Location, Operand, Place, TerminatorKind, *}; use rustc_middle::ty::{TyCtxt, TypingEnv}; @@ -31,13 +31,13 @@ impl<'tcx> crate::MirPass<'tcx> for SsaRangePropagation { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let typing_env = body.typing_env(tcx); let ssa = SsaLocals::new(tcx, body, typing_env); - // Clone dominators because we need them while mutating the body. - let dominators = body.basic_blocks.dominators().clone(); + let cfg_cache = body.basic_blocks.cfg_cache(); + let dominators = cfg_cache.dominators(&body.basic_blocks); + let predecessors = cfg_cache.predecessors(&body.basic_blocks); let mut range_set = - RangeSet::new(tcx, typing_env, body, &ssa, &body.local_decls, dominators); + RangeSet::new(tcx, typing_env, &ssa, &body.local_decls, dominators, predecessors); - let reverse_postorder = body.basic_blocks.reverse_postorder().to_vec(); - for bb in reverse_postorder { + for &bb in cfg_cache.reverse_postorder(&body.basic_blocks) { let data = &mut body.basic_blocks.as_mut_preserves_cfg()[bb]; range_set.visit_basic_block_data(bb, data); } @@ -53,29 +53,21 @@ struct RangeSet<'tcx, 'body, 'a> { typing_env: TypingEnv<'tcx>, ssa: &'a SsaLocals, local_decls: &'body LocalDecls<'tcx>, - dominators: Dominators, + dominators: &'a Dominators, /// Known ranges at each locations. ranges: FxHashMap, Vec<(Location, WrappingRange)>>, - /// Determines if the basic block has a single unique predecessor. - unique_predecessors: DenseBitSet, + predecessors: &'a Predecessors, } impl<'tcx, 'body, 'a> RangeSet<'tcx, 'body, 'a> { fn new( tcx: TyCtxt<'tcx>, typing_env: TypingEnv<'tcx>, - body: &Body<'tcx>, ssa: &'a SsaLocals, local_decls: &'body LocalDecls<'tcx>, - dominators: Dominators, + dominators: &'a Dominators, + predecessors: &'a Predecessors, ) -> Self { - let predecessors = body.basic_blocks.predecessors(); - let mut unique_predecessors = DenseBitSet::new_empty(body.basic_blocks.len()); - for bb in body.basic_blocks.indices() { - if predecessors[bb].len() == 1 { - unique_predecessors.insert(bb); - } - } RangeSet { tcx, typing_env, @@ -83,10 +75,14 @@ impl<'tcx, 'body, 'a> RangeSet<'tcx, 'body, 'a> { local_decls, dominators, ranges: FxHashMap::default(), - unique_predecessors, + predecessors, } } + fn has_unique_predecessor(&self, bb: BasicBlock) -> bool { + self.predecessors[bb].len() == 1 + } + /// Create a new known range at the location. fn insert_range(&mut self, place: Place<'tcx>, location: Location, range: WrappingRange) { assert!(self.is_ssa(place)); @@ -186,7 +182,7 @@ impl<'tcx> MutVisitor<'tcx> for RangeSet<'tcx, '_, '_> { continue; } let successor = Location { block: target, statement_index: 0 }; - if self.unique_predecessors.contains(successor.block) { + if self.has_unique_predecessor(successor.block) { assert_ne!(location.block, successor.block); let range = WrappingRange { start: val, end: val }; self.insert_range(place, successor, range); @@ -198,7 +194,7 @@ impl<'tcx> MutVisitor<'tcx> for RangeSet<'tcx, '_, '_> { let otherwise = Location { block: targets.otherwise(), statement_index: 0 }; if place.ty(self.local_decls, self.tcx).ty.is_bool() && let [val] = targets.all_values() - && self.unique_predecessors.contains(otherwise.block) + && self.has_unique_predecessor(otherwise.block) { assert_ne!(location.block, otherwise.block); let range = if val.get() == 0 {