Avoid cloning CFG-related analysis data.

compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs

@@ -3708,8 +3708,11 @@ impl<'infcx, 'tcx> MirBorrowckCtxt<'_, 'infcx, 'tcx> {
             location: Location,
         ) -> impl Iterator<Item = Location> {
             if location.statement_index == 0 {
-                let predecessors = body.basic_blocks.predecessors()[location.block].to_vec();
-                Either::Left(predecessors.into_iter().map(move |bb| body.terminator_loc(bb)))
+                Either::Left(
+                    body.basic_blocks.predecessors()[location.block]
+                        .iter()
+                        .map(move |&bb| body.terminator_loc(bb)),
+                )
             } else {
                 Either::Right(std::iter::once(Location {
                     statement_index: location.statement_index - 1,

compiler/rustc_middle/src/mir/basic_blocks.rs

@@ -15,44 +15,72 @@ use crate::mir::{BasicBlock, BasicBlockData, START_BLOCK};
 pub struct BasicBlocks<'tcx> {
     basic_blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>,
     /// Use an `Arc` so we can share the cache when we clone the MIR body, as borrowck does.
-    cache: Arc<Cache>,
+    cache: Arc<CfgCache>,
 }
 
 // Typically 95%+ of basic blocks have 4 or fewer predecessors.
-type Predecessors = IndexVec<BasicBlock, SmallVec<[BasicBlock; 4]>>;
+pub type Predecessors = IndexVec<BasicBlock, SmallVec<[BasicBlock; 4]>>;
 
 #[derive(Clone, Default, Debug)]
-struct Cache {
+pub struct CfgCache {
     predecessors: OnceLock<Predecessors>,
-    reverse_postorder: OnceLock<Vec<BasicBlock>>,
+    reverse_postorder: OnceLock<IndexVec<BasicBlock, BasicBlock>>,
     dominators: OnceLock<Dominators<BasicBlock>>,
 }
 
+impl CfgCache {
+    #[inline]
+    pub fn dominators(&self, bbs: &'_ BasicBlocks<'_>) -> &Dominators<BasicBlock> {
+        self.dominators.get_or_init(|| dominators(bbs))
+    }
+
+    #[inline]
+    pub fn reverse_postorder(
+        &self,
+        bbs: &'_ BasicBlocks<'_>,
+    ) -> &IndexSlice<BasicBlock, BasicBlock> {
+        self.reverse_postorder.get_or_init(|| {
+            let mut rpo: IndexVec<BasicBlock, BasicBlock> =
+                Postorder::new(&bbs.basic_blocks, START_BLOCK, None).collect();
+            rpo.raw.reverse();
+            rpo
+        })
+    }
+
+    #[inline]
+    pub fn predecessors(&self, bbs: &'_ BasicBlocks<'_>) -> &Predecessors {
+        self.predecessors.get_or_init(|| {
+            let mut preds = IndexVec::from_elem(SmallVec::new(), &bbs.basic_blocks);
+            for (bb, data) in bbs.basic_blocks.iter_enumerated() {
+                for succ in data.terminator().successors() {
+                    preds[succ].push(bb);
+                }
+            }
+            preds
+        })
+    }
+}
+
 impl<'tcx> BasicBlocks<'tcx> {
     #[inline]
     pub fn new(basic_blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>) -> Self {
-        BasicBlocks { basic_blocks, cache: Arc::new(Cache::default()) }
+        BasicBlocks { basic_blocks, cache: Arc::new(CfgCache::default()) }
+    }
+
+    #[inline]
+    pub fn cfg_cache(&self) -> Arc<CfgCache> {
+        self.cache.clone()
     }
 
     #[inline]
     pub fn dominators(&self) -> &Dominators<BasicBlock> {
-        self.cache.dominators.get_or_init(|| dominators(self))
+        self.cache.dominators(self)
     }
 
     /// Returns predecessors for each basic block.
     #[inline]
     pub fn predecessors(&self) -> &Predecessors {
-        self.cache.predecessors.get_or_init(|| {
-            let mut preds = IndexVec::from_elem(SmallVec::new(), &self.basic_blocks);
-            for (bb, data) in self.basic_blocks.iter_enumerated() {
-                if let Some(term) = &data.terminator {
-                    for succ in term.successors() {
-                        preds[succ].push(bb);
-                    }
-                }
-            }
-            preds
-        })
+        self.cache.predecessors(self)
     }
 
     /// Returns basic blocks in a reverse postorder.
@@ -61,12 +89,8 @@ impl<'tcx> BasicBlocks<'tcx> {
     ///
     /// [`traversal::reverse_postorder`]: crate::mir::traversal::reverse_postorder
     #[inline]
-    pub fn reverse_postorder(&self) -> &[BasicBlock] {
-        self.cache.reverse_postorder.get_or_init(|| {
-            let mut rpo: Vec<_> = Postorder::new(&self.basic_blocks, START_BLOCK, None).collect();
-            rpo.reverse();
-            rpo
-        })
+    pub fn reverse_postorder(&self) -> &IndexSlice<BasicBlock, BasicBlock> {
+        self.cache.reverse_postorder(self)
     }
 
     /// Returns mutable reference to basic blocks. Invalidates CFG cache.
@@ -100,11 +124,11 @@ impl<'tcx> BasicBlocks<'tcx> {
     pub fn invalidate_cfg_cache(&mut self) {
         if let Some(cache) = Arc::get_mut(&mut self.cache) {
             // If we only have a single reference to this cache, clear it.
-            *cache = Cache::default();
+            *cache = CfgCache::default();
         } else {
             // If we have several references to this cache, overwrite the pointer itself so other
             // users can continue to use their (valid) cache.
-            self.cache = Arc::new(Cache::default());
+            self.cache = Arc::new(CfgCache::default());
         }
     }
 }
@@ -149,21 +173,21 @@ impl<'tcx> graph::Predecessors for BasicBlocks<'tcx> {
 }
 
 // Done here instead of in `structural_impls.rs` because `Cache` is private, as is `basic_blocks`.
-TrivialTypeTraversalImpls! { Cache }
+TrivialTypeTraversalImpls! { CfgCache }
 
-impl<S: Encoder> Encodable<S> for Cache {
+impl<S: Encoder> Encodable<S> for CfgCache {
     #[inline]
     fn encode(&self, _s: &mut S) {}
 }
 
-impl<D: Decoder> Decodable<D> for Cache {
+impl<D: Decoder> Decodable<D> for CfgCache {
     #[inline]
     fn decode(_: &mut D) -> Self {
         Default::default()
     }
 }
 
-impl StableHash for Cache {
+impl StableHash for CfgCache {
     #[inline]
     fn stable_hash<Hcx: StableHashCtxt>(&self, _: &mut Hcx, _: &mut StableHasher) {}
 }

compiler/rustc_middle/src/mir/mod.rs

@@ -36,7 +36,7 @@ use crate::ty::{
     TypingEnv, UserTypeAnnotationIndex,
 };
 
-mod basic_blocks;
+pub mod basic_blocks;
 mod consts;
 pub mod coverage;
 mod generic_graph;

compiler/rustc_mir_transform/src/gvn.rs

@@ -137,8 +137,8 @@ impl<'tcx> crate::MirPass<'tcx> for GVN {
 
         let typing_env = body.typing_env(tcx);
         let ssa = SsaLocals::new(tcx, body, typing_env);
-        // Clone dominators because we need them while mutating the body.
-        let dominators = body.basic_blocks.dominators().clone();
+        let cfg_cache = body.basic_blocks.cfg_cache();
+        let dominators = cfg_cache.dominators(&body.basic_blocks);
 
         let arena = DroplessArena::default();
         let mut state =
@@ -149,8 +149,7 @@ impl<'tcx> crate::MirPass<'tcx> for GVN {
             state.assign(local, opaque);
         }
 
-        let reverse_postorder = body.basic_blocks.reverse_postorder().to_vec();
-        for bb in reverse_postorder {
+        for &bb in cfg_cache.reverse_postorder(&body.basic_blocks) {
             let data = &mut body.basic_blocks.as_mut_preserves_cfg()[bb];
             state.visit_basic_block_data(bb, data);
         }
@@ -399,7 +398,7 @@ struct VnState<'body, 'a, 'tcx> {
     /// - `Some(Some(op))` are successful computations.
     evaluated: IndexVec<VnIndex, Option<Option<&'a OpTy<'tcx>>>>,
     ssa: &'body SsaLocals,
-    dominators: Dominators<BasicBlock>,
+    dominators: &'a Dominators<BasicBlock>,
     reused_locals: DenseBitSet<Local>,
     arena: &'a DroplessArena,
 }
@@ -410,7 +409,7 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> {
         body: &Body<'tcx>,
         typing_env: ty::TypingEnv<'tcx>,
         ssa: &'body SsaLocals,
-        dominators: Dominators<BasicBlock>,
+        dominators: &'a Dominators<BasicBlock>,
         local_decls: &'body LocalDecls<'tcx>,
         arena: &'a DroplessArena,
     ) -> Self {

compiler/rustc_mir_transform/src/prettify.rs

@@ -23,8 +23,7 @@ impl<'tcx> crate::MirPass<'tcx> for ReorderBasicBlocks {
     }
 
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
-        let rpo: IndexVec<BasicBlock, BasicBlock> =
-            body.basic_blocks.reverse_postorder().iter().copied().collect();
+        let rpo = body.basic_blocks.reverse_postorder();
         if rpo.iter().is_sorted() {
             return;
         }

compiler/rustc_mir_transform/src/ssa_range_prop.rs

@@ -13,7 +13,7 @@ use rustc_abi::WrappingRange;
 use rustc_const_eval::interpret::Scalar;
 use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::graph::dominators::Dominators;
-use rustc_index::bit_set::DenseBitSet;
+use rustc_middle::mir::basic_blocks::Predecessors;
 use rustc_middle::mir::visit::MutVisitor;
 use rustc_middle::mir::{BasicBlock, Body, Location, Operand, Place, TerminatorKind, *};
 use rustc_middle::ty::{TyCtxt, TypingEnv};
@@ -31,13 +31,13 @@ impl<'tcx> crate::MirPass<'tcx> for SsaRangePropagation {
     fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
         let typing_env = body.typing_env(tcx);
         let ssa = SsaLocals::new(tcx, body, typing_env);
-        // Clone dominators because we need them while mutating the body.
-        let dominators = body.basic_blocks.dominators().clone();
+        let cfg_cache = body.basic_blocks.cfg_cache();
+        let dominators = cfg_cache.dominators(&body.basic_blocks);
+        let predecessors = cfg_cache.predecessors(&body.basic_blocks);
         let mut range_set =
-            RangeSet::new(tcx, typing_env, body, &ssa, &body.local_decls, dominators);
+            RangeSet::new(tcx, typing_env, &ssa, &body.local_decls, dominators, predecessors);
 
-        let reverse_postorder = body.basic_blocks.reverse_postorder().to_vec();
-        for bb in reverse_postorder {
+        for &bb in cfg_cache.reverse_postorder(&body.basic_blocks) {
             let data = &mut body.basic_blocks.as_mut_preserves_cfg()[bb];
             range_set.visit_basic_block_data(bb, data);
         }
@@ -53,40 +53,36 @@ struct RangeSet<'tcx, 'body, 'a> {
     typing_env: TypingEnv<'tcx>,
     ssa: &'a SsaLocals,
     local_decls: &'body LocalDecls<'tcx>,
-    dominators: Dominators<BasicBlock>,
+    dominators: &'a Dominators<BasicBlock>,
     /// Known ranges at each locations.
     ranges: FxHashMap<Place<'tcx>, Vec<(Location, WrappingRange)>>,
-    /// Determines if the basic block has a single unique predecessor.
-    unique_predecessors: DenseBitSet<BasicBlock>,
+    predecessors: &'a Predecessors,
 }
 
 impl<'tcx, 'body, 'a> RangeSet<'tcx, 'body, 'a> {
     fn new(
         tcx: TyCtxt<'tcx>,
         typing_env: TypingEnv<'tcx>,
-        body: &Body<'tcx>,
         ssa: &'a SsaLocals,
         local_decls: &'body LocalDecls<'tcx>,
-        dominators: Dominators<BasicBlock>,
+        dominators: &'a Dominators<BasicBlock>,
+        predecessors: &'a Predecessors,
     ) -> Self {
-        let predecessors = body.basic_blocks.predecessors();
-        let mut unique_predecessors = DenseBitSet::new_empty(body.basic_blocks.len());
-        for bb in body.basic_blocks.indices() {
-            if predecessors[bb].len() == 1 {
-                unique_predecessors.insert(bb);
-            }
-        }
         RangeSet {
             tcx,
             typing_env,
             ssa,
             local_decls,
             dominators,
             ranges: FxHashMap::default(),
-            unique_predecessors,
+            predecessors,
         }
     }
 
+    fn has_unique_predecessor(&self, bb: BasicBlock) -> bool {
+        self.predecessors[bb].len() == 1
+    }
+
     /// Create a new known range at the location.
     fn insert_range(&mut self, place: Place<'tcx>, location: Location, range: WrappingRange) {
         assert!(self.is_ssa(place));
@@ -186,7 +182,7 @@ impl<'tcx> MutVisitor<'tcx> for RangeSet<'tcx, '_, '_> {
                         continue;
                     }
                     let successor = Location { block: target, statement_index: 0 };
-                    if self.unique_predecessors.contains(successor.block) {
+                    if self.has_unique_predecessor(successor.block) {
                         assert_ne!(location.block, successor.block);
                         let range = WrappingRange { start: val, end: val };
                         self.insert_range(place, successor, range);
@@ -198,7 +194,7 @@ impl<'tcx> MutVisitor<'tcx> for RangeSet<'tcx, '_, '_> {
                 let otherwise = Location { block: targets.otherwise(), statement_index: 0 };
                 if place.ty(self.local_decls, self.tcx).ty.is_bool()
                     && let [val] = targets.all_values()
-                    && self.unique_predecessors.contains(otherwise.block)
+                    && self.has_unique_predecessor(otherwise.block)
                 {
                     assert_ne!(location.block, otherwise.block);
                     let range = if val.get() == 0 {