[Wall time +0.25% over carry prerequisite; paired PGO estimate] Decode cache footer tables directly and borrow carried maps

compiler/rustc_incremental/src/persist/file_format.rs

@@ -26,7 +26,11 @@ use crate::diagnostics;
 const FILE_MAGIC: &[u8] = b"RSIC";
 
 /// Change this if the header format changes.
-const HEADER_FORMAT_VERSION: u16 = 0;
+const HEADER_FORMAT_VERSION: u16 = 1;
+
+pub(crate) fn file_header_len(sess: &Session) -> usize {
+    FILE_MAGIC.len() + size_of::<u16>() + size_of::<u8>() + rustc_version(sess).len()
+}
 
 pub(crate) fn write_file_header(stream: &mut FileEncoder<'_>, sess: &Session) {
     stream.emit_raw_bytes(FILE_MAGIC);
@@ -50,8 +54,7 @@ where
     // truncate and overwrite it, since it might be a shared hard-link, the
     // underlying data of which we don't want to modify.
     //
-    // We have to ensure we have dropped the memory maps to this file
-    // before performing this removal.
+    // On platforms that cannot unlink a mapped file, drop its mappings first.
     match fs::remove_file(&path_buf) {
         Ok(()) => {
             debug!("save: remove old file");
@@ -121,10 +124,10 @@ pub(crate) fn open_incremental_file(
         }
     })?;
 
-    // SAFETY: This process must not modify nor remove the backing file while the memory map lives.
+    // SAFETY: This process must not modify the backing file while the memory map lives.
     // For the dep-graph and the work product index, it is as soon as the decoding is done.
-    // For the query result cache, the memory map is dropped in save_dep_graph before calling
-    // save_in and trying to remove the backing file.
+    // The query cache can keep its mapping through serialization on Unix, where unlinking
+    // preserves the mapped file's contents. Other platforms drop the mapping before removal.
     //
     // There is no way to prevent another process from modifying this file.
     let mmap = unsafe { Mmap::map(file) }?;

compiler/rustc_incremental/src/persist/save.rs

@@ -62,25 +62,21 @@ pub(crate) fn save_dep_graph(tcx: TyCtxt<'_>) {
                     // even if there was no previous session.
                     let on_disk_cache = tcx.query_system.on_disk_cache.as_ref().unwrap();
 
-                    // For every green dep node that has a disk-cached value from the
-                    // previous session, make sure the value is loaded into the memory
-                    // cache, so that it will be serialized as part of this session.
-                    //
-                    // This reads data from the previous session, so it needs to happen
-                    // before dropping the mmap.
-                    //
-                    // FIXME(Zalathar): This step is intended to be cheap, but still does
-                    // quite a lot of work, especially in builds with few or no changes.
-                    // Can we be smarter about how we identify values that need promotion?
-                    // Can we promote values without decoding them into the memory cache?
-                    tcx.dep_graph.exec_cache_promotions(tcx);
-
-                    // Drop the memory map so that we can remove the file and write to it.
-                    on_disk_cache.close_serialized_data_mmap();
+                    let carried_data =
+                        if on_disk_cache.can_carry_forward(file_format::file_header_len(sess)) {
+                            on_disk_cache.take_serialized_data_mmap()
+                        } else {
+                            // Compact periodically, and preserve the existing path on platforms
+                            // that do not support unlinking a mapped file. Load every eligible
+                            // green value before dropping the old mapping and re-encoding it.
+                            tcx.dep_graph.exec_cache_promotions(tcx);
+                            on_disk_cache.close_serialized_data_mmap();
+                            None
+                        };
 
                     file_format::save_in(sess, query_cache_path, "query cache", |encoder| {
                         tcx.sess.time("incr_comp_serialize_result_cache", || {
-                            on_disk_cache::OnDiskCache::serialize(tcx, encoder)
+                            on_disk_cache::OnDiskCache::serialize(tcx, encoder, carried_data)
                         })
                     });
                 });

compiler/rustc_middle/src/dep_graph/graph.rs

@@ -700,6 +700,11 @@ impl DepGraphData {
         matches!(self.colors.get(prev_index), DepNodeColor::Green(_))
     }
 
+    #[inline]
+    pub fn prev_key_fingerprint_of(&self, prev_index: SerializedDepNodeIndex) -> PackedFingerprint {
+        self.previous.index_to_node(prev_index).key_fingerprint
+    }
+
     #[inline]
     pub fn prev_value_fingerprint_of(&self, prev_index: SerializedDepNodeIndex) -> Fingerprint {
         self.previous.value_fingerprint_for_index(prev_index)
@@ -1090,6 +1095,18 @@ impl DepGraph {
         }
     }
 
+    pub fn for_each_green_prev_index(
+        &self,
+        f: &mut dyn FnMut(SerializedDepNodeIndex, DepNodeIndex),
+    ) {
+        let data = self.data.as_ref().unwrap();
+        for prev_index in data.colors.values.indices() {
+            if let DepNodeColor::Green(dep_node_index) = data.colors.get(prev_index) {
+                f(prev_index, dep_node_index);
+            }
+        }
+    }
+
     pub(crate) fn finish_encoding(&self) -> FileEncodeResult {
         if let Some(data) = &self.data { data.current.encoder.finish(&data.current) } else { Ok(0) }
     }