diff --git a/src/compiler/c.rs b/src/compiler/c.rs index 1206225cf..ab51c0161 100644 --- a/src/compiler/c.rs +++ b/src/compiler/c.rs @@ -29,6 +29,7 @@ use crate::util::{ Digest, HashToDigest, MetadataCtimeExt, TimeMacroFinder, Timestamp, decode_path, encode_path, hash_all, strip_basedirs, strip_basedirs_from_arg, }; +use crate::{compiler::DirectCacheType, errors::*}; use async_trait::async_trait; use fs_err as fs; use std::borrow::Cow; @@ -42,8 +43,6 @@ use std::path::{Path, PathBuf}; use std::process; use std::sync::{Arc, LazyLock}; -use crate::errors::*; - use super::CacheControl; use super::preprocessor_cache::PreprocessorCacheEntry; @@ -529,6 +528,7 @@ where env_vars: env_vars.clone(), }), weak_toolchain_key, + hash_key_type: DirectCacheType::Hit, }); } else { debug!("Preprocessor cache miss: {preprocessor_key}"); @@ -686,6 +686,11 @@ where env_vars, }), weak_toolchain_key, + hash_key_type: if needs_preprocessing { + DirectCacheType::Miss + } else { + DirectCacheType::NotAttempted + } }) } diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index 56dea23c0..6ccf98a6e 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -571,7 +571,7 @@ where out_pretty, fmt_duration_as_secs(&start.elapsed()) ); - let (key, compilation, weak_toolchain_key) = match result { + let (key, compilation, weak_toolchain_key, hash_key_type) = match result { Err(e) => { return match e.downcast::() { Ok(ProcessError(output)) => { @@ -585,7 +585,8 @@ where key, compilation, weak_toolchain_key, - }) => (key, compilation, weak_toolchain_key), + hash_key_type, + }) => (key, compilation, weak_toolchain_key, hash_key_type), }; debug!("[{}]: Hash key: {}", out_pretty, key); // If `ForceRecache` is enabled, we won't check the cache. @@ -652,7 +653,7 @@ where outputs.clone() }; - let hit = CompileResult::CacheHit(duration); + let hit = CompileResult::CacheHit(duration, hash_key_type); match entry.extract_objects(filtered_outputs, &pool).await { Ok(()) => Ok(CacheLookupResult::Success(hit, output)), Err(e) => { @@ -1170,6 +1171,7 @@ where pub compilation: Box + 'static>, /// A weak key that may be used to identify the toolchain pub weak_toolchain_key: String, + pub hash_key_type: DirectCacheType, } /// Possible results of parsing compiler arguments. @@ -1212,6 +1214,14 @@ pub enum DistType { Error, } +/// Specifics about cache misses. +#[derive(Debug, PartialEq, Eq)] +pub enum DirectCacheType { + Hit, + Miss, + NotAttempted, +} + /// Specifics about cache misses. #[derive(Debug, PartialEq, Eq)] pub enum MissType { @@ -1238,7 +1248,7 @@ pub enum CompileResult { /// An error made the compilation not possible. Error, /// Result was found in cache. - CacheHit(Duration), + CacheHit(Duration, DirectCacheType), /// Result was not found in cache. /// /// The `CacheWriteFuture` will resolve when the result is finished @@ -1271,7 +1281,9 @@ impl fmt::Debug for CompileResult { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match *self { CompileResult::Error => write!(f, "CompileResult::Error"), - CompileResult::CacheHit(ref d) => write!(f, "CompileResult::CacheHit({:?})", d), + CompileResult::CacheHit(ref d, ref ht) => { + write!(f, "CompileResult::CacheHit({:?}, {:?})", d, ht) + } CompileResult::CacheMiss(ref m, ref dt, ref d, _) => { write!(f, "CompileResult::CacheMiss({:?}, {:?}, {:?}, _)", d, m, dt) } @@ -1293,7 +1305,7 @@ impl PartialEq for CompileResult { fn eq(&self, other: &CompileResult) -> bool { match (self, other) { (&CompileResult::Error, &CompileResult::Error) => true, - (&CompileResult::CacheHit(_), &CompileResult::CacheHit(_)) => true, + (CompileResult::CacheHit(_, ht), CompileResult::CacheHit(_, ht2)) => ht == ht2, (CompileResult::CacheMiss(m, dt, _, _), CompileResult::CacheMiss(n, dt2, _, _)) => { m == n && dt == dt2 } @@ -2960,7 +2972,10 @@ LLVM version: 6.0", .unwrap(); // Ensure that the object file was created. assert!(fs::metadata(&obj).map(|m| m.len() > 0).unwrap()); - assert_eq!(CompileResult::CacheHit(Duration::new(0, 0)), cached); + assert_eq!( + CompileResult::CacheHit(Duration::new(0, 0), DirectCacheType::Miss), + cached + ); assert_eq!(exit_status(0), res.status); assert_eq!(COMPILER_STDOUT, res.stdout.as_slice()); assert_eq!(COMPILER_STDERR, res.stderr.as_slice()); @@ -3091,7 +3106,10 @@ LLVM version: 6.0", .unwrap(); // Ensure that the object file was created. assert!(fs::metadata(&obj).map(|m| m.len() > 0).unwrap()); - assert_eq!(CompileResult::CacheHit(Duration::new(0, 0)), cached); + assert_eq!( + CompileResult::CacheHit(Duration::new(0, 0), DirectCacheType::Miss), + cached + ); assert_eq!(exit_status(0), res.status); assert_eq!(COMPILER_STDOUT, res.stdout.as_slice()); assert_eq!(COMPILER_STDERR, res.stderr.as_slice()); @@ -3267,7 +3285,7 @@ LLVM version: 6.0", )) .unwrap(); match cached { - CompileResult::CacheHit(duration) => { + CompileResult::CacheHit(duration, _) => { assert!(duration >= storage_delay); } _ => panic!("Unexpected compile result: {:?}", cached), diff --git a/src/compiler/rust.rs b/src/compiler/rust.rs index 21b7e9450..21d48a0f8 100644 --- a/src/compiler/rust.rs +++ b/src/compiler/rust.rs @@ -15,7 +15,7 @@ use crate::cache::{FileObjectSource, Storage}; use crate::compiler::args::*; use crate::compiler::{ - CCompileCommand, Cacheable, ColorMode, Compilation, CompileCommand, Compiler, + CCompileCommand, DirectCacheType, Cacheable, ColorMode, Compilation, CompileCommand, Compiler, CompilerArguments, CompilerHasher, CompilerKind, CompilerProxy, HashResult, Language, SingleCompileCommand, c::ArtifactDescriptor, }; @@ -1739,6 +1739,7 @@ where rlib_dep_reader: self.rlib_dep_reader.clone(), }), weak_toolchain_key, + hash_key_type: DirectCacheType::NotAttempted, }) } diff --git a/src/server.rs b/src/server.rs index 2f142684a..f68740ddb 100644 --- a/src/server.rs +++ b/src/server.rs @@ -14,11 +14,11 @@ use crate::cache::readonly::ReadOnlyStorage; use crate::cache::{CacheMode, Storage, storage_from_config}; -use crate::compiler::PreprocessorCacheEntry; use crate::compiler::{ CacheControl, CompileResult, Compiler, CompilerArguments, CompilerHasher, CompilerKind, CompilerProxy, DistType, Language, MissType, get_compiler_info, }; +use crate::compiler::{DirectCacheType, PreprocessorCacheEntry}; #[cfg(feature = "dist-client")] use crate::config; use crate::config::Config; @@ -1502,7 +1502,11 @@ where }; let mut stats = me.stats.lock().await; - + let direct_mode_capable = me + .storage + .preprocessor_cache_mode_config() + .use_preprocessor_cache_mode + && lang.needs_c_preprocessing(); match result { Ok((compiled, out)) => { let mut dist_type = DistType::NoDist; @@ -1513,10 +1517,14 @@ where stats.cache_errors.increment(&kind, &lang); } - CompileResult::CacheHit(duration) => { + CompileResult::CacheHit(duration, cache_type) => { debug!("[{}]: compile result: cache hit", out_pretty); - stats.cache_hits.increment(&kind, &lang); + if cache_type == DirectCacheType::Hit { + stats.direct_cache_hits.increment(&kind, &lang); + } else if cache_type == DirectCacheType::Miss && direct_mode_capable { + stats.direct_cache_misses.increment(&kind, &lang); + } stats.cache_read_hit_duration += duration; } CompileResult::CacheMiss(miss_type, dt, duration, future) => { @@ -1538,6 +1546,9 @@ where } stats.compilations += 1; stats.cache_misses.increment(&kind, &lang); + if direct_mode_capable { + stats.direct_cache_misses.increment(&kind, &lang); + } stats.compiler_write_duration += duration; debug!("stats after compile result: {stats:?}"); cache_write = Some(future); @@ -1732,8 +1743,12 @@ pub struct ServerStats { pub cache_errors: PerLanguageCount, /// The count of cache hits for handled compile requests (per language). pub cache_hits: PerLanguageCount, + /// The count of direct cache hits for handled compile requests (per language). + pub direct_cache_hits: PerLanguageCount, /// The count of cache misses for handled compile requests (per language). pub cache_misses: PerLanguageCount, + /// The count of direct cache misses for handled compile requests (per language). + pub direct_cache_misses: PerLanguageCount, /// The count of cache misses because the cache took too long to respond. pub cache_timeouts: u64, /// The count of errors reading cache entries. @@ -1838,7 +1853,9 @@ impl Default for ServerStats { requests_executed: u64::default(), cache_errors: PerLanguageCount::new(), cache_hits: PerLanguageCount::new(), + direct_cache_hits: PerLanguageCount::new(), cache_misses: PerLanguageCount::new(), + direct_cache_misses: PerLanguageCount::new(), cache_timeouts: u64::default(), cache_read_errors: u64::default(), non_cacheable_compilations: u64::default(), @@ -1925,10 +1942,14 @@ impl ServerStats { ); if advanced { set_compiler_stat!(stats_vec, self.cache_hits, "Cache hits"); + set_compiler_stat!(stats_vec, self.direct_cache_hits, "Direct cache hits"); set_compiler_stat!(stats_vec, self.cache_misses, "Cache misses"); + set_compiler_stat!(stats_vec, self.direct_cache_misses, "Cache misses"); } else { set_lang_stat!(stats_vec, self.cache_hits, "Cache hits"); + set_lang_stat!(stats_vec, self.direct_cache_hits, "Direct cache hits"); set_lang_stat!(stats_vec, self.cache_misses, "Cache misses"); + set_lang_stat!(stats_vec, self.direct_cache_misses, "Direct cache hits"); } self.set_percentage_stats(&mut stats_vec, advanced); @@ -2057,6 +2078,13 @@ impl ServerStats { "Cache hits rate", ); + set_percentage_stat( + stats_vec, + self.direct_cache_hits.all(), + self.direct_cache_misses.all() + self.direct_cache_hits.all(), + "Direct cache hits rate", + ); + let (stats_hits, stats_misses): (Vec<_>, Vec<_>) = if advanced { ( self.cache_hits.adv_counts.iter().collect(), @@ -2489,6 +2517,7 @@ mod tests { let output = writer.get_output(); assert!(output.contains("Cache hits rate -")); + assert!(output.contains("Direct cache hits rate -")); } #[test] @@ -2519,6 +2548,7 @@ mod tests { let output = writer.get_output(); assert!(output.contains("Cache hits rate 46.15 %")); + assert!(output.contains("Direct cache hits rate -")); assert!(output.contains("Cache hits rate (C/C++) 100.00 %")); assert!(output.contains("Cache hits rate (Cuda) 0.00 %")); assert!(output.contains("Cache hits rate (Rust) 66.67 %")); @@ -2552,6 +2582,7 @@ mod tests { let output = writer.get_output(); assert!(output.contains("Cache hits rate -")); + assert!(output.contains("Direct cache hits rate -")); assert!(output.contains("Cache hits rate (c/c++ [clang]) 100.00 %")); assert!(output.contains("Cache hits rate (cuda) 0.00 %")); assert!(output.contains("Cache hits rate (rust) 33.33 %"));