From 04b02f3a113fe9436962997d7472124898668dc3 Mon Sep 17 00:00:00 2001 From: KK Date: Fri, 11 Sep 2026 15:19:23 +0530 Subject: [PATCH 1/4] Fix parse_rustc_z_ls for deps without -HASH suffix Modern rustc (1.75+) emits extended metadata per dependency line in `rustc -Z ls=root` output: N libname[-hash] hash HASH host_hash ... kind ... public Crates built without a content hash suffix (e.g. crc_fast, a build-script artifact) appear as: N crc_fast hash 05bce6... host_hash None kind Unconditional public parse_rustc_z_ls used splitn(2, ' ') which yields libstring as the entire trailing metadata ("crc_fast hash 05bce6... host_hash ..."). The subsequent rsplitn(2, '-') finds no '-' and libname becomes the whole string, so it never matches the bare "crc_fast" parsed from rmeta filenames during crate_link_paths scanning. The unmatched dep is silently skipped, its rmeta is not packaged for the dist worker, and the worker fails with E0463: can't find crate for `crc_fast`. Fix: take only the first whitespace-delimited token of libstring before the existing rsplitn('-') logic runs, so crates with and without a -HASH suffix both resolve to the bare crate name. Signed-off-by: KK --- src/compiler/rust.rs | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/src/compiler/rust.rs b/src/compiler/rust.rs index 779e9dd79..f7e5c2785 100644 --- a/src/compiler/rust.rs +++ b/src/compiler/rust.rs @@ -2676,6 +2676,15 @@ fn parse_rustc_z_ls(stdout: &str) -> Result> { let libstring = line_splits .next() .context("No lib string on line from rustc -Z ls")?; + // The libstring may contain additional metadata after the crate name + // (e.g., "crc_fast hash 05bce6... host_hash None kind Unconditional public" + // when the crate has no -HASH suffix). Take only the first + // whitespace-delimited token so rsplitn(-) below operates on just + // the crate name, not the entire trailing metadata. + let libstring = libstring + .split_whitespace() + .next() + .context("No lib string on line from rustc -Z ls")?; if num != dep_names.len() + 1 { bail!( "Unexpected numbering of {} in rustc -Z ls output", @@ -3359,6 +3368,39 @@ proc_macro false assert_eq!(res[2], "lucet_runtime_macros"); } + #[cfg(feature = "dist-client")] + #[test] + fn test_parse_rustc_z_ls_modern_no_hash_suffix() { + // Modern rustc (1.75+) prints extended metadata per dep line: + // N libname[-hash] hash HASH host_hash ... kind ... public + // Crates without a -HASH suffix (e.g. crc_fast, a proc-macro-style + // library built with a deterministic hash) appear as: + // N crc_fast hash 05bce6... host_hash None kind Unconditional public + // The parser must extract just "crc_fast", not the entire trailing + // metadata, so that RustInputsPackager can match it against rmeta + // files in crate_link_paths. + let output = "Crate info: +name xai_file_utils +hash 42fac6f0 stable_crate_id StableCrateId(12206970385906972588) +=External Dependencies= +1 std-453218b5e9634890 hash c76be37888b32288681053863554e618 host_hash None kind Unconditional public +2 core-5f5c0031517c19c4 hash 4e0d60221dfd8f9efa10e2a33c921b61 host_hash None kind Unconditional public +3 crc_fast hash 05bce60290e56777e3ae6d3ddcc01e6c host_hash None kind Unconditional public +4 crc-8c7d86e779319534 hash 49e85fac7c830ee1def0dd1674e740b0 host_hash None kind Unconditional public +5 aws_sdk_s3-c49a342c963fe6e9 hash 119191ef3d4a8059aee3abebc4c4b5bf host_hash None kind Unconditional public + +"; + let res = parse_rustc_z_ls(output); + assert!(res.is_ok()); + let res = res.unwrap(); + assert_eq!(res.len(), 5); + assert_eq!(res[0], "std"); + assert_eq!(res[1], "core"); + assert_eq!(res[2], "crc_fast"); + assert_eq!(res[3], "crc"); + assert_eq!(res[4], "aws_sdk_s3"); + } + #[cfg(feature = "dist-client")] #[test] fn test_rlib_dep_reader_call() { From 14b1d48688794b126c287969f1c8dbee4e1eeac5 Mon Sep 17 00:00:00 2001 From: KK Date: Fri, 11 Sep 2026 16:06:15 +0530 Subject: [PATCH 2/4] Fix crate_link_paths scan for no-hash dep files The RustInputsPackager scans crate_link_paths for dependency rmeta and rlib files, extracting the crate name from each filename via rsplitn(2, '-'). This assumes every file has a - suffix. When a file has no hash (e.g. libcrc_fast.rmeta produced by a crate with crate-type = ["lib", "cdylib", "staticlib"]), rsplitn yields a single element and the code hits continue, skipping the file entirely. The file is never packaged and sent to the dist worker, which then fails with E0463: can't find crate for crc_fast. Extract the crate-name-and-extension logic into a testable helper, crate_name_and_ext_from_lib_path, that uses file_stem() to strip the extension before rsplitn, then falls back to the whole stem as the libname when no - separator is found. Signed-off-by: KK --- src/compiler/rust.rs | 108 ++++++++++++++++++++++++++++++++----------- 1 file changed, 82 insertions(+), 26 deletions(-) diff --git a/src/compiler/rust.rs b/src/compiler/rust.rs index f7e5c2785..695d6bca4 100644 --- a/src/compiler/rust.rs +++ b/src/compiler/rust.rs @@ -2006,6 +2006,83 @@ fn test_can_trim_this() { assert!(!can_trim_this(&rlib_file)); } +/// Extract the crate name and extension from a library file path in a crate +/// link directory. +/// +/// Filenames are normally `lib-.`, but cdylib and +/// staticlib outputs can have no metadata hash (e.g. `libcrc_fast.rmeta` from a +/// crate with `crate-type = ["lib", "cdylib", "staticlib"]`). In that case +/// `rsplitn(2, '-')` yields a single element, so the whole filename is the +/// libname. +#[cfg(feature = "dist-client")] +fn crate_name_and_ext_from_lib_path(path: &Path) -> Option<(&str, &str)> { + let ext = path.extension()?.to_str()?; + // file_stem strips the extension, giving e.g. `libcrc_fast` from + // `libcrc_fast.rmeta`. rsplitn on the stem then correctly handles both + // `libfoo-HASH` (normal) and `libcrc_fast` (cdylib, no hash) cases. + let stem = path.file_stem()?.to_str()?; + let mut rev_name_split = stem.rsplitn(2, '-'); + let _extra_filename_and_ext = rev_name_split.next(); + let libname = rev_name_split.next().unwrap_or(stem); + if libname.starts_with(DLL_PREFIX) && ext == DLL_EXTENSION { + Some((&libname[DLL_PREFIX.len()..], ext)) + } else if libname.starts_with(RLIB_PREFIX) && (ext == RLIB_EXTENSION || ext == RMETA_EXTENSION) { + Some((&libname[RLIB_PREFIX.len()..], ext)) + } else { + None + } +} + +#[test] +#[cfg(feature = "dist-client")] +fn test_crate_name_and_ext_from_lib_path() { + use std::path::Path; + + // Normal rlib with metadata hash suffix + let p = Path::new("libfoo-abc123.rlib"); + assert_eq!(crate_name_and_ext_from_lib_path(p), Some(("foo", "rlib"))); + + // Normal rmeta with metadata hash suffix + let p = Path::new("libfoo-abc123.rmeta"); + assert_eq!(crate_name_and_ext_from_lib_path(p), Some(("foo", "rmeta"))); + + // cdylib output with no metadata hash suffix (crate-type includes cdylib) + let p = Path::new("libcrc_fast.rmeta"); + assert_eq!( + crate_name_and_ext_from_lib_path(p), + Some(("crc_fast", "rmeta")) + ); + + // cdylib output with no metadata hash suffix, rlib variant + let p = Path::new("libcrc_fast.rlib"); + assert_eq!( + crate_name_and_ext_from_lib_path(p), + Some(("crc_fast", "rlib")) + ); + + // Dynamic library with metadata hash + let p = Path::new("libfoo-abc123.so"); + assert_eq!(crate_name_and_ext_from_lib_path(p), Some(("foo", "so"))); + + // Dynamic library without metadata hash (cdylib) + let p = Path::new("libcrc_fast.so"); + assert_eq!( + crate_name_and_ext_from_lib_path(p), + Some(("crc_fast", "so")) + ); + + // Not a library file + let p = Path::new("foo.txt"); + assert_eq!(crate_name_and_ext_from_lib_path(p), None); + + // Path with multiple dashes in crate name + let p = Path::new("libxai_file_utils-abc123.rmeta"); + assert_eq!( + crate_name_and_ext_from_lib_path(p), + Some(("xai_file_utils", "rmeta")) + ); +} + #[cfg(feature = "dist-client")] fn maybe_add_cargo_toml(input_path: &Path, verify: bool) -> Option { let lib_rs = PathBuf::new().join("src").join("lib.rs"); @@ -2158,32 +2235,11 @@ impl pkg::InputsPackager for RustInputsPackager { { // Take a look at the path and see if it's something we care about - let libname: &str = match path.file_name().and_then(|s| s.to_str()) { - Some(name) => { - let mut rev_name_split = name.rsplitn(2, '-'); - let _extra_filename_and_ext = rev_name_split.next(); - let libname = if let Some(libname) = rev_name_split.next() { - libname - } else { - continue; - }; - assert!(rev_name_split.next().is_none()); - libname - } - None => continue, - }; - let (crate_name, ext): (&str, _) = match path.extension() { - Some(ext) if libname.starts_with(DLL_PREFIX) && ext == DLL_EXTENSION => { - (&libname[DLL_PREFIX.len()..], ext) - } - Some(ext) if libname.starts_with(RLIB_PREFIX) && ext == RLIB_EXTENSION => { - (&libname[RLIB_PREFIX.len()..], ext) - } - Some(ext) if libname.starts_with(RLIB_PREFIX) && ext == RMETA_EXTENSION => { - (&libname[RLIB_PREFIX.len()..], ext) - } - _ => continue, - }; + let (crate_name, ext): (&str, &str) = + match crate_name_and_ext_from_lib_path(&path) { + Some(c) => c, + None => continue, + }; if let Some((_, ref dep_crate_names)) = rlib_dep_reader_and_names { // We have a list of crate names we care about, see if this lib is a candidate if !dep_crate_names.contains(crate_name) { From 9497df82d9914fa925da491968b309029d00c5bf Mon Sep 17 00:00:00 2001 From: KK Date: Fri, 11 Sep 2026 16:03:48 +0530 Subject: [PATCH 3/4] Fix OverlayBuilder on persistent toolchain dirs When sccache-dist restarts, the in-memory toolchain_dir_map is wiped but the on-disk toolchain directories survive. On the next build, prepare_overlay_dirs hits the else branch (toolchain not in the map) and calls fs::create_dir on the existing directory, which returns an AlreadyExists error and fails the build. Guard create_dir with an exists() check. The archive_id is a content hash, so a pre-existing directory for the same id already holds the correct unpacked toolchain and is safe to reuse. Signed-off-by: KK --- src/bin/sccache-dist/build.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/bin/sccache-dist/build.rs b/src/bin/sccache-dist/build.rs index bfd7a980e..2f3ad2c4b 100644 --- a/src/bin/sccache-dist/build.rs +++ b/src/bin/sccache-dist/build.rs @@ -190,7 +190,12 @@ impl OverlayBuilder { entry.clone() } else { trace!("Creating toolchain directory for {}", tc.archive_id); - fs::create_dir(&toolchain_dir)?; + // The directory can persist on disk from a previous server + // process (or a toolchain evicted from the in-memory map + // without its directory). Treat it as already-prepared. + if !toolchain_dir.exists() { + fs::create_dir(&toolchain_dir)?; + } let mut tccache = tccache.lock().unwrap(); let toolchain_rdr = match tccache.get(tc) { From 11f83ad2e234e0ccd75604f9ad4db0c5b35f395b Mon Sep 17 00:00:00 2001 From: KK Date: Fri, 11 Sep 2026 16:03:48 +0530 Subject: [PATCH 4/4] Package rustlib/ not whole sysroot lib dir On distros where rustc --print sysroot reports /usr (e.g. Void Linux), the ToolchainPackager joins LIBS_DIR ("lib") to the sysroot and packages the entire /usr/lib tree. This drags the full system library set (several GiB) into every distributed toolchain archive, blowing past the toolchain cache size and making toolchain packaging prohibitively slow. Package the rustlib/ subtree instead, which is the actual Rust standard library. Fall back to the old behavior (whole lib dir) if rustlib/ does not exist, preserving compatibility with non-standard sysroot layouts. Signed-off-by: KK --- src/compiler/rust.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/compiler/rust.rs b/src/compiler/rust.rs index 695d6bca4..932a05029 100644 --- a/src/compiler/rust.rs +++ b/src/compiler/rust.rs @@ -2341,10 +2341,18 @@ impl pkg::ToolchainPackager for RustToolchainPackager { let sysroot_executable = bins_path.join("rustc").with_extension(EXE_EXTENSION); package_builder.add_executable_and_deps(sysroot_executable)?; - package_builder.add_dir_contents(&bins_path)?; + // Package the Rust standard library subtree instead of the entire + // sysroot lib directory. Distros like Void Linux report `/usr` as the + // sysroot, so packaging `$sysroot/lib` would drag in the whole system + // library tree (several GiB) rather than just the Rust stdlib. if BINS_DIR != LIBS_DIR { let libs_path = sysroot.join(LIBS_DIR); - package_builder.add_dir_contents(&libs_path)?; + let rustlib_path = libs_path.join("rustlib"); + if rustlib_path.is_dir() { + package_builder.add_dir_contents(&rustlib_path)?; + } else { + package_builder.add_dir_contents(&libs_path)?; + } } package_builder.into_compressed_tar(f)