diff --git a/Cargo.lock b/Cargo.lock index 8b027fe..549b53a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -32,7 +32,7 @@ checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -346,6 +346,19 @@ version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +[[package]] +name = "blake3" +version = "1.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d9e454fc11f76977dc803893aff6304ed33d6a26efae8696573bea74baa27ae" +dependencies = [ + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.1", +] + [[package]] name = "block" version = "0.1.6" @@ -626,6 +639,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + [[package]] name = "core-foundation" version = "0.9.4" @@ -675,6 +694,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + [[package]] name = "crc" version = "3.4.0" @@ -1386,7 +1414,9 @@ dependencies = [ name = "haru-core" version = "0.1.0" dependencies = [ + "blake3", "image", + "lz4_flex", "serde", "serde_json", "tapline", @@ -1822,6 +1852,15 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "lz4_flex" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" +dependencies = [ + "twox-hash", +] + [[package]] name = "lzma-rs" version = "0.3.0" @@ -3057,7 +3096,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -3068,7 +3107,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] diff --git a/Cargo.toml b/Cargo.toml index 84157f7..f4036f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ tapline-ids = { path = "tapline/crates/tapline-ids" } tapline-steamworks = { path = "tapline/crates/tapline-steamworks" } anyhow = "1" +blake3 = "1" eframe = { version = "0.29", default-features = false, features = [ "default_fonts", "wgpu", @@ -37,6 +38,7 @@ eframe = { version = "0.29", default-features = false, features = [ "x11", ] } egui = "0.29" +lz4_flex = "0.11" sha2 = { version = "0.10", default-features = false } image = { version = "0.25", default-features = false, features = [ "gif", diff --git a/crates/haru-apply/src/lib.rs b/crates/haru-apply/src/lib.rs index 6549af0..7ff4602 100644 --- a/crates/haru-apply/src/lib.rs +++ b/crates/haru-apply/src/lib.rs @@ -9,6 +9,7 @@ pub mod launch; mod monitors; mod offscreen; mod relaunch; +pub mod repack; mod socket; pub mod startup; mod stream; diff --git a/crates/haru-apply/src/relaunch.rs b/crates/haru-apply/src/relaunch.rs index 7dc1cb1..d77ebbf 100644 --- a/crates/haru-apply/src/relaunch.rs +++ b/crates/haru-apply/src/relaunch.rs @@ -33,7 +33,7 @@ impl Relaunch { .lock() .ok() .and_then(|showing| showing.clone()) - .filter(|dir| dir.is_dir()); + .filter(|dir| haru_core::library::still_there(dir)); let Some(dir) = showing else { return Ok(()); }; diff --git a/crates/haru-apply/src/repack.rs b/crates/haru-apply/src/repack.rs new file mode 100644 index 0000000..c635656 --- /dev/null +++ b/crates/haru-apply/src/repack.rs @@ -0,0 +1,203 @@ +//! Turning downloaded Wallpaper Engine items into kirie packages. +//! +//! Each item is handed to `kirie convert`, which writes a `.kpk` holding the +//! item's files as they are. An item haru fetched itself is repacked and its +//! folder removed, so it lives on disk as one file. An item the Steam client +//! downloaded keeps its folder, since Steam fetches a missing one again, and +//! gets a copy in `haru_core::package::copies` instead (see `mirror`). If +//! kirie is missing or too old to convert, the folders work as before. + +use std::path::{Path, PathBuf}; + +/// Repack the item in `item_dir` into haru's package folder, and remove the +/// folder once the package is there. The package's path, or why it could not +/// be made, in which case the folder is untouched. +pub fn repack(item_dir: &Path) -> Result { + let kirie = crate::install::installed().ok_or("kirie is not installed")?; + let into = + haru_core::package::home().ok_or("there is no data directory to keep packages in")?; + std::fs::create_dir_all(&into).map_err(|e| format!("cannot create {}: {e}", into.display()))?; + let name = item_dir + .file_name() + .ok_or("the item has no folder name")? + .to_string_lossy() + .into_owned(); + let package = into.join(format!("{name}.{}", haru_core::package::EXTENSION)); + convert(&kirie, item_dir, &package)?; + // The package is complete, and the library prefers it to a folder with + // the same id, so a folder that cannot be removed costs only space. + let _ = std::fs::remove_dir_all(item_dir); + Ok(package) +} + +/// Run `kirie convert` on `item_dir`, writing `package`, and check haru can +/// read what it wrote. +fn convert(kirie: &Path, item_dir: &Path, package: &Path) -> Result<(), String> { + let output = crate::child::quiet(kirie) + .arg("convert") + .arg(item_dir) + .arg("-o") + .arg(package) + .output() + .map_err(|e| format!("cannot run kirie: {e}"))?; + if !output.status.success() { + let said = String::from_utf8_lossy(&output.stderr); + return Err(format!("kirie could not repack it: {}", said.trim())); + } + // Only a package haru can read stands in for the folder. + if haru_core::package::Package::open(package).is_none() { + let _ = std::fs::remove_file(package); + return Err("kirie wrote a package haru cannot read".to_owned()); + } + Ok(()) +} + +/// Copy every Wallpaper Engine item in the Workshop folders under `roots` +/// into a package, leaving the folders where they are, and remove copies +/// whose folder is gone. A copy is made again when its folder changes. +/// +/// An item kirie cannot convert (an application wallpaper, or a kirie too old +/// to have `convert`) is noted beside the copies and not tried again until the +/// item or kirie changes. Returns whether any copy was made or removed. +#[must_use] +pub fn mirror(roots: &[PathBuf]) -> bool { + let Some(kirie) = crate::install::installed() else { + return false; + }; + let Some(into) = haru_core::package::copies() else { + return false; + }; + let items = haru_core::library::workshop_items(roots); + if items.is_empty() && !into.is_dir() { + return false; + } + if std::fs::create_dir_all(&into).is_err() { + return false; + } + let kirie_changed = modified(&kirie); + let mut changed = false; + + for item in &items { + let package = into.join(format!("{}.{}", item.id, haru_core::package::EXTENSION)); + if haru_core::library::copy_is_current(&package, &item.dir) { + continue; + } + let refused = refusal(&into, &item.id); + if haru_core::library::copy_is_current(&refused, &item.dir) + && modified(&refused) >= kirie_changed + { + continue; + } + match convert(&kirie, &item.dir, &package) { + Ok(()) => { + let _ = std::fs::remove_file(&refused); + changed = true; + } + Err(why) => { + let _ = std::fs::write(&refused, why); + } + } + } + + changed |= forget(&into, &items); + changed +} + +/// Where `mirror` notes why it could not copy item `id`. +fn refusal(into: &Path, id: &str) -> PathBuf { + into.join(format!(".{id}.refused")) +} + +fn modified(path: &Path) -> Option { + std::fs::metadata(path) + .and_then(|meta| meta.modified()) + .ok() +} + +/// Remove the copies, and the notes, of items no longer among `items`. +fn forget(into: &Path, items: &[haru_core::Installed]) -> bool { + let Ok(entries) = std::fs::read_dir(into) else { + return false; + }; + let mut removed = false; + for path in entries.flatten().map(|entry| entry.path()) { + let Some(name) = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + else { + continue; + }; + let id = + if let Some(stem) = name.strip_suffix(&format!(".{}", haru_core::package::EXTENSION)) { + stem + } else if let Some(id) = name + .strip_prefix('.') + .and_then(|rest| rest.strip_suffix(".refused")) + { + id + } else { + // kirie's half-written packages and the previews folder. + continue; + }; + if path.is_file() + && !items.iter().any(|item| item.id == id) + && std::fs::remove_file(&path).is_ok() + { + removed = true; + } + } + removed +} + +#[cfg(test)] +mod tests { + use super::*; + + fn scratch(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("haru-repack-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + dir + } + + fn item(id: &str, dir: &Path) -> haru_core::Installed { + haru_core::Installed { + id: id.to_owned(), + dir: dir.join(id), + title: id.to_owned(), + kind: "scene".to_owned(), + preview: None, + size: 0, + installed: std::time::UNIX_EPOCH, + source: None, + } + } + + #[test] + fn copies_and_notes_of_items_that_are_gone_are_removed() + -> Result<(), Box> { + let dir = scratch("forget"); + std::fs::create_dir_all(dir.join(".previews"))?; + for name in [ + "1.kpk", + "2.kpk", + ".2.refused", + ".3.refused", + ".4.1234.partial", + "notes.txt", + ] { + std::fs::write(dir.join(name), b"")?; + } + + assert!(forget(&dir, &[item("1", &dir)])); + + let mut left: Vec = std::fs::read_dir(&dir)? + .flatten() + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .collect(); + left.sort(); + assert_eq!(left, [".4.1234.partial", ".previews", "1.kpk", "notes.txt"]); + assert!(!forget(&dir, &[item("1", &dir)]), "nothing more to remove"); + std::fs::remove_dir_all(&dir)?; + Ok(()) + } +} diff --git a/crates/haru-core/Cargo.toml b/crates/haru-core/Cargo.toml index cd513ff..0471827 100644 --- a/crates/haru-core/Cargo.toml +++ b/crates/haru-core/Cargo.toml @@ -8,7 +8,9 @@ repository.workspace = true rust-version.workspace = true [dependencies] +blake3 = { workspace = true } image = { workspace = true } +lz4_flex = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } tapline = { workspace = true } diff --git a/crates/haru-core/src/lib.rs b/crates/haru-core/src/lib.rs index 39496b5..c86d95b 100644 --- a/crates/haru-core/src/lib.rs +++ b/crates/haru-core/src/lib.rs @@ -3,6 +3,7 @@ pub mod engine; pub mod library; pub mod overrides; pub mod own; +pub mod package; pub mod properties; pub mod renderer; diff --git a/crates/haru-core/src/library.rs b/crates/haru-core/src/library.rs index 42f57d1..712bf3c 100644 --- a/crates/haru-core/src/library.rs +++ b/crates/haru-core/src/library.rs @@ -11,6 +11,9 @@ pub struct Installed { pub preview: Option, pub size: u64, pub installed: std::time::SystemTime, + /// For a package copied from a folder the Steam client keeps, that + /// folder: removing the item removes both. + pub source: Option, } #[must_use] @@ -92,11 +95,163 @@ pub fn scan(roots: &[PathBuf]) -> Vec { scan_dirs(&content_dirs(roots)) } -/// Everything the library shows: the Workshop items under `roots`, and the -/// user's own pictures and videos (see `crate::own`). +/// Everything the library shows: the packages haru made from the items it +/// downloaded (see `crate::package`), the Workshop items under `roots`, and +/// the user's own pictures and videos (see `crate::own`). #[must_use] pub fn scan_all(roots: &[PathBuf]) -> Vec { - scan_dirs(&all_dirs(roots)) + let packages = crate::package::home().map_or_else(Vec::new, |dir| scan_packages(&dir)); + let folders = scan_dirs(&all_dirs(roots)); + let copies = + crate::package::copies().map_or_else(Vec::new, |dir| current_copies(&dir, &folders)); + merge(packages, merge(copies, folders)) +} + +/// The Wallpaper Engine items in the Workshop folders under `roots`, which +/// the Steam client (or haru, before it repacks them) keeps as folders. +#[must_use] +pub fn workshop_items(roots: &[PathBuf]) -> Vec { + scan_dirs(&content_dirs(roots)) +} + +/// Whether the package at `copy` was made from `item_dir` as it is now: not +/// older than the folder or its `project.json`. Steam rewrites an item's +/// files when it updates one, so an older copy is out of date. +#[must_use] +pub fn copy_is_current(copy: &Path, item_dir: &Path) -> bool { + let modified = |path: &Path| { + std::fs::metadata(path) + .and_then(|meta| meta.modified()) + .ok() + }; + let Some(made) = modified(copy) else { + return false; + }; + [item_dir.to_owned(), item_dir.join("project.json")] + .iter() + .filter_map(|path| modified(path)) + .all(|changed| changed <= made) +} + +/// The packages in `dir` copied from one of `folders` and still current, each +/// taking its folder's place in the library. A copy whose folder is gone or +/// newer is left out, so the folder shows until the copy is made again. +fn current_copies(dir: &Path, folders: &[Installed]) -> Vec { + scan_packages(dir) + .into_iter() + .filter_map(|copy| { + let folder = folders.iter().find(|folder| folder.id == copy.id)?; + copy_is_current(©.dir, &folder.dir).then(|| Installed { + installed: folder.installed, + source: Some(folder.dir.clone()), + ..copy + }) + }) + .collect() +} + +/// Whether a wallpaper haru remembered is still there to put up: an item's +/// folder, or the package it was repacked into. +#[must_use] +pub fn still_there(wallpaper: &Path) -> bool { + wallpaper.is_dir() || (wallpaper.is_file() && crate::package::is_package(wallpaper)) +} + +/// `first`, then whatever of `rest` has an id `first` lacks, newest first. +fn merge(first: Vec, rest: Vec) -> Vec { + let mut items = first; + for item in rest { + if !items.iter().any(|known| known.id == item.id) { + items.push(item); + } + } + items.sort_by_key(|item| std::cmp::Reverse(item.installed)); + items +} + +/// The packages directly inside `dir`. A package's preview is copied out +/// once, into `.previews` beside them, for the picture decoder to read. +#[must_use] +pub fn scan_packages(dir: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(dir) else { + return Vec::new(); + }; + let mut items: Vec = Vec::new(); + for path in entries.flatten().map(|entry| entry.path()) { + let hidden = path + .file_name() + .is_none_or(|name| name.to_string_lossy().starts_with('.')); + if hidden || !path.is_file() || !crate::package::is_package(&path) { + continue; + } + if let Some(item) = read_package(&path) + && !items.iter().any(|known| known.id == item.id) + { + items.push(item); + } + } + items.sort_by_key(|item| std::cmp::Reverse(item.installed)); + items +} + +fn read_package(path: &Path) -> Option { + let package = crate::package::Package::open(path)?; + let id = package.library_id(); + let project = package.project(); + let title = Some(package.manifest.title.trim()) + .filter(|title| !title.is_empty()) + .map_or_else(|| id.clone(), crate::plain_text); + let kind = match &project { + Some(project) => project + .get("type") + .and_then(serde_json::Value::as_str) + .map_or_else(|| "scene".to_owned(), str::to_lowercase), + None => package.manifest.kind.clone(), + }; + let preview = package + .manifest + .preview + .as_deref() + .and_then(|name| package_preview(&package, path, name)); + let meta = std::fs::metadata(path).ok()?; + Some(Installed { + id, + title, + kind, + preview, + size: meta.len(), + installed: meta.modified().unwrap_or(std::time::UNIX_EPOCH), + dir: path.to_owned(), + source: None, + }) +} + +/// The package's preview as a file, copied out the first time it is asked +/// for and named after the package's fingerprint, so a changed package gets +/// a fresh copy. +fn package_preview(package: &crate::package::Package, path: &Path, name: &str) -> Option { + let ext = Path::new(name) + .extension()? + .to_string_lossy() + .to_ascii_lowercase(); + if !ext.bytes().all(|b| b.is_ascii_alphanumeric()) { + return None; + } + let previews = path.parent()?.join(".previews"); + let file = previews.join(format!("{}.{ext}", package.fingerprint())); + if file.is_file() { + return Some(file); + } + let bytes = package.read(name)?; + std::fs::create_dir_all(&previews).ok()?; + let partial = previews.join(format!( + ".{}.{}.partial", + package.fingerprint(), + std::process::id() + )); + std::fs::write(&partial, bytes).ok()?; + std::fs::rename(&partial, &file).ok()?; + Some(file) } /// The item directories under `roots` that are not among `known`, which is @@ -215,6 +370,7 @@ fn read(dir: &Path, id: String) -> Option { size: directory_size(dir), installed, dir: dir.to_owned(), + source: None, }) } @@ -244,6 +400,90 @@ fn directory_size(dir: &Path) -> u64 { mod tests { use super::*; + #[test] + fn a_copy_stands_in_for_its_folder_only_while_current() -> Result<(), Box> + { + use crate::package::tests::{CONVERTED, PROJECT, write_package}; + + let dir = std::env::temp_dir().join(format!("haru-copies-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let content = dir.join("content"); + let copies = dir.join("copies"); + std::fs::create_dir_all(content.join("1388331347"))?; + std::fs::create_dir_all(&copies)?; + std::fs::write(content.join("1388331347/project.json"), PROJECT)?; + let package = copies.join("1388331347.kpk"); + write_package(&package, CONVERTED, &[("project.json", PROJECT, true)])?; + let folders = scan_dirs(std::slice::from_ref(&content)); + + let listed = current_copies(&copies, &folders); + let [copy] = listed.as_slice() else { + return Err("the current copy was not listed".into()); + }; + assert_eq!(copy.dir, package); + assert_eq!(copy.source.as_ref(), Some(&content.join("1388331347"))); + assert_eq!(merge(vec![copy.clone()], folders.clone()).len(), 1); + + // Steam updates the item after the copy was made. + let later = std::time::SystemTime::now() + std::time::Duration::from_secs(60); + std::fs::File::options() + .write(true) + .open(content.join("1388331347/project.json"))? + .set_modified(later)?; + assert!(current_copies(&copies, &folders).is_empty()); + + // And a copy whose folder is gone is not listed either. + assert!(current_copies(&copies, &[]).is_empty()); + std::fs::remove_dir_all(&dir)?; + Ok(()) + } + + #[test] + fn a_package_lists_under_its_workshop_id_with_its_preview() + -> Result<(), Box> { + use crate::package::tests::{CONVERTED, PROJECT, write_package}; + + let dir = std::env::temp_dir().join(format!("haru-packages-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir)?; + write_package( + &dir.join("1388331347.kpk"), + CONVERTED, + &[ + ("preview.jpg", b"jpeg", false), + ("project.json", PROJECT, true), + ], + )?; + std::fs::write(dir.join("notes.txt"), b"not a package")?; + + let items = scan_packages(&dir); + let [item] = items.as_slice() else { + return Err(format!("expected one item, found {}", items.len()).into()); + }; + assert_eq!(item.id, "1388331347"); + assert_eq!(item.title, "Rainy street"); + assert_eq!(item.kind, "scene"); + assert_eq!(item.dir, dir.join("1388331347.kpk")); + let preview = item.preview.as_ref().ok_or("no preview")?; + assert_eq!(std::fs::read(preview)?, b"jpeg"); + + // The package wins over a folder with the same id. + let folder = Installed { + dir: dir.join("1388331347"), + ..item.clone() + }; + let merged = merge(items.clone(), vec![folder]); + assert_eq!(merged.len(), 1); + assert_eq!( + merged.first().map(|item| &item.dir), + Some(&dir.join("1388331347.kpk")) + ); + + assert_eq!(crate::properties::read(&item.dir).len(), 1); + std::fs::remove_dir_all(&dir)?; + Ok(()) + } + #[test] fn the_steam_root_matches_where_this_os_keeps_it() { let roots = home_relative_roots(); diff --git a/crates/haru-core/src/package.rs b/crates/haru-core/src/package.rs new file mode 100644 index 0000000..229ae44 --- /dev/null +++ b/crates/haru-core/src/package.rs @@ -0,0 +1,321 @@ +//! Reading kirie's `.kpk` packages, enough to list them. +//! +//! kirie writes and plays packages; haru only needs a package's manifest, its +//! preview and, for a repacked Wallpaper Engine item, its `project.json`. The +//! format is read here rather than through kirie's code, as haru shares no +//! code with kirie. See kirie's `docs/PACKAGE.md` and `crates/kirie-pack`. +//! +//! Layout: a 64-byte header (magic `KIRIEPKG`, version 1, flags 0, then the +//! manifest's and the index's offset and length as little-endian u64s, then +//! the first 16 bytes of blake3 over both), entries, then the manifest and +//! index as JSON. Every entry carries the blake3 hash of its stored bytes. + +use std::io::{Read, Seek, SeekFrom}; +use std::path::{Path, PathBuf}; + +use serde::Deserialize; + +/// The extension a package file carries. +pub const EXTENSION: &str = "kpk"; + +const MAGIC: &[u8; 8] = b"KIRIEPKG"; +const VERSION: u16 = 1; +const HEADER_LEN: usize = 64; +const MAX_MANIFEST_LEN: u64 = 1 << 20; +const MAX_INDEX_LEN: u64 = 64 << 20; +/// The most an entry read here may be: previews and project files are small, +/// and a package claiming more is not decompressed into memory. +const MAX_READ_LEN: u64 = 64 << 20; + +/// Where haru keeps the packages it makes from the items it downloads. +#[must_use] +pub fn home() -> Option { + crate::data_home().map(|base| base.join("haru/packages")) +} + +/// Where haru keeps packages copied from items the Steam client downloaded. +/// Steam fetches a folder again if it goes missing, so those folders stay and +/// the copy sits beside them; see `crate::library::copy_is_current`. +#[must_use] +pub fn copies() -> Option { + home().map(|dir| dir.join("steam")) +} + +/// Whether `path` names a package rather than an item folder. +#[must_use] +pub fn is_package(path: &Path) -> bool { + path.extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case(EXTENSION)) +} + +#[derive(Debug, Clone, Deserialize)] +pub struct Manifest { + pub id: String, + pub title: String, + pub kind: String, + pub entry: String, + #[serde(default)] + pub preview: Option, + #[serde(default)] + pub provenance: Provenance, +} + +#[derive(Debug, Clone, Default, Deserialize)] +pub struct Provenance { + #[serde(default)] + pub origin: String, + #[serde(default)] + pub source: String, + #[serde(default)] + pub source_id: String, +} + +#[derive(Debug, Clone, Deserialize)] +struct Index { + entries: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +struct Entry { + path: String, + offset: u64, + stored_len: u64, + len: u64, + compression: String, + #[serde(default = "none")] + cipher: String, + blake3: String, +} + +fn none() -> String { + "none".to_owned() +} + +/// An opened package: its manifest and where its entries are. +pub struct Package { + path: PathBuf, + pub manifest: Manifest, + entries: Vec, + fingerprint: String, +} + +impl Package { + /// Open `path` and read its manifest and index. Nothing is returned for a + /// file that is not a version 1 package or whose directory is damaged. + #[must_use] + pub fn open(path: &Path) -> Option { + let mut file = std::fs::File::open(path).ok()?; + let file_len = file.metadata().ok()?.len(); + let mut header = [0_u8; HEADER_LEN]; + file.read_exact(&mut header).ok()?; + let bytes = |at: usize, len: usize| header.get(at..at + len); + let u16_at = |at: usize| Some(u16::from_le_bytes(bytes(at, 2)?.try_into().ok()?)); + let u64_at = |at: usize| Some(u64::from_le_bytes(bytes(at, 8)?.try_into().ok()?)); + if bytes(0, 8)? != MAGIC || u16_at(8)? != VERSION || u16_at(10)? != 0 { + return None; + } + let (manifest_offset, manifest_len) = (u64_at(16)?, u64_at(24)?); + let (index_offset, index_len) = (u64_at(32)?, u64_at(40)?); + let directory_hash = bytes(48, 16)?; + if manifest_len > MAX_MANIFEST_LEN + || index_len > MAX_INDEX_LEN + || index_offset != manifest_offset.checked_add(manifest_len)? + || index_offset.checked_add(index_len)? != file_len + { + return None; + } + let manifest_bytes = read_at(&mut file, manifest_offset, manifest_len)?; + let index_bytes = read_at(&mut file, index_offset, index_len)?; + let mut hasher = blake3::Hasher::new(); + hasher.update(&manifest_bytes); + hasher.update(&index_bytes); + if hasher.finalize().as_bytes().get(..16)? != directory_hash { + return None; + } + let manifest: Manifest = serde_json::from_slice(&manifest_bytes).ok()?; + let index: Index = serde_json::from_slice(&index_bytes).ok()?; + Some(Self { + path: path.to_owned(), + manifest, + entries: index.entries, + fingerprint: directory_hash.iter().map(|b| format!("{b:02x}")).collect(), + }) + } + + /// 32 hex digits that change whenever the package's contents do. + #[must_use] + pub fn fingerprint(&self) -> &str { + &self.fingerprint + } + + /// The id the library knows this package by: the Workshop id of the item + /// it was made from, so it lines up with the Workshop, or else its own. + #[must_use] + pub fn library_id(&self) -> String { + let provenance = &self.manifest.provenance; + if provenance.source == "wallpaper_engine" && !provenance.source_id.is_empty() { + provenance.source_id.clone() + } else { + self.manifest.id.clone() + } + } + + /// One entry's bytes, checked against its hash and decompressed. + #[must_use] + pub fn read(&self, path: &str) -> Option> { + let entry = self.entries.iter().find(|entry| entry.path == path)?; + if entry.cipher != "none" || entry.len > MAX_READ_LEN || entry.stored_len > MAX_READ_LEN { + return None; + } + let mut file = std::fs::File::open(&self.path).ok()?; + let stored = read_at(&mut file, entry.offset, entry.stored_len)?; + if blake3::hash(&stored).to_hex().as_str() != entry.blake3 { + return None; + } + let bytes = match entry.compression.as_str() { + "none" => stored, + "lz4" => lz4_flex::decompress(&stored, usize::try_from(entry.len).ok()?).ok()?, + _ => return None, + }; + (bytes.len() as u64 == entry.len).then_some(bytes) + } + + /// The `project.json` of a repacked Wallpaper Engine item. + #[must_use] + pub fn project(&self) -> Option { + if self.manifest.kind != "wallpaper_engine" { + return None; + } + serde_json::from_slice(&self.read(&self.manifest.entry)?).ok() + } +} + +fn read_at(file: &mut std::fs::File, offset: u64, len: u64) -> Option> { + file.seek(SeekFrom::Start(offset)).ok()?; + let mut buf = vec![0_u8; usize::try_from(len).ok()?]; + file.read_exact(&mut buf).ok()?; + Some(buf) +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + + type Outcome = Result<(), Box>; + + /// A package as kirie writes one, built by hand. Each entry is stored as + /// is, or compressed with LZ4 when its flag is set. + pub(crate) fn write_package( + path: &Path, + manifest: &str, + entries: &[(&str, &[u8], bool)], + ) -> std::io::Result<()> { + let mut entry_area = Vec::new(); + let mut index = Vec::new(); + for (name, bytes, compress) in entries { + let stored = if *compress { + lz4_flex::compress(bytes) + } else { + bytes.to_vec() + }; + let offset = 4096 + entry_area.len() as u64; + entry_area.extend_from_slice(&stored); + entry_area.resize(entry_area.len().div_ceil(4096) * 4096, 0); + index.push(serde_json::json!({ + "path": name, + "offset": offset, + "stored_len": stored.len(), + "len": bytes.len(), + "compression": if *compress { "lz4" } else { "none" }, + "cipher": "none", + "blake3": blake3::hash(&stored).to_hex().to_string(), + })); + } + let manifest = manifest.as_bytes(); + let index = serde_json::to_vec(&serde_json::json!({ "entries": index }))?; + let manifest_offset = 4096 + entry_area.len() as u64; + let mut hasher = blake3::Hasher::new(); + hasher.update(manifest); + hasher.update(&index); + + let mut header = Vec::with_capacity(4096); + header.extend_from_slice(MAGIC); + header.extend_from_slice(&VERSION.to_le_bytes()); + header.extend_from_slice(&[0; 6]); + header.extend_from_slice(&manifest_offset.to_le_bytes()); + header.extend_from_slice(&(manifest.len() as u64).to_le_bytes()); + header.extend_from_slice(&(manifest_offset + manifest.len() as u64).to_le_bytes()); + header.extend_from_slice(&(index.len() as u64).to_le_bytes()); + header.extend_from_slice(hasher.finalize().as_bytes().get(..16).unwrap_or_default()); + header.resize(4096, 0); + + let mut out = header; + out.extend_from_slice(&entry_area); + out.extend_from_slice(manifest); + out.extend_from_slice(&index); + std::fs::write(path, out) + } + + pub(crate) const CONVERTED: &str = r#"{"id":"we-1388331347","title":"Rainy street","kind":"wallpaper_engine", + "entry":"project.json","preview":"preview.jpg", + "provenance":{"origin":"converted","source":"wallpaper_engine","source_id":"1388331347"}}"#; + + pub(crate) const PROJECT: &[u8] = + br#"{"title":"Rainy street","file":"scene.json","type":"Scene","general":{"properties":{"rain":{"type":"bool","text":"Rain","value":true,"order":1}}}}"#; + + #[test] + fn a_converted_item_reads_back_with_its_workshop_id() -> Outcome { + let path = std::env::temp_dir().join(format!("haru-package-{}.kpk", std::process::id())); + write_package( + &path, + CONVERTED, + &[ + ("preview.jpg", b"jpeg", false), + ("project.json", PROJECT, true), + ], + )?; + + let package = Package::open(&path).ok_or("did not open")?; + assert_eq!(package.library_id(), "1388331347"); + assert_eq!(package.manifest.title, "Rainy street"); + assert_eq!(package.read("preview.jpg").as_deref(), Some(&b"jpeg"[..])); + let project = package.project().ok_or("no project.json")?; + assert_eq!(project.get("type"), Some(&serde_json::json!("Scene"))); + assert_eq!(package.fingerprint().len(), 32); + std::fs::remove_file(&path)?; + Ok(()) + } + + #[test] + fn a_damaged_package_is_refused() -> Outcome { + let path = + std::env::temp_dir().join(format!("haru-package-bad-{}.kpk", std::process::id())); + write_package(&path, CONVERTED, &[("preview.jpg", b"jpeg", false)])?; + let mut bytes = std::fs::read(&path)?; + + // A changed entry byte: the package opens, the entry does not read. + if let Some(byte) = bytes.get_mut(4096) { + *byte ^= 1; + } + std::fs::write(&path, &bytes)?; + assert!( + Package::open(&path) + .ok_or("did not open")? + .read("preview.jpg") + .is_none() + ); + + // A changed manifest byte: the package does not open. + let at = bytes + .windows(5) + .position(|w| w == b"Rainy") + .ok_or("no title")?; + if let Some(byte) = bytes.get_mut(at) { + *byte = b'X'; + } + std::fs::write(&path, &bytes)?; + assert!(Package::open(&path).is_none()); + std::fs::remove_file(&path)?; + Ok(()) + } +} diff --git a/crates/haru-core/src/properties.rs b/crates/haru-core/src/properties.rs index 5ccfba7..254260d 100644 --- a/crates/haru-core/src/properties.rs +++ b/crates/haru-core/src/properties.rs @@ -61,12 +61,19 @@ impl Property { } } +/// An item's `project.json`, from its folder or from the package it was +/// repacked into. +fn project(item: &Path) -> Option { + if crate::package::is_package(item) { + return crate::package::Package::open(item)?.project(); + } + let text = std::fs::read_to_string(item.join("project.json")).ok()?; + serde_json::from_str(&text).ok() +} + #[must_use] pub fn read(dir: &Path) -> Vec { - let Ok(text) = std::fs::read_to_string(dir.join("project.json")) else { - return Vec::new(); - }; - let Ok(parsed) = serde_json::from_str::(&text) else { + let Some(parsed) = project(dir) else { return Vec::new(); }; let Some(properties) = parsed diff --git a/crates/haru-ui/src/app.rs b/crates/haru-ui/src/app.rs index 5fab788..1f0e9bd 100644 --- a/crates/haru-ui/src/app.rs +++ b/crates/haru-ui/src/app.rs @@ -408,7 +408,9 @@ impl Haru { match reply { haru_workshop::Reply::Progress { done, total, .. } => { let share = done.saturating_mul(100).checked_div(total).unwrap_or(0); - self.assets_note = format!("fetching Wallpaper Engine's assets… {share}%"); + let megabytes = total.div_ceil(1_000_000); + self.assets_note = + format!("fetching Wallpaper Engine's assets… {share}% of {megabytes} MB"); return; } haru_workshop::Reply::EngineAssets { dir } => { @@ -583,8 +585,7 @@ impl Haru { let wallpaper = found .current .or_else(|| self.config.screens.get(&found.name).cloned())?; - wallpaper - .is_dir() + haru_core::library::still_there(&wallpaper) .then(|| haru_apply::launch::Plan::showing(found.name, wallpaper)) }) .collect() @@ -626,7 +627,7 @@ impl Haru { .find(|(had, _)| *had == name) .map(|(_, wallpaper)| wallpaper.clone()) .or_else(|| self.config.screens.get(&name).cloned()) - .filter(|wallpaper| wallpaper.is_dir()); + .filter(|wallpaper| haru_core::library::still_there(wallpaper)); haru_apply::launch::Plan { screen: name, wallpaper, @@ -669,7 +670,7 @@ impl Haru { .config .screens .iter() - .find(|(_, wallpaper)| wallpaper.is_dir()) + .find(|(_, wallpaper)| haru_core::library::still_there(wallpaper)) .map(|(screen, wallpaper)| (screen.clone(), wallpaper.clone())); match last { Some((screen, wallpaper)) => self.start_renderer(&screen, &wallpaper), diff --git a/crates/haru-ui/src/lib.rs b/crates/haru-ui/src/lib.rs index 487dd8c..6ae2bff 100644 --- a/crates/haru-ui/src/lib.rs +++ b/crates/haru-ui/src/lib.rs @@ -66,6 +66,9 @@ pub struct Browser { fetching: Option, installed: Vec, landed: Option, + /// An item haru downloaded being repacked into one package; it lands + /// as the package, or as its folder if that fails. + repacking: Option<(u64, std::sync::mpsc::Receiver)>, fit: bool, settling: Option<(u32, f64)>, settings: crate::props::Panel, @@ -120,6 +123,7 @@ impl Browser { fetching: None, installed: Vec::new(), landed: None, + repacking: None, fit: true, settling: None, settings: crate::props::Panel::default(), @@ -164,7 +168,7 @@ impl Browser { self.collect(); self.check_files(ctx); - if self.awaiting.is_some() || self.fetching.is_some() { + if self.awaiting.is_some() || self.fetching.is_some() || self.repacking.is_some() { ctx.request_repaint_after(std::time::Duration::from_millis(120)); } @@ -221,6 +225,18 @@ impl Browser { } } + if let Some((item, landed)) = &self.repacking { + match landed.try_recv() { + Ok(path) => { + self.installed.push(*item); + self.landed = Some(path); + self.repacking = None; + } + Err(std::sync::mpsc::TryRecvError::Empty) => {} + Err(std::sync::mpsc::TryRecvError::Disconnected) => self.repacking = None, + } + } + while let Some(id) = self.fetching && let Some(reply) = self.workshop.take(id) { @@ -235,9 +251,12 @@ impl Browser { Reply::Installed { id: item, dir } => { self.fetching = None; self.downloading = None; - self.installed.push(item); self.status = Status::Idle; - self.landed = Some(dir); + let (done, landed) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = done.send(haru_apply::repack::repack(&dir).unwrap_or(dir)); + }); + self.repacking = Some((item, landed)); } Reply::Subscribed => { self.fetching = None; @@ -520,6 +539,11 @@ impl Browser { .join("project.json") .is_file() }) + || haru_core::package::home().is_some_and(|packages| { + packages + .join(format!("{id}.{}", haru_core::package::EXTENSION)) + .is_file() + }) } fn download_row(&mut self, ui: &mut egui::Ui, found: &BrowseResult) { @@ -537,6 +561,20 @@ impl Browser { }); return; } + if let Some((repacking, _)) = &self.repacking + && *repacking == id + { + ui.horizontal(|ui| { + ui.spinner(); + ui.label( + RichText::new("Packing it into one file\u{2026}") + .small() + .color(theme::MUTED), + ); + }); + return; + } + if let Some((waiting, _, _)) = self.landing && waiting == id { diff --git a/crates/haru-ui/src/library.rs b/crates/haru-ui/src/library.rs index 92d3f07..003b794 100644 --- a/crates/haru-ui/src/library.rs +++ b/crates/haru-ui/src/library.rs @@ -73,6 +73,9 @@ pub struct Library { adding: Option>, /// Files dropped while an add was running, added once it lands. queued: Vec, + /// Steam client items being copied into packages, with the roots to + /// rescan if any copy changed. Converting a large item takes a while. + mirroring: Option<(Receiver, Vec)>, } /// What adding files produced: the folders made, and why any were refused. @@ -123,6 +126,7 @@ impl Library { removed: Vec::new(), adding: None, queued: Vec::new(), + mirroring: None, confirming: None, settings: crate::props::Panel::default(), workshop, @@ -154,12 +158,59 @@ impl Library { pub fn refresh(&mut self, config: &Config, engine: &Engine) { let roots = config.libraries(); + if self.scanning.is_some() { + self.rescan = Some(roots.clone()); + } else { + self.start_scan(roots.clone()); + } + self.start_mirror(roots); + self.sync(engine); + } + + /// Copies the Steam client's items into packages on a thread of its own, + /// unless that is already running. haru's own downloads are left out: + /// they are repacked as they land. + fn start_mirror(&mut self, roots: Vec) { + if self.mirroring.is_some() { + return; + } + let own = haru_core::engine::library_home(); + let steam: Vec = roots + .iter() + .filter(|root| Some(*root) != own.as_ref()) + .cloned() + .collect(); + let (tell, heard) = channel(); + let started = std::thread::Builder::new() + .name("haru-library-mirror".to_owned()) + .spawn(move || { + let _ = tell.send(haru_apply::repack::mirror(&steam)); + }); + if started.is_ok() { + self.mirroring = Some((heard, roots)); + } + } + + fn collect_mirror(&mut self) { + let Some((mirroring, _)) = self.mirroring.as_ref() else { + return; + }; + let changed = match mirroring.try_recv() { + Ok(changed) => changed, + Err(std::sync::mpsc::TryRecvError::Empty) => return, + Err(std::sync::mpsc::TryRecvError::Disconnected) => false, + }; + let Some((_, roots)) = self.mirroring.take() else { + return; + }; + if !changed { + return; + } if self.scanning.is_some() { self.rescan = Some(roots); } else { self.start_scan(roots); } - self.sync(engine); } fn start_scan(&mut self, roots: Vec) { @@ -326,6 +377,7 @@ impl Library { ) { self.collect(); self.collect_scan(); + self.collect_mirror(); self.collect_adding(config, engine); self.sync(engine); @@ -341,6 +393,8 @@ impl Library { if self.unsubscribing.is_some() || self.scanning.is_some() || self.adding.is_some() { ctx.request_repaint_after(std::time::Duration::from_millis(120)); + } else if self.mirroring.is_some() { + ctx.request_repaint_after(std::time::Duration::from_secs(1)); } if sidebar { @@ -730,7 +784,17 @@ impl Library { })); } - self.status = match std::fs::remove_dir_all(&item.dir) { + let removed = if item.dir.is_file() { + std::fs::remove_file(&item.dir) + } else { + std::fs::remove_dir_all(&item.dir) + }; + // A copy's folder goes too, or it would come back as the item. + let removed = removed.and_then(|()| match &item.source { + Some(folder) if folder.is_dir() => std::fs::remove_dir_all(folder), + _ => Ok(()), + }); + self.status = match removed { Ok(()) => { self.items.retain(|other| other.id != item.id); self.selected = None; @@ -973,6 +1037,7 @@ mod tests { preview: None, size, installed: std::time::UNIX_EPOCH, + source: None, } } diff --git a/crates/haru-ui/src/settings.rs b/crates/haru-ui/src/settings.rs index 8afb13b..99f40f7 100644 --- a/crates/haru-ui/src/settings.rs +++ b/crates/haru-ui/src/settings.rs @@ -362,8 +362,11 @@ impl Settings { ); ui.add_space(4.0); if ui - .button("Fetch engine assets (377 MB)") - .on_hover_text("Downloaded with your own account; no Steam client needed") + .button("Fetch engine assets") + .on_hover_text( + "Only the assets folder kirie draws with, downloaded with your own \ + account; no Steam client needed", + ) .clicked() { actions.fetch_assets = true; diff --git a/crates/haru-workshop/src/lib.rs b/crates/haru-workshop/src/lib.rs index 2ba3c55..90fd39a 100644 --- a/crates/haru-workshop/src/lib.rs +++ b/crates/haru-workshop/src/lib.rs @@ -6,6 +6,9 @@ use tapline::{BrowsePage, BrowseQuery, Session, WorkshopItem}; pub const WALLPAPER_ENGINE: tapline_ids::AppId = tapline_ids::AppId(431_960); +/// The one folder of Wallpaper Engine's install that kirie draws scenes with. +const ENGINE_ASSETS_FOLDER: &str = "assets"; + #[derive(Debug, Clone)] pub enum Request { Browse(BrowseQuery), @@ -229,8 +232,11 @@ async fn engine_assets( id: RequestId, ) -> Reply { let app = tapline_ids::AppId(haru_core::engine::WALLPAPER_ENGINE_APP); + // kirie reads nothing of Wallpaper Engine but its `assets` folder, so the + // program itself, its web runtime and its UI are left on Steam's servers. let options = tapline::InstallOptions { install_dir: into.to_owned(), + paths: vec![ENGINE_ASSETS_FOLDER.to_owned()], ..tapline::InstallOptions::default() }; @@ -260,7 +266,7 @@ async fn engine_assets( match session.install_observed(app, &options, &mut observe).await { Ok(_) => Reply::EngineAssets { - dir: into.join("assets"), + dir: into.join(ENGINE_ASSETS_FOLDER), }, Err(error) if error.needs_login() => Reply::NeedsAccount, Err(error) => Reply::Failed(error.to_string()), diff --git a/tapline b/tapline index 8c439f1..f541366 160000 --- a/tapline +++ b/tapline @@ -1 +1 @@ -Subproject commit 8c439f13307fe126c602bbc2074064ccbab4fc17 +Subproject commit f54136611519bef50cf31336e0c28a78eb175657