From 449c4d4fb1be8b8b46dca266860ad9713d3e7f3d Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Mon, 28 Sep 2026 12:39:15 +0200 Subject: [PATCH 01/75] fix(file): keep the image layout when writing a partition back write_partition ran `fallocate -d` on the whole image, which turned every zero-filled block into a hole. A bmap created afterwards with `-b` then skipped those blocks, e.g. ext4 journals that the image deliberately writes as zeros, so a bmap-based flash left the old journal data on the device. Write only the blocks that differ from the image instead. Unchanged blocks keep their allocation, and a block that changed to all zeros is now written too (`dd conv=sparse` skipped it and kept the old content). Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- Cargo.lock | 2 +- Cargo.toml | 2 +- Dockerfile | 1 - project-context.md | 2 +- src/file/functions.rs | 68 ++++++++++++++++++++++++++++---------- tests/integration_tests.rs | 54 +++++++++++++++++++++++++++++- 6 files changed, 106 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5d4efd9a..8b7d028d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1789,7 +1789,7 @@ dependencies = [ [[package]] name = "omnect-cli" -version = "0.28.2" +version = "0.28.3" dependencies = [ "actix-web", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index c2c160d4..f2f9ebf2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ license = "MIT OR Apache-2.0" name = "omnect-cli" readme = "README.md" repository = "https://github.com/omnect/omnect-cli" -version = "0.28.2" +version = "0.28.3" [dependencies] actix-web = "4.11" diff --git a/Dockerfile b/Dockerfile index 989ae539..8eb12929 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,7 +37,6 @@ RUN < Result<()> { - let mut dd = Command::new("dd"); - dd.arg(format!("if={partition_file}")) - .arg(format!("of={image_file}")) - .arg("bs=512") - .arg(format!("seek={}", partition_info.start)) - .arg(format!("count={}", partition_info.count)) - .arg("conv=notrunc,sparse") - .arg("status=none"); - exec_cmd!(dd); - - let mut fallocate = Command::new("fallocate"); - fallocate.arg("-d").arg(image_file); - exec_cmd!(fallocate); - - let mut sync = Command::new("sync"); - exec_cmd!(sync); + let partition = fs::File::open(partition_file) + .context(format!("write_partition: cannot open {partition_file}"))?; + let image = fs::OpenOptions::new() + .read(true) + .write(true) + .open(image_file) + .context(format!("write_partition: cannot open {image_file}"))?; + + let image_offset = partition_info.start * SECTOR_SIZE; + let len = partition + .metadata() + .context("write_partition: cannot get partition file size")? + .len() + .min(partition_info.count * SECTOR_SIZE); + + // Only write blocks that differ: unchanged blocks keep their allocation, so a bmap + // created from the result still maps every block the source image mapped. + let mut src = vec![0u8; COPY_CHUNK_SIZE]; + let mut dst = vec![0u8; COPY_CHUNK_SIZE]; + let mut pos = 0; + while pos < len { + let n = usize::try_from((len - pos).min(COPY_CHUNK_SIZE as u64))?; + partition + .read_exact_at(&mut src[..n], pos) + .context("write_partition: cannot read partition file")?; + image + .read_exact_at(&mut dst[..n], image_offset + pos) + .context("write_partition: cannot read image")?; + + let blocks = src[..n] + .chunks(COMPARE_BLOCK_SIZE) + .zip(dst[..n].chunks(COMPARE_BLOCK_SIZE)); + for (i, (s, d)) in blocks.enumerate() { + if s != d { + image + .write_all_at(s, image_offset + pos + (i * COMPARE_BLOCK_SIZE) as u64) + .context("write_partition: cannot write image")?; + } + } + pos += n as u64; + } - Ok(()) + image + .sync_all() + .context("write_partition: cannot sync image") } pub fn generate_bmap_file(image_file: &str) -> Result<()> { diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index ac3e7167..e51b00c0 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -5,7 +5,7 @@ use common::Testrunner; use httpmock::prelude::*; use omnect_cli::ssh; use predicates::prelude::*; -use std::{fs::create_dir_all, path::PathBuf}; +use std::{fs::create_dir_all, os::unix::fs::MetadataExt, path::PathBuf}; use stdext::function_name; #[test] @@ -895,6 +895,58 @@ fn check_file_copy(tr: Testrunner, partition: &str) { assert!(file_diff::diff(in_file4, out_file4)); } +// unit of `st_blocks` +const STAT_BLOCK_SIZE: u64 = 512; + +fn bmap_value(bmap: &str, tag: &str) -> u64 { + // the header comment of a bmap file mentions the tags too; only the closing tag is unique + let open_tag = format!("<{tag}>"); + let end = bmap + .find(&format!("")) + .expect("closing tag not found in bmap"); + let start = bmap[..end] + .rfind(&open_tag) + .expect("opening tag not found in bmap") + + open_tag.len(); + bmap[start..end] + .trim() + .parse() + .expect("tag value is no number") +} + +#[test] +fn check_file_copy_keeps_image_layout() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let bmap_path = PathBuf::from(format!("{}.bmap", image_path.to_string_lossy())); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + + let meta = image_path.metadata().expect("image metadata"); + assert!( + meta.blocks() * STAT_BLOCK_SIZE >= meta.len(), + "test image must be fully allocated" + ); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},factory:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .arg("-b") + .assert() + .success(); + + // zero blocks of the input must stay mapped, e.g. zeroed ext4 journals + let bmap = std::fs::read_to_string(&bmap_path).expect("read bmap"); + assert_eq!( + bmap_value(&bmap, "MappedBlocksCount"), + bmap_value(&bmap, "BlocksCount") + ); +} + #[test] fn check_bmap_generation_wic() { let tr = Testrunner::new(function_name!().split("::").last().unwrap()); From 5186d4b34f36de75cfe5544fb8c8e95190746087 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Mon, 28 Sep 2026 13:10:51 +0200 Subject: [PATCH 02/75] chore: bump dependencies including major versions - all direct dependencies to their latest version, omnect-crypto to 0.5.0 (create_cert_and_key no longer takes the unused extensions argument; the extensions of a certificate without CSR are the same) - actix-web without default features: the OAuth callback server only serves plain HTTP on localhost, and the http2 feature pulled in h2 0.3 (RUSTSEC-2026-0258, no fixed 0.3 release) - flate2 selects its rust_backend itself instead of relying on actix-web's compression features to enable one Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- Cargo.lock | 1307 ++++++++++++++++++++++------------------------------ Cargo.toml | 26 +- src/lib.rs | 2 +- 3 files changed, 571 insertions(+), 764 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8b7d028d..a6f3a345 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 4 [[package]] name = "actix-codec" -version = "0.5.2" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" +checksum = "4c13df95297bcf9014dc89162b0cc69431e192e34e3b419612fc124cfcd45dbf" dependencies = [ "bitflags", "bytes", @@ -21,51 +21,43 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.12.1" +version = "3.18.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93acb4a42f64936f9b8cae4a433b237599dd6eb6ed06124eb67132ef8cc90662" +checksum = "e2ab7d72c0bb26f10df574ea560177b4de2f27ee8f421b2c1e9b75628bd9db23" dependencies = [ "actix-codec", - "actix-rt", "actix-service", "actix-utils", - "base64", "bitflags", - "brotli", "bytes", "bytestring", "derive_more", "encoding_rs", - "flate2", "foldhash", "futures-core", - "h2 0.3.27", "http 0.2.12", "httparse", "httpdate", "itoa", "language-tags", - "local-channel", "mime", "percent-encoding", "pin-project-lite", - "rand 0.10.1", - "sha1 0.11.0", "smallvec", "tokio", "tokio-util", "tracing", - "zstd", ] [[package]] name = "actix-macros" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" +checksum = "367f814ad4afbac74f07df5001214da65f65e185c90ef56c4dd8df23f8695b9b" dependencies = [ + "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] @@ -77,7 +69,6 @@ dependencies = [ "bytestring", "cfg-if", "http 0.2.12", - "regex", "regex-lite", "serde", "tracing", @@ -85,9 +76,9 @@ dependencies = [ [[package]] name = "actix-rt" -version = "2.11.0" +version = "2.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92589714878ca59a7626ea19734f0e07a6a875197eec751bb5d3f99e64998c63" +checksum = "e5f794807f82bbd36430c12cd600c73bbab0f52fdde4f0ed49978df113f4807f" dependencies = [ "futures-core", "tokio", @@ -95,17 +86,16 @@ dependencies = [ [[package]] name = "actix-server" -version = "2.6.0" +version = "2.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a65064ea4a457eaf07f2fba30b4c695bf43b721790e9530d26cb6f9019ff7502" +checksum = "8db2cf6f034e227c0c79825fbf11f3cad48569109515f62542b84d8abedb9376" dependencies = [ "actix-rt", "actix-service", - "actix-utils", "futures-core", "futures-util", "mio", - "socket2 0.5.10", + "socket2", "tokio", "tracing", ] @@ -122,9 +112,9 @@ dependencies = [ [[package]] name = "actix-utils" -version = "3.0.1" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88a1dcdff1466e3c2488e1cb5c36a71822750ad43839937f85d2f4d9f8b705d8" +checksum = "0128396dd7313f697ad05b21b1a7be7d4cbb81888704f55996e4a27db196bb4d" dependencies = [ "local-waker", "pin-project-lite", @@ -132,9 +122,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.13.0" +version = "4.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff87453bc3b56e9b2b23c1cc0b1be8797184accf51d2abe0f8a33ec275d316bf" +checksum = "bbacab3593b6b4f7be815076fc52d60a83c873426824675417e2abdd229e2e36" dependencies = [ "actix-codec", "actix-http", @@ -148,7 +138,6 @@ dependencies = [ "bytes", "bytestring", "cfg-if", - "cookie", "derive_more", "encoding_rs", "foldhash", @@ -161,13 +150,12 @@ dependencies = [ "mime", "once_cell", "pin-project-lite", - "regex", "regex-lite", "serde", "serde_json", "serde_urlencoded", "smallvec", - "socket2 0.6.4", + "socket2", "time", "tracing", "url", @@ -175,14 +163,14 @@ dependencies = [ [[package]] name = "actix-web-codegen" -version = "4.3.0" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f591380e2e68490b5dfaf1dd1aa0ebe78d84ba7067078512b4ea6e4492d622b8" +checksum = "b96b09c4878563f8ab4a5fd0c59f9f0d6e0e9f60eb9b748526a0b9604fd89c50" dependencies = [ "actix-router", "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] @@ -193,33 +181,18 @@ checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] -[[package]] -name = "alloc-no-stdlib" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" - -[[package]] -name = "alloc-stdlib" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" -dependencies = [ - "alloc-no-stdlib", -] - [[package]] name = "android_system_properties" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" dependencies = [ "libc", ] @@ -232,9 +205,48 @@ checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "asn1-rs" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6fd5ddaf0351dff5b8da21b2fb4ff8e08ddd02857f0bf69c47639106c0fff0" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "synstructure 0.12.6", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] [[package]] name = "assert-json-diff" @@ -284,13 +296,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] @@ -311,6 +323,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "bincode" version = "2.0.1" @@ -333,15 +351,15 @@ dependencies = [ [[package]] name = "bitflags" -version = "2.13.0" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "bitvec" -version = "1.0.1" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" dependencies = [ "funty", "radium", @@ -367,36 +385,15 @@ dependencies = [ "hybrid-array", ] -[[package]] -name = "brotli" -version = "8.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor", -] - -[[package]] -name = "brotli-decompressor" -version = "5.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", -] - [[package]] name = "bstr" -version = "1.12.1" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" dependencies = [ "memchr", "regex-automata", - "serde", + "serde_core", ] [[package]] @@ -405,11 +402,17 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" dependencies = [ "serde", ] @@ -434,32 +437,19 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.64" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f" +checksum = "f360145194ee8e21db5ee7f3fcd4fe52210864c75c985dae33218202c8bbe040" dependencies = [ "find-msvc-tools", - "jobserver", - "libc", "shlex", ] [[package]] name = "cfg-if" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" - -[[package]] -name = "chacha20" -version = "0.10.0" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" -dependencies = [ - "cfg-if", - "cpufeatures 0.3.0", - "rand_core 0.10.1", -] +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" [[package]] name = "chrono" @@ -477,9 +467,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.1" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" dependencies = [ "clap_builder", "clap_derive", @@ -487,9 +477,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" dependencies = [ "anstyle", "clap_lex", @@ -497,21 +487,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "clap_lex" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" [[package]] name = "cmov" @@ -519,21 +509,6 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" -[[package]] -name = "concurrent-queue" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "const-oid" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" - [[package]] name = "convert_case" version = "0.10.0" @@ -543,17 +518,6 @@ dependencies = [ "unicode-segmentation", ] -[[package]] -name = "cookie" -version = "0.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" -dependencies = [ - "percent-encoding", - "time", - "version_check", -] - [[package]] name = "core-foundation" version = "0.10.1" @@ -570,6 +534,12 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -581,9 +551,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" dependencies = [ "libc", ] @@ -605,18 +575,18 @@ checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "crc32fast" -version = "1.5.0" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-utils" -version = "0.8.21" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] name = "crypto-common" @@ -648,79 +618,98 @@ dependencies = [ [[package]] name = "darling" -version = "0.20.11" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" dependencies = [ - "darling_core 0.20.11", - "darling_macro 0.20.11", + "darling_core 0.21.3", + "darling_macro 0.21.3", ] [[package]] name = "darling" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ - "darling_core 0.21.3", - "darling_macro 0.21.3", + "darling_core 0.23.0", + "darling_macro 0.23.0", ] [[package]] name = "darling_core" -version = "0.20.11" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.119", ] [[package]] name = "darling_core" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" dependencies = [ - "fnv", "ident_case", "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.119", ] [[package]] name = "darling_macro" -version = "0.20.11" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ - "darling_core 0.20.11", + "darling_core 0.21.3", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "darling_macro" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ - "darling_core 0.21.3", + "darling_core 0.23.0", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "data-encoding" -version = "2.11.0" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + +[[package]] +name = "der" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" + +[[package]] +name = "der-parser" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbd676fbbab537128ef0278adb5576cf363cff6aa22a7b24effe97347cfab61e" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] [[package]] name = "deranged" @@ -747,7 +736,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn", + "syn 2.0.119", "unicode-xid", ] @@ -774,7 +763,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer 0.12.1", - "const-oid", "crypto-common 0.2.2", "ctutils", ] @@ -802,38 +790,44 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "encoding_rs" -version = "0.8.35" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679" dependencies = [ "cfg-if", + "core_detect", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", ] [[package]] name = "env_filter" -version = "1.0.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" dependencies = [ "log", + "regex", ] [[package]] name = "env_logger" -version = "0.11.10" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" dependencies = [ "env_filter", "log", @@ -857,11 +851,10 @@ dependencies = [ [[package]] name = "event-listener" -version = "5.4.1" +version = "5.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" dependencies = [ - "concurrent-queue", "parking", "pin-project-lite", ] @@ -878,9 +871,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "file_diff" @@ -911,15 +904,15 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" [[package]] name = "flate2" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ "crc32fast", "miniz_oxide", @@ -942,9 +935,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "foldhash" -version = "0.1.5" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] name = "foreign-types" @@ -978,41 +971,41 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", ] [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-macro" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-timer" @@ -1022,9 +1015,9 @@ checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-core", "futures-macro", @@ -1058,28 +1051,13 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" -dependencies = [ - "cfg-if", - "libc", - "r-efi 5.3.0", - "wasip2", -] - -[[package]] -name = "getrandom" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", - "r-efi 6.0.0", - "rand_core 0.10.1", - "wasip2", - "wasip3", + "r-efi", ] [[package]] @@ -1089,41 +1067,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13bd684d0cdb92e5993c63ef59bb8d1e959ff32ae131e5614fc143612f8544cc" dependencies = [ "crc", - "thiserror 2.0.18", + "thiserror 2.0.21", "wincode", ] [[package]] name = "h2" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http 0.2.12", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "h2" -version = "0.4.15" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" dependencies = [ "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "http 1.4.2", + "http 1.5.0", "indexmap", "slab", "tokio", @@ -1131,15 +1090,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" -dependencies = [ - "foldhash", -] - [[package]] name = "hashbrown" version = "0.17.1" @@ -1148,17 +1098,17 @@ checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "headers" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb" +checksum = "fc2700e3fe555c3310aa7286cac6167449f2c87e0eb58769c9208a1c58a1d106" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "headers-core", - "http 1.4.2", + "http 1.5.0", "httpdate", "mime", - "sha1 0.10.6", + "sha1", ] [[package]] @@ -1167,7 +1117,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" dependencies = [ - "http 1.4.2", + "http 1.5.0", ] [[package]] @@ -1178,9 +1128,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" [[package]] name = "hmac" @@ -1204,9 +1154,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -1214,23 +1164,23 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", - "http 1.4.2", + "http 1.5.0", ] [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", - "http 1.4.2", + "http 1.5.0", "http-body", "pin-project-lite", ] @@ -1256,14 +1206,14 @@ dependencies = [ "assert-json-diff", "async-object-pool", "async-trait", - "base64", + "base64 0.22.1", "bytes", "crossbeam-utils", "form_urlencoded", "futures-timer", "futures-util", "headers", - "http 1.4.2", + "http 1.5.0", "http-body-util", "hyper", "hyper-util", @@ -1275,7 +1225,7 @@ dependencies = [ "similar", "stringmetrics", "tabwriter", - "thiserror 2.0.18", + "thiserror 2.0.21", "tokio", "tracing", "url", @@ -1283,25 +1233,25 @@ dependencies = [ [[package]] name = "hybrid-array" -version = "0.4.12" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" dependencies = [ "typenum", ] [[package]] name = "hyper" -version = "1.10.1" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", - "h2 0.4.15", - "http 1.4.2", + "h2", + "http 1.5.0", "http-body", "httparse", "httpdate", @@ -1330,22 +1280,23 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" dependencies = [ - "base64", + "base64 0.23.1", "bytes", "futures-channel", "futures-util", - "http 1.4.2", + "http 1.5.0", "http-body", + "httparse", "hyper", "ipnet", "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.4", + "socket2", "tokio", "tower-service", "tracing", @@ -1377,9 +1328,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", @@ -1391,9 +1342,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", @@ -1404,9 +1355,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -1418,16 +1369,17 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -1438,15 +1390,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", @@ -1457,12 +1409,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - [[package]] name = "ident_case" version = "1.0.1" @@ -1492,27 +1438,25 @@ dependencies = [ [[package]] name = "impl-more" -version = "0.1.9" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8a5a9a0ff0086c7a148acb942baaabeadf9504d10400b5a05645853729b9cd2" +checksum = "30c0cddce6b7505483307994f60d97c4b156020e1504d0cf1f05a21b1b325e64" [[package]] name = "indexmap" -version = "2.14.0" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", - "hashbrown 0.17.1", - "serde", - "serde_core", + "hashbrown", ] [[package]] name = "ipnet" -version = "2.12.0" +version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" [[package]] name = "is-docker" @@ -1539,21 +1483,11 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" -[[package]] -name = "jobserver" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" -dependencies = [ - "getrandom 0.3.4", - "libc", -] - [[package]] name = "js-sys" -version = "0.3.102" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" dependencies = [ "cfg-if", "futures-util", @@ -1576,10 +1510,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" [[package]] -name = "leb128fmt" -version = "0.1.0" +name = "lazy_static" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libbz2-rs-sys" @@ -1589,9 +1523,9 @@ checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfs" @@ -1604,15 +1538,15 @@ dependencies = [ "linux-raw-sys", "log", "rustix", - "thiserror 2.0.18", + "thiserror 2.0.21", "xattr", ] [[package]] name = "libredox" -version = "0.1.17" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" dependencies = [ "libc", ] @@ -1625,20 +1559,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" - -[[package]] -name = "local-channel" -version = "0.1.5" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6cbc85e69b8df4b8bb8b89ec634e7189099cea8927a276b7384ce5488e53ec8" -dependencies = [ - "futures-core", - "futures-sink", - "local-waker", -] +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" [[package]] name = "local-waker" @@ -1657,9 +1580,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.32" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lzma-sys" @@ -1682,14 +1605,14 @@ dependencies = [ "bitvec", "serde", "serde-big-array", - "thiserror 2.0.18", + "thiserror 2.0.21", ] [[package]] name = "memchr" -version = "2.8.2" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "mime" @@ -1697,11 +1620,17 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" -version = "0.8.9" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" dependencies = [ "adler2", "simd-adler32", @@ -1709,9 +1638,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.1" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" dependencies = [ "libc", "log", @@ -1719,6 +1648,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + [[package]] name = "native-tls" version = "0.2.18" @@ -1736,18 +1671,47 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "normalize-line-endings" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1773,11 +1737,11 @@ version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" dependencies = [ - "base64", + "base64 0.22.1", "chrono", "getrandom 0.2.17", - "http 1.4.2", - "rand 0.8.6", + "http 1.5.0", + "rand", "reqwest 0.12.28", "serde", "serde_json", @@ -1787,6 +1751,15 @@ dependencies = [ "url", ] +[[package]] +name = "oid-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bedf36ffb6ba96c2eb7144ef6270557b52e54b20c0a8e1eb2ff99a6c6959bff" +dependencies = [ + "asn1-rs", +] + [[package]] name = "omnect-cli" version = "0.28.3" @@ -1795,7 +1768,7 @@ dependencies = [ "anyhow", "assert-json-diff", "assert_cmd", - "base64", + "base64 0.23.1", "bzip2", "clap", "data-encoding", @@ -1817,7 +1790,7 @@ dependencies = [ "open", "predicates", "regex", - "reqwest 0.13.4", + "reqwest 0.13.5", "ring", "serde", "serde_json", @@ -1839,13 +1812,17 @@ dependencies = [ [[package]] name = "omnect-crypto" -version = "0.4.0" -source = "git+https://github.com/omnect/omnect-crypto.git?tag=0.4.0#2cf5fb05c514c9a8d3d7c4f2cdd4d4eeaaeef485" +version = "0.5.0" +source = "git+https://github.com/omnect/omnect-crypto.git?tag=0.5.0#128777f9946d9a6d31546da36fd500a700ecdee9" dependencies = [ "anyhow", + "byteorder", + "der", + "der-parser", "log", "openssl", "openssl-sys", + "x509-parser", ] [[package]] @@ -1856,13 +1833,12 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "open" -version = "5.3.5" +version = "5.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fbaa89d2ddc8473c78a3adf69eea8cffa28c483b8e02a971ef31527cd0fc92c" +checksum = "aa576c76302b7b808eecc68061e67336c47833ef9d22caa74dda10fa9675eebc" dependencies = [ "is-wsl", "libc", - "pathdiff", ] [[package]] @@ -1887,7 +1863,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1952,12 +1928,6 @@ dependencies = [ "smallvec", ] -[[package]] -name = "pathdiff" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" - [[package]] name = "percent-encoding" version = "2.3.2" @@ -1972,15 +1942,15 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pkg-config" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "potential_utf" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ "zerovec", ] @@ -2031,61 +2001,45 @@ dependencies = [ ] [[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn", -] - -[[package]] -name = "proc-macro-error-attr2" -version = "2.0.0" +name = "proc-macro-error-attr3" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +checksum = "9e564d14133360e1ae169ffde5da25881b5fa47261665b8e5713c212c27799da" dependencies = [ "proc-macro2", "quote", ] [[package]] -name = "proc-macro-error2" -version = "2.0.1" +name = "proc-macro-error3" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +checksum = "8f0d4471b3436c22106b21913b1dda531558918ae9b7ec55d58aa84b43552233" dependencies = [ - "proc-macro-error-attr2", + "proc-macro-error-attr3", "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.45" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - [[package]] name = "r-efi" version = "6.0.0" @@ -2100,24 +2054,13 @@ checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" [[package]] name = "rand" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" dependencies = [ "libc", "rand_chacha", - "rand_core 0.6.4", -] - -[[package]] -name = "rand" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" -dependencies = [ - "chacha20", - "getrandom 0.4.2", - "rand_core 0.10.1", + "rand_core", ] [[package]] @@ -2127,7 +2070,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.4", + "rand_core", ] [[package]] @@ -2139,12 +2082,6 @@ dependencies = [ "getrandom 0.2.17", ] -[[package]] -name = "rand_core" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" - [[package]] name = "redox_syscall" version = "0.5.18" @@ -2156,20 +2093,19 @@ dependencies = [ [[package]] name = "redox_users" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +checksum = "60dc65c0ff1a7ae1294b0c67b9f14baf70b644404010370171787bfac1038fc0" dependencies = [ - "getrandom 0.2.17", "libredox", - "thiserror 2.0.18", + "thiserror 2.0.21", ] [[package]] name = "regex" -version = "1.12.4" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -2179,9 +2115,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -2206,10 +2142,10 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", - "http 1.4.2", + "http 1.5.0", "http-body", "http-body-util", "hyper", @@ -2238,14 +2174,14 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" dependencies = [ - "base64", + "base64 0.23.1", "bytes", "futures-core", - "http 1.4.2", + "http 1.5.0", "http-body", "http-body-util", "hyper", @@ -2294,11 +2230,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + [[package]] name = "rustix" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" dependencies = [ "bitflags", "errno", @@ -2309,18 +2254,18 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.1" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ "zeroize", ] [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "ryu" @@ -2374,9 +2319,9 @@ checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -2393,29 +2338,29 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "itoa", "memchr", @@ -2468,26 +2413,15 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" dependencies = [ "cfg-if", "cpufeatures 0.2.17", "digest 0.10.7", ] -[[package]] -name = "sha1" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" -dependencies = [ - "cfg-if", - "cpufeatures 0.3.0", - "digest 0.11.3", -] - [[package]] name = "sha2" version = "0.10.9" @@ -2506,7 +2440,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", "digest 0.11.3", ] @@ -2528,9 +2462,15 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.9" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simdutf8" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "similar" @@ -2546,25 +2486,15 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.2" +version = "1.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" [[package]] name = "socket2" -version = "0.5.10" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - -[[package]] -name = "socket2" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", "windows-sys 0.61.2", @@ -2609,14 +2539,36 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.119", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", ] [[package]] name = "syn" -version = "2.0.117" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" dependencies = [ "proc-macro2", "quote", @@ -2634,13 +2586,25 @@ dependencies = [ [[package]] name = "synstructure" -version = "0.13.2" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "unicode-xid", +] + +[[package]] +name = "synstructure" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] @@ -2676,7 +2640,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", @@ -2699,11 +2663,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" dependencies = [ - "thiserror-impl 2.0.18", + "thiserror-impl 2.0.21", ] [[package]] @@ -2714,25 +2678,25 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "time" -version = "0.3.49" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" dependencies = [ "deranged", "num-conv", @@ -2750,9 +2714,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.29" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71c652a3727a9cbb9a02f707f530b618ce00d0ccd762009c8c23bd191df3c17d" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" dependencies = [ "num-conv", "time-core", @@ -2760,9 +2724,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", "zerovec", @@ -2770,9 +2734,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -2780,20 +2744,20 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.4", + "socket2", "tokio-macros", "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.7.0" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] @@ -2808,22 +2772,23 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", "pin-project-lite", "tokio", ] [[package]] name = "toml" -version = "1.1.2+spec-1.1.0" +version = "1.1.6+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" +checksum = "920602543f0911ab71da12c50d59701da54c196d1a2bf5cb4b75667f137a406a" dependencies = [ "serde_core", "serde_spanned", @@ -2843,9 +2808,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.1.2+spec-1.1.0" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" dependencies = [ "winnow", ] @@ -2874,7 +2839,7 @@ dependencies = [ "bitflags", "bytes", "futures-util", - "http 1.4.2", + "http 1.5.0", "http-body", "pin-project-lite", "tower", @@ -2915,7 +2880,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2941,9 +2906,9 @@ checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "unicode-ident" -version = "1.0.24" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" [[package]] name = "unicode-segmentation" @@ -2996,21 +2961,20 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.23.3" +version = "1.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" +checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" dependencies = [ - "getrandom 0.4.2", + "getrandom 0.4.3", ] [[package]] name = "validator" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43fb22e1a008ece370ce08a3e9e4447a910e92621bb49b85d6e48a45397e7cfa" +checksum = "c3d68c6633c483df6780cc5277a417c7c2d1bceee2649d06c8ab6b0fd2dd3c81" dependencies = [ "idna", - "once_cell", "regex", "serde", "serde_derive", @@ -3021,16 +2985,15 @@ dependencies = [ [[package]] name = "validator_derive" -version = "0.20.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7df16e474ef958526d1205f6dda359fdfab79d9aa6d54bafcb92dcd07673dca" +checksum = "240e4b81c20a1d6d50d1d7265c658dfbd204e8b9ac4d80f3c931f39462196335" dependencies = [ - "darling 0.20.11", - "once_cell", - "proc-macro-error2", + "darling 0.23.0", + "proc-macro-error3", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3075,29 +3038,11 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" -[[package]] -name = "wasip2" -version = "1.0.4+wasi-0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" -dependencies = [ - "wit-bindgen 0.57.1", -] - -[[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" -dependencies = [ - "wit-bindgen 0.51.0", -] - [[package]] name = "wasm-bindgen" -version = "0.2.125" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" dependencies = [ "cfg-if", "once_cell", @@ -3108,19 +3053,20 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.75" +version = "0.4.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" dependencies = [ "js-sys", + "tokio", "wasm-bindgen", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.125" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3128,65 +3074,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.125" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 3.0.6", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.125" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" dependencies = [ "unicode-ident", ] -[[package]] -name = "wasm-encoder" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" -dependencies = [ - "leb128fmt", - "wasmparser", -] - -[[package]] -name = "wasm-metadata" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" -dependencies = [ - "anyhow", - "indexmap", - "wasm-encoder", - "wasmparser", -] - -[[package]] -name = "wasmparser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" -dependencies = [ - "bitflags", - "hashbrown 0.15.5", - "indexmap", - "semver", -] - [[package]] name = "web-sys" -version = "0.3.102" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" dependencies = [ "js-sys", "wasm-bindgen", @@ -3200,7 +3112,7 @@ checksum = "d5cec722a3274e47d1524cbe2cea762f2c19d615bd9d73ada21db9066349d57e" dependencies = [ "proc-macro2", "quote", - "thiserror 2.0.18", + "thiserror 2.0.21", "wincode-derive", ] @@ -3213,7 +3125,7 @@ dependencies = [ "darling 0.21.3", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3237,7 +3149,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3248,7 +3160,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3359,109 +3271,15 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" - -[[package]] -name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - -[[package]] -name = "wit-bindgen" -version = "0.57.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" - -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck", - "indexmap", - "prettyplease", - "syn", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags", - "indexmap", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.244.0" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" -dependencies = [ - "anyhow", - "id-arena", - "indexmap", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", -] +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" [[package]] name = "writeable" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "wyz" @@ -3472,6 +3290,23 @@ dependencies = [ "tap", ] +[[package]] +name = "x509-parser" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7069fba5b66b9193bd2c5d3d4ff12b839118f6bcbef5328efafafb5395cf63da" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + [[package]] name = "xattr" version = "1.6.1" @@ -3504,34 +3339,34 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78" dependencies = [ "proc-macro2", "quote", - "syn", - "synstructure", + "syn 3.0.6", + "synstructure 0.14.0", ] [[package]] name = "zerocopy" -version = "0.8.52" +version = "0.8.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +checksum = "6df92bf3d9227be3d53173901ddbffac2babc27ae50f397776ffd6dc33f800cb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.52" +version = "0.8.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +checksum = "ac4f328cf2f05d084e496c3e9c3f33ed0a183656a16e1fcec4d464d8373aec82" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3545,14 +3380,14 @@ dependencies = [ [[package]] name = "zerofrom-derive" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" dependencies = [ "proc-macro2", "quote", - "syn", - "synstructure", + "syn 3.0.6", + "synstructure 0.14.0", ] [[package]] @@ -3563,9 +3398,9 @@ checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zerotrie" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", "yoke", @@ -3574,9 +3409,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ "yoke", "zerofrom", @@ -3585,45 +3420,17 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "zmij" -version = "1.0.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" - -[[package]] -name = "zstd" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "7.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" -dependencies = [ - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" -dependencies = [ - "cc", - "pkg-config", -] +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml index f2f9ebf2..e1cccc79 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,11 +10,11 @@ repository = "https://github.com/omnect/omnect-cli" version = "0.28.3" [dependencies] -actix-web = "4.11" +actix-web = { version = "4.15", default-features = false, features = ["macros"] } anyhow = "1" -base64 = { version = "0.22", default-features = false } +base64 = { version = "0.23", default-features = false } bzip2 = { version = "0.6", default-features = true } -clap = { version = "4.5", default-features = false, features = [ +clap = { version = "4.6", default-features = false, features = [ "derive", "help", "std", @@ -25,11 +25,11 @@ env_logger = { version = "0.11", default-features = false } filemagic = { version = "0.13", default-features = false, features = [ "pkg-config", ] } -flate2 = { version = "1.1", default-features = false } +flate2 = { version = "1.1", default-features = false, features = ["rust_backend"] } gptman = { version = "3.1", default-features = false } hmac = { version = "0.13", default-features = false } mbrman = { version = "0.6", default-features = false } -omnect-crypto = { git = "https://github.com/omnect/omnect-crypto.git", tag = "0.4.0" } +omnect-crypto = { git = "https://github.com/omnect/omnect-crypto.git", tag = "0.5.0" } keyring-core = { version = "1.0", default-features = false } libfs = { version = "0.9", default-features = false } log = { version = "0.4", default-features = false } @@ -38,8 +38,8 @@ oauth2 = { version = "5.0", default-features = false, features = [ "reqwest", "native-tls", ] } -open = { version = "5.3", default-features = false } -regex = { version = "1.11", default-features = false } +open = { version = "5.4", default-features = false } +regex = { version = "1.13", default-features = false } reqwest = { version = "0.13", default-features = false, features = [ "json", "native-tls", @@ -51,28 +51,28 @@ sha2 = { version = "0.11", default-features = false } stdext = { version = "0.3", default-features = false } strum = { version = "0.28", default-features = false } strum_macros = { version = "0.28", default-features = false } -tempfile = { version = "3.20", default-features = false } +tempfile = { version = "3.27", default-features = false } time = { version = "0.3", default-features = false } tokio = { version = "1", default-features = false, features = [ "macros", "fs", "rt-multi-thread", ] } -toml = { version = "1.0", default-features = false, features = [ +toml = { version = "1.1", default-features = false, features = [ "parse", "serde", ] } -uuid = { version = "1.17", default-features = false, features = ["v4"] } +uuid = { version = "1.26", default-features = false, features = ["v4"] } url = { version = "2.5", default-features = false } -validator = { version = "0.20", default-features = false, features = [ +validator = { version = "0.21", default-features = false, features = [ "derive", ] } xz2 = { version = "0.1", default-features = false } [dev-dependencies] -assert_cmd = "2.0" +assert_cmd = "2.2" assert-json-diff = "2.0" -data-encoding = "2.9" +data-encoding = "2.11" file_diff = "1.0" httpmock = "0.8" predicates = "3.1" diff --git a/src/lib.rs b/src/lib.rs index 11eeaa62..343f912e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -175,7 +175,7 @@ fn create_image_cert(image: &Path, cert_opts: CertificateOptions) -> Result Date: Mon, 28 Sep 2026 13:12:50 +0200 Subject: [PATCH 03/75] style(device_update): drop redundant borrows in info! arguments Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/device_update/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/device_update/mod.rs b/src/device_update/mod.rs index 8e2f6da3..761aa5c0 100644 --- a/src/device_update/mod.rs +++ b/src/device_update/mod.rs @@ -292,7 +292,7 @@ pub async fn import_update( debug!("import update: {import_update}"); let import_update_response = adu_client.import_update(instance_id, import_update).await?; - info!("Result of import update: {:?}", &import_update_response); + info!("Result of import update: {:?}", import_update_response); Ok(()) } @@ -321,7 +321,7 @@ pub async fn remove_update( let remove_update_response = adu_client .delete_update(instance_id, provider, name, version) .await?; - info!("Result of remove update: {:?}", &remove_update_response); + info!("Result of remove update: {:?}", remove_update_response); Ok(()) } From e9163a560b190610973acbcd3902f41a046001d1 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:01:01 +0200 Subject: [PATCH 04/75] fix(file): keep holes when copying the image libfs was built with `default-features = false`, which also drops its `use_linux` feature. Without it libfs copies byte by byte, so every image omnect-cli works on came back fully allocated, and a bmap created with `-b` mapped the whole image. The `fallocate -d` that write_partition used to run hid this. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- Cargo.toml | 2 +- tests/integration_tests.rs | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index e1cccc79..6451b69d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,7 @@ hmac = { version = "0.13", default-features = false } mbrman = { version = "0.6", default-features = false } omnect-crypto = { git = "https://github.com/omnect/omnect-crypto.git", tag = "0.5.0" } keyring-core = { version = "1.0", default-features = false } -libfs = { version = "0.9", default-features = false } +libfs = { version = "0.9", default-features = false, features = ["use_linux"] } log = { version = "0.4", default-features = false } num_cpus = { version = "1.17", default-features = false } oauth2 = { version = "5.0", default-features = false, features = [ diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index e51b00c0..4052768b 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -947,6 +947,40 @@ fn check_file_copy_keeps_image_layout() { ); } +#[test] +fn check_file_copy_keeps_holes() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let sparse_path = image_path.with_file_name("sparse.wic"); + let bmap_path = PathBuf::from(format!("{}.bmap", sparse_path.to_string_lossy())); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + + let status = std::process::Command::new("cp") + .arg("--sparse=always") + .arg(&image_path) + .arg(&sparse_path) + .status() + .expect("run cp"); + assert!(status.success(), "sparse copy of the test image failed"); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},factory:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&sparse_path) + .arg("-b") + .assert() + .success(); + + let meta = sparse_path.metadata().expect("image metadata"); + assert!(meta.blocks() * STAT_BLOCK_SIZE < meta.len()); + let bmap = std::fs::read_to_string(&bmap_path).expect("read bmap"); + assert!(bmap_value(&bmap, "MappedBlocksCount") < bmap_value(&bmap, "BlocksCount")); +} + #[test] fn check_bmap_generation_wic() { let tr = Testrunner::new(function_name!().split("::").last().unwrap()); From e2443b265d70789ae496d5b05d6c048f16f7d6ea Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:01:02 +0200 Subject: [PATCH 05/75] fix(file): map the ext4 journals in a generated bmap A fresh ext4 journal is all zeros, and a sparse image, e.g. one that xz decompressed into a file, holds it as holes. A bmap then skips the journal, so a bmap-based flash leaves the journal of the previous installation on the device, and after a power cut its replay can apply an old transaction to the new filesystem. Before creating the bmap, rewrite the journal blocks of every ext4 partition with their own content. Holes become allocated zero blocks and all other blocks stay unchanged. debugfs lists the journal blocks, so the package now depends on e2fsprogs. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- Cargo.toml | 2 +- src/file/functions.rs | 226 ++++++++++++++++++++++++++++++++++++++++++ src/file/partition.rs | 36 +++++++ 3 files changed, 263 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 6451b69d..b9efc7d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -81,5 +81,5 @@ tar = "0.4" # metadata for building with cargo-deb (https://crates.io/crates/cargo-deb) [package.metadata.deb] -depends = "bmap-tools, e2tools, keychain, libc6 (>= 2.34), libmagic1, libssl3 (>= 3.0.0), mtools" +depends = "bmap-tools, e2fsprogs, e2tools, keychain, libc6 (>= 2.34), libmagic1, libssl3 (>= 3.0.0), mtools" revision = "" diff --git a/src/file/functions.rs b/src/file/functions.rs index 49a486c7..800ff5a4 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -14,6 +14,16 @@ const SECTOR_SIZE: u64 = 512; const COMPARE_BLOCK_SIZE: usize = 4096; const COPY_CHUNK_SIZE: usize = 1024 * 1024; +const EXT4_SUPERBLOCK_OFFSET: u64 = 1024; +const EXT4_SUPERBLOCK_SIZE: usize = 1024; +const EXT4_S_LOG_BLOCK_SIZE: usize = 0x18; +const EXT4_S_MAGIC: usize = 0x38; +const EXT4_S_FEATURE_COMPAT: usize = 0x5c; +const EXT4_S_JOURNAL_INUM: usize = 0xe0; +const EXT4_MAGIC: u16 = 0xef53; +const EXT4_FEATURE_COMPAT_HAS_JOURNAL: u32 = 0x4; +const EXT4_MIN_BLOCK_SIZE: u64 = 1024; + #[derive(clap::ValueEnum, Debug, Clone, Eq, Hash, PartialEq)] #[clap(rename_all = "verbatim")] #[allow(non_camel_case_types)] @@ -485,7 +495,113 @@ fn write_partition( .context("write_partition: cannot sync image") } +struct Ext4Journal { + inode: u32, + block_size: u64, +} + +fn read_ext4_journal(image: &fs::File, offset: u64) -> Result> { + let mut sb = [0u8; EXT4_SUPERBLOCK_SIZE]; + image + .read_exact_at(&mut sb, offset + EXT4_SUPERBLOCK_OFFSET) + .context("read_ext4_journal: cannot read superblock")?; + let le16 = |o: usize| u16::from_le_bytes([sb[o], sb[o + 1]]); + let le32 = |o: usize| u32::from_le_bytes([sb[o], sb[o + 1], sb[o + 2], sb[o + 3]]); + + if le16(EXT4_S_MAGIC) != EXT4_MAGIC + || le32(EXT4_S_FEATURE_COMPAT) & EXT4_FEATURE_COMPAT_HAS_JOURNAL == 0 + { + return Ok(None); + } + + // an external journal has no inode in this filesystem + let inode = le32(EXT4_S_JOURNAL_INUM); + if inode == 0 { + return Ok(None); + } + + let block_size = EXT4_MIN_BLOCK_SIZE + .checked_shl(le32(EXT4_S_LOG_BLOCK_SIZE)) + .context("read_ext4_journal: invalid block size")?; + + Ok(Some(Ext4Journal { inode, block_size })) +} + +fn ext4_journal_blocks(image_file: &str, offset: u64, inode: u32) -> Result> { + let mut debugfs = Command::new("debugfs"); + debugfs + .arg("-R") + .arg(format!("blocks <{inode}>")) + .arg(format!("{image_file}?offset={offset}")); + let output = debugfs + .output() + .context(format!("ext4_journal_blocks: cannot run {debugfs:?}"))?; + anyhow::ensure!( + output.status.success(), + "ext4_journal_blocks: {debugfs:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let blocks = String::from_utf8(output.stdout) + .context("ext4_journal_blocks: invalid debugfs output")? + .split_whitespace() + .map(|b| { + b.parse::() + .context(format!("ext4_journal_blocks: invalid block number {b}")) + }) + .collect::>>()?; + anyhow::ensure!( + !blocks.is_empty(), + "ext4_journal_blocks: no journal blocks at offset {offset}" + ); + + Ok(blocks) +} + +/// Rewrites the journal blocks of every ext4 partition with their own content. Blocks that +/// are holes become allocated, so a bmap maps the whole journal and a bmap-based flash +/// overwrites the journal an earlier installation left on the device. +fn allocate_ext4_journals(image_file: &str) -> Result<()> { + use crate::file::partition::get_partitions; + + let image = fs::OpenOptions::new() + .read(true) + .write(true) + .open(image_file) + .context(format!("allocate_ext4_journals: cannot open {image_file}"))?; + + for partition in get_partitions(image_file)? { + let offset = partition.start * SECTOR_SIZE; + let Some(journal) = read_ext4_journal(&image, offset)? else { + continue; + }; + let blocks = ext4_journal_blocks(image_file, offset, journal.inode)?; + debug!( + "allocate_ext4_journals: partition {}: {} journal blocks", + partition.num, + blocks.len() + ); + + let mut buf = vec![0u8; usize::try_from(journal.block_size)?]; + for block in blocks { + let pos = offset + block * journal.block_size; + image + .read_exact_at(&mut buf, pos) + .context("allocate_ext4_journals: cannot read journal block")?; + image + .write_all_at(&buf, pos) + .context("allocate_ext4_journals: cannot write journal block")?; + } + } + + image + .sync_all() + .context("allocate_ext4_journals: cannot sync image") +} + pub fn generate_bmap_file(image_file: &str) -> Result<()> { + allocate_ext4_journals(image_file)?; + let mut bmaptool = Command::new("bmaptool"); bmaptool .arg("create") @@ -496,3 +612,113 @@ pub fn generate_bmap_file(image_file: &str) -> Result<()> { Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + const PART_START_LBA: u32 = 2048; + const PART_SECTORS: u32 = 16384; + const FS_BLOCK_SIZE: u64 = 1024; + const MBR_DISK_SIGNATURE: [u8; 4] = [1, 2, 3, 4]; + const MBR_TYPE_LINUX: u8 = 0x83; + const BMAP_BLOCK_SIZE: u64 = 4096; + + fn run(cmd: &mut Command) { + let status = cmd.status().expect("run command"); + assert!(status.success(), "{cmd:?} failed"); + } + + fn create_image_with_ext4_journal(image: &Path) { + let raw = image.with_extension("raw"); + let mut f = fs::File::create(&raw).expect("create image"); + f.set_len(u64::from(PART_START_LBA + PART_SECTORS) * SECTOR_SIZE) + .expect("set image size"); + let mut mbr = mbrman::MBR::new_from(&mut f, SECTOR_SIZE as u32, MBR_DISK_SIGNATURE) + .expect("create mbr"); + mbr[1] = mbrman::MBRPartitionEntry { + boot: mbrman::BOOT_INACTIVE, + first_chs: mbrman::CHS::empty(), + sys: MBR_TYPE_LINUX, + last_chs: mbrman::CHS::empty(), + starting_lba: PART_START_LBA, + sectors: PART_SECTORS, + }; + mbr.write_into(&mut f).expect("write mbr"); + + let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; + let fs_blocks = u64::from(PART_SECTORS) * SECTOR_SIZE / FS_BLOCK_SIZE; + run(Command::new("mkfs.ext4") + .arg("-q") + .arg("-F") + .arg("-b") + .arg(FS_BLOCK_SIZE.to_string()) + .arg("-J") + .arg("size=1") + .arg("-E") + .arg(format!("offset={offset}")) + .arg(&raw) + .arg(fs_blocks.to_string())); + // mke2fs zeroes the journal as unwritten extents, which a bmap maps; a sparse copy + // turns the zeros into holes, as a sparse decompress does + run(Command::new("cp") + .arg("--sparse=always") + .arg(&raw) + .arg(image)); + } + + fn bmap_ranges(bmap: &str) -> Vec<(u64, u64)> { + bmap.lines() + .filter_map(|l| { + let range = l.split_once("')?.1; + let range = range.split_once(" usize { + let ranges = bmap_ranges(bmap); + let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; + let file = fs::File::open(image).expect("open image"); + let journal = read_ext4_journal(&file, offset) + .expect("read superblock") + .expect("ext4 with journal"); + ext4_journal_blocks(image, offset, journal.inode) + .expect("journal blocks") + .iter() + .map(|b| (offset + b * journal.block_size) / BMAP_BLOCK_SIZE) + .filter(|b| !ranges.iter().any(|(s, e)| s <= b && b <= e)) + .count() + } + + #[test] + fn generate_bmap_file_maps_ext4_journal() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_path = dir.path().join("image.wic"); + create_image_with_ext4_journal(&image_path); + let image = image_path.to_str().expect("image path"); + let bmap_path = format!("{image}.bmap"); + + run(Command::new("bmaptool") + .arg("create") + .arg("-o") + .arg(&bmap_path) + .arg(image)); + let bmap = fs::read_to_string(&bmap_path).expect("read bmap"); + assert!(unmapped_journal_blocks(image, &bmap) > 0); + + generate_bmap_file(image).expect("generate bmap"); + let bmap = fs::read_to_string(&bmap_path).expect("read bmap"); + assert_eq!(unmapped_journal_blocks(image, &bmap), 0); + + let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; + run(Command::new("e2fsck") + .arg("-fn") + .arg(format!("{image}?offset={offset}"))); + } +} diff --git a/src/file/partition.rs b/src/file/partition.rs index 04d2fc6d..cc7b28a3 100644 --- a/src/file/partition.rs +++ b/src/file/partition.rs @@ -55,6 +55,42 @@ pub fn get_partition_data>(path: P, partition_num: u32) -> Result anyhow::bail!("partition {partition_num} not found in image") } +pub fn get_partitions>(path: P) -> Result> { + let path = path.as_ref(); + let mut file = + File::open(path).with_context(|| format!("failed to open image: {}", path.display()))?; + + let gpt_err = match gptman::GPT::find_from(&mut file) { + Ok(gpt) => { + return Ok(gpt + .iter() + .filter(|(_, e)| e.is_used() && e.ending_lba >= e.starting_lba) + .map(|(num, e)| PartitionData { + num, + start: e.starting_lba, + count: e.ending_lba - e.starting_lba + 1, + }) + .collect()); + } + Err(e) => e, + }; + + file.seek(SeekFrom::Start(0)) + .context("failed to seek to start of image")?; + let mbr = mbrman::MBR::read_from(&mut file, 512) + .with_context(|| format!("image is neither valid GPT nor MBR (GPT error: {gpt_err})"))?; + mbr.iter() + .filter(|(_, p)| p.is_used()) + .map(|(num, p)| { + Ok(PartitionData { + num: u32::try_from(num).context("MBR partition number out of range")?, + start: p.starting_lba as u64, + count: p.sectors as u64, + }) + }) + .collect() +} + pub fn is_gpt>(path: P) -> Result { let mut file = File::open(path.as_ref()).context("is_gpt: failed to open image")?; Ok(gptman::GPT::find_from(&mut file).is_ok()) From 52fe21d5cbfb87bed1ceaae052b09f0fd38a57da Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:05:32 +0200 Subject: [PATCH 06/75] fix: copy image data segments larger than 2 GiB completely libfs::copy_file copies each data segment with one copy_file_range call and ignores a short copy, so a segment over 2 GiB was cut off without error. copy_image keeps the holes and loops until each segment is copied. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/lib.rs | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 343f912e..a8ce6bf4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,7 +22,7 @@ use cli::{ }; use file::{compression::Compression, functions::FileCopyToParams}; use log::error; -use std::{fs, path::Path, path::PathBuf}; +use std::{fs, io::Read, path::Path, path::PathBuf}; use tokio::fs::remove_dir_all; use uuid::Uuid; @@ -93,9 +93,8 @@ where tmp_image_file = compression::decompress(&tmp_image_file, &source_compression)?; dest_image_file.set_extension(""); } else { - // copy sparse file (std::fs::copy isn't able) - libfs::copy_file(&image_file, &tmp_image_file).context(format!( - "error: libfs::copy_file({:?}, {:?})", + copy_image(&image_file, &tmp_image_file).context(format!( + "error: copy_image({:?}, {:?})", image_file, tmp_image_file ))?; } @@ -140,9 +139,8 @@ where tmp_image_file, dest_image_file ))?; } else { - // copy sparse file (std::fs::copy isn't able) - libfs::copy_file(&tmp_image_file, &dest_image_file).context(format!( - "error: libfs::copy_file({:?}, {:?})", + copy_image(&tmp_image_file, &dest_image_file).context(format!( + "error: copy_image({:?}, {:?})", tmp_image_file, dest_image_file ))?; } @@ -150,6 +148,35 @@ where Ok(()) } +/// Copies `from` to `to` and keeps the holes of a sparse `from`. +fn copy_image(from: &Path, to: &Path) -> Result<()> { + let src = fs::File::open(from).context("copy_image: cannot open source")?; + let len = src + .metadata() + .context("copy_image: cannot get source size")? + .len(); + let dst = fs::File::create(to).context("copy_image: cannot create destination")?; + dst.set_len(len) + .context("copy_image: cannot set destination size")?; + + // one copy_file_range call copies at most 2 GiB; io::copy repeats it until a segment is done + let sparse = libfs::probably_sparse(&src)?; + let mut pos = 0; + while pos < len { + let (data, hole) = if sparse { + libfs::next_sparse_segments(&src, &dst, pos)? + } else { + (pos, len) + }; + let copied = std::io::copy(&mut (&src).take(hole - data), &mut &dst) + .context("copy_image: cannot copy data")?; + anyhow::ensure!(copied == hole - data, "copy_image: source ended early"); + pos = hole; + } + + Ok(()) +} + struct CertInfo { cert_path: PathBuf, key_path: PathBuf, From 19bcd306f3966d384bb029b9b99de70515a7078c Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:05:33 +0200 Subject: [PATCH 07/75] fix(file): harden ext4 journal lookup for bmap generation - fall back to /usr/sbin/debugfs when debugfs is not in PATH - reject an invalid s_log_block_size before it is used as a shift - skip partitions whose superblock is past the end of the image - skip the MBR extended container entry - share SECTOR_SIZE between partition and functions Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/functions.rs | 49 +++++++++++++++++++++++++++++++------------ src/file/partition.rs | 8 ++++--- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/src/file/functions.rs b/src/file/functions.rs index 800ff5a4..241b0dd2 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -1,3 +1,4 @@ +use crate::file::partition::SECTOR_SIZE; use anyhow::{Context, Result}; use log::{debug, warn}; use std::collections::HashMap; @@ -10,7 +11,6 @@ use std::str::FromStr; use stdext::function_name; use uuid::Uuid; -const SECTOR_SIZE: u64 = 512; const COMPARE_BLOCK_SIZE: usize = 4096; const COPY_CHUNK_SIZE: usize = 1024 * 1024; @@ -23,6 +23,11 @@ const EXT4_S_JOURNAL_INUM: usize = 0xe0; const EXT4_MAGIC: u16 = 0xef53; const EXT4_FEATURE_COMPAT_HAS_JOURNAL: u32 = 0x4; const EXT4_MIN_BLOCK_SIZE: u64 = 1024; +const EXT4_MAX_LOG_BLOCK_SIZE: u32 = 6; + +const DEBUGFS: &str = "debugfs"; +// debugfs is installed to sbin, which is not always in the PATH of a normal user +const DEBUGFS_SBIN: &str = "/usr/sbin/debugfs"; #[derive(clap::ValueEnum, Debug, Clone, Eq, Hash, PartialEq)] #[clap(rename_all = "verbatim")] @@ -520,22 +525,33 @@ fn read_ext4_journal(image: &fs::File, offset: u64) -> Result Result> { - let mut debugfs = Command::new("debugfs"); - debugfs - .arg("-R") - .arg(format!("blocks <{inode}>")) - .arg(format!("{image_file}?offset={offset}")); - let output = debugfs - .output() - .context(format!("ext4_journal_blocks: cannot run {debugfs:?}"))?; + let debugfs_cmd = |program| { + let mut cmd = Command::new(program); + cmd.arg("-R") + .arg(format!("blocks <{inode}>")) + .arg(format!("{image_file}?offset={offset}")); + cmd + }; + let mut debugfs = debugfs_cmd(DEBUGFS); + let output = match debugfs.output() { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + debugfs = debugfs_cmd(DEBUGFS_SBIN); + debugfs.output() + } + output => output, + } + .context(format!("ext4_journal_blocks: cannot run {debugfs:?}"))?; anyhow::ensure!( output.status.success(), "ext4_journal_blocks: {debugfs:?} failed: {}", @@ -569,9 +585,16 @@ fn allocate_ext4_journals(image_file: &str) -> Result<()> { .write(true) .open(image_file) .context(format!("allocate_ext4_journals: cannot open {image_file}"))?; + let image_len = image + .metadata() + .context("allocate_ext4_journals: cannot get image size")? + .len(); for partition in get_partitions(image_file)? { let offset = partition.start * SECTOR_SIZE; + if offset + EXT4_SUPERBLOCK_OFFSET + EXT4_SUPERBLOCK_SIZE as u64 > image_len { + continue; + } let Some(journal) = read_ext4_journal(&image, offset)? else { continue; }; @@ -615,7 +638,7 @@ pub fn generate_bmap_file(image_file: &str) -> Result<()> { #[cfg(test)] mod tests { - use super::*; + use crate::file::functions::*; const PART_START_LBA: u32 = 2048; const PART_SECTORS: u32 = 16384; diff --git a/src/file/partition.rs b/src/file/partition.rs index cc7b28a3..9d9d0b8c 100644 --- a/src/file/partition.rs +++ b/src/file/partition.rs @@ -4,6 +4,8 @@ use std::io::Seek; use std::io::SeekFrom; use std::path::Path; +pub const SECTOR_SIZE: u64 = 512; + pub struct PartitionData { pub num: u32, pub start: u64, @@ -41,7 +43,7 @@ pub fn get_partition_data>(path: P, partition_num: u32) -> Result // Try MBR — iter() includes logical partitions (5, 6, ...) file.seek(SeekFrom::Start(0)) .context("failed to seek to start of image")?; - let mbr = mbrman::MBR::read_from(&mut file, 512) + let mbr = mbrman::MBR::read_from(&mut file, SECTOR_SIZE as u32) .with_context(|| format!("image is neither valid GPT nor MBR (GPT error: {gpt_err})"))?; for (i, p) in mbr.iter() { if u32::try_from(i).ok() == Some(partition_num) && p.is_used() { @@ -77,10 +79,10 @@ pub fn get_partitions>(path: P) -> Result> { file.seek(SeekFrom::Start(0)) .context("failed to seek to start of image")?; - let mbr = mbrman::MBR::read_from(&mut file, 512) + let mbr = mbrman::MBR::read_from(&mut file, SECTOR_SIZE as u32) .with_context(|| format!("image is neither valid GPT nor MBR (GPT error: {gpt_err})"))?; mbr.iter() - .filter(|(_, p)| p.is_used()) + .filter(|(_, p)| p.is_used() && !p.is_extended()) .map(|(num, p)| { Ok(PartitionData { num: u32::try_from(num).context("MBR partition number out of range")?, From d7e724244c2ef4aa10a162ba7ab4ceff2b205186 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:05:34 +0200 Subject: [PATCH 08/75] build(docker): install e2fsprogs in the builder The deb package depends on e2fsprogs for debugfs. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 8eb12929..4fc20c1a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends \ bmap-tools \ ca-certificates \ + e2fsprogs \ e2tools \ fdisk \ keychain \ From 2912452cebd3eb6d2eeeae94d11149d998f708a2 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:13:38 +0200 Subject: [PATCH 09/75] fix(file): keep zero blocks as holes when decompressing Decompress wrote every zero block as data, so a bmap of a compressed input mapped the whole image. The decompressed file now seeks over zero blocks and keeps them as holes. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/compression.rs | 97 ++++++++++++++++++++++++++++++++++++-- tests/integration_tests.rs | 3 ++ 2 files changed, 96 insertions(+), 4 deletions(-) diff --git a/src/file/compression.rs b/src/file/compression.rs index 16f18626..fca98d49 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -3,11 +3,14 @@ use filemagic::Magic; use log::debug; use std::env; use std::fs::File; +use std::io::{Seek, SeekFrom, Write}; use std::path::PathBuf; use std::str::FromStr; use strum::IntoEnumIterator; use strum_macros::EnumIter; +const SPARSE_BLOCK_SIZE: u64 = 4096; + #[derive(Clone, Debug, EnumIter)] #[allow(non_camel_case_types)] pub enum Compression { @@ -76,15 +79,21 @@ impl Compression { source: &mut std::fs::File, destination: &mut std::fs::File, ) -> std::io::Result { - let mut dec: Box = match &self { - Compression::bzip2 => Box::new(bzip2::write::BzDecoder::new(destination)), - Compression::gzip => Box::new(flate2::write::GzDecoder::new(destination)), - Compression::xz { .. } => Box::new(xz2::write::XzDecoder::new(destination)), + let sparse = SparseWriter(destination); + let mut dec: Box = match &self { + Compression::bzip2 => Box::new(bzip2::write::BzDecoder::new(sparse)), + Compression::gzip => Box::new(flate2::write::GzDecoder::new(sparse)), + Compression::xz { .. } => Box::new(xz2::write::XzDecoder::new(sparse)), }; let bytes_written = std::io::copy(source, &mut dec)?; dec.write_all(&[])?; dec.flush()?; + drop(dec); + + // zero blocks at the end were only seeked over + let len = destination.stream_position()?; + destination.set_len(len)?; Ok(bytes_written) } @@ -126,6 +135,27 @@ impl Compression { } } +/// Seeks over zero blocks instead of writing them, so they stay holes in a new file. +struct SparseWriter<'a>(&'a mut File); + +impl Write for SparseWriter<'_> { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let pos = self.0.stream_position()?; + let to_block_end = SPARSE_BLOCK_SIZE - pos % SPARSE_BLOCK_SIZE; + let chunk = &buf[..buf.len().min(to_block_end as usize)]; + if chunk.iter().all(|b| *b == 0) { + self.0.seek(SeekFrom::Current(chunk.len() as i64))?; + Ok(chunk.len()) + } else { + self.0.write(chunk) + } + } + + fn flush(&mut self) -> std::io::Result<()> { + self.0.flush() + } +} + pub fn decompress(image_file_name: &PathBuf, compression: &Compression) -> Result { let mut new_image_file = PathBuf::from(image_file_name); @@ -157,3 +187,62 @@ pub fn compress(image_file_name: &PathBuf, compression: &Compression) -> Result< debug!("image::compress: copied {} bytes.", bytes_written); Ok(new_image_file) } + +#[cfg(test)] +mod tests { + use crate::file::compression::*; + use std::io::Read; + use std::os::unix::fs::MetadataExt; + + // unit of `st_blocks` + const STAT_BLOCK_SIZE: u64 = 512; + const DATA_OFFSET: usize = 1024 * 1024 + 100; + const IMAGE_LEN: usize = 3 * 1024 * 1024 + 123; + + fn roundtrip(compression: Compression, image: &[u8]) -> File { + let mut source = tempfile::tempfile().expect("create source"); + source.write_all(image).expect("write source"); + source.rewind().expect("rewind source"); + + let mut compressed = tempfile::tempfile().expect("create compressed"); + compression + .compress(&mut source, &mut compressed) + .expect("compress"); + compressed.rewind().expect("rewind compressed"); + + let mut decompressed = tempfile::tempfile().expect("create decompressed"); + compression + .decompress(&mut compressed, &mut decompressed) + .expect("decompress"); + decompressed + } + + #[test] + fn decompress_keeps_zero_blocks_as_holes() { + let mut image = vec![0u8; IMAGE_LEN]; + image[DATA_OFFSET..DATA_OFFSET + 3].copy_from_slice(b"abc"); + + for compression in [ + Compression::bzip2, + Compression::gzip, + Compression::xz { + compression_level: 1, + }, + ] { + let mut decompressed = roundtrip(compression.clone(), &image); + + let mut content = Vec::new(); + decompressed.rewind().expect("rewind decompressed"); + decompressed + .read_to_end(&mut content) + .expect("read decompressed"); + assert!(content == image, "{compression:?}: content differs"); + + let meta = decompressed.metadata().expect("metadata"); + assert!( + meta.blocks() * STAT_BLOCK_SIZE < meta.len(), + "{compression:?}: no holes" + ); + } + } +} diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 4052768b..c807429d 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1051,6 +1051,9 @@ fn check_bmap_generation_wic_xz() { .assert(); assert.success(); + let bmap = std::fs::read_to_string(&image_path_bmap).expect("read bmap"); + assert!(bmap_value(&bmap, "MappedBlocksCount") < bmap_value(&bmap, "BlocksCount")); + // use bmaptool to verify that the checksum of the bmap file and the image // still match after the copy operations let assert = Command::new("bmaptool") From 84ba27e6d68445033af07aa6222b7dec49d873e9 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:18:24 +0200 Subject: [PATCH 10/75] refactor(file): move bmap generation into its own module The ext4 journal lookup is only used for bmap generation, so it moves with generate_bmap_file from functions.rs to bmap.rs. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/bmap.rs | 269 ++++++++++++++++++++++++++++++++++++++++++ src/file/functions.rs | 261 ---------------------------------------- src/file/mod.rs | 1 + src/lib.rs | 2 +- 4 files changed, 271 insertions(+), 262 deletions(-) create mode 100644 src/file/bmap.rs diff --git a/src/file/bmap.rs b/src/file/bmap.rs new file mode 100644 index 00000000..e6b6352b --- /dev/null +++ b/src/file/bmap.rs @@ -0,0 +1,269 @@ +use crate::file::partition::{SECTOR_SIZE, get_partitions}; +use anyhow::{Context, Result}; +use log::debug; +use std::fs; +use std::os::unix::fs::FileExt; +use std::process::Command; + +const EXT4_SUPERBLOCK_OFFSET: u64 = 1024; +const EXT4_SUPERBLOCK_SIZE: usize = 1024; +const EXT4_S_LOG_BLOCK_SIZE: usize = 0x18; +const EXT4_S_MAGIC: usize = 0x38; +const EXT4_S_FEATURE_COMPAT: usize = 0x5c; +const EXT4_S_JOURNAL_INUM: usize = 0xe0; +const EXT4_MAGIC: u16 = 0xef53; +const EXT4_FEATURE_COMPAT_HAS_JOURNAL: u32 = 0x4; +const EXT4_MIN_BLOCK_SIZE: u64 = 1024; +const EXT4_MAX_LOG_BLOCK_SIZE: u32 = 6; + +const DEBUGFS: &str = "debugfs"; +// debugfs is installed to sbin, which is not always in the PATH of a normal user +const DEBUGFS_SBIN: &str = "/usr/sbin/debugfs"; + +struct Ext4Journal { + inode: u32, + block_size: u64, +} + +fn read_ext4_journal(image: &fs::File, offset: u64) -> Result> { + let mut sb = [0u8; EXT4_SUPERBLOCK_SIZE]; + image + .read_exact_at(&mut sb, offset + EXT4_SUPERBLOCK_OFFSET) + .context("read_ext4_journal: cannot read superblock")?; + let le16 = |o: usize| u16::from_le_bytes([sb[o], sb[o + 1]]); + let le32 = |o: usize| u32::from_le_bytes([sb[o], sb[o + 1], sb[o + 2], sb[o + 3]]); + + if le16(EXT4_S_MAGIC) != EXT4_MAGIC + || le32(EXT4_S_FEATURE_COMPAT) & EXT4_FEATURE_COMPAT_HAS_JOURNAL == 0 + { + return Ok(None); + } + + // an external journal has no inode in this filesystem + let inode = le32(EXT4_S_JOURNAL_INUM); + if inode == 0 { + return Ok(None); + } + + let log_block_size = le32(EXT4_S_LOG_BLOCK_SIZE); + anyhow::ensure!( + log_block_size <= EXT4_MAX_LOG_BLOCK_SIZE, + "read_ext4_journal: invalid block size" + ); + let block_size = EXT4_MIN_BLOCK_SIZE << log_block_size; + + Ok(Some(Ext4Journal { inode, block_size })) +} + +fn ext4_journal_blocks(image_file: &str, offset: u64, inode: u32) -> Result> { + let debugfs_cmd = |program| { + let mut cmd = Command::new(program); + cmd.arg("-R") + .arg(format!("blocks <{inode}>")) + .arg(format!("{image_file}?offset={offset}")); + cmd + }; + let mut debugfs = debugfs_cmd(DEBUGFS); + let output = match debugfs.output() { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + debugfs = debugfs_cmd(DEBUGFS_SBIN); + debugfs.output() + } + output => output, + } + .context(format!("ext4_journal_blocks: cannot run {debugfs:?}"))?; + anyhow::ensure!( + output.status.success(), + "ext4_journal_blocks: {debugfs:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let blocks = String::from_utf8(output.stdout) + .context("ext4_journal_blocks: invalid debugfs output")? + .split_whitespace() + .map(|b| { + b.parse::() + .context(format!("ext4_journal_blocks: invalid block number {b}")) + }) + .collect::>>()?; + anyhow::ensure!( + !blocks.is_empty(), + "ext4_journal_blocks: no journal blocks at offset {offset}" + ); + + Ok(blocks) +} + +/// Rewrites the journal blocks of every ext4 partition with their own content. Blocks that +/// are holes become allocated, so a bmap maps the whole journal and a bmap-based flash +/// overwrites the journal an earlier installation left on the device. +fn allocate_ext4_journals(image_file: &str) -> Result<()> { + let image = fs::OpenOptions::new() + .read(true) + .write(true) + .open(image_file) + .context(format!("allocate_ext4_journals: cannot open {image_file}"))?; + let image_len = image + .metadata() + .context("allocate_ext4_journals: cannot get image size")? + .len(); + + for partition in get_partitions(image_file)? { + let offset = partition.start * SECTOR_SIZE; + if offset + EXT4_SUPERBLOCK_OFFSET + EXT4_SUPERBLOCK_SIZE as u64 > image_len { + continue; + } + let Some(journal) = read_ext4_journal(&image, offset)? else { + continue; + }; + let blocks = ext4_journal_blocks(image_file, offset, journal.inode)?; + debug!( + "allocate_ext4_journals: partition {}: {} journal blocks", + partition.num, + blocks.len() + ); + + let mut buf = vec![0u8; usize::try_from(journal.block_size)?]; + for block in blocks { + let pos = offset + block * journal.block_size; + image + .read_exact_at(&mut buf, pos) + .context("allocate_ext4_journals: cannot read journal block")?; + image + .write_all_at(&buf, pos) + .context("allocate_ext4_journals: cannot write journal block")?; + } + } + + image + .sync_all() + .context("allocate_ext4_journals: cannot sync image") +} + +pub fn generate_bmap_file(image_file: &str) -> Result<()> { + allocate_ext4_journals(image_file)?; + + let mut bmaptool = Command::new("bmaptool"); + bmaptool + .arg("create") + .arg("-o") + .arg(format!("{image_file}.bmap")) + .arg(image_file); + let status = bmaptool + .status() + .context(format!("generate_bmap_file: cannot run {bmaptool:?}"))?; + anyhow::ensure!(status.success(), "generate_bmap_file: {bmaptool:?} failed"); + + Ok(()) +} + +#[cfg(test)] +mod tests { + use crate::file::bmap::*; + use std::path::Path; + + const PART_START_LBA: u32 = 2048; + const PART_SECTORS: u32 = 16384; + const FS_BLOCK_SIZE: u64 = 1024; + const MBR_DISK_SIGNATURE: [u8; 4] = [1, 2, 3, 4]; + const MBR_TYPE_LINUX: u8 = 0x83; + const BMAP_BLOCK_SIZE: u64 = 4096; + + fn run(cmd: &mut Command) { + let status = cmd.status().expect("run command"); + assert!(status.success(), "{cmd:?} failed"); + } + + fn create_image_with_ext4_journal(image: &Path) { + let raw = image.with_extension("raw"); + let mut f = fs::File::create(&raw).expect("create image"); + f.set_len(u64::from(PART_START_LBA + PART_SECTORS) * SECTOR_SIZE) + .expect("set image size"); + let mut mbr = mbrman::MBR::new_from(&mut f, SECTOR_SIZE as u32, MBR_DISK_SIGNATURE) + .expect("create mbr"); + mbr[1] = mbrman::MBRPartitionEntry { + boot: mbrman::BOOT_INACTIVE, + first_chs: mbrman::CHS::empty(), + sys: MBR_TYPE_LINUX, + last_chs: mbrman::CHS::empty(), + starting_lba: PART_START_LBA, + sectors: PART_SECTORS, + }; + mbr.write_into(&mut f).expect("write mbr"); + + let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; + let fs_blocks = u64::from(PART_SECTORS) * SECTOR_SIZE / FS_BLOCK_SIZE; + run(Command::new("mkfs.ext4") + .arg("-q") + .arg("-F") + .arg("-b") + .arg(FS_BLOCK_SIZE.to_string()) + .arg("-J") + .arg("size=1") + .arg("-E") + .arg(format!("offset={offset}")) + .arg(&raw) + .arg(fs_blocks.to_string())); + // mke2fs zeroes the journal as unwritten extents, which a bmap maps; a sparse copy + // turns the zeros into holes, as a sparse decompress does + run(Command::new("cp") + .arg("--sparse=always") + .arg(&raw) + .arg(image)); + } + + fn bmap_ranges(bmap: &str) -> Vec<(u64, u64)> { + bmap.lines() + .filter_map(|l| { + let range = l.split_once("')?.1; + let range = range.split_once(" usize { + let ranges = bmap_ranges(bmap); + let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; + let file = fs::File::open(image).expect("open image"); + let journal = read_ext4_journal(&file, offset) + .expect("read superblock") + .expect("ext4 with journal"); + ext4_journal_blocks(image, offset, journal.inode) + .expect("journal blocks") + .iter() + .map(|b| (offset + b * journal.block_size) / BMAP_BLOCK_SIZE) + .filter(|b| !ranges.iter().any(|(s, e)| s <= b && b <= e)) + .count() + } + + #[test] + fn generate_bmap_file_maps_ext4_journal() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_path = dir.path().join("image.wic"); + create_image_with_ext4_journal(&image_path); + let image = image_path.to_str().expect("image path"); + let bmap_path = format!("{image}.bmap"); + + run(Command::new("bmaptool") + .arg("create") + .arg("-o") + .arg(&bmap_path) + .arg(image)); + let bmap = fs::read_to_string(&bmap_path).expect("read bmap"); + assert!(unmapped_journal_blocks(image, &bmap) > 0); + + generate_bmap_file(image).expect("generate bmap"); + let bmap = fs::read_to_string(&bmap_path).expect("read bmap"); + assert_eq!(unmapped_journal_blocks(image, &bmap), 0); + + let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; + run(Command::new("e2fsck") + .arg("-fn") + .arg(format!("{image}?offset={offset}"))); + } +} diff --git a/src/file/functions.rs b/src/file/functions.rs index 241b0dd2..d6591e9f 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -14,21 +14,6 @@ use uuid::Uuid; const COMPARE_BLOCK_SIZE: usize = 4096; const COPY_CHUNK_SIZE: usize = 1024 * 1024; -const EXT4_SUPERBLOCK_OFFSET: u64 = 1024; -const EXT4_SUPERBLOCK_SIZE: usize = 1024; -const EXT4_S_LOG_BLOCK_SIZE: usize = 0x18; -const EXT4_S_MAGIC: usize = 0x38; -const EXT4_S_FEATURE_COMPAT: usize = 0x5c; -const EXT4_S_JOURNAL_INUM: usize = 0xe0; -const EXT4_MAGIC: u16 = 0xef53; -const EXT4_FEATURE_COMPAT_HAS_JOURNAL: u32 = 0x4; -const EXT4_MIN_BLOCK_SIZE: u64 = 1024; -const EXT4_MAX_LOG_BLOCK_SIZE: u32 = 6; - -const DEBUGFS: &str = "debugfs"; -// debugfs is installed to sbin, which is not always in the PATH of a normal user -const DEBUGFS_SBIN: &str = "/usr/sbin/debugfs"; - #[derive(clap::ValueEnum, Debug, Clone, Eq, Hash, PartialEq)] #[clap(rename_all = "verbatim")] #[allow(non_camel_case_types)] @@ -499,249 +484,3 @@ fn write_partition( .sync_all() .context("write_partition: cannot sync image") } - -struct Ext4Journal { - inode: u32, - block_size: u64, -} - -fn read_ext4_journal(image: &fs::File, offset: u64) -> Result> { - let mut sb = [0u8; EXT4_SUPERBLOCK_SIZE]; - image - .read_exact_at(&mut sb, offset + EXT4_SUPERBLOCK_OFFSET) - .context("read_ext4_journal: cannot read superblock")?; - let le16 = |o: usize| u16::from_le_bytes([sb[o], sb[o + 1]]); - let le32 = |o: usize| u32::from_le_bytes([sb[o], sb[o + 1], sb[o + 2], sb[o + 3]]); - - if le16(EXT4_S_MAGIC) != EXT4_MAGIC - || le32(EXT4_S_FEATURE_COMPAT) & EXT4_FEATURE_COMPAT_HAS_JOURNAL == 0 - { - return Ok(None); - } - - // an external journal has no inode in this filesystem - let inode = le32(EXT4_S_JOURNAL_INUM); - if inode == 0 { - return Ok(None); - } - - let log_block_size = le32(EXT4_S_LOG_BLOCK_SIZE); - anyhow::ensure!( - log_block_size <= EXT4_MAX_LOG_BLOCK_SIZE, - "read_ext4_journal: invalid block size" - ); - let block_size = EXT4_MIN_BLOCK_SIZE << log_block_size; - - Ok(Some(Ext4Journal { inode, block_size })) -} - -fn ext4_journal_blocks(image_file: &str, offset: u64, inode: u32) -> Result> { - let debugfs_cmd = |program| { - let mut cmd = Command::new(program); - cmd.arg("-R") - .arg(format!("blocks <{inode}>")) - .arg(format!("{image_file}?offset={offset}")); - cmd - }; - let mut debugfs = debugfs_cmd(DEBUGFS); - let output = match debugfs.output() { - Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - debugfs = debugfs_cmd(DEBUGFS_SBIN); - debugfs.output() - } - output => output, - } - .context(format!("ext4_journal_blocks: cannot run {debugfs:?}"))?; - anyhow::ensure!( - output.status.success(), - "ext4_journal_blocks: {debugfs:?} failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - - let blocks = String::from_utf8(output.stdout) - .context("ext4_journal_blocks: invalid debugfs output")? - .split_whitespace() - .map(|b| { - b.parse::() - .context(format!("ext4_journal_blocks: invalid block number {b}")) - }) - .collect::>>()?; - anyhow::ensure!( - !blocks.is_empty(), - "ext4_journal_blocks: no journal blocks at offset {offset}" - ); - - Ok(blocks) -} - -/// Rewrites the journal blocks of every ext4 partition with their own content. Blocks that -/// are holes become allocated, so a bmap maps the whole journal and a bmap-based flash -/// overwrites the journal an earlier installation left on the device. -fn allocate_ext4_journals(image_file: &str) -> Result<()> { - use crate::file::partition::get_partitions; - - let image = fs::OpenOptions::new() - .read(true) - .write(true) - .open(image_file) - .context(format!("allocate_ext4_journals: cannot open {image_file}"))?; - let image_len = image - .metadata() - .context("allocate_ext4_journals: cannot get image size")? - .len(); - - for partition in get_partitions(image_file)? { - let offset = partition.start * SECTOR_SIZE; - if offset + EXT4_SUPERBLOCK_OFFSET + EXT4_SUPERBLOCK_SIZE as u64 > image_len { - continue; - } - let Some(journal) = read_ext4_journal(&image, offset)? else { - continue; - }; - let blocks = ext4_journal_blocks(image_file, offset, journal.inode)?; - debug!( - "allocate_ext4_journals: partition {}: {} journal blocks", - partition.num, - blocks.len() - ); - - let mut buf = vec![0u8; usize::try_from(journal.block_size)?]; - for block in blocks { - let pos = offset + block * journal.block_size; - image - .read_exact_at(&mut buf, pos) - .context("allocate_ext4_journals: cannot read journal block")?; - image - .write_all_at(&buf, pos) - .context("allocate_ext4_journals: cannot write journal block")?; - } - } - - image - .sync_all() - .context("allocate_ext4_journals: cannot sync image") -} - -pub fn generate_bmap_file(image_file: &str) -> Result<()> { - allocate_ext4_journals(image_file)?; - - let mut bmaptool = Command::new("bmaptool"); - bmaptool - .arg("create") - .arg("-o") - .arg(format!("{image_file}.bmap")) - .arg(image_file); - exec_cmd!(bmaptool); - - Ok(()) -} - -#[cfg(test)] -mod tests { - use crate::file::functions::*; - - const PART_START_LBA: u32 = 2048; - const PART_SECTORS: u32 = 16384; - const FS_BLOCK_SIZE: u64 = 1024; - const MBR_DISK_SIGNATURE: [u8; 4] = [1, 2, 3, 4]; - const MBR_TYPE_LINUX: u8 = 0x83; - const BMAP_BLOCK_SIZE: u64 = 4096; - - fn run(cmd: &mut Command) { - let status = cmd.status().expect("run command"); - assert!(status.success(), "{cmd:?} failed"); - } - - fn create_image_with_ext4_journal(image: &Path) { - let raw = image.with_extension("raw"); - let mut f = fs::File::create(&raw).expect("create image"); - f.set_len(u64::from(PART_START_LBA + PART_SECTORS) * SECTOR_SIZE) - .expect("set image size"); - let mut mbr = mbrman::MBR::new_from(&mut f, SECTOR_SIZE as u32, MBR_DISK_SIGNATURE) - .expect("create mbr"); - mbr[1] = mbrman::MBRPartitionEntry { - boot: mbrman::BOOT_INACTIVE, - first_chs: mbrman::CHS::empty(), - sys: MBR_TYPE_LINUX, - last_chs: mbrman::CHS::empty(), - starting_lba: PART_START_LBA, - sectors: PART_SECTORS, - }; - mbr.write_into(&mut f).expect("write mbr"); - - let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; - let fs_blocks = u64::from(PART_SECTORS) * SECTOR_SIZE / FS_BLOCK_SIZE; - run(Command::new("mkfs.ext4") - .arg("-q") - .arg("-F") - .arg("-b") - .arg(FS_BLOCK_SIZE.to_string()) - .arg("-J") - .arg("size=1") - .arg("-E") - .arg(format!("offset={offset}")) - .arg(&raw) - .arg(fs_blocks.to_string())); - // mke2fs zeroes the journal as unwritten extents, which a bmap maps; a sparse copy - // turns the zeros into holes, as a sparse decompress does - run(Command::new("cp") - .arg("--sparse=always") - .arg(&raw) - .arg(image)); - } - - fn bmap_ranges(bmap: &str) -> Vec<(u64, u64)> { - bmap.lines() - .filter_map(|l| { - let range = l.split_once("')?.1; - let range = range.split_once(" usize { - let ranges = bmap_ranges(bmap); - let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; - let file = fs::File::open(image).expect("open image"); - let journal = read_ext4_journal(&file, offset) - .expect("read superblock") - .expect("ext4 with journal"); - ext4_journal_blocks(image, offset, journal.inode) - .expect("journal blocks") - .iter() - .map(|b| (offset + b * journal.block_size) / BMAP_BLOCK_SIZE) - .filter(|b| !ranges.iter().any(|(s, e)| s <= b && b <= e)) - .count() - } - - #[test] - fn generate_bmap_file_maps_ext4_journal() { - let dir = tempfile::tempdir().expect("create temp dir"); - let image_path = dir.path().join("image.wic"); - create_image_with_ext4_journal(&image_path); - let image = image_path.to_str().expect("image path"); - let bmap_path = format!("{image}.bmap"); - - run(Command::new("bmaptool") - .arg("create") - .arg("-o") - .arg(&bmap_path) - .arg(image)); - let bmap = fs::read_to_string(&bmap_path).expect("read bmap"); - assert!(unmapped_journal_blocks(image, &bmap) > 0); - - generate_bmap_file(image).expect("generate bmap"); - let bmap = fs::read_to_string(&bmap_path).expect("read bmap"); - assert_eq!(unmapped_journal_blocks(image, &bmap), 0); - - let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; - run(Command::new("e2fsck") - .arg("-fn") - .arg(format!("{image}?offset={offset}"))); - } -} diff --git a/src/file/mod.rs b/src/file/mod.rs index 6f9db481..e9ccd43e 100644 --- a/src/file/mod.rs +++ b/src/file/mod.rs @@ -1,3 +1,4 @@ +pub mod bmap; pub mod compression; pub mod functions; mod partition; diff --git a/src/lib.rs b/src/lib.rs index a8ce6bf4..c487ceda 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -114,7 +114,7 @@ where .to_str() .context("cannot get image file path")? )); - file::functions::generate_bmap_file( + file::bmap::generate_bmap_file( tmp_image_file .to_str() .context("cannot get image file path")?, From 9842fef98392f3df43fb94fb84e25f6e66e24103 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:35:36 +0200 Subject: [PATCH 11/75] test(file): cover ext4 journal edge cases and a flash onto old content - a bmap flash onto a device with old content now overwrites the journal - an invalid s_log_block_size is rejected - a partition that starts past the end of the image is skipped - the MBR extended container entry is not returned as a partition Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/bmap.rs | 99 ++++++++++++++++++++++++++++++++++++++++--- src/file/partition.rs | 19 +++++++++ 2 files changed, 113 insertions(+), 5 deletions(-) diff --git a/src/file/bmap.rs b/src/file/bmap.rs index e6b6352b..f31259b7 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -168,17 +168,19 @@ mod tests { const MBR_DISK_SIGNATURE: [u8; 4] = [1, 2, 3, 4]; const MBR_TYPE_LINUX: u8 = 0x83; const BMAP_BLOCK_SIZE: u64 = 4096; + // content an earlier installation left on the device + const OLD_CONTENT_BYTE: u8 = 0xa5; + const JOURNAL_INODE: u32 = 8; + const INVALID_LOG_BLOCK_SIZE: u32 = 40; fn run(cmd: &mut Command) { let status = cmd.status().expect("run command"); assert!(status.success(), "{cmd:?} failed"); } - fn create_image_with_ext4_journal(image: &Path) { - let raw = image.with_extension("raw"); - let mut f = fs::File::create(&raw).expect("create image"); - f.set_len(u64::from(PART_START_LBA + PART_SECTORS) * SECTOR_SIZE) - .expect("set image size"); + fn create_image_with_mbr(image: &Path, len: u64) { + let mut f = fs::File::create(image).expect("create image"); + f.set_len(len).expect("set image size"); let mut mbr = mbrman::MBR::new_from(&mut f, SECTOR_SIZE as u32, MBR_DISK_SIGNATURE) .expect("create mbr"); mbr[1] = mbrman::MBRPartitionEntry { @@ -190,6 +192,11 @@ mod tests { sectors: PART_SECTORS, }; mbr.write_into(&mut f).expect("write mbr"); + } + + fn create_image_with_ext4_journal(image: &Path) { + let raw = image.with_extension("raw"); + create_image_with_mbr(&raw, u64::from(PART_START_LBA + PART_SECTORS) * SECTOR_SIZE); let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; let fs_blocks = u64::from(PART_SECTORS) * SECTOR_SIZE / FS_BLOCK_SIZE; @@ -241,6 +248,55 @@ mod tests { .count() } + /// Writes the blocks `bmap` maps onto a device that still holds old content, as a bmap + /// flash does. + fn bmap_flash(image: &str, bmap: &str, device: &Path) { + let image = fs::File::open(image).expect("open image"); + let len = image.metadata().expect("image metadata").len(); + fs::write( + device, + vec![OLD_CONTENT_BYTE; usize::try_from(len).expect("image size")], + ) + .expect("write old content"); + let device = fs::OpenOptions::new() + .write(true) + .open(device) + .expect("open device"); + + let mut buf = vec![0u8; usize::try_from(BMAP_BLOCK_SIZE).expect("block size")]; + for (first, last) in bmap_ranges(bmap) { + for block in first..=last { + image + .read_exact_at(&mut buf, block * BMAP_BLOCK_SIZE) + .expect("read image block"); + device + .write_all_at(&buf, block * BMAP_BLOCK_SIZE) + .expect("write device block"); + } + } + } + + fn journal_on_device_matches_image(image: &str, device: &Path) -> bool { + let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; + let image_file = fs::File::open(image).expect("open image"); + let device = fs::File::open(device).expect("open device"); + let journal = read_ext4_journal(&image_file, offset) + .expect("read superblock") + .expect("ext4 with journal"); + + let block_size = usize::try_from(journal.block_size).expect("block size"); + let (mut a, mut b) = (vec![0u8; block_size], vec![0u8; block_size]); + ext4_journal_blocks(image, offset, journal.inode) + .expect("journal blocks") + .iter() + .all(|block| { + let pos = offset + block * journal.block_size; + image_file.read_exact_at(&mut a, pos).expect("read image"); + device.read_exact_at(&mut b, pos).expect("read device"); + a == b + }) + } + #[test] fn generate_bmap_file_maps_ext4_journal() { let dir = tempfile::tempdir().expect("create temp dir"); @@ -256,14 +312,47 @@ mod tests { .arg(image)); let bmap = fs::read_to_string(&bmap_path).expect("read bmap"); assert!(unmapped_journal_blocks(image, &bmap) > 0); + let device = dir.path().join("device"); + bmap_flash(image, &bmap, &device); + assert!(!journal_on_device_matches_image(image, &device)); generate_bmap_file(image).expect("generate bmap"); let bmap = fs::read_to_string(&bmap_path).expect("read bmap"); assert_eq!(unmapped_journal_blocks(image, &bmap), 0); + bmap_flash(image, &bmap, &device); + assert!(journal_on_device_matches_image(image, &device)); let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; run(Command::new("e2fsck") .arg("-fn") .arg(format!("{image}?offset={offset}"))); } + + #[test] + fn read_ext4_journal_rejects_invalid_block_size() { + let mut sb = [0u8; EXT4_SUPERBLOCK_SIZE]; + let mut set = |o: usize, v: &[u8]| sb[o..o + v.len()].copy_from_slice(v); + set(EXT4_S_MAGIC, &EXT4_MAGIC.to_le_bytes()); + set( + EXT4_S_FEATURE_COMPAT, + &EXT4_FEATURE_COMPAT_HAS_JOURNAL.to_le_bytes(), + ); + set(EXT4_S_JOURNAL_INUM, &JOURNAL_INODE.to_le_bytes()); + set(EXT4_S_LOG_BLOCK_SIZE, &INVALID_LOG_BLOCK_SIZE.to_le_bytes()); + + let image = tempfile::tempfile().expect("create image"); + image + .write_all_at(&sb, EXT4_SUPERBLOCK_OFFSET) + .expect("write superblock"); + assert!(read_ext4_journal(&image, 0).is_err()); + } + + #[test] + fn allocate_ext4_journals_skips_partition_past_image_end() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + create_image_with_mbr(&image, u64::from(PART_START_LBA) * SECTOR_SIZE); + + allocate_ext4_journals(image.to_str().expect("image path")).expect("allocate journals"); + } } diff --git a/src/file/partition.rs b/src/file/partition.rs index 9d9d0b8c..12cbc5e0 100644 --- a/src/file/partition.rs +++ b/src/file/partition.rs @@ -97,3 +97,22 @@ pub fn is_gpt>(path: P) -> Result { let mut file = File::open(path.as_ref()).context("is_gpt: failed to open image")?; Ok(gptman::GPT::find_from(&mut file).is_ok()) } + +#[cfg(test)] +mod tests { + use crate::file::partition::*; + + const TEST_IMAGE: &str = "testfiles/image.wic"; + const MBR_EXTENDED_PARTITION: u32 = 4; + + #[test] + fn get_partitions_skips_mbr_extended_entry() { + let nums: Vec = get_partitions(TEST_IMAGE) + .expect("read partitions") + .iter() + .map(|p| p.num) + .collect(); + assert!(!nums.contains(&MBR_EXTENDED_PARTITION)); + assert!(nums.contains(&(MBR_EXTENDED_PARTITION + 1))); + } +} From 68f21f4e1292a217c077e5a128739526cdf9103c Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Fri, 23 Oct 2026 15:51:40 +0200 Subject: [PATCH 12/75] fix(deps): resolve RUSTSEC-2026-0385 (ctutils) Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f70b2cb6..fdcfe234 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -610,9 +610,9 @@ dependencies = [ [[package]] name = "ctutils" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" +checksum = "38d28fb6aa7dd680b3ffd1a3e64e115c177bdba63c0ae7f9cd48129ff05c4fa0" dependencies = [ "cmov", ] From 7e7505261c80299c3a11c64b63b8ddbc61445e16 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:28:36 +0100 Subject: [PATCH 13/75] fix(file): detect truncated input and harden sparse decompress - return decoder errors at the end of the stream - truncate the destination before a sparse decompress - write runs of blocks in one call in SparseWriter - find e2fsprogs tools in sbin, reuse get_partitions, share the image block size Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- project-context.md | 3 +- src/file/bmap.rs | 44 ++++----- src/file/compression.rs | 197 +++++++++++++++++++++++++++++----------- src/file/functions.rs | 48 +++++++++- src/file/mod.rs | 3 + src/file/partition.rs | 49 ++-------- 6 files changed, 221 insertions(+), 123 deletions(-) diff --git a/project-context.md b/project-context.md index 8f3b61d6..3e1c93ad 100644 --- a/project-context.md +++ b/project-context.md @@ -50,7 +50,8 @@ - `src/docker.rs` — `docker pull --platform` + `docker save` for multi-arch images - `src/image.rs` — firmware image architecture detection (ARM32/ARM64/x86_64) - `src/file/mod.rs` — high-level image operations: identity config, certs, hostname patching -- `src/file/functions.rs` — partition read/modify/write via dd + e2cp/mcopy +- `src/file/functions.rs` — partition read via dd, modify via e2cp/mcopy, write back of changed blocks only +- `src/file/bmap.rs` — bmap generation; allocates ext4 journals first (needs `debugfs`) - `src/file/partition.rs` — GPT/MBR partition table parsing - `src/file/compression.rs` — xz/bzip2/gzip compress/decompress with auto-detection - `src/validators/` — validation for identity config (TOML), device-update config (JSON), SSH keys diff --git a/src/file/bmap.rs b/src/file/bmap.rs index f31259b7..8b13b2c2 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -1,9 +1,7 @@ use crate::file::partition::{SECTOR_SIZE, get_partitions}; use anyhow::{Context, Result}; use log::debug; -use std::fs; -use std::os::unix::fs::FileExt; -use std::process::Command; +use std::{env, fs, os::unix::fs::FileExt, process::Command}; const EXT4_SUPERBLOCK_OFFSET: u64 = 1024; const EXT4_SUPERBLOCK_SIZE: usize = 1024; @@ -17,8 +15,8 @@ const EXT4_MIN_BLOCK_SIZE: u64 = 1024; const EXT4_MAX_LOG_BLOCK_SIZE: u32 = 6; const DEBUGFS: &str = "debugfs"; -// debugfs is installed to sbin, which is not always in the PATH of a normal user -const DEBUGFS_SBIN: &str = "/usr/sbin/debugfs"; +// e2fsprogs installs to sbin, which is not always in the PATH of a normal user +const SBIN_DIRS: &str = "/usr/sbin:/sbin"; struct Ext4Journal { inode: u32, @@ -55,26 +53,24 @@ fn read_ext4_journal(image: &fs::File, offset: u64) -> Result Command { + let path = + env::var("PATH").map_or_else(|_| SBIN_DIRS.to_string(), |p| format!("{p}:{SBIN_DIRS}")); + let mut cmd = Command::new(program); + cmd.env("PATH", path); + cmd +} + fn ext4_journal_blocks(image_file: &str, offset: u64, inode: u32) -> Result> { - let debugfs_cmd = |program| { - let mut cmd = Command::new(program); - cmd.arg("-R") - .arg(format!("blocks <{inode}>")) - .arg(format!("{image_file}?offset={offset}")); - cmd - }; - let mut debugfs = debugfs_cmd(DEBUGFS); - let output = match debugfs.output() { - Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - debugfs = debugfs_cmd(DEBUGFS_SBIN); - debugfs.output() - } - output => output, - } - .context(format!("ext4_journal_blocks: cannot run {debugfs:?}"))?; + let output = sbin_command(DEBUGFS) + .arg("-R") + .arg(format!("blocks <{inode}>")) + .arg(format!("{image_file}?offset={offset}")) + .output() + .context(format!("ext4_journal_blocks: cannot run {DEBUGFS}"))?; anyhow::ensure!( output.status.success(), - "ext4_journal_blocks: {debugfs:?} failed: {}", + "ext4_journal_blocks: {DEBUGFS} failed: {}", String::from_utf8_lossy(&output.stderr) ); @@ -200,7 +196,7 @@ mod tests { let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; let fs_blocks = u64::from(PART_SECTORS) * SECTOR_SIZE / FS_BLOCK_SIZE; - run(Command::new("mkfs.ext4") + run(sbin_command("mkfs.ext4") .arg("-q") .arg("-F") .arg("-b") @@ -323,7 +319,7 @@ mod tests { assert!(journal_on_device_matches_image(image, &device)); let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; - run(Command::new("e2fsck") + run(sbin_command("e2fsck") .arg("-fn") .arg(format!("{image}?offset={offset}"))); } diff --git a/src/file/compression.rs b/src/file/compression.rs index fca98d49..ba5a43d0 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -1,16 +1,16 @@ +use crate::file::IMAGE_BLOCK_SIZE; use anyhow::{Context, Result}; use filemagic::Magic; use log::debug; use std::env; use std::fs::File; use std::io::{Seek, SeekFrom, Write}; +use std::os::unix::fs::FileExt; use std::path::PathBuf; use std::str::FromStr; use strum::IntoEnumIterator; use strum_macros::EnumIter; -const SPARSE_BLOCK_SIZE: u64 = 4096; - #[derive(Clone, Debug, EnumIter)] #[allow(non_camel_case_types)] pub enum Compression { @@ -79,22 +79,24 @@ impl Compression { source: &mut std::fs::File, destination: &mut std::fs::File, ) -> std::io::Result { - let sparse = SparseWriter(destination); - let mut dec: Box = match &self { - Compression::bzip2 => Box::new(bzip2::write::BzDecoder::new(sparse)), - Compression::gzip => Box::new(flate2::write::GzDecoder::new(sparse)), - Compression::xz { .. } => Box::new(xz2::write::XzDecoder::new(sparse)), + let sparse = SparseWriter::new(destination)?; + // only `finish` detects a truncated or corrupt stream + let (bytes_read, sparse) = match &self { + Compression::bzip2 => { + let mut dec = bzip2::write::BzDecoder::new(sparse); + (std::io::copy(source, &mut dec)?, dec.finish()?) + } + Compression::gzip => { + let mut dec = flate2::write::GzDecoder::new(sparse); + (std::io::copy(source, &mut dec)?, dec.finish()?) + } + Compression::xz { .. } => { + let mut dec = xz2::write::XzDecoder::new(sparse); + (std::io::copy(source, &mut dec)?, dec.finish()?) + } }; - - let bytes_written = std::io::copy(source, &mut dec)?; - dec.write_all(&[])?; - dec.flush()?; - drop(dec); - - // zero blocks at the end were only seeked over - let len = destination.stream_position()?; - destination.set_len(len)?; - Ok(bytes_written) + sparse.finish()?; + Ok(bytes_read) } fn marker(&self) -> &'static str { @@ -135,24 +137,55 @@ impl Compression { } } -/// Seeks over zero blocks instead of writing them, so they stay holes in a new file. -struct SparseWriter<'a>(&'a mut File); +/// Skips zero blocks, so they stay holes in the file. +struct SparseWriter<'a> { + file: &'a mut File, + pos: u64, +} + +impl<'a> SparseWriter<'a> { + fn new(file: &'a mut File) -> std::io::Result { + let pos = file.stream_position()?; + // skipped zero blocks must read back as zeros + file.set_len(pos)?; + Ok(Self { file, pos }) + } + + fn finish(self) -> std::io::Result<()> { + // extends the file over zero blocks at the end + self.file.set_len(self.pos)?; + self.file.seek(SeekFrom::Start(self.pos))?; + Ok(()) + } +} impl Write for SparseWriter<'_> { fn write(&mut self, buf: &[u8]) -> std::io::Result { - let pos = self.0.stream_position()?; - let to_block_end = SPARSE_BLOCK_SIZE - pos % SPARSE_BLOCK_SIZE; - let chunk = &buf[..buf.len().min(to_block_end as usize)]; - if chunk.iter().all(|b| *b == 0) { - self.0.seek(SeekFrom::Current(chunk.len() as i64))?; - Ok(chunk.len()) - } else { - self.0.write(chunk) + let is_zero = |b: &[u8]| b.iter().all(|x| *x == 0); + let block_offset = (self.pos % IMAGE_BLOCK_SIZE as u64) as usize; + + // one call handles a run of blocks that are all zero or all data + let mut end = buf.len().min(IMAGE_BLOCK_SIZE - block_offset); + let zero = is_zero(&buf[..end]); + while end < buf.len() { + let next = buf.len().min(end + IMAGE_BLOCK_SIZE); + if is_zero(&buf[end..next]) != zero { + break; + } + end = next; } + + let written = if zero { + end + } else { + self.file.write_at(&buf[..end], self.pos)? + }; + self.pos += written as u64; + Ok(written) } fn flush(&mut self) -> std::io::Result<()> { - self.0.flush() + self.file.flush() } } @@ -197,9 +230,31 @@ mod tests { // unit of `st_blocks` const STAT_BLOCK_SIZE: u64 = 512; const DATA_OFFSET: usize = 1024 * 1024 + 100; + const DATA: &[u8] = b"abc"; const IMAGE_LEN: usize = 3 * 1024 * 1024 + 123; + const XZ_TEST_LEVEL: u32 = 1; + // cuts into the trailer, which every format checks only at the end of the stream + const TRUNCATED_BYTES: u64 = 4; + const OLD_CONTENT_BYTE: u8 = 0xa5; + const OLD_CONTENT_EXTRA_LEN: usize = 4096; + + fn all_compressions() -> [Compression; 3] { + [ + Compression::bzip2, + Compression::gzip, + Compression::xz { + compression_level: XZ_TEST_LEVEL, + }, + ] + } + + fn test_image() -> Vec { + let mut image = vec![0u8; IMAGE_LEN]; + image[DATA_OFFSET..DATA_OFFSET + DATA.len()].copy_from_slice(DATA); + image + } - fn roundtrip(compression: Compression, image: &[u8]) -> File { + fn compress_image(compression: &Compression, image: &[u8]) -> File { let mut source = tempfile::tempfile().expect("create source"); source.write_all(image).expect("write source"); source.rewind().expect("rewind source"); @@ -209,35 +264,31 @@ mod tests { .compress(&mut source, &mut compressed) .expect("compress"); compressed.rewind().expect("rewind compressed"); + compressed + } - let mut decompressed = tempfile::tempfile().expect("create decompressed"); - compression - .decompress(&mut compressed, &mut decompressed) - .expect("decompress"); - decompressed + fn read_all(file: &mut File) -> Vec { + let mut content = Vec::new(); + file.rewind().expect("rewind"); + file.read_to_end(&mut content).expect("read"); + content } #[test] fn decompress_keeps_zero_blocks_as_holes() { - let mut image = vec![0u8; IMAGE_LEN]; - image[DATA_OFFSET..DATA_OFFSET + 3].copy_from_slice(b"abc"); + let image = test_image(); - for compression in [ - Compression::bzip2, - Compression::gzip, - Compression::xz { - compression_level: 1, - }, - ] { - let mut decompressed = roundtrip(compression.clone(), &image); - - let mut content = Vec::new(); - decompressed.rewind().expect("rewind decompressed"); - decompressed - .read_to_end(&mut content) - .expect("read decompressed"); - assert!(content == image, "{compression:?}: content differs"); + for compression in all_compressions() { + let mut compressed = compress_image(&compression, &image); + let mut decompressed = tempfile::tempfile().expect("create decompressed"); + compression + .decompress(&mut compressed, &mut decompressed) + .expect("decompress"); + assert!( + read_all(&mut decompressed) == image, + "{compression:?}: content differs" + ); let meta = decompressed.metadata().expect("metadata"); assert!( meta.blocks() * STAT_BLOCK_SIZE < meta.len(), @@ -245,4 +296,48 @@ mod tests { ); } } + + #[test] + fn decompress_replaces_old_content() { + let image = test_image(); + + for compression in all_compressions() { + let mut compressed = compress_image(&compression, &image); + let mut decompressed = tempfile::tempfile().expect("create decompressed"); + decompressed + .write_all(&vec![OLD_CONTENT_BYTE; IMAGE_LEN + OLD_CONTENT_EXTRA_LEN]) + .expect("write old content"); + decompressed.rewind().expect("rewind decompressed"); + + compression + .decompress(&mut compressed, &mut decompressed) + .expect("decompress"); + + assert!( + read_all(&mut decompressed) == image, + "{compression:?}: content differs" + ); + } + } + + #[test] + fn decompress_rejects_truncated_input() { + let image = test_image(); + + for compression in all_compressions() { + let mut compressed = compress_image(&compression, &image); + let len = compressed.metadata().expect("metadata").len(); + compressed + .set_len(len - TRUNCATED_BYTES) + .expect("truncate compressed"); + + let mut decompressed = tempfile::tempfile().expect("create decompressed"); + assert!( + compression + .decompress(&mut compressed, &mut decompressed) + .is_err(), + "{compression:?}: truncated input accepted" + ); + } + } } diff --git a/src/file/functions.rs b/src/file/functions.rs index d6591e9f..5f8eb6d0 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -1,4 +1,4 @@ -use crate::file::partition::SECTOR_SIZE; +use crate::file::{IMAGE_BLOCK_SIZE, partition::SECTOR_SIZE}; use anyhow::{Context, Result}; use log::{debug, warn}; use std::collections::HashMap; @@ -11,7 +11,6 @@ use std::str::FromStr; use stdext::function_name; use uuid::Uuid; -const COMPARE_BLOCK_SIZE: usize = 4096; const COPY_CHUNK_SIZE: usize = 1024 * 1024; #[derive(clap::ValueEnum, Debug, Clone, Eq, Hash, PartialEq)] @@ -468,12 +467,12 @@ fn write_partition( .context("write_partition: cannot read image")?; let blocks = src[..n] - .chunks(COMPARE_BLOCK_SIZE) - .zip(dst[..n].chunks(COMPARE_BLOCK_SIZE)); + .chunks(IMAGE_BLOCK_SIZE) + .zip(dst[..n].chunks(IMAGE_BLOCK_SIZE)); for (i, (s, d)) in blocks.enumerate() { if s != d { image - .write_all_at(s, image_offset + pos + (i * COMPARE_BLOCK_SIZE) as u64) + .write_all_at(s, image_offset + pos + (i * IMAGE_BLOCK_SIZE) as u64) .context("write_partition: cannot write image")?; } } @@ -484,3 +483,42 @@ fn write_partition( .sync_all() .context("write_partition: cannot sync image") } + +#[cfg(test)] +mod tests { + use crate::file::functions::*; + + const PART_NUM: u32 = 1; + const PART_START: u64 = 8; + const PART_SECTORS: u64 = 64; + const OLD_CONTENT_BYTE: u8 = 0xa5; + const ZEROED_BLOCK: usize = 2; + + #[test] + fn write_partition_writes_block_changed_to_zeros() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + let partition = dir.path().join("partition.img"); + let image_len = usize::try_from((PART_START + PART_SECTORS) * SECTOR_SIZE).expect("size"); + let part_offset = usize::try_from(PART_START * SECTOR_SIZE).expect("offset"); + + fs::write(&image, vec![OLD_CONTENT_BYTE; image_len]).expect("write image"); + let mut content = vec![OLD_CONTENT_BYTE; image_len - part_offset]; + content[ZEROED_BLOCK * IMAGE_BLOCK_SIZE..(ZEROED_BLOCK + 1) * IMAGE_BLOCK_SIZE].fill(0); + fs::write(&partition, &content).expect("write partition"); + + write_partition( + image.to_str().expect("image path"), + partition.to_str().expect("partition path"), + &PartitionInfo { + num: PART_NUM, + start: PART_START, + count: PART_SECTORS, + }, + ) + .expect("write partition"); + + let written = fs::read(&image).expect("read image"); + assert!(written[part_offset..] == content[..], "partition differs"); + } +} diff --git a/src/file/mod.rs b/src/file/mod.rs index e9ccd43e..073eb7dd 100644 --- a/src/file/mod.rs +++ b/src/file/mod.rs @@ -2,6 +2,9 @@ pub mod bmap; pub mod compression; pub mod functions; mod partition; + +/// Granularity at which zero data in an image file is kept as a hole. +const IMAGE_BLOCK_SIZE: usize = 4096; use super::validators::{ device_update, identity::{IdentityConfig, IdentityType, validate_identity}, diff --git a/src/file/partition.rs b/src/file/partition.rs index 12cbc5e0..36fe983d 100644 --- a/src/file/partition.rs +++ b/src/file/partition.rs @@ -13,48 +13,10 @@ pub struct PartitionData { } pub fn get_partition_data>(path: P, partition_num: u32) -> Result { - let path = path.as_ref(); - let mut file = - File::open(path).with_context(|| format!("failed to open image: {}", path.display()))?; - - // Try GPT first (validates CRC32 — more robust than signature check). - // Capture any error so we can attach it as context if MBR also fails. - let gpt_err = match gptman::GPT::find_from(&mut file) { - Ok(gpt) => { - let entry = gpt - .iter() - .find(|(i, _)| *i == partition_num) - .map(|(_, e)| e) - .with_context(|| format!("GPT partition {partition_num} out of range"))?; - anyhow::ensure!(entry.is_used(), "GPT partition {partition_num} is not used"); - anyhow::ensure!( - entry.ending_lba >= entry.starting_lba, - "GPT partition {partition_num} has invalid LBA range (ending_lba < starting_lba)" - ); - return Ok(PartitionData { - num: partition_num, - start: entry.starting_lba, - count: entry.ending_lba - entry.starting_lba + 1, - }); - } - Err(e) => e, - }; - - // Try MBR — iter() includes logical partitions (5, 6, ...) - file.seek(SeekFrom::Start(0)) - .context("failed to seek to start of image")?; - let mbr = mbrman::MBR::read_from(&mut file, SECTOR_SIZE as u32) - .with_context(|| format!("image is neither valid GPT nor MBR (GPT error: {gpt_err})"))?; - for (i, p) in mbr.iter() { - if u32::try_from(i).ok() == Some(partition_num) && p.is_used() { - return Ok(PartitionData { - num: partition_num, - start: p.starting_lba as u64, - count: p.sectors as u64, - }); - } - } - anyhow::bail!("partition {partition_num} not found in image") + get_partitions(path)? + .into_iter() + .find(|p| p.num == partition_num) + .with_context(|| format!("partition {partition_num} not found in image")) } pub fn get_partitions>(path: P) -> Result> { @@ -62,6 +24,8 @@ pub fn get_partitions>(path: P) -> Result> { let mut file = File::open(path).with_context(|| format!("failed to open image: {}", path.display()))?; + // Try GPT first (validates CRC32 — more robust than signature check). + // Capture any error so we can attach it as context if MBR also fails. let gpt_err = match gptman::GPT::find_from(&mut file) { Ok(gpt) => { return Ok(gpt @@ -77,6 +41,7 @@ pub fn get_partitions>(path: P) -> Result> { Err(e) => e, }; + // Try MBR — iter() includes logical partitions (5, 6, ...) file.seek(SeekFrom::Start(0)) .context("failed to seek to start of image")?; let mbr = mbrman::MBR::read_from(&mut file, SECTOR_SIZE as u32) From a0da23c701028c0e513d2dd2a39e5b51b8e68438 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:48:18 +0100 Subject: [PATCH 14/75] fix(file): keep the ranges of the input bmap mapped Sparse decompress turns every zero block into a hole, and a bmap created from it skips them: the GPT entries, FAT sectors of boot and ext4 journals and metadata. A bmap-based flash then keeps what an earlier installation left there. Take the bmap of the input image (next to it, or --bmap) and write the zero blocks of its ranges as data. Without an input bmap, every block is written. This replaces the ext4 journal handling, so debugfs and e2fsprogs are no longer needed. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- Cargo.lock | 10 + Cargo.toml | 3 +- Dockerfile | 1 - README.md | 6 + project-context.md | 2 +- src/cli.rs | 30 ++ src/file/bmap.rs | 580 ++++++++++++++++++++----------------- src/file/functions.rs | 4 +- src/file/mod.rs | 1 + src/lib.rs | 110 +++++-- tests/integration_tests.rs | 86 ++++-- 11 files changed, 501 insertions(+), 332 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fdcfe234..b824097f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1793,6 +1793,7 @@ dependencies = [ "regex", "reqwest 0.13.5", "ring", + "roxmltree", "serde", "serde_json", "serde_path_to_error", @@ -2222,6 +2223,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "roxmltree" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9da7f59328b09405b181fee1018925fad50f157afb43c0c0f5c522c256dd5907" +dependencies = [ + "memchr", +] + [[package]] name = "rustc_version" version = "0.4.1" diff --git a/Cargo.toml b/Cargo.toml index 8e90f87f..35d5c6eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,7 @@ reqwest = { version = "0.13", default-features = false, features = [ "json", "native-tls", ] } +roxmltree = { version = "0.21", default-features = false, features = ["std"] } serde = { version = "1.0", default-features = false, features = ["derive"] } serde_json = { version = "1.0", default-features = false } serde_path_to_error = { version = "0.1", default-features = false } @@ -81,5 +82,5 @@ tar = "0.4" # metadata for building with cargo-deb (https://crates.io/crates/cargo-deb) [package.metadata.deb] -depends = "bmap-tools, e2fsprogs, e2tools, keychain, libc6 (>= 2.39), libmagic1, libssl3 (>= 3.0.0), mtools" +depends = "bmap-tools, e2tools, keychain, libc6 (>= 2.39), libmagic1, libssl3 (>= 3.0.0), mtools" revision = "" diff --git a/Dockerfile b/Dockerfile index 03389a9a..9300a868 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,6 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends \ bmap-tools \ ca-certificates \ - e2fsprogs \ e2tools \ fdisk \ keychain \ diff --git a/README.md b/README.md index a3cc9c44..18133e7c 100644 --- a/README.md +++ b/README.md @@ -184,6 +184,12 @@ omnect-cli file copy-to-image --help - File permissions: inject `systemd-tmpfiles.d` - Wifi: inject `wpa_supplicant-wlan0.conf` +## bmap files + +Commands that change an image create a bmap file with `-b`. The bmap of the input image is the base for it: `omnect-cli` takes `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it, or the file given with `--bmap`. Every range the input bmap maps stays mapped, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). The created bmap maps these ranges plus the blocks `omnect-cli` changed. + +Without an input bmap, the created bmap maps the whole image, and a flash writes every block. + ## ssh tunnel ### Inject ssh tunnel credentials diff --git a/project-context.md b/project-context.md index 3e1c93ad..554c6b73 100644 --- a/project-context.md +++ b/project-context.md @@ -51,7 +51,7 @@ - `src/image.rs` — firmware image architecture detection (ARM32/ARM64/x86_64) - `src/file/mod.rs` — high-level image operations: identity config, certs, hostname patching - `src/file/functions.rs` — partition read via dd, modify via e2cp/mcopy, write back of changed blocks only -- `src/file/bmap.rs` — bmap generation; allocates ext4 journals first (needs `debugfs`) +- `src/file/bmap.rs` — input bmap lookup; maps its ranges in the working image, or the whole image without one; bmap generation via `bmaptool create` - `src/file/partition.rs` — GPT/MBR partition table parsing - `src/file/compression.rs` — xz/bzip2/gzip compress/decompress with auto-detection - `src/validators/` — validation for identity config (TOML), device-update config (JSON), SSH keys diff --git a/src/cli.rs b/src/cli.rs index 746e2537..6b104ac4 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -37,6 +37,9 @@ pub enum Docker { /// optional: generate bmap file (currently not working in docker image) #[arg(short = 'b', long = "generate-bmap-file")] generate_bmap: bool, + /// optional: bmap of the input image; default: .bmap or .bmap next to the image + #[arg(long = "bmap")] + bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -58,6 +61,9 @@ pub enum File { /// optional: generate bmap file (currently not working in docker image) #[arg(short = 'b', long = "generate-bmap-file")] generate_bmap: bool, + /// optional: bmap of the input image; default: .bmap or .bmap next to the image + #[arg(long = "bmap")] + bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -91,6 +97,9 @@ pub enum IdentityConfig { /// optional: generate bmap file (currently not working in docker image) #[arg(short = 'b', long = "generate-bmap-file")] generate_bmap: bool, + /// optional: bmap of the input image; default: .bmap or .bmap next to the image + #[arg(long = "bmap")] + bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -115,6 +124,9 @@ pub enum IdentityConfig { /// optional: generate bmap file (currently not working in docker image) #[arg(short = 'b', long = "generate-bmap-file")] generate_bmap: bool, + /// optional: bmap of the input image; default: .bmap or .bmap next to the image + #[arg(long = "bmap")] + bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -133,6 +145,9 @@ pub enum IdentityConfig { /// optional: generate bmap file (currently not working in docker image) #[arg(short = 'b', long = "generate-bmap-file")] generate_bmap: bool, + /// optional: bmap of the input image; default: .bmap or .bmap next to the image + #[arg(long = "bmap")] + bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -157,6 +172,9 @@ pub enum IdentityConfig { /// optional: generate bmap file (currently not working in docker image) #[arg(short = 'b', long = "generate-bmap-file")] generate_bmap: bool, + /// optional: bmap of the input image; default: .bmap or .bmap next to the image + #[arg(long = "bmap")] + bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -175,6 +193,9 @@ pub enum IdentityConfig { /// optional: generate bmap file (currently not working in docker image) #[arg(short = 'b', long = "generate-bmap-file")] generate_bmap: bool, + /// optional: bmap of the input image; default: .bmap or .bmap next to the image + #[arg(long = "bmap")] + bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -199,6 +220,9 @@ pub enum IdentityConfig { /// optional: generate bmap file (currently not working in docker image) #[arg(short = 'b', long = "generate-bmap-file")] generate_bmap: bool, + /// optional: bmap of the input image; default: .bmap or .bmap next to the image + #[arg(long = "bmap")] + bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -220,6 +244,9 @@ pub enum IotHubDeviceUpdate { /// optional: generate bmap file (currently not working in docker image) #[arg(short = 'b', long = "generate-bmap-file")] generate_bmap: bool, + /// optional: bmap of the input image; default: .bmap or .bmap next to the image + #[arg(long = "bmap")] + bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -339,6 +366,9 @@ pub enum SshConfig { /// optional: generate bmap file (currently not working in docker image) #[arg(short = 'b', long = "generate-bmap-file")] generate_bmap: bool, + /// optional: bmap of the input image; default: .bmap or .bmap next to the image + #[arg(long = "bmap")] + bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, diff --git a/src/file/bmap.rs b/src/file/bmap.rs index 8b13b2c2..605c394b 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -1,149 +1,194 @@ -use crate::file::partition::{SECTOR_SIZE, get_partitions}; +use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE}; use anyhow::{Context, Result}; -use log::debug; -use std::{env, fs, os::unix::fs::FileExt, process::Command}; - -const EXT4_SUPERBLOCK_OFFSET: u64 = 1024; -const EXT4_SUPERBLOCK_SIZE: usize = 1024; -const EXT4_S_LOG_BLOCK_SIZE: usize = 0x18; -const EXT4_S_MAGIC: usize = 0x38; -const EXT4_S_FEATURE_COMPAT: usize = 0x5c; -const EXT4_S_JOURNAL_INUM: usize = 0xe0; -const EXT4_MAGIC: u16 = 0xef53; -const EXT4_FEATURE_COMPAT_HAS_JOURNAL: u32 = 0x4; -const EXT4_MIN_BLOCK_SIZE: u64 = 1024; -const EXT4_MAX_LOG_BLOCK_SIZE: u32 = 6; - -const DEBUGFS: &str = "debugfs"; -// e2fsprogs installs to sbin, which is not always in the PATH of a normal user -const SBIN_DIRS: &str = "/usr/sbin:/sbin"; - -struct Ext4Journal { - inode: u32, - block_size: u64, +use std::{ + fs, + ops::RangeInclusive, + os::unix::fs::FileExt, + path::{Path, PathBuf}, + process::Command, +}; + +const BMAP_EXTENSION: &str = "bmap"; +const COMPRESSION_EXTENSIONS: [&str; 5] = ["xz", "gz", "gzip", "bz2", "bzip2"]; + +/// The block ranges of an image that a bmap file maps. +#[derive(Debug)] +pub struct Bmap { + pub block_size: u64, + pub image_size: u64, + pub ranges: Vec>, } -fn read_ext4_journal(image: &fs::File, offset: u64) -> Result> { - let mut sb = [0u8; EXT4_SUPERBLOCK_SIZE]; - image - .read_exact_at(&mut sb, offset + EXT4_SUPERBLOCK_OFFSET) - .context("read_ext4_journal: cannot read superblock")?; - let le16 = |o: usize| u16::from_le_bytes([sb[o], sb[o + 1]]); - let le32 = |o: usize| u32::from_le_bytes([sb[o], sb[o + 1], sb[o + 2], sb[o + 3]]); - - if le16(EXT4_S_MAGIC) != EXT4_MAGIC - || le32(EXT4_S_FEATURE_COMPAT) & EXT4_FEATURE_COMPAT_HAS_JOURNAL == 0 - { - return Ok(None); - } +impl Bmap { + pub fn read(path: &Path) -> Result { + let text = fs::read_to_string(path) + .context(format!("Bmap::read: cannot read {}", path.display()))?; + let doc = roxmltree::Document::parse(&text) + .context(format!("Bmap::read: invalid xml in {}", path.display()))?; + let root = doc.root_element(); + let child = |name: &str| { + root.children() + .find(|n| n.has_tag_name(name)) + .context(format!("Bmap::read: no {name} in {}", path.display())) + }; + let number = |name: &str| -> Result { + child(name)? + .text() + .unwrap_or_default() + .trim() + .parse() + .context(format!("Bmap::read: invalid {name} in {}", path.display())) + }; - // an external journal has no inode in this filesystem - let inode = le32(EXT4_S_JOURNAL_INUM); - if inode == 0 { - return Ok(None); + let block_size = number("BlockSize")?; + anyhow::ensure!(block_size > 0, "Bmap::read: BlockSize is 0"); + let image_size = number("ImageSize")?; + let ranges = child("BlockMap")? + .children() + .filter(|n| n.has_tag_name("Range")) + .map(|n| parse_range(n.text().unwrap_or_default())) + .collect::>()?; + + Ok(Self { + block_size, + image_size, + ranges, + }) } - let log_block_size = le32(EXT4_S_LOG_BLOCK_SIZE); - anyhow::ensure!( - log_block_size <= EXT4_MAX_LOG_BLOCK_SIZE, - "read_ext4_journal: invalid block size" - ); - let block_size = EXT4_MIN_BLOCK_SIZE << log_block_size; + pub fn whole_image(image_size: u64) -> Self { + let block_size = IMAGE_BLOCK_SIZE as u64; + let ranges = match image_size { + 0 => vec![], + size => vec![RangeInclusive::new(0, (size - 1) / block_size)], + }; - Ok(Some(Ext4Journal { inode, block_size })) + Self { + block_size, + image_size, + ranges, + } + } } -fn sbin_command(program: &str) -> Command { - let path = - env::var("PATH").map_or_else(|_| SBIN_DIRS.to_string(), |p| format!("{p}:{SBIN_DIRS}")); - let mut cmd = Command::new(program); - cmd.env("PATH", path); - cmd +fn parse_range(text: &str) -> Result> { + let text = text.trim(); + let (first, last) = text.split_once('-').unwrap_or((text, text)); + let parse = |s: &str| { + s.trim() + .parse::() + .context(format!("parse_range: invalid range {text}")) + }; + let (first, last) = (parse(first)?, parse(last)?); + anyhow::ensure!(first <= last, "parse_range: invalid range {text}"); + + Ok(first..=last) } -fn ext4_journal_blocks(image_file: &str, offset: u64, inode: u32) -> Result> { - let output = sbin_command(DEBUGFS) - .arg("-R") - .arg(format!("blocks <{inode}>")) - .arg(format!("{image_file}?offset={offset}")) - .output() - .context(format!("ext4_journal_blocks: cannot run {DEBUGFS}"))?; - anyhow::ensure!( - output.status.success(), - "ext4_journal_blocks: {DEBUGFS} failed: {}", - String::from_utf8_lossy(&output.stderr) - ); +/// Returns `explicit`, else `.bmap`, else `.bmap`, +/// whichever exists first. +pub fn find_input_bmap(image: &Path, explicit: Option<&Path>) -> Result> { + if let Some(path) = explicit { + anyhow::ensure!( + path.is_file(), + "find_input_bmap: {} not found", + path.display() + ); + return Ok(Some(path.to_path_buf())); + } - let blocks = String::from_utf8(output.stdout) - .context("ext4_journal_blocks: invalid debugfs output")? - .split_whitespace() - .map(|b| { - b.parse::() - .context(format!("ext4_journal_blocks: invalid block number {b}")) - }) - .collect::>>()?; - anyhow::ensure!( - !blocks.is_empty(), - "ext4_journal_blocks: no journal blocks at offset {offset}" - ); + let with_bmap_extension = |p: &Path| { + let mut path = p.as_os_str().to_owned(); + path.push(format!(".{BMAP_EXTENSION}")); + PathBuf::from(path) + }; + let mut candidates = vec![with_bmap_extension(image)]; + if image + .extension() + .and_then(|e| e.to_str()) + .is_some_and(|e| COMPRESSION_EXTENSIONS.contains(&e)) + { + candidates.push(with_bmap_extension(&image.with_extension(""))); + } - Ok(blocks) + Ok(candidates.into_iter().find(|p| p.is_file())) } -/// Rewrites the journal blocks of every ext4 partition with their own content. Blocks that -/// are holes become allocated, so a bmap maps the whole journal and a bmap-based flash -/// overwrites the journal an earlier installation left on the device. -fn allocate_ext4_journals(image_file: &str) -> Result<()> { +/// Writes the zero blocks in the ranges of `bmap` as data. A zero block can be a hole, and a +/// bmap created from the image would skip it, so a bmap-based flash would keep what an earlier +/// installation left there. +pub fn allocate_ranges(image_file: &Path, bmap: &Bmap) -> Result<()> { let image = fs::OpenOptions::new() .read(true) .write(true) .open(image_file) - .context(format!("allocate_ext4_journals: cannot open {image_file}"))?; + .context(format!( + "allocate_ranges: cannot open {}", + image_file.display() + ))?; let image_len = image .metadata() - .context("allocate_ext4_journals: cannot get image size")? + .context("allocate_ranges: cannot get image size")? .len(); + anyhow::ensure!( + bmap.image_size == image_len, + "allocate_ranges: bmap is for an image of {} bytes, the image has {image_len}", + bmap.image_size + ); - for partition in get_partitions(image_file)? { - let offset = partition.start * SECTOR_SIZE; - if offset + EXT4_SUPERBLOCK_OFFSET + EXT4_SUPERBLOCK_SIZE as u64 > image_len { - continue; - } - let Some(journal) = read_ext4_journal(&image, offset)? else { - continue; - }; - let blocks = ext4_journal_blocks(image_file, offset, journal.inode)?; - debug!( - "allocate_ext4_journals: partition {}: {} journal blocks", - partition.num, - blocks.len() + let image_blocks = image_len.div_ceil(bmap.block_size); + let mut buf = vec![0u8; COPY_CHUNK_SIZE]; + for range in &bmap.ranges { + anyhow::ensure!( + *range.end() < image_blocks, + "allocate_ranges: range {range:?} is outside the image" ); + let start = range.start() * bmap.block_size; + let end = ((range.end() + 1) * bmap.block_size).min(image_len); - let mut buf = vec![0u8; usize::try_from(journal.block_size)?]; - for block in blocks { - let pos = offset + block * journal.block_size; + let mut pos = start; + while pos < end { + let n = usize::try_from((end - pos).min(COPY_CHUNK_SIZE as u64))?; + let chunk = &mut buf[..n]; image - .read_exact_at(&mut buf, pos) - .context("allocate_ext4_journals: cannot read journal block")?; - image - .write_all_at(&buf, pos) - .context("allocate_ext4_journals: cannot write journal block")?; + .read_exact_at(chunk, pos) + .context("allocate_ranges: cannot read image")?; + write_zero_blocks(&image, chunk, pos)?; + pos += n as u64; } } image .sync_all() - .context("allocate_ext4_journals: cannot sync image") + .context("allocate_ranges: cannot sync image") } -pub fn generate_bmap_file(image_file: &str) -> Result<()> { - allocate_ext4_journals(image_file)?; +fn write_zero_blocks(image: &fs::File, chunk: &[u8], pos: u64) -> Result<()> { + let is_zero = |b: &[u8]| b.iter().all(|x| *x == 0); + let blocks: Vec<(u64, &[u8])> = if is_zero(chunk) { + vec![(pos, chunk)] + } else { + chunk + .chunks(IMAGE_BLOCK_SIZE) + .enumerate() + .filter(|(_, block)| is_zero(block)) + .map(|(i, block)| (pos + (i * IMAGE_BLOCK_SIZE) as u64, block)) + .collect() + }; + + blocks.into_iter().try_for_each(|(pos, block)| { + image + .write_all_at(block, pos) + .context("allocate_ranges: cannot write image") + }) +} +pub fn generate_bmap_file(image_file: &str) -> Result<()> { let mut bmaptool = Command::new("bmaptool"); bmaptool .arg("create") .arg("-o") - .arg(format!("{image_file}.bmap")) + .arg(format!("{image_file}.{BMAP_EXTENSION}")) .arg(image_file); let status = bmaptool .status() @@ -156,199 +201,194 @@ pub fn generate_bmap_file(image_file: &str) -> Result<()> { #[cfg(test)] mod tests { use crate::file::bmap::*; - use std::path::Path; - - const PART_START_LBA: u32 = 2048; - const PART_SECTORS: u32 = 16384; - const FS_BLOCK_SIZE: u64 = 1024; - const MBR_DISK_SIGNATURE: [u8; 4] = [1, 2, 3, 4]; - const MBR_TYPE_LINUX: u8 = 0x83; - const BMAP_BLOCK_SIZE: u64 = 4096; + + const BLOCK_SIZE: u64 = IMAGE_BLOCK_SIZE as u64; + const IMAGE_BLOCKS: u64 = 16; + const DATA_BLOCK: u64 = 0; + const ZERO_BLOCKS: RangeInclusive = 2..=3; + const UNMAPPED_BLOCK: u64 = 5; + const DATA_BYTE: u8 = 0x5a; // content an earlier installation left on the device const OLD_CONTENT_BYTE: u8 = 0xa5; - const JOURNAL_INODE: u32 = 8; - const INVALID_LOG_BLOCK_SIZE: u32 = 40; - - fn run(cmd: &mut Command) { - let status = cmd.status().expect("run command"); - assert!(status.success(), "{cmd:?} failed"); + const YOCTO_BMAP: &str = "testfiles/image.wic.bmap"; + const YOCTO_IMAGE_SIZE: u64 = 30408704; + const YOCTO_MAPPED_BLOCKS: u64 = 77; + + fn bmap_xml(image_size: u64, block_map: &str) -> String { + format!( + "{image_size}\ + {BLOCK_SIZE}{block_map}" + ) } - fn create_image_with_mbr(image: &Path, len: u64) { - let mut f = fs::File::create(image).expect("create image"); - f.set_len(len).expect("set image size"); - let mut mbr = mbrman::MBR::new_from(&mut f, SECTOR_SIZE as u32, MBR_DISK_SIGNATURE) - .expect("create mbr"); - mbr[1] = mbrman::MBRPartitionEntry { - boot: mbrman::BOOT_INACTIVE, - first_chs: mbrman::CHS::empty(), - sys: MBR_TYPE_LINUX, - last_chs: mbrman::CHS::empty(), - starting_lba: PART_START_LBA, - sectors: PART_SECTORS, - }; - mbr.write_into(&mut f).expect("write mbr"); + fn write_bmap(path: &Path, image_size: u64, ranges: &[RangeInclusive]) { + let block_map: String = ranges + .iter() + .map(|r| format!("{}-{}", r.start(), r.end())) + .collect(); + fs::write(path, bmap_xml(image_size, &block_map)).expect("write bmap"); } - fn create_image_with_ext4_journal(image: &Path) { - let raw = image.with_extension("raw"); - create_image_with_mbr(&raw, u64::from(PART_START_LBA + PART_SECTORS) * SECTOR_SIZE); - - let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; - let fs_blocks = u64::from(PART_SECTORS) * SECTOR_SIZE / FS_BLOCK_SIZE; - run(sbin_command("mkfs.ext4") - .arg("-q") - .arg("-F") - .arg("-b") - .arg(FS_BLOCK_SIZE.to_string()) - .arg("-J") - .arg("size=1") - .arg("-E") - .arg(format!("offset={offset}")) - .arg(&raw) - .arg(fs_blocks.to_string())); - // mke2fs zeroes the journal as unwritten extents, which a bmap maps; a sparse copy - // turns the zeros into holes, as a sparse decompress does - run(Command::new("cp") - .arg("--sparse=always") - .arg(&raw) - .arg(image)); + /// A sparse image with data in `DATA_BLOCK` and holes everywhere else. + fn sparse_image(dir: &Path) -> PathBuf { + let path = dir.join("image.wic"); + let image = fs::File::create(&path).expect("create image"); + image + .set_len(IMAGE_BLOCKS * BLOCK_SIZE) + .expect("set image size"); + image + .write_all_at(&[DATA_BYTE; IMAGE_BLOCK_SIZE], DATA_BLOCK * BLOCK_SIZE) + .expect("write data block"); + path } - fn bmap_ranges(bmap: &str) -> Vec<(u64, u64)> { - bmap.lines() - .filter_map(|l| { - let range = l.split_once("')?.1; - let range = range.split_once(" Bmap { + let image = image.to_str().expect("image path"); + generate_bmap_file(image).expect("generate bmap"); + Bmap::read(Path::new(&format!("{image}.{BMAP_EXTENSION}"))).expect("read bmap") } - fn unmapped_journal_blocks(image: &str, bmap: &str) -> usize { - let ranges = bmap_ranges(bmap); - let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; - let file = fs::File::open(image).expect("open image"); - let journal = read_ext4_journal(&file, offset) - .expect("read superblock") - .expect("ext4 with journal"); - ext4_journal_blocks(image, offset, journal.inode) - .expect("journal blocks") - .iter() - .map(|b| (offset + b * journal.block_size) / BMAP_BLOCK_SIZE) - .filter(|b| !ranges.iter().any(|(s, e)| s <= b && b <= e)) - .count() + fn is_mapped(bmap: &Bmap, block: u64) -> bool { + bmap.ranges.iter().any(|r| r.contains(&block)) } /// Writes the blocks `bmap` maps onto a device that still holds old content, as a bmap /// flash does. - fn bmap_flash(image: &str, bmap: &str, device: &Path) { - let image = fs::File::open(image).expect("open image"); - let len = image.metadata().expect("image metadata").len(); - fs::write( - device, - vec![OLD_CONTENT_BYTE; usize::try_from(len).expect("image size")], - ) - .expect("write old content"); - let device = fs::OpenOptions::new() - .write(true) - .open(device) - .expect("open device"); - - let mut buf = vec![0u8; usize::try_from(BMAP_BLOCK_SIZE).expect("block size")]; - for (first, last) in bmap_ranges(bmap) { - for block in first..=last { - image - .read_exact_at(&mut buf, block * BMAP_BLOCK_SIZE) - .expect("read image block"); - device - .write_all_at(&buf, block * BMAP_BLOCK_SIZE) - .expect("write device block"); - } + fn bmap_flash(image: &Path, bmap: &Bmap, device: &Path) { + let image = fs::read(image).expect("read image"); + let mut content = vec![OLD_CONTENT_BYTE; image.len()]; + for range in &bmap.ranges { + let start = usize::try_from(range.start() * bmap.block_size).expect("start"); + let end = usize::try_from((range.end() + 1) * bmap.block_size).expect("end"); + let end = end.min(image.len()); + content[start..end].copy_from_slice(&image[start..end]); } + fs::write(device, content).expect("write device"); } - fn journal_on_device_matches_image(image: &str, device: &Path) -> bool { - let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; - let image_file = fs::File::open(image).expect("open image"); - let device = fs::File::open(device).expect("open device"); - let journal = read_ext4_journal(&image_file, offset) - .expect("read superblock") - .expect("ext4 with journal"); - - let block_size = usize::try_from(journal.block_size).expect("block size"); - let (mut a, mut b) = (vec![0u8; block_size], vec![0u8; block_size]); - ext4_journal_blocks(image, offset, journal.inode) - .expect("journal blocks") - .iter() - .all(|block| { - let pos = offset + block * journal.block_size; - image_file.read_exact_at(&mut a, pos).expect("read image"); - device.read_exact_at(&mut b, pos).expect("read device"); - a == b - }) + #[test] + fn read_parses_yocto_bmap() { + let bmap = Bmap::read(Path::new(YOCTO_BMAP)).expect("read bmap"); + + assert_eq!(bmap.block_size, BLOCK_SIZE); + assert_eq!(bmap.image_size, YOCTO_IMAGE_SIZE); + let mapped: u64 = bmap.ranges.iter().map(|r| r.end() - r.start() + 1).sum(); + assert_eq!(mapped, YOCTO_MAPPED_BLOCKS); } #[test] - fn generate_bmap_file_maps_ext4_journal() { + fn read_parses_single_blocks_and_spans() { let dir = tempfile::tempdir().expect("create temp dir"); - let image_path = dir.path().join("image.wic"); - create_image_with_ext4_journal(&image_path); - let image = image_path.to_str().expect("image path"); - let bmap_path = format!("{image}.bmap"); - - run(Command::new("bmaptool") - .arg("create") - .arg("-o") - .arg(&bmap_path) - .arg(image)); - let bmap = fs::read_to_string(&bmap_path).expect("read bmap"); - assert!(unmapped_journal_blocks(image, &bmap) > 0); - let device = dir.path().join("device"); - bmap_flash(image, &bmap, &device); - assert!(!journal_on_device_matches_image(image, &device)); + let path = dir.path().join("image.wic.bmap"); + let block_map = " 3 5-7 "; + fs::write(&path, bmap_xml(BLOCK_SIZE, block_map)).expect("write bmap"); - generate_bmap_file(image).expect("generate bmap"); - let bmap = fs::read_to_string(&bmap_path).expect("read bmap"); - assert_eq!(unmapped_journal_blocks(image, &bmap), 0); - bmap_flash(image, &bmap, &device); - assert!(journal_on_device_matches_image(image, &device)); - - let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; - run(sbin_command("e2fsck") - .arg("-fn") - .arg(format!("{image}?offset={offset}"))); + let bmap = Bmap::read(&path).expect("read bmap"); + assert_eq!(bmap.ranges, vec![3..=3, 5..=7]); + } + + #[test] + fn read_rejects_invalid_bmaps() { + let dir = tempfile::tempdir().expect("create temp dir"); + let path = dir.path().join("image.wic.bmap"); + let invalid = [ + "1".to_string(), + bmap_xml(BLOCK_SIZE, "7-5"), + bmap_xml(BLOCK_SIZE, "x"), + "".to_string(), + ]; + + for xml in invalid { + fs::write(&path, &xml).expect("write bmap"); + assert!(Bmap::read(&path).is_err(), "accepted {xml}"); + } } #[test] - fn read_ext4_journal_rejects_invalid_block_size() { - let mut sb = [0u8; EXT4_SUPERBLOCK_SIZE]; - let mut set = |o: usize, v: &[u8]| sb[o..o + v.len()].copy_from_slice(v); - set(EXT4_S_MAGIC, &EXT4_MAGIC.to_le_bytes()); - set( - EXT4_S_FEATURE_COMPAT, - &EXT4_FEATURE_COMPAT_HAS_JOURNAL.to_le_bytes(), + fn find_input_bmap_takes_explicit_path() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + let explicit = dir.path().join("other.bmap"); + + assert!(find_input_bmap(&image, Some(&explicit)).is_err()); + fs::write(&explicit, "").expect("write bmap"); + assert_eq!( + find_input_bmap(&image, Some(&explicit)).expect("find bmap"), + Some(explicit) ); - set(EXT4_S_JOURNAL_INUM, &JOURNAL_INODE.to_le_bytes()); - set(EXT4_S_LOG_BLOCK_SIZE, &INVALID_LOG_BLOCK_SIZE.to_le_bytes()); + } - let image = tempfile::tempfile().expect("create image"); - image - .write_all_at(&sb, EXT4_SUPERBLOCK_OFFSET) - .expect("write superblock"); - assert!(read_ext4_journal(&image, 0).is_err()); + #[test] + fn find_input_bmap_looks_next_to_the_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic.xz"); + let uncompressed_bmap = dir.path().join("image.wic.bmap"); + let compressed_bmap = dir.path().join("image.wic.xz.bmap"); + + assert_eq!(find_input_bmap(&image, None).expect("find bmap"), None); + fs::write(&uncompressed_bmap, "").expect("write bmap"); + assert_eq!( + find_input_bmap(&image, None).expect("find bmap"), + Some(uncompressed_bmap) + ); + fs::write(&compressed_bmap, "").expect("write bmap"); + assert_eq!( + find_input_bmap(&image, None).expect("find bmap"), + Some(compressed_bmap) + ); } #[test] - fn allocate_ext4_journals_skips_partition_past_image_end() { + fn allocate_ranges_maps_zero_ranges_of_the_input_bmap() { let dir = tempfile::tempdir().expect("create temp dir"); - let image = dir.path().join("image.wic"); - create_image_with_mbr(&image, u64::from(PART_START_LBA) * SECTOR_SIZE); + let image = sparse_image(dir.path()); + let input_bmap = dir.path().join("input.bmap"); + write_bmap( + &input_bmap, + IMAGE_BLOCKS * BLOCK_SIZE, + &[DATA_BLOCK..=DATA_BLOCK, ZERO_BLOCKS], + ); + let input = Bmap::read(&input_bmap).expect("read bmap"); + + let before = created_bmap(&image); + assert!(ZERO_BLOCKS.clone().all(|b| !is_mapped(&before, b))); - allocate_ext4_journals(image.to_str().expect("image path")).expect("allocate journals"); + allocate_ranges(&image, &input).expect("allocate ranges"); + let after = created_bmap(&image); + assert!(ZERO_BLOCKS.clone().all(|b| is_mapped(&after, b))); + assert!(!is_mapped(&after, UNMAPPED_BLOCK)); + + let device = dir.path().join("device"); + bmap_flash(&image, &after, &device); + let device = fs::read(&device).expect("read device"); + let zero_start = usize::try_from(ZERO_BLOCKS.start() * BLOCK_SIZE).expect("start"); + let zero_end = usize::try_from((ZERO_BLOCKS.end() + 1) * BLOCK_SIZE).expect("end"); + assert!(device[zero_start..zero_end].iter().all(|b| *b == 0)); + } + + #[test] + fn allocate_ranges_maps_whole_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + + allocate_ranges(&image, &Bmap::whole_image(IMAGE_BLOCKS * BLOCK_SIZE)) + .expect("allocate ranges"); + assert_eq!(created_bmap(&image).ranges, vec![0..=IMAGE_BLOCKS - 1]); + } + + #[test] + fn allocate_ranges_rejects_bmap_of_another_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + + let other_size = Bmap::whole_image((IMAGE_BLOCKS + 1) * BLOCK_SIZE); + assert!(allocate_ranges(&image, &other_size).is_err()); + + let outside = Bmap { + block_size: BLOCK_SIZE, + image_size: IMAGE_BLOCKS * BLOCK_SIZE, + ranges: vec![RangeInclusive::new(IMAGE_BLOCKS, IMAGE_BLOCKS)], + }; + assert!(allocate_ranges(&image, &outside).is_err()); } } diff --git a/src/file/functions.rs b/src/file/functions.rs index 5f8eb6d0..95855d29 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -1,4 +1,4 @@ -use crate::file::{IMAGE_BLOCK_SIZE, partition::SECTOR_SIZE}; +use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, partition::SECTOR_SIZE}; use anyhow::{Context, Result}; use log::{debug, warn}; use std::collections::HashMap; @@ -11,8 +11,6 @@ use std::str::FromStr; use stdext::function_name; use uuid::Uuid; -const COPY_CHUNK_SIZE: usize = 1024 * 1024; - #[derive(clap::ValueEnum, Debug, Clone, Eq, Hash, PartialEq)] #[clap(rename_all = "verbatim")] #[allow(non_camel_case_types)] diff --git a/src/file/mod.rs b/src/file/mod.rs index 073eb7dd..d5920523 100644 --- a/src/file/mod.rs +++ b/src/file/mod.rs @@ -5,6 +5,7 @@ mod partition; /// Granularity at which zero data in an image file is kept as a hole. const IMAGE_BLOCK_SIZE: usize = 4096; +const COPY_CHUNK_SIZE: usize = 1024 * 1024; use super::validators::{ device_update, identity::{IdentityConfig, IdentityType, validate_identity}, diff --git a/src/lib.rs b/src/lib.rs index c487ceda..fcb5e4e0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,8 +20,8 @@ use cli::{ IotHubDeviceUpdate::{self, SetDeviceConfig as IotHubDeviceUpdateSet}, SshConfig::{SetCertificate, SetConnection}, }; -use file::{compression::Compression, functions::FileCopyToParams}; -use log::error; +use file::{bmap::Bmap, compression::Compression, functions::FileCopyToParams}; +use log::{error, info}; use std::{fs, io::Read, path::Path, path::PathBuf}; use tokio::fs::remove_dir_all; use uuid::Uuid; @@ -50,6 +50,7 @@ impl Drop for TempDirGuard { fn run_image_command( image_file: PathBuf, + input_bmap: Option, generate_bmap: bool, target_compression: Option, command: F, @@ -70,6 +71,7 @@ where image_file.to_str().context("cannot get image file path")? ); + let input_bmap = file::bmap::find_input_bmap(&image_file, input_bmap.as_deref())?; let mut dest_image_file = image_file.clone(); // create /tmp/{uuid}/ and copy image into @@ -99,6 +101,22 @@ where ))?; } + let bmap = match input_bmap { + Some(path) => { + info!("keep the ranges of {} mapped", path.display()); + Bmap::read(&path)? + } + None => { + info!("no input bmap found: every block of the image gets mapped"); + Bmap::whole_image( + fs::metadata(&tmp_image_file) + .context("run_image_command: cannot get image size")? + .len(), + ) + } + }; + file::bmap::allocate_ranges(&tmp_image_file, &bmap)?; + // run command command(&tmp_image_file)?; @@ -225,8 +243,9 @@ pub fn run() -> Result<()> { partition, dest, generate_bmap, + bmap, compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img| { + }) => run_image_command(image, bmap, generate_bmap, compress_image, |img| { anyhow::ensure!( dest.to_string_lossy().ends_with(".tar.gz"), format!( @@ -265,8 +284,9 @@ pub fn run() -> Result<()> { image, payload, generate_bmap, + bmap, compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img| { + }) => run_image_command(image, bmap, generate_bmap, compress_image, |img| { file::set_identity_config(&config, img, payload.as_deref()) })?, Command::Identity(SetDeviceCertificate { @@ -276,6 +296,7 @@ pub fn run() -> Result<()> { device_id, days, generate_bmap, + bmap, compress_image, }) => { let cert_info = create_image_cert( @@ -291,7 +312,7 @@ pub fn run() -> Result<()> { ) .context("set_device_certificate: could not create certificate")?; - run_image_command(image, generate_bmap, compress_image, |img| { + run_image_command(image, bmap, generate_bmap, compress_image, |img| { file::set_device_cert( Some(&intermediate_full_chain_cert), &cert_info.cert_path, @@ -307,6 +328,7 @@ pub fn run() -> Result<()> { device_id, days, generate_bmap, + bmap, compress_image, }) => { let cert_info = create_image_cert( @@ -322,7 +344,7 @@ pub fn run() -> Result<()> { ) .context("set_edge_ca_certificate: could not create certificate")?; - run_image_command(image, generate_bmap, compress_image, |img| { + run_image_command(image, bmap, generate_bmap, compress_image, |img| { file::set_edge_ca_cert( Some(&intermediate_full_chain_cert), &cert_info.cert_path, @@ -336,8 +358,9 @@ pub fn run() -> Result<()> { device_key: device_key_pem, image, generate_bmap, + bmap, compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img| { + }) => run_image_command(image, bmap, generate_bmap, compress_image, |img| { file::set_device_cert(None, &device_cert_pem, &device_key_pem, img) })?, Command::Identity(SetIotedgeGatewayConfig { @@ -347,41 +370,65 @@ pub fn run() -> Result<()> { device_identity, device_identity_key, generate_bmap, + bmap, compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img: &PathBuf| { - file::set_iotedge_gateway_config( - &config, - img, - &root_ca, - &device_identity, - &device_identity_key, - ) - })?, + }) => run_image_command( + image, + bmap, + generate_bmap, + compress_image, + |img: &PathBuf| { + file::set_iotedge_gateway_config( + &config, + img, + &root_ca, + &device_identity, + &device_identity_key, + ) + }, + )?, Command::Identity(SetIotLeafSasConfig { config, image, root_ca, generate_bmap, + bmap, compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img: &PathBuf| { - file::set_iot_leaf_sas_config(&config, img, &root_ca) - })?, + }) => run_image_command( + image, + bmap, + generate_bmap, + compress_image, + |img: &PathBuf| file::set_iot_leaf_sas_config(&config, img, &root_ca), + )?, Command::Ssh(SetCertificate { image, root_ca, generate_bmap, + bmap, compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img: &PathBuf| { - file::set_ssh_tunnel_certificate(img, &root_ca) - })?, + }) => run_image_command( + image, + bmap, + generate_bmap, + compress_image, + |img: &PathBuf| file::set_ssh_tunnel_certificate(img, &root_ca), + )?, Command::IotHubDeviceUpdate(IotHubDeviceUpdateSet { iot_hub_device_update_config, image, generate_bmap, + bmap, compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img: &PathBuf| { - file::set_iot_hub_device_update_config(&iot_hub_device_update_config, img) - })?, + }) => run_image_command( + image, + bmap, + generate_bmap, + compress_image, + |img: &PathBuf| { + file::set_iot_hub_device_update_config(&iot_hub_device_update_config, img) + }, + )?, Command::IotHubDeviceUpdate(IotHubDeviceUpdate::ImportUpdate { import_manifest: import_manifest_path, storage_container_name, @@ -495,14 +542,19 @@ pub fn run() -> Result<()> { file_copy_params, image, generate_bmap, + bmap, compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img: &PathBuf| { - file::copy_to_image(&file_copy_params, img) - })?, + }) => run_image_command( + image, + bmap, + generate_bmap, + compress_image, + |img: &PathBuf| file::copy_to_image(&file_copy_params, img), + )?, Command::File(CopyFromImage { file_copy_params, image, - }) => run_image_command(image, false, None, |img: &PathBuf| { + }) => run_image_command(image, None, false, None, |img: &PathBuf| { file::copy_from_image(&file_copy_params, img) })?, } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index c807429d..cdf20263 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -3,9 +3,13 @@ use assert_cmd::Command; use assert_json_diff::assert_json_eq; use common::Testrunner; use httpmock::prelude::*; -use omnect_cli::ssh; +use omnect_cli::{file::bmap::Bmap, ssh}; use predicates::prelude::*; -use std::{fs::create_dir_all, os::unix::fs::MetadataExt, path::PathBuf}; +use std::{ + fs::create_dir_all, + os::unix::fs::MetadataExt, + path::{Path, PathBuf}, +}; use stdext::function_name; #[test] @@ -898,20 +902,24 @@ fn check_file_copy(tr: Testrunner, partition: &str) { // unit of `st_blocks` const STAT_BLOCK_SIZE: u64 = 512; -fn bmap_value(bmap: &str, tag: &str) -> u64 { - // the header comment of a bmap file mentions the tags too; only the closing tag is unique - let open_tag = format!("<{tag}>"); - let end = bmap - .find(&format!("")) - .expect("closing tag not found in bmap"); - let start = bmap[..end] - .rfind(&open_tag) - .expect("opening tag not found in bmap") - + open_tag.len(); - bmap[start..end] - .trim() - .parse() - .expect("tag value is no number") +fn read_bmap(path: &Path) -> Bmap { + Bmap::read(path).expect("read bmap") +} + +fn mapped_blocks(bmap: &Bmap) -> u64 { + bmap.ranges.iter().map(|r| r.end() - r.start() + 1).sum() +} + +fn maps_whole_image(bmap: &Bmap) -> bool { + mapped_blocks(bmap) == bmap.image_size.div_ceil(bmap.block_size) +} + +fn maps_all_ranges_of(bmap: &Bmap, input: &Bmap) -> bool { + input + .ranges + .iter() + .flat_map(|r| r.clone()) + .all(|b| bmap.ranges.iter().any(|r| r.contains(&b))) } #[test] @@ -939,18 +947,14 @@ fn check_file_copy_keeps_image_layout() { .assert() .success(); - // zero blocks of the input must stay mapped, e.g. zeroed ext4 journals - let bmap = std::fs::read_to_string(&bmap_path).expect("read bmap"); - assert_eq!( - bmap_value(&bmap, "MappedBlocksCount"), - bmap_value(&bmap, "BlocksCount") - ); + assert!(maps_whole_image(&read_bmap(&bmap_path))); } #[test] fn check_file_copy_keeps_holes() { let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); let image_path = tr.to_pathbuf("testfiles/image.wic"); + let input_bmap_path = tr.to_pathbuf("testfiles/image.wic.bmap"); let sparse_path = image_path.with_file_name("sparse.wic"); let bmap_path = PathBuf::from(format!("{}.bmap", sparse_path.to_string_lossy())); let in_file = tr.to_pathbuf("testfiles/boot.scr"); @@ -971,14 +975,17 @@ fn check_file_copy_keeps_holes() { .arg(format!("{},factory:/my-file", in_file.to_string_lossy())) .arg("-i") .arg(&sparse_path) + .arg("--bmap") + .arg(&input_bmap_path) .arg("-b") .assert() .success(); let meta = sparse_path.metadata().expect("image metadata"); assert!(meta.blocks() * STAT_BLOCK_SIZE < meta.len()); - let bmap = std::fs::read_to_string(&bmap_path).expect("read bmap"); - assert!(bmap_value(&bmap, "MappedBlocksCount") < bmap_value(&bmap, "BlocksCount")); + let bmap = read_bmap(&bmap_path); + assert!(!maps_whole_image(&bmap)); + assert!(maps_all_ranges_of(&bmap, &read_bmap(&input_bmap_path))); } #[test] @@ -1035,7 +1042,9 @@ fn check_bmap_generation_wic_xz() { let image_path_wic_xz = tr.to_pathbuf("testfiles/image.wic.xz"); let image_path_wic = image_path_wic_xz.with_extension(""); let image_path_wic_copy = image_path_wic_xz.with_extension("copy"); - let image_path_bmap = image_path_wic_xz.with_extension("bmap"); + // the Yocto bmap; omnect-cli replaces it with the one it creates + let image_path_bmap = tr.to_pathbuf("testfiles/image.wic.bmap"); + let input_bmap = read_bmap(&image_path_bmap); let in_file = tr.to_pathbuf("testfiles/boot.scr"); let in_file = in_file.to_str().unwrap(); @@ -1051,8 +1060,9 @@ fn check_bmap_generation_wic_xz() { .assert(); assert.success(); - let bmap = std::fs::read_to_string(&image_path_bmap).expect("read bmap"); - assert!(bmap_value(&bmap, "MappedBlocksCount") < bmap_value(&bmap, "BlocksCount")); + let bmap = read_bmap(&image_path_bmap); + assert!(!maps_whole_image(&bmap)); + assert!(maps_all_ranges_of(&bmap, &input_bmap)); // use bmaptool to verify that the checksum of the bmap file and the image // still match after the copy operations @@ -1065,6 +1075,28 @@ fn check_bmap_generation_wic_xz() { assert.success(); } +#[test] +fn check_bmap_generation_wic_xz_without_input_bmap() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path_wic_xz = tr.to_pathbuf("testfiles/image.wic.xz"); + let image_path_bmap = image_path_wic_xz.with_extension("bmap"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path_wic_xz) + .arg("-b") + .assert() + .success(); + + assert!(maps_whole_image(&read_bmap(&image_path_bmap))); +} + #[test] fn check_image_compression() { let tr = Testrunner::new(function_name!().split("::").last().unwrap()); From fb4441d4afab6f703455044fcc23c14d9c95ec49 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:25:55 +0100 Subject: [PATCH 15/75] fix(file): address review findings on input bmap handling - widen bmap ranges to whole 4 KiB blocks, so a hole is not missed when the bmap block size is not a multiple of 4096 - copy-from-image does not read or enforce an input bmap - read the input bmap before the image is copied, and name it in errors - finish the compress stream, so a failed trailer write is an error - revert the unused partition table split - test a flash onto old content and a bmap of another image - README: describe where the created bmap goes and when it maps the whole image Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 4 +- src/file/bmap.rs | 32 +++++++++++---- src/file/compression.rs | 31 +++++++------- src/file/functions.rs | 41 ++++++++++++++++--- src/file/mod.rs | 11 +++-- src/file/partition.rs | 64 ++++++++++------------------- src/lib.rs | 75 +++++++++++++++++++++------------- tests/integration_tests.rs | 83 +++++++++++++++++++++++++++++++++----- 8 files changed, 230 insertions(+), 111 deletions(-) diff --git a/README.md b/README.md index 18133e7c..53b90332 100644 --- a/README.md +++ b/README.md @@ -186,7 +186,9 @@ omnect-cli file copy-to-image --help ## bmap files -Commands that change an image create a bmap file with `-b`. The bmap of the input image is the base for it: `omnect-cli` takes `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it, or the file given with `--bmap`. Every range the input bmap maps stays mapped, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). The created bmap maps these ranges plus the blocks `omnect-cli` changed. +Commands that change an image create a bmap file with `-b`. It is written next to the image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic.xz`, and replaces a file with this name. + +The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--bmap`, else `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image, otherwise the command fails. Every range the input bmap maps stays mapped, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). The created bmap maps these ranges, the blocks `omnect-cli` changed, and all other blocks that the image file stores as data. For an uncompressed image file that is not sparse, this is the whole image. Without an input bmap, the created bmap maps the whole image, and a flash writes every block. diff --git a/src/file/bmap.rs b/src/file/bmap.rs index 605c394b..918660e9 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -1,4 +1,4 @@ -use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE}; +use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, is_zero}; use anyhow::{Context, Result}; use std::{ fs, @@ -137,14 +137,18 @@ pub fn allocate_ranges(image_file: &Path, bmap: &Bmap) -> Result<()> { ); let image_blocks = image_len.div_ceil(bmap.block_size); + let image_block_size = IMAGE_BLOCK_SIZE as u64; let mut buf = vec![0u8; COPY_CHUNK_SIZE]; for range in &bmap.ranges { anyhow::ensure!( *range.end() < image_blocks, "allocate_ranges: range {range:?} is outside the image" ); - let start = range.start() * bmap.block_size; - let end = ((range.end() + 1) * bmap.block_size).min(image_len); + // check whole image blocks, also when the bmap uses a smaller block size + let start = range.start() * bmap.block_size / image_block_size * image_block_size; + let end = ((range.end() + 1) * bmap.block_size) + .next_multiple_of(image_block_size) + .min(image_len); let mut pos = start; while pos < end { @@ -158,13 +162,10 @@ pub fn allocate_ranges(image_file: &Path, bmap: &Bmap) -> Result<()> { } } - image - .sync_all() - .context("allocate_ranges: cannot sync image") + Ok(()) } fn write_zero_blocks(image: &fs::File, chunk: &[u8], pos: u64) -> Result<()> { - let is_zero = |b: &[u8]| b.iter().all(|x| *x == 0); let blocks: Vec<(u64, &[u8])> = if is_zero(chunk) { vec![(pos, chunk)] } else { @@ -207,6 +208,7 @@ mod tests { const DATA_BLOCK: u64 = 0; const ZERO_BLOCKS: RangeInclusive = 2..=3; const UNMAPPED_BLOCK: u64 = 5; + const SMALL_BLOCK_SIZE: u64 = 512; const DATA_BYTE: u8 = 0x5a; // content an earlier installation left on the device const OLD_CONTENT_BYTE: u8 = 0xa5; @@ -376,6 +378,22 @@ mod tests { assert_eq!(created_bmap(&image).ranges, vec![0..=IMAGE_BLOCKS - 1]); } + #[test] + fn allocate_ranges_maps_whole_blocks_for_a_smaller_bmap_block_size() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + // starts in DATA_BLOCK and ends in the hole after it + let first = (DATA_BLOCK + 1) * BLOCK_SIZE / SMALL_BLOCK_SIZE - 1; + let input = Bmap { + block_size: SMALL_BLOCK_SIZE, + image_size: IMAGE_BLOCKS * BLOCK_SIZE, + ranges: vec![RangeInclusive::new(first, first + 2)], + }; + + allocate_ranges(&image, &input).expect("allocate ranges"); + assert!(is_mapped(&created_bmap(&image), DATA_BLOCK + 1)); + } + #[test] fn allocate_ranges_rejects_bmap_of_another_image() { let dir = tempfile::tempdir().expect("create temp dir"); diff --git a/src/file/compression.rs b/src/file/compression.rs index ba5a43d0..2af9cefc 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -1,4 +1,4 @@ -use crate::file::IMAGE_BLOCK_SIZE; +use crate::file::{IMAGE_BLOCK_SIZE, is_zero}; use anyhow::{Context, Result}; use filemagic::Magic; use log::debug; @@ -49,15 +49,17 @@ impl Compression { source: &mut std::fs::File, destination: &mut std::fs::File, ) -> std::io::Result { - let mut enc: Box = match &self { - Compression::bzip2 => Box::new(bzip2::write::BzEncoder::new( - destination, - bzip2::Compression::best(), - )), - Compression::gzip => Box::new(flate2::write::GzEncoder::new( - destination, - flate2::Compression::best(), - )), + // only `finish` reports an error while writing the trailer + let (bytes_read, _) = match &self { + Compression::bzip2 => { + let mut enc = bzip2::write::BzEncoder::new(destination, bzip2::Compression::best()); + (std::io::copy(source, &mut enc)?, enc.finish()?) + } + Compression::gzip => { + let mut enc = + flate2::write::GzEncoder::new(destination, flate2::Compression::best()); + (std::io::copy(source, &mut enc)?, enc.finish()?) + } Compression::xz { compression_level: level, } => { @@ -65,13 +67,11 @@ impl Compression { .threads(num_cpus::get() as u32) .preset(*level) .encoder()?; - Box::new(xz2::write::XzEncoder::new_stream(destination, stream)) + let mut enc = xz2::write::XzEncoder::new_stream(destination, stream); + (std::io::copy(source, &mut enc)?, enc.finish()?) } }; - - let bytes_written = std::io::copy(source, &mut enc)?; - enc.flush()?; - Ok(bytes_written) + Ok(bytes_read) } pub fn decompress( @@ -161,7 +161,6 @@ impl<'a> SparseWriter<'a> { impl Write for SparseWriter<'_> { fn write(&mut self, buf: &[u8]) -> std::io::Result { - let is_zero = |b: &[u8]| b.iter().all(|x| *x == 0); let block_offset = (self.pos % IMAGE_BLOCK_SIZE as u64) as usize; // one call handles a run of blocks that are all zero or all data diff --git a/src/file/functions.rs b/src/file/functions.rs index 95855d29..b85b969e 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -485,6 +485,7 @@ fn write_partition( #[cfg(test)] mod tests { use crate::file::functions::*; + use std::os::unix::fs::MetadataExt; const PART_NUM: u32 = 1; const PART_START: u64 = 8; @@ -492,6 +493,14 @@ mod tests { const OLD_CONTENT_BYTE: u8 = 0xa5; const ZEROED_BLOCK: usize = 2; + fn partition_info() -> PartitionInfo { + PartitionInfo { + num: PART_NUM, + start: PART_START, + count: PART_SECTORS, + } + } + #[test] fn write_partition_writes_block_changed_to_zeros() { let dir = tempfile::tempdir().expect("create temp dir"); @@ -508,15 +517,37 @@ mod tests { write_partition( image.to_str().expect("image path"), partition.to_str().expect("partition path"), - &PartitionInfo { - num: PART_NUM, - start: PART_START, - count: PART_SECTORS, - }, + &partition_info(), ) .expect("write partition"); let written = fs::read(&image).expect("read image"); assert!(written[part_offset..] == content[..], "partition differs"); } + + #[test] + fn write_partition_keeps_holes_of_unchanged_blocks() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + let partition = dir.path().join("partition.img"); + fs::File::create(&image) + .expect("create image") + .set_len((PART_START + PART_SECTORS) * SECTOR_SIZE) + .expect("set image size"); + fs::File::create(&partition) + .expect("create partition") + .set_len(PART_SECTORS * SECTOR_SIZE) + .expect("set partition size"); + let allocated = || fs::metadata(&image).expect("image metadata").blocks(); + let before = allocated(); + + write_partition( + image.to_str().expect("image path"), + partition.to_str().expect("partition path"), + &partition_info(), + ) + .expect("write partition"); + + assert_eq!(allocated(), before, "unchanged zero blocks were written"); + } } diff --git a/src/file/mod.rs b/src/file/mod.rs index d5920523..6d334287 100644 --- a/src/file/mod.rs +++ b/src/file/mod.rs @@ -3,9 +3,6 @@ pub mod compression; pub mod functions; mod partition; -/// Granularity at which zero data in an image file is kept as a hole. -const IMAGE_BLOCK_SIZE: usize = 4096; -const COPY_CHUNK_SIZE: usize = 1024 * 1024; use super::validators::{ device_update, identity::{IdentityConfig, IdentityType, validate_identity}, @@ -20,6 +17,14 @@ use std::{ path::{Path, PathBuf}, }; +/// Granularity at which zero data in an image file is kept as a hole. +const IMAGE_BLOCK_SIZE: usize = 4096; +const COPY_CHUNK_SIZE: usize = 1024 * 1024; + +fn is_zero(data: &[u8]) -> bool { + data.iter().all(|b| *b == 0) +} + pub fn set_iotedge_gateway_config( config_file: &Path, image_file: &Path, diff --git a/src/file/partition.rs b/src/file/partition.rs index 36fe983d..7a617fcd 100644 --- a/src/file/partition.rs +++ b/src/file/partition.rs @@ -13,13 +13,6 @@ pub struct PartitionData { } pub fn get_partition_data>(path: P, partition_num: u32) -> Result { - get_partitions(path)? - .into_iter() - .find(|p| p.num == partition_num) - .with_context(|| format!("partition {partition_num} not found in image")) -} - -pub fn get_partitions>(path: P) -> Result> { let path = path.as_ref(); let mut file = File::open(path).with_context(|| format!("failed to open image: {}", path.display()))?; @@ -28,15 +21,21 @@ pub fn get_partitions>(path: P) -> Result> { // Capture any error so we can attach it as context if MBR also fails. let gpt_err = match gptman::GPT::find_from(&mut file) { Ok(gpt) => { - return Ok(gpt + let entry = gpt .iter() - .filter(|(_, e)| e.is_used() && e.ending_lba >= e.starting_lba) - .map(|(num, e)| PartitionData { - num, - start: e.starting_lba, - count: e.ending_lba - e.starting_lba + 1, - }) - .collect()); + .find(|(i, _)| *i == partition_num) + .map(|(_, e)| e) + .with_context(|| format!("GPT partition {partition_num} out of range"))?; + anyhow::ensure!(entry.is_used(), "GPT partition {partition_num} is not used"); + anyhow::ensure!( + entry.ending_lba >= entry.starting_lba, + "GPT partition {partition_num} has invalid LBA range (ending_lba < starting_lba)" + ); + return Ok(PartitionData { + num: partition_num, + start: entry.starting_lba, + count: entry.ending_lba - entry.starting_lba + 1, + }); } Err(e) => e, }; @@ -46,38 +45,19 @@ pub fn get_partitions>(path: P) -> Result> { .context("failed to seek to start of image")?; let mbr = mbrman::MBR::read_from(&mut file, SECTOR_SIZE as u32) .with_context(|| format!("image is neither valid GPT nor MBR (GPT error: {gpt_err})"))?; - mbr.iter() - .filter(|(_, p)| p.is_used() && !p.is_extended()) - .map(|(num, p)| { - Ok(PartitionData { - num: u32::try_from(num).context("MBR partition number out of range")?, + for (i, p) in mbr.iter() { + if u32::try_from(i).ok() == Some(partition_num) && p.is_used() { + return Ok(PartitionData { + num: partition_num, start: p.starting_lba as u64, count: p.sectors as u64, - }) - }) - .collect() + }); + } + } + anyhow::bail!("partition {partition_num} not found in image") } pub fn is_gpt>(path: P) -> Result { let mut file = File::open(path.as_ref()).context("is_gpt: failed to open image")?; Ok(gptman::GPT::find_from(&mut file).is_ok()) } - -#[cfg(test)] -mod tests { - use crate::file::partition::*; - - const TEST_IMAGE: &str = "testfiles/image.wic"; - const MBR_EXTENDED_PARTITION: u32 = 4; - - #[test] - fn get_partitions_skips_mbr_extended_entry() { - let nums: Vec = get_partitions(TEST_IMAGE) - .expect("read partitions") - .iter() - .map(|p| p.num) - .collect(); - assert!(!nums.contains(&MBR_EXTENDED_PARTITION)); - assert!(nums.contains(&(MBR_EXTENDED_PARTITION + 1))); - } -} diff --git a/src/lib.rs b/src/lib.rs index fcb5e4e0..b1861e15 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,7 +22,11 @@ use cli::{ }; use file::{bmap::Bmap, compression::Compression, functions::FileCopyToParams}; use log::{error, info}; -use std::{fs, io::Read, path::Path, path::PathBuf}; +use std::{ + fs, + io::Read, + path::{Path, PathBuf}, +}; use tokio::fs::remove_dir_all; use uuid::Uuid; @@ -48,9 +52,17 @@ impl Drop for TempDirGuard { } } +/// What an image command does with the holes and mapped blocks of the image. +enum Layout { + /// The command does not change the image. + Unchanged, + /// Keep the ranges of this bmap, or of the one found next to the image, mapped. + Keep(Option), +} + fn run_image_command( image_file: PathBuf, - input_bmap: Option, + layout: Layout, generate_bmap: bool, target_compression: Option, command: F, @@ -71,7 +83,14 @@ where image_file.to_str().context("cannot get image file path")? ); - let input_bmap = file::bmap::find_input_bmap(&image_file, input_bmap.as_deref())?; + let input_bmap = match layout { + Layout::Unchanged => None, + Layout::Keep(explicit) => Some( + file::bmap::find_input_bmap(&image_file, explicit.as_deref())? + .map(|path| Bmap::read(&path).map(|bmap| (path, bmap))) + .transpose()?, + ), + }; let mut dest_image_file = image_file.clone(); // create /tmp/{uuid}/ and copy image into @@ -101,21 +120,21 @@ where ))?; } - let bmap = match input_bmap { - Some(path) => { - info!("keep the ranges of {} mapped", path.display()); - Bmap::read(&path)? + match input_bmap { + None => {} + Some(Some((path, bmap))) => { + info!("keeping the ranges of {} mapped", path.display()); + file::bmap::allocate_ranges(&tmp_image_file, &bmap) + .context(format!("cannot use input bmap {}", path.display()))?; } - None => { - info!("no input bmap found: every block of the image gets mapped"); - Bmap::whole_image( - fs::metadata(&tmp_image_file) - .context("run_image_command: cannot get image size")? - .len(), - ) + Some(None) => { + info!("no input bmap found, so every block of the image is allocated"); + let image_size = fs::metadata(&tmp_image_file) + .context("run_image_command: cannot get image size")? + .len(); + file::bmap::allocate_ranges(&tmp_image_file, &Bmap::whole_image(image_size))?; } - }; - file::bmap::allocate_ranges(&tmp_image_file, &bmap)?; + } // run command command(&tmp_image_file)?; @@ -236,6 +255,8 @@ fn create_image_cert(image: &Path, cert_opts: CertificateOptions) -> Result Result<()> { + use Layout::{Keep, Unchanged}; + match cli::from_args() { Command::Docker(Inject { docker_image, @@ -245,7 +266,7 @@ pub fn run() -> Result<()> { generate_bmap, bmap, compress_image, - }) => run_image_command(image, bmap, generate_bmap, compress_image, |img| { + }) => run_image_command(image, Keep(bmap), generate_bmap, compress_image, |img| { anyhow::ensure!( dest.to_string_lossy().ends_with(".tar.gz"), format!( @@ -286,7 +307,7 @@ pub fn run() -> Result<()> { generate_bmap, bmap, compress_image, - }) => run_image_command(image, bmap, generate_bmap, compress_image, |img| { + }) => run_image_command(image, Keep(bmap), generate_bmap, compress_image, |img| { file::set_identity_config(&config, img, payload.as_deref()) })?, Command::Identity(SetDeviceCertificate { @@ -312,7 +333,7 @@ pub fn run() -> Result<()> { ) .context("set_device_certificate: could not create certificate")?; - run_image_command(image, bmap, generate_bmap, compress_image, |img| { + run_image_command(image, Keep(bmap), generate_bmap, compress_image, |img| { file::set_device_cert( Some(&intermediate_full_chain_cert), &cert_info.cert_path, @@ -344,7 +365,7 @@ pub fn run() -> Result<()> { ) .context("set_edge_ca_certificate: could not create certificate")?; - run_image_command(image, bmap, generate_bmap, compress_image, |img| { + run_image_command(image, Keep(bmap), generate_bmap, compress_image, |img| { file::set_edge_ca_cert( Some(&intermediate_full_chain_cert), &cert_info.cert_path, @@ -360,7 +381,7 @@ pub fn run() -> Result<()> { generate_bmap, bmap, compress_image, - }) => run_image_command(image, bmap, generate_bmap, compress_image, |img| { + }) => run_image_command(image, Keep(bmap), generate_bmap, compress_image, |img| { file::set_device_cert(None, &device_cert_pem, &device_key_pem, img) })?, Command::Identity(SetIotedgeGatewayConfig { @@ -374,7 +395,7 @@ pub fn run() -> Result<()> { compress_image, }) => run_image_command( image, - bmap, + Keep(bmap), generate_bmap, compress_image, |img: &PathBuf| { @@ -396,7 +417,7 @@ pub fn run() -> Result<()> { compress_image, }) => run_image_command( image, - bmap, + Keep(bmap), generate_bmap, compress_image, |img: &PathBuf| file::set_iot_leaf_sas_config(&config, img, &root_ca), @@ -409,7 +430,7 @@ pub fn run() -> Result<()> { compress_image, }) => run_image_command( image, - bmap, + Keep(bmap), generate_bmap, compress_image, |img: &PathBuf| file::set_ssh_tunnel_certificate(img, &root_ca), @@ -422,7 +443,7 @@ pub fn run() -> Result<()> { compress_image, }) => run_image_command( image, - bmap, + Keep(bmap), generate_bmap, compress_image, |img: &PathBuf| { @@ -546,7 +567,7 @@ pub fn run() -> Result<()> { compress_image, }) => run_image_command( image, - bmap, + Keep(bmap), generate_bmap, compress_image, |img: &PathBuf| file::copy_to_image(&file_copy_params, img), @@ -554,7 +575,7 @@ pub fn run() -> Result<()> { Command::File(CopyFromImage { file_copy_params, image, - }) => run_image_command(image, None, false, None, |img: &PathBuf| { + }) => run_image_command(image, Unchanged, false, None, |img: &PathBuf| { file::copy_from_image(&file_copy_params, img) })?, } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index cdf20263..d0ffca0c 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -6,7 +6,7 @@ use httpmock::prelude::*; use omnect_cli::{file::bmap::Bmap, ssh}; use predicates::prelude::*; use std::{ - fs::create_dir_all, + fs::{self, create_dir_all}, os::unix::fs::MetadataExt, path::{Path, PathBuf}, }; @@ -901,6 +901,10 @@ fn check_file_copy(tr: Testrunner, partition: &str) { // unit of `st_blocks` const STAT_BLOCK_SIZE: u64 = 512; +// content an earlier installation left on the device +const OLD_CONTENT_BYTE: u8 = 0xa5; +const OTHER_IMAGE_BMAP: &str = "40964096\ + 0"; fn read_bmap(path: &Path) -> Bmap { Bmap::read(path).expect("read bmap") @@ -922,19 +926,26 @@ fn maps_all_ranges_of(bmap: &Bmap, input: &Bmap) -> bool { .all(|b| bmap.ranges.iter().any(|r| r.contains(&b))) } +/// Writes the blocks `bmap` maps onto a device that still holds old content, as a bmap flash +/// does. +fn bmap_flash(image: &[u8], bmap: &Bmap) -> Vec { + let mut device = vec![OLD_CONTENT_BYTE; image.len()]; + for range in &bmap.ranges { + let start = usize::try_from(range.start() * bmap.block_size).expect("start"); + let end = usize::try_from((range.end() + 1) * bmap.block_size).expect("end"); + let end = end.min(image.len()); + device[start..end].copy_from_slice(&image[start..end]); + } + device +} + #[test] -fn check_file_copy_keeps_image_layout() { +fn check_file_copy_without_input_bmap_maps_whole_image() { let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); let image_path = tr.to_pathbuf("testfiles/image.wic"); let bmap_path = PathBuf::from(format!("{}.bmap", image_path.to_string_lossy())); let in_file = tr.to_pathbuf("testfiles/boot.scr"); - let meta = image_path.metadata().expect("image metadata"); - assert!( - meta.blocks() * STAT_BLOCK_SIZE >= meta.len(), - "test image must be fully allocated" - ); - Command::cargo_bin("omnect-cli") .expect("omnect-cli binary") .arg("file") @@ -984,8 +995,59 @@ fn check_file_copy_keeps_holes() { let meta = sparse_path.metadata().expect("image metadata"); assert!(meta.blocks() * STAT_BLOCK_SIZE < meta.len()); let bmap = read_bmap(&bmap_path); + let input_bmap = read_bmap(&input_bmap_path); assert!(!maps_whole_image(&bmap)); - assert!(maps_all_ranges_of(&bmap, &read_bmap(&input_bmap_path))); + assert!(maps_all_ranges_of(&bmap, &input_bmap)); + + // every data block and every block of the input bmap must replace the old content + let image = fs::read(&sparse_path).expect("read image"); + let device = bmap_flash(&image, &bmap); + let block_size = usize::try_from(input_bmap.block_size).expect("block size"); + let blocks = image.chunks(block_size).zip(device.chunks(block_size)); + for (block, (image_block, device_block)) in (0u64..).zip(blocks) { + let needed = image_block.iter().any(|b| *b != 0) + || input_bmap.ranges.iter().any(|r| r.contains(&block)); + assert!( + !needed || image_block == device_block, + "block {block} keeps old content" + ); + } +} + +#[test] +fn check_input_bmap_of_another_image() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let bmap_path = PathBuf::from(format!("{}.bmap", image_path.to_string_lossy())); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + let out_file = tr.pathbuf().join("hosts"); + fs::write(&bmap_path, OTHER_IMAGE_BMAP).expect("write bmap"); + let image_hash = Testrunner::file_hash(&image_path); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .assert() + .failure() + .stderr(predicate::str::contains("cannot use input bmap")); + assert_eq!(Testrunner::file_hash(&image_path), image_hash); + + // reading from the image does not use the bmap + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-from-image") + .arg("-f") + .arg(format!("rootA:/etc/hosts,{}", out_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .assert() + .success(); } #[test] @@ -1042,7 +1104,8 @@ fn check_bmap_generation_wic_xz() { let image_path_wic_xz = tr.to_pathbuf("testfiles/image.wic.xz"); let image_path_wic = image_path_wic_xz.with_extension(""); let image_path_wic_copy = image_path_wic_xz.with_extension("copy"); - // the Yocto bmap; omnect-cli replaces it with the one it creates + // a Yocto bmap of an image with the same layout; omnect-cli replaces it with the one it + // creates let image_path_bmap = tr.to_pathbuf("testfiles/image.wic.bmap"); let input_bmap = read_bmap(&image_path_bmap); let in_file = tr.to_pathbuf("testfiles/boot.scr"); From cb3c4765a2a4437166ae039dbb956a1ee68c37a8 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:34:49 +0100 Subject: [PATCH 16/75] feat(cli)!: rename bmap options to --input-bmap and --output-bmap The short option -b stays. BREAKING CHANGE: --generate-bmap-file is now --output-bmap. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 6 +- src/cli.rs | 100 +++++++++---------- src/lib.rs | 200 +++++++++++++++++++++---------------- tests/integration_tests.rs | 2 +- 6 files changed, 169 insertions(+), 143 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b824097f..6bf25d20 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1763,7 +1763,7 @@ dependencies = [ [[package]] name = "omnect-cli" -version = "0.28.6" +version = "0.29.0" dependencies = [ "actix-web", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 35d5c6eb..3443bc09 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ license = "MIT OR Apache-2.0" name = "omnect-cli" readme = "README.md" repository = "https://github.com/omnect/omnect-cli" -version = "0.28.6" +version = "0.29.0" [dependencies] actix-web = { version = "4.17", default-features = false, features = ["macros"] } diff --git a/README.md b/README.md index 53b90332..8062664f 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ docker run --rm -it \ omnect/omnect-cli:latest file copy-to-image --files /source/my-source-file,boot:/my-dest-file -i /source/my-image.wic ``` - **Note1**: `-b` option to create bmap file is not supported by omnect-cli docker image.
+ **Note1**: `-b` (`--output-bmap`) option to create a bmap file is not supported by omnect-cli docker image.
**Note2**: The ssh tunnel option requires some additional settings. See [here](Usage-with-docker) for more details.
**Note3**: The docker inject command is not supported by omnect-cli docker image.
. @@ -186,9 +186,9 @@ omnect-cli file copy-to-image --help ## bmap files -Commands that change an image create a bmap file with `-b`. It is written next to the image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic.xz`, and replaces a file with this name. +Commands that change an image create a bmap file with `-b` (`--output-bmap`). It is written next to the image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic.xz`, and replaces a file with this name. -The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--bmap`, else `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image, otherwise the command fails. Every range the input bmap maps stays mapped, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). The created bmap maps these ranges, the blocks `omnect-cli` changed, and all other blocks that the image file stores as data. For an uncompressed image file that is not sparse, this is the whole image. +The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image, otherwise the command fails. Every range the input bmap maps stays mapped, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). The created bmap maps these ranges, the blocks `omnect-cli` changed, and all other blocks that the image file stores as data. For an uncompressed image file that is not sparse, this is the whole image. Without an input bmap, the created bmap maps the whole image, and a flash writes every block. diff --git a/src/cli.rs b/src/cli.rs index 6b104ac4..8d7278a7 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -34,12 +34,12 @@ pub enum Docker { /// destination path of the docker image in the firmware image (must end in ".tar.gz") #[clap(short = 'e', long = "dest")] dest: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, + /// optional: create a bmap of the output image (currently not working in docker image) + #[arg(short = 'b', long = "output-bmap")] + output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "bmap")] - bmap: Option, + #[arg(long = "input-bmap")] + input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -58,12 +58,12 @@ pub enum File { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, + /// optional: create a bmap of the output image (currently not working in docker image) + #[arg(short = 'b', long = "output-bmap")] + output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "bmap")] - bmap: Option, + #[arg(long = "input-bmap")] + input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -94,12 +94,12 @@ pub enum IdentityConfig { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, + /// optional: create a bmap of the output image (currently not working in docker image) + #[arg(short = 'b', long = "output-bmap")] + output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "bmap")] - bmap: Option, + #[arg(long = "input-bmap")] + input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -121,12 +121,12 @@ pub enum IdentityConfig { /// path to device identity certificate key file #[arg(short = 'k', long = "device_identity_key")] device_identity_key: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, + /// optional: create a bmap of the output image (currently not working in docker image) + #[arg(short = 'b', long = "output-bmap")] + output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "bmap")] - bmap: Option, + #[arg(long = "input-bmap")] + input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -142,12 +142,12 @@ pub enum IdentityConfig { /// path to root ca certificate file #[arg(short = 'r', long = "root_ca")] root_ca: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, + /// optional: create a bmap of the output image (currently not working in docker image) + #[arg(short = 'b', long = "output-bmap")] + output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "bmap")] - bmap: Option, + #[arg(long = "input-bmap")] + input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -169,12 +169,12 @@ pub enum IdentityConfig { /// period of validity in days #[arg(short = 'D', long = "days")] days: u32, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, + /// optional: create a bmap of the output image (currently not working in docker image) + #[arg(short = 'b', long = "output-bmap")] + output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "bmap")] - bmap: Option, + #[arg(long = "input-bmap")] + input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -190,12 +190,12 @@ pub enum IdentityConfig { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, + /// optional: create a bmap of the output image (currently not working in docker image) + #[arg(short = 'b', long = "output-bmap")] + output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "bmap")] - bmap: Option, + #[arg(long = "input-bmap")] + input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -217,12 +217,12 @@ pub enum IdentityConfig { /// period of validity in days #[arg(short = 'D', long = "days")] days: u32, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, + /// optional: create a bmap of the output image (currently not working in docker image) + #[arg(short = 'b', long = "output-bmap")] + output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "bmap")] - bmap: Option, + #[arg(long = "input-bmap")] + input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -241,12 +241,12 @@ pub enum IotHubDeviceUpdate { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, + /// optional: create a bmap of the output image (currently not working in docker image) + #[arg(short = 'b', long = "output-bmap")] + output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "bmap")] - bmap: Option, + #[arg(long = "input-bmap")] + input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -363,12 +363,12 @@ pub enum SshConfig { /// path to public key of the ssh root ca #[arg(short = 'r', long = "root_ca")] root_ca: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, + /// optional: create a bmap of the output image (currently not working in docker image) + #[arg(short = 'b', long = "output-bmap")] + output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "bmap")] - bmap: Option, + #[arg(long = "input-bmap")] + input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, diff --git a/src/lib.rs b/src/lib.rs index b1861e15..db5c98a0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -63,7 +63,7 @@ enum Layout { fn run_image_command( image_file: PathBuf, layout: Layout, - generate_bmap: bool, + output_bmap: bool, target_compression: Option, command: F, ) -> Result<()> @@ -72,7 +72,7 @@ where { if let Ok("true") | Ok("1") = std::env::var("CONTAINERIZED").as_deref() { anyhow::ensure!( - !generate_bmap, + !output_bmap, "run_image_command: generating bmap file is not supported in containerized environments." ); } @@ -140,7 +140,7 @@ where command(&tmp_image_file)?; // create and copy back bmap file if one was created - if generate_bmap { + if output_bmap { let mut target_bmap = image_file .parent() .context("cannot get parent dir of image path")? @@ -263,61 +263,71 @@ pub fn run() -> Result<()> { image, partition, dest, - generate_bmap, - bmap, + output_bmap, + input_bmap, compress_image, - }) => run_image_command(image, Keep(bmap), generate_bmap, compress_image, |img| { - anyhow::ensure!( - dest.to_string_lossy().ends_with(".tar.gz"), - format!( - "invalid destination file path \"{}\". Must end in \".tar.gz\".", - dest.to_string_lossy(), - ), - ); - - let arch = image::image_arch(img)?; - - let docker_path = docker::pull_image(&docker_image, arch)?; - - let result = file::copy_to_image( - &[FileCopyToParams::new( - &docker_path, - partition.clone(), - &dest, - )], - img, - ); - std::fs::remove_file(docker_path)?; - - if result.is_ok() { - println!( - "Stored {} to {}:{}", - docker_image, - partition, - dest.to_string_lossy(), + }) => run_image_command( + image, + Keep(input_bmap), + output_bmap, + compress_image, + |img| { + anyhow::ensure!( + dest.to_string_lossy().ends_with(".tar.gz"), + format!( + "invalid destination file path \"{}\". Must end in \".tar.gz\".", + dest.to_string_lossy(), + ), ); - } - result - })?, + let arch = image::image_arch(img)?; + + let docker_path = docker::pull_image(&docker_image, arch)?; + + let result = file::copy_to_image( + &[FileCopyToParams::new( + &docker_path, + partition.clone(), + &dest, + )], + img, + ); + std::fs::remove_file(docker_path)?; + + if result.is_ok() { + println!( + "Stored {} to {}:{}", + docker_image, + partition, + dest.to_string_lossy(), + ); + } + + result + }, + )?, Command::Identity(SetConfig { config, image, payload, - generate_bmap, - bmap, + output_bmap, + input_bmap, compress_image, - }) => run_image_command(image, Keep(bmap), generate_bmap, compress_image, |img| { - file::set_identity_config(&config, img, payload.as_deref()) - })?, + }) => run_image_command( + image, + Keep(input_bmap), + output_bmap, + compress_image, + |img| file::set_identity_config(&config, img, payload.as_deref()), + )?, Command::Identity(SetDeviceCertificate { intermediate_full_chain_cert, intermediate_key, image, device_id, days, - generate_bmap, - bmap, + output_bmap, + input_bmap, compress_image, }) => { let cert_info = create_image_cert( @@ -333,14 +343,20 @@ pub fn run() -> Result<()> { ) .context("set_device_certificate: could not create certificate")?; - run_image_command(image, Keep(bmap), generate_bmap, compress_image, |img| { - file::set_device_cert( - Some(&intermediate_full_chain_cert), - &cert_info.cert_path, - &cert_info.key_path, - img, - ) - })? + run_image_command( + image, + Keep(input_bmap), + output_bmap, + compress_image, + |img| { + file::set_device_cert( + Some(&intermediate_full_chain_cert), + &cert_info.cert_path, + &cert_info.key_path, + img, + ) + }, + )? } Command::Identity(SetEdgeCaCertificate { intermediate_full_chain_cert, @@ -348,8 +364,8 @@ pub fn run() -> Result<()> { image, device_id, days, - generate_bmap, - bmap, + output_bmap, + input_bmap, compress_image, }) => { let cert_info = create_image_cert( @@ -365,38 +381,48 @@ pub fn run() -> Result<()> { ) .context("set_edge_ca_certificate: could not create certificate")?; - run_image_command(image, Keep(bmap), generate_bmap, compress_image, |img| { - file::set_edge_ca_cert( - Some(&intermediate_full_chain_cert), - &cert_info.cert_path, - &cert_info.key_path, - img, - ) - })? + run_image_command( + image, + Keep(input_bmap), + output_bmap, + compress_image, + |img| { + file::set_edge_ca_cert( + Some(&intermediate_full_chain_cert), + &cert_info.cert_path, + &cert_info.key_path, + img, + ) + }, + )? } Command::Identity(SetDeviceCertificateNoEst { device_cert: device_cert_pem, device_key: device_key_pem, image, - generate_bmap, - bmap, + output_bmap, + input_bmap, compress_image, - }) => run_image_command(image, Keep(bmap), generate_bmap, compress_image, |img| { - file::set_device_cert(None, &device_cert_pem, &device_key_pem, img) - })?, + }) => run_image_command( + image, + Keep(input_bmap), + output_bmap, + compress_image, + |img| file::set_device_cert(None, &device_cert_pem, &device_key_pem, img), + )?, Command::Identity(SetIotedgeGatewayConfig { config, image, root_ca, device_identity, device_identity_key, - generate_bmap, - bmap, + output_bmap, + input_bmap, compress_image, }) => run_image_command( image, - Keep(bmap), - generate_bmap, + Keep(input_bmap), + output_bmap, compress_image, |img: &PathBuf| { file::set_iotedge_gateway_config( @@ -412,39 +438,39 @@ pub fn run() -> Result<()> { config, image, root_ca, - generate_bmap, - bmap, + output_bmap, + input_bmap, compress_image, }) => run_image_command( image, - Keep(bmap), - generate_bmap, + Keep(input_bmap), + output_bmap, compress_image, |img: &PathBuf| file::set_iot_leaf_sas_config(&config, img, &root_ca), )?, Command::Ssh(SetCertificate { image, root_ca, - generate_bmap, - bmap, + output_bmap, + input_bmap, compress_image, }) => run_image_command( image, - Keep(bmap), - generate_bmap, + Keep(input_bmap), + output_bmap, compress_image, |img: &PathBuf| file::set_ssh_tunnel_certificate(img, &root_ca), )?, Command::IotHubDeviceUpdate(IotHubDeviceUpdateSet { iot_hub_device_update_config, image, - generate_bmap, - bmap, + output_bmap, + input_bmap, compress_image, }) => run_image_command( image, - Keep(bmap), - generate_bmap, + Keep(input_bmap), + output_bmap, compress_image, |img: &PathBuf| { file::set_iot_hub_device_update_config(&iot_hub_device_update_config, img) @@ -562,13 +588,13 @@ pub fn run() -> Result<()> { Command::File(CopyToImage { file_copy_params, image, - generate_bmap, - bmap, + output_bmap, + input_bmap, compress_image, }) => run_image_command( image, - Keep(bmap), - generate_bmap, + Keep(input_bmap), + output_bmap, compress_image, |img: &PathBuf| file::copy_to_image(&file_copy_params, img), )?, diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index d0ffca0c..fb7f165e 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -986,7 +986,7 @@ fn check_file_copy_keeps_holes() { .arg(format!("{},factory:/my-file", in_file.to_string_lossy())) .arg("-i") .arg(&sparse_path) - .arg("--bmap") + .arg("--input-bmap") .arg(&input_bmap_path) .arg("-b") .assert() From 46d03622f658b7e8672718fe052272fd85b377c5 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:32:22 +0100 Subject: [PATCH 17/75] fix(file): map zero blocks that a copy writes into a hole read_partition turned every zero block into a hole, and write_partition skipped blocks that are zero in both files. A zero block of a copied file that landed in free space stayed a hole in the image, so a bmap created from it did not map the block, and a bmap flash kept the old content there. read_partition now keeps the holes of the image, and write_partition also writes a block that is data in the partition file but a hole in the image. The sparse copy of copy_image moves to a shared helper that takes offsets. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- Cargo.lock | 17 +---- Cargo.toml | 2 +- src/file/functions.rs | 96 +++++++++++++++++++----- src/file/mod.rs | 1 + src/file/sparse.rs | 150 +++++++++++++++++++++++++++++++++++++ src/lib.rs | 18 +---- tests/integration_tests.rs | 80 ++++++++++++++++++-- 7 files changed, 305 insertions(+), 59 deletions(-) create mode 100644 src/file/sparse.rs diff --git a/Cargo.lock b/Cargo.lock index 6bf25d20..38a63979 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1528,21 +1528,6 @@ version = "0.2.190" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" -[[package]] -name = "libfs" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b79c3145dcaad30b6d7b1e30c9d2d23799b882e9a2e9d1fc7bcc0699c5279ff9" -dependencies = [ - "cfg-if", - "libc", - "linux-raw-sys", - "log", - "rustix", - "thiserror 2.0.21", - "xattr", -] - [[package]] name = "libredox" version = "0.1.25" @@ -1782,7 +1767,6 @@ dependencies = [ "hmac", "httpmock", "keyring-core", - "libfs", "log", "mbrman", "num_cpus", @@ -1794,6 +1778,7 @@ dependencies = [ "reqwest 0.13.5", "ring", "roxmltree", + "rustix", "serde", "serde_json", "serde_path_to_error", diff --git a/Cargo.toml b/Cargo.toml index 3443bc09..e9081bae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,6 @@ hmac = { version = "0.13", default-features = false } mbrman = { version = "0.6", default-features = false } omnect-crypto = { git = "https://github.com/omnect/omnect-crypto.git", tag = "0.5.0" } keyring-core = { version = "1.0", default-features = false } -libfs = { version = "0.9", default-features = false, features = ["use_linux"] } log = { version = "0.4", default-features = false } num_cpus = { version = "1.17", default-features = false } oauth2 = { version = "5.0", default-features = false, features = [ @@ -45,6 +44,7 @@ reqwest = { version = "0.13", default-features = false, features = [ "native-tls", ] } roxmltree = { version = "0.21", default-features = false, features = ["std"] } +rustix = { version = "1.1", default-features = false, features = ["fs", "std"] } serde = { version = "1.0", default-features = false, features = ["derive"] } serde_json = { version = "1.0", default-features = false } serde_path_to_error = { version = "0.1", default-features = false } diff --git a/src/file/functions.rs b/src/file/functions.rs index b85b969e..22c3a15e 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -1,9 +1,10 @@ -use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, partition::SECTOR_SIZE}; +use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, partition::SECTOR_SIZE, sparse}; use anyhow::{Context, Result}; use log::{debug, warn}; use std::collections::HashMap; use std::fmt::{self, Display}; use std::fs; +use std::ops::Range; use std::os::unix::fs::FileExt; use std::path::{Path, PathBuf}; use std::process::Command; @@ -414,20 +415,22 @@ fn read_partition( return Ok(()); } - let mut dd = Command::new("dd"); - dd.arg(format!("if={image_file}")) - .arg(format!("of={partition_file}")) - .arg(format!("bs={SECTOR_SIZE}")) - .arg(format!("skip={}", partition_info.start)) - .arg(format!("count={}", partition_info.count)) - .arg("conv=sparse") - .arg("status=none"); - exec_cmd!(dd); - - let mut sync = Command::new("sync"); - exec_cmd!(sync); - - Ok(()) + let image = + fs::File::open(image_file).context(format!("read_partition: cannot open {image_file}"))?; + let partition = fs::File::create(partition_file) + .context(format!("read_partition: cannot create {partition_file}"))?; + let len = partition_info.count * SECTOR_SIZE; + partition + .set_len(len) + .context("read_partition: cannot set partition file size")?; + + sparse::copy_range( + &image, + partition_info.start * SECTOR_SIZE, + &partition, + 0, + len, + ) } fn write_partition( @@ -451,7 +454,11 @@ fn write_partition( .min(partition_info.count * SECTOR_SIZE); // Only write blocks that differ: unchanged blocks keep their allocation, so a bmap - // created from the result still maps every block the source image mapped. + // created from the result still maps every block the source image mapped. A zero block + // that a tool wrote into a hole is data in the partition file and must become data in the + // image too, else a bmap flash keeps the old content there. + let partition_data = sparse::data_ranges(&partition, 0, len)?; + let image_data = sparse::data_ranges(&image, image_offset, image_offset + len)?; let mut src = vec![0u8; COPY_CHUNK_SIZE]; let mut dst = vec![0u8; COPY_CHUNK_SIZE]; let mut pos = 0; @@ -468,9 +475,13 @@ fn write_partition( .chunks(IMAGE_BLOCK_SIZE) .zip(dst[..n].chunks(IMAGE_BLOCK_SIZE)); for (i, (s, d)) in blocks.enumerate() { - if s != d { + let block_pos = pos + (i * IMAGE_BLOCK_SIZE) as u64; + let block = block_pos..block_pos + s.len() as u64; + let image_block = image_offset + block.start..image_offset + block.end; + if s != d || (has_data(&partition_data, &block) && has_hole(&image_data, &image_block)) + { image - .write_all_at(s, image_offset + pos + (i * IMAGE_BLOCK_SIZE) as u64) + .write_all_at(s, image_block.start) .context("write_partition: cannot write image")?; } } @@ -482,6 +493,18 @@ fn write_partition( .context("write_partition: cannot sync image") } +fn has_data(data_ranges: &[Range], block: &Range) -> bool { + let i = data_ranges.partition_point(|r| r.end <= block.start); + data_ranges.get(i).is_some_and(|r| r.start < block.end) +} + +fn has_hole(data_ranges: &[Range], block: &Range) -> bool { + let i = data_ranges.partition_point(|r| r.end <= block.start); + !data_ranges + .get(i) + .is_some_and(|r| r.start <= block.start && block.end <= r.end) +} + #[cfg(test)] mod tests { use crate::file::functions::*; @@ -550,4 +573,41 @@ mod tests { assert_eq!(allocated(), before, "unchanged zero blocks were written"); } + + #[test] + fn write_partition_maps_zero_block_written_into_a_hole() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + let partition = dir.path().join("partition.img"); + let image_file = fs::File::create(&image).expect("create image"); + image_file + .set_len((PART_START + PART_SECTORS) * SECTOR_SIZE) + .expect("set image size"); + let partition_file = fs::File::create(&partition).expect("create partition"); + partition_file + .set_len(PART_SECTORS * SECTOR_SIZE) + .expect("set partition size"); + let block = (ZEROED_BLOCK * IMAGE_BLOCK_SIZE) as u64; + partition_file + .write_all_at(&[0; IMAGE_BLOCK_SIZE], block) + .expect("write zero block"); + + write_partition( + image.to_str().expect("image path"), + partition.to_str().expect("partition path"), + &partition_info(), + ) + .expect("write partition"); + + let block = PART_START * SECTOR_SIZE + block; + assert_eq!( + sparse::data_ranges( + &image_file, + 0, + image_file.metadata().expect("metadata").len() + ) + .expect("data ranges"), + vec![block..block + IMAGE_BLOCK_SIZE as u64] + ); + } } diff --git a/src/file/mod.rs b/src/file/mod.rs index 6d334287..f8999cef 100644 --- a/src/file/mod.rs +++ b/src/file/mod.rs @@ -2,6 +2,7 @@ pub mod bmap; pub mod compression; pub mod functions; mod partition; +pub(crate) mod sparse; use super::validators::{ device_update, diff --git a/src/file/sparse.rs b/src/file/sparse.rs new file mode 100644 index 00000000..1c9d62bd --- /dev/null +++ b/src/file/sparse.rs @@ -0,0 +1,150 @@ +use anyhow::{Context, Result}; +use rustix::{ + fs::{SeekFrom, seek}, + io::Errno, +}; +use std::{ + fs::File, + io::{self, Read, Seek}, + ops::Range, +}; + +/// Returns the byte ranges of `file` between `start` and `end` that hold data. +pub fn data_ranges(file: &File, start: u64, end: u64) -> Result>> { + let mut ranges = vec![]; + let mut pos = start; + while pos < end { + let data = match seek(file, SeekFrom::Data(pos)) { + Ok(data) => data, + // no data after pos + Err(Errno::NXIO) => break, + Err(e) => return Err(e).context("data_ranges: cannot find data"), + }; + if data >= end { + break; + } + let hole = seek(file, SeekFrom::Hole(data)).context("data_ranges: cannot find hole")?; + ranges.push(data..hole.min(end)); + pos = hole; + } + + Ok(ranges) +} + +/// Copies `len` bytes of `src` at `src_offset` to `dst` at `dst_offset`. Holes of `src` are +/// skipped, so they stay holes in a new `dst`. +pub fn copy_range( + src: &File, + src_offset: u64, + dst: &File, + dst_offset: u64, + len: u64, +) -> Result<()> { + let src_len = src + .metadata() + .context("copy_range: cannot get source size")? + .len(); + anyhow::ensure!( + src_offset + len <= src_len, + "copy_range: source has {src_len} bytes, cannot copy {len} bytes at {src_offset}" + ); + + for data in data_ranges(src, src_offset, src_offset + len)? { + let mut src = src; + let mut dst = dst; + src.seek(io::SeekFrom::Start(data.start)) + .context("copy_range: cannot seek source")?; + dst.seek(io::SeekFrom::Start(data.start - src_offset + dst_offset)) + .context("copy_range: cannot seek destination")?; + // one copy_file_range call copies at most 2 GiB; io::copy repeats it until a range is done + let copied = io::copy(&mut src.take(data.end - data.start), &mut dst) + .context("copy_range: cannot copy data")?; + anyhow::ensure!( + copied == data.end - data.start, + "copy_range: source ended early" + ); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use crate::file::sparse::*; + use std::os::unix::fs::FileExt; + + const BLOCK_SIZE: u64 = 4096; + const FILE_BLOCKS: u64 = 8; + const DATA_BLOCK: u64 = 2; + const DATA_BYTE: u8 = 0x5a; + const DST_OFFSET_BLOCKS: u64 = 1; + + /// A sparse file with data in `DATA_BLOCK` and holes everywhere else. + fn sparse_file(dir: &std::path::Path) -> File { + let file = File::options() + .read(true) + .write(true) + .create_new(true) + .open(dir.join("src")) + .expect("create file"); + file.set_len(FILE_BLOCKS * BLOCK_SIZE).expect("set size"); + file.write_all_at(&[DATA_BYTE; BLOCK_SIZE as usize], DATA_BLOCK * BLOCK_SIZE) + .expect("write data block"); + file + } + + #[test] + fn data_ranges_finds_data_between_holes() { + let dir = tempfile::tempdir().expect("create temp dir"); + let file = sparse_file(dir.path()); + let data = DATA_BLOCK * BLOCK_SIZE..(DATA_BLOCK + 1) * BLOCK_SIZE; + + assert_eq!( + data_ranges(&file, 0, FILE_BLOCKS * BLOCK_SIZE).expect("data ranges"), + vec![data.clone()] + ); + assert_eq!( + data_ranges(&file, 0, data.start).expect("data ranges"), + vec![] + ); + assert_eq!( + data_ranges(&file, data.start + 1, data.end - 1).expect("data ranges"), + vec![data.start + 1..data.end - 1] + ); + } + + #[test] + fn copy_range_keeps_holes() { + let dir = tempfile::tempdir().expect("create temp dir"); + let src = sparse_file(dir.path()); + let dst = File::options() + .read(true) + .write(true) + .create_new(true) + .open(dir.path().join("dst")) + .expect("create file"); + let len = (FILE_BLOCKS - DST_OFFSET_BLOCKS) * BLOCK_SIZE; + dst.set_len(len).expect("set size"); + + copy_range(&src, DST_OFFSET_BLOCKS * BLOCK_SIZE, &dst, 0, len).expect("copy range"); + + let data_start = (DATA_BLOCK - DST_OFFSET_BLOCKS) * BLOCK_SIZE; + assert_eq!( + data_ranges(&dst, 0, len).expect("data ranges"), + vec![data_start..data_start + BLOCK_SIZE] + ); + let mut block = [0u8; BLOCK_SIZE as usize]; + dst.read_exact_at(&mut block, data_start) + .expect("read block"); + assert!(block.iter().all(|b| *b == DATA_BYTE)); + } + + #[test] + fn copy_range_rejects_range_after_source_end() { + let dir = tempfile::tempdir().expect("create temp dir"); + let src = sparse_file(dir.path()); + let dst = File::create(dir.path().join("dst")).expect("create file"); + + assert!(copy_range(&src, BLOCK_SIZE, &dst, 0, FILE_BLOCKS * BLOCK_SIZE).is_err()); + } +} diff --git a/src/lib.rs b/src/lib.rs index db5c98a0..9cc77c8c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -24,7 +24,6 @@ use file::{bmap::Bmap, compression::Compression, functions::FileCopyToParams}; use log::{error, info}; use std::{ fs, - io::Read, path::{Path, PathBuf}, }; use tokio::fs::remove_dir_all; @@ -196,22 +195,7 @@ fn copy_image(from: &Path, to: &Path) -> Result<()> { dst.set_len(len) .context("copy_image: cannot set destination size")?; - // one copy_file_range call copies at most 2 GiB; io::copy repeats it until a segment is done - let sparse = libfs::probably_sparse(&src)?; - let mut pos = 0; - while pos < len { - let (data, hole) = if sparse { - libfs::next_sparse_segments(&src, &dst, pos)? - } else { - (pos, len) - }; - let copied = std::io::copy(&mut (&src).take(hole - data), &mut &dst) - .context("copy_image: cannot copy data")?; - anyhow::ensure!(copied == hole - data, "copy_image: source ended early"); - pos = hole; - } - - Ok(()) + file::sparse::copy_range(&src, 0, &dst, 0, len) } struct CertInfo { diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index fb7f165e..e570738a 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -903,6 +903,9 @@ fn check_file_copy(tr: Testrunner, partition: &str) { const STAT_BLOCK_SIZE: u64 = 512; // content an earlier installation left on the device const OLD_CONTENT_BYTE: u8 = 0xa5; +// a file with a zero range that spans whole image blocks +const ZERO_BLOCKS_FILE_EDGE: usize = 4096; +const ZERO_BLOCKS_FILE_ZEROS: usize = 64 * 1024; const OTHER_IMAGE_BMAP: &str = "40964096\ 0"; @@ -939,6 +942,16 @@ fn bmap_flash(image: &[u8], bmap: &Bmap) -> Vec { device } +fn sparse_copy(from: &Path, to: &Path) { + let status = std::process::Command::new("cp") + .arg("--sparse=always") + .arg(from) + .arg(to) + .status() + .expect("run cp"); + assert!(status.success(), "sparse copy of the test image failed"); +} + #[test] fn check_file_copy_without_input_bmap_maps_whole_image() { let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); @@ -970,13 +983,7 @@ fn check_file_copy_keeps_holes() { let bmap_path = PathBuf::from(format!("{}.bmap", sparse_path.to_string_lossy())); let in_file = tr.to_pathbuf("testfiles/boot.scr"); - let status = std::process::Command::new("cp") - .arg("--sparse=always") - .arg(&image_path) - .arg(&sparse_path) - .status() - .expect("run cp"); - assert!(status.success(), "sparse copy of the test image failed"); + sparse_copy(&image_path, &sparse_path); Command::cargo_bin("omnect-cli") .expect("omnect-cli binary") @@ -1014,6 +1021,65 @@ fn check_file_copy_keeps_holes() { } } +#[test] +fn check_file_copy_maps_zero_blocks_of_copied_files() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let input_bmap_path = tr.to_pathbuf("testfiles/image.wic.bmap"); + let sparse_path = image_path.with_file_name("sparse.wic"); + let bmap_path = PathBuf::from(format!("{}.bmap", sparse_path.to_string_lossy())); + let device_path = tr.pathbuf().join("device.wic"); + let in_file = tr.pathbuf().join("zero-blocks.bin"); + let in_content = [ + vec![0x11; ZERO_BLOCKS_FILE_EDGE], + vec![0; ZERO_BLOCKS_FILE_ZEROS], + vec![0x22; ZERO_BLOCKS_FILE_EDGE], + ] + .concat(); + fs::write(&in_file, &in_content).expect("write input file"); + sparse_copy(&image_path, &sparse_path); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},factory:/my-file", in_file.to_string_lossy())) + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&sparse_path) + .arg("--input-bmap") + .arg(&input_bmap_path) + .arg("-b") + .assert() + .success(); + + let image = fs::read(&sparse_path).expect("read image"); + fs::write(&device_path, bmap_flash(&image, &read_bmap(&bmap_path))).expect("write device"); + + for partition in ["factory", "boot"] { + let out_file = tr.pathbuf().join(format!("{partition}-out.bin")); + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-from-image") + .arg("-f") + .arg(format!( + "{partition}:/my-file,{}", + out_file.to_string_lossy() + )) + .arg("-i") + .arg(&device_path) + .assert() + .success(); + assert!( + fs::read(&out_file).expect("read output file") == in_content, + "{partition}: file differs after a bmap flash" + ); + } +} + #[test] fn check_input_bmap_of_another_image() { let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); From 02a712bba1057d3ddb5c87d25f040d30d73cce40 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:36:06 +0100 Subject: [PATCH 18/75] fix(file): reject an input bmap whose checksums do not match the image All omnect partitions have a fixed size, so builds of one machine have the same image size, and the bmap of another build passed the size check. It was then used without a warning, and a bmap flash kept old content in blocks the image needs. Each range of the input bmap is now checked against its sha256 before the command changes the image. A mismatch, or a bmap without sha256 range checksums, stops the command with a hint to --input-bmap. The test bmap was not in git, and its checksums matched neither test image. It is now added, with checksums for image.wic, and image.wic.xz is rebuilt from image.wic, so the bmap fits both. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/bmap.rs | 151 ++++++++++++++++++++++++++++++++++--- src/lib.rs | 3 +- testfiles/image.wic.bmap | 69 +++++++++++++++++ testfiles/image.wic.xz | Bin 7868 -> 7228 bytes tests/integration_tests.rs | 29 +++++++ 5 files changed, 240 insertions(+), 12 deletions(-) create mode 100644 testfiles/image.wic.bmap diff --git a/src/file/bmap.rs b/src/file/bmap.rs index 918660e9..c7125a65 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -1,5 +1,6 @@ use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, is_zero}; use anyhow::{Context, Result}; +use sha2::{Digest, Sha256}; use std::{ fs, ops::RangeInclusive, @@ -10,6 +11,7 @@ use std::{ const BMAP_EXTENSION: &str = "bmap"; const COMPRESSION_EXTENSIONS: [&str; 5] = ["xz", "gz", "gzip", "bz2", "bzip2"]; +const SHA256_CHECKSUM_TYPE: &str = "sha256"; /// The block ranges of an image that a bmap file maps. #[derive(Debug)] @@ -17,6 +19,8 @@ pub struct Bmap { pub block_size: u64, pub image_size: u64, pub ranges: Vec>, + /// The sha256 of each range, if the bmap has them. + pub sha256: Option>, } impl Bmap { @@ -43,16 +47,31 @@ impl Bmap { let block_size = number("BlockSize")?; anyhow::ensure!(block_size > 0, "Bmap::read: BlockSize is 0"); let image_size = number("ImageSize")?; - let ranges = child("BlockMap")? + let range_nodes: Vec<_> = child("BlockMap")? .children() .filter(|n| n.has_tag_name("Range")) + .collect(); + let ranges = range_nodes + .iter() .map(|n| parse_range(n.text().unwrap_or_default())) .collect::>()?; + let sha256 = child("ChecksumType") + .ok() + .and_then(|n| n.text()) + .is_some_and(|t| t.trim() == SHA256_CHECKSUM_TYPE) + .then(|| { + range_nodes + .iter() + .map(|n| n.attribute("chksum").map(str::to_owned)) + .collect::>() + }) + .flatten(); Ok(Self { block_size, image_size, ranges, + sha256, }) } @@ -67,6 +86,7 @@ impl Bmap { block_size, image_size, ranges, + sha256: None, } } } @@ -114,6 +134,66 @@ pub fn find_input_bmap(image: &Path, explicit: Option<&Path>) -> Result Result<()> { + let sha256 = bmap + .sha256 + .as_ref() + .context("verify_checksums: the bmap has no sha256 range checksums")?; + let image = fs::File::open(image_file).context(format!( + "verify_checksums: cannot open {}", + image_file.display() + ))?; + let image_len = image + .metadata() + .context("verify_checksums: cannot get image size")? + .len(); + check_layout(bmap, image_len)?; + + let mut buf = vec![0u8; COPY_CHUNK_SIZE]; + for (range, expected) in bmap.ranges.iter().zip(sha256) { + let end = ((range.end() + 1) * bmap.block_size).min(image_len); + let mut pos = range.start() * bmap.block_size; + let mut hasher = Sha256::new(); + while pos < end { + let n = usize::try_from((end - pos).min(COPY_CHUNK_SIZE as u64))?; + image + .read_exact_at(&mut buf[..n], pos) + .context("verify_checksums: cannot read image")?; + hasher.update(&buf[..n]); + pos += n as u64; + } + let actual = hex(&hasher.finalize()); + anyhow::ensure!( + actual.eq_ignore_ascii_case(expected.trim()), + "verify_checksums: range {range:?} of the image does not match the bmap; \ + the bmap is for another image, pass the right one with --input-bmap" + ); + } + + Ok(()) +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// Fails if `bmap` is for an image of another size or maps blocks outside the image. +fn check_layout(bmap: &Bmap, image_len: u64) -> Result<()> { + anyhow::ensure!( + bmap.image_size == image_len, + "check_layout: bmap is for an image of {} bytes, the image has {image_len}", + bmap.image_size + ); + let image_blocks = image_len.div_ceil(bmap.block_size); + if let Some(range) = bmap.ranges.iter().find(|r| *r.end() >= image_blocks) { + anyhow::bail!("check_layout: range {range:?} is outside the image"); + } + + Ok(()) +} + /// Writes the zero blocks in the ranges of `bmap` as data. A zero block can be a hole, and a /// bmap created from the image would skip it, so a bmap-based flash would keep what an earlier /// installation left there. @@ -130,20 +210,11 @@ pub fn allocate_ranges(image_file: &Path, bmap: &Bmap) -> Result<()> { .metadata() .context("allocate_ranges: cannot get image size")? .len(); - anyhow::ensure!( - bmap.image_size == image_len, - "allocate_ranges: bmap is for an image of {} bytes, the image has {image_len}", - bmap.image_size - ); + check_layout(bmap, image_len)?; - let image_blocks = image_len.div_ceil(bmap.block_size); let image_block_size = IMAGE_BLOCK_SIZE as u64; let mut buf = vec![0u8; COPY_CHUNK_SIZE]; for range in &bmap.ranges { - anyhow::ensure!( - *range.end() < image_blocks, - "allocate_ranges: range {range:?} is outside the image" - ); // check whole image blocks, also when the bmap uses a smaller block size let start = range.start() * bmap.block_size / image_block_size * image_block_size; let end = ((range.end() + 1) * bmap.block_size) @@ -209,6 +280,7 @@ mod tests { const ZERO_BLOCKS: RangeInclusive = 2..=3; const UNMAPPED_BLOCK: u64 = 5; const SMALL_BLOCK_SIZE: u64 = 512; + const PARTIAL_BLOCK_LEN: usize = 100; const DATA_BYTE: u8 = 0x5a; // content an earlier installation left on the device const OLD_CONTENT_BYTE: u8 = 0xa5; @@ -276,6 +348,11 @@ mod tests { assert_eq!(bmap.image_size, YOCTO_IMAGE_SIZE); let mapped: u64 = bmap.ranges.iter().map(|r| r.end() - r.start() + 1).sum(); assert_eq!(mapped, YOCTO_MAPPED_BLOCKS); + assert_eq!( + bmap.sha256.map(|c| c.len()), + Some(bmap.ranges.len()), + "one checksum per range" + ); } #[test] @@ -289,6 +366,15 @@ mod tests { assert_eq!(bmap.ranges, vec![3..=3, 5..=7]); } + #[test] + fn read_has_no_checksums_without_sha256_checksum_type() { + let dir = tempfile::tempdir().expect("create temp dir"); + let path = dir.path().join("image.wic.bmap"); + fs::write(&path, bmap_xml(BLOCK_SIZE, "0")).expect("write bmap"); + + assert_eq!(Bmap::read(&path).expect("read bmap").sha256, None); + } + #[test] fn read_rejects_invalid_bmaps() { let dir = tempfile::tempdir().expect("create temp dir"); @@ -388,12 +474,54 @@ mod tests { block_size: SMALL_BLOCK_SIZE, image_size: IMAGE_BLOCKS * BLOCK_SIZE, ranges: vec![RangeInclusive::new(first, first + 2)], + sha256: None, }; allocate_ranges(&image, &input).expect("allocate ranges"); assert!(is_mapped(&created_bmap(&image), DATA_BLOCK + 1)); } + #[test] + fn verify_checksums_accepts_bmap_of_the_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + // a mapped last block that is shorter than the block size + fs::OpenOptions::new() + .write(true) + .open(&image) + .expect("open image") + .write_all_at(&[DATA_BYTE; PARTIAL_BLOCK_LEN], IMAGE_BLOCKS * BLOCK_SIZE) + .expect("write partial block"); + + verify_checksums(&image, &created_bmap(&image)).expect("verify checksums"); + } + + #[test] + fn verify_checksums_rejects_changed_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + let bmap = created_bmap(&image); + fs::OpenOptions::new() + .write(true) + .open(&image) + .expect("open image") + .write_all_at(&[!DATA_BYTE], DATA_BLOCK * BLOCK_SIZE) + .expect("change data block"); + + let err = verify_checksums(&image, &bmap).expect_err("changed image accepted"); + assert!(err.to_string().contains("--input-bmap"), "{err}"); + } + + #[test] + fn verify_checksums_rejects_bmap_without_checksums() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + let mut bmap = created_bmap(&image); + bmap.sha256 = None; + + assert!(verify_checksums(&image, &bmap).is_err()); + } + #[test] fn allocate_ranges_rejects_bmap_of_another_image() { let dir = tempfile::tempdir().expect("create temp dir"); @@ -406,6 +534,7 @@ mod tests { block_size: BLOCK_SIZE, image_size: IMAGE_BLOCKS * BLOCK_SIZE, ranges: vec![RangeInclusive::new(IMAGE_BLOCKS, IMAGE_BLOCKS)], + sha256: None, }; assert!(allocate_ranges(&image, &outside).is_err()); } diff --git a/src/lib.rs b/src/lib.rs index 9cc77c8c..d7dc96f3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -123,7 +123,8 @@ where None => {} Some(Some((path, bmap))) => { info!("keeping the ranges of {} mapped", path.display()); - file::bmap::allocate_ranges(&tmp_image_file, &bmap) + file::bmap::verify_checksums(&tmp_image_file, &bmap) + .and_then(|_| file::bmap::allocate_ranges(&tmp_image_file, &bmap)) .context(format!("cannot use input bmap {}", path.display()))?; } Some(None) => { diff --git a/testfiles/image.wic.bmap b/testfiles/image.wic.bmap new file mode 100644 index 00000000..7b84de94 --- /dev/null +++ b/testfiles/image.wic.bmap @@ -0,0 +1,69 @@ + + + + + + 30408704 + + + 4096 + + + 7424 + + + 77 + + + sha256 + + + 43b997b9550b07ad422eddddba268d0f0607660f664592a7d749034a4f8b9eea + + + + 0-1 + 1024-1025 + 2048-2055 + 2058-2059 + 2062-2063 + 2190-2191 + 3072-3079 + 3082-3083 + 4094-4103 + 4106-4107 + 5104-5105 + 5120-5127 + 5130-5131 + 6128-6129 + 6144-6151 + 6154-6155 + 7152-7153 + 7168-7175 + 7178-7179 + 7423 + + diff --git a/testfiles/image.wic.xz b/testfiles/image.wic.xz index b86e9327a458826a63bbc40906595a2e692e93c2..6053d9aeffdce7a0b021c338ac0971d6b949b7a2 100644 GIT binary patch delta 2903 zcmV-d3#jzGJ-j#={Wp48S^xk5{}S0j0stWa8~^|S5YJf5u^>za0S=RG1{;4R(zOkZ z7QxXzq#AUP@D6n848He@xtep0KVxkbTN5p!jApms0-6`*l{&w;p@;6@e5}BvbNVg1 zfYj04-{F`Zv}c;saDeR>2rA)B@YfDC;x-(!=#nW)fmy&c0N<^>O-VC(EhH1FUXWM8 z=PdES8CXJ0Gxb#P6+b6~o^^leGG@UqR*ngpmtD`PRc{|RD@BIzG&b(VM?8C0bo2(P zEoM4V@g~EH(Zc#r>~n-2g-rxScj80Hg#uul0VSv-dzlQB4f9J4-i)$*c9d{lq1 zy0j+7Q)o<4fE7Yo+KIENM%H!wO_2u#Eq#5})#Swor&vPvJA^mLeO+RXUyY(J0rLKQz$cnkuIP!f>5$OkM4qsok z)JTSsar~?3hLZ|5G6P@^Mrb(7obdWPR2H}55qM`t!vzUlxrwEPA1z90hT6aW&V=GQ z2l@OWsvalz_~!)Tw_Sh=%5YzFs+TG6*s1DM@>Gapf_;A274m<}ESv?D{{Y=mR7v@l zyk=+*-=*-XDZBFF?tp@pTq84dq4H`rR}yl3@S0oZ9dE)Cbs12O#0~)FXLSf=S^&sw z@Ai5;aP!3_WS+DdqS$m@S|5AO%m>GnDI;UFiCAvvi*sqDl!WV~vLsS8KJzU3vc^8o z!ig+*G=5kMq!NFvPuo4WgLfF>F6J*1LgXwLlfx2d?^z1>bHddD&$@$DF(y6%o&St6Hp9i}$s9XmQ|WiZaVxgi1O- zUsRu`DR$qD5VG%*w?-&Kg*IAU>w_Nn?C6FFB6%?T(R&!YFg_f;JiaVL8sRL5@gS5|HSyp!N1QW zAXMcER24qZ+kBi>ol~G;bb|K1n3KLeCU6 z#}ZyG^juO$lI=G6iKqXaUYFaIQSsSf`dZd*DI|Zil`0oTh)Krt9qpVOV6+G3M@<6- zev>|wgPsx&cFTXywwv8kNqT*dO49jN)!r=j_f584;qDysai3IXAL^Igv6C(U)+Fks;$x62oYre|6O3p$K z_a@X#s@)KB!Y$mn8>3MK35!i;G}|=kl>|Q)FNTOt87p#NDa97?n2xfVo%^1Q+BVyw%AjLB8P1=etFOMYQlRp-{1VG?0 z^RsXbGy#7B_W;E;CGqb&(3Mc_QwnEgX=11`fPH`;l&qeEN%s%ke=`upV5BT@79I&< zxE-Czf6iXV1ZDHdM0_-p94OwKw96$73Q*#7)Djb@?yPm*wAg8Lj*tQ%fvj~2(*r4P zk<1~iyVEQA3VNg+`Jzj+x`+L4+<78ln5Y;=xTSyFLZ>6qB?-AJ#x#AiMOR?I&DT8N zF3;%ZT-Yjbikp+6wAOA{7|#w8(n!HGJFFQ&Ha(r1n-wykx+`~TUH4`JpN`^U7jOG+ z^)`~m0LLioD0iR-v$zh^D&LLuX~uLum^~2SoPOApaUVU8B;jzLrz2nND9eS>m6(ct zDv^^<7Wf3s143=Ha1Ap7f87D$0D)u^$rjwSxcTrhhwOEzx`ni+u&C(AZv8Eeg0SYF zGS0}1hf_3;HvIU{n5A;#?8Uk3;&Oz5K}YsQaL^!ow=3TB=8;7GqHgWI?g7EVtp=xu zjQ$Fk{nkn71*VN2OEc1M4V@KNPv|_wR^C3$3kR3Rlj$BG_UmkMe??5Oi_yW^4+b^Vh!dVH6#?}NB?5V_sY z?TszfQSN>}kA=koX|(cy9_xhoFFM{dS18*-1kb|lVSU#y>PUz|zGvPlYTVX+7MNoXN~xD@6g zH|{omS*||?i$WfpeK#^akc1Femjg#J+%Hz~LKUdK<8EM%eN?C=o8yg4#33niB9-sf%0XefT79atC4Li97|A*W=X-z_B2MIy> z0QFsLY(4VpH2a_b6#=aP*Wdiz<2&d~zbvO<>Z#0P;ro{q z+|Wz8gxV=W@|grlECkm&w`hrt<#{WO06(GIS7G%II2?72jshj`?nHT5nRNm44}e2OI)wt1aUVU8B;m6^76<_Z-Q5^{vn?DY0e@AX>ai>6-D0<%A^U3X z$x+;bKsO)7kOl*0C11jF?r>uo85aUH-+Q}QVI5c z&P%T}uMwiV_GVZ_d=zH;GpF&QV2$fwyh#sf1@bnjk2@29xW}wZ<{y&@{9(97duf2oPoe87PkQGK{6CW&hkRz%4`R~$t)=F%riY5AOc?^QPgz0|KI=s00001mNx4R+Ug25-i=vh8_0wxQBwLKnO@Tg2`o5wLi?`RXddn{1pq3e*H z!-TgNYC|qT%?o9W2oGAm@|DYmfWFPtGOxPQU>NWOgLv-HF{So;n9T3%&o@!Sj|M0nL;>c2 zgS96%kh#ju)&G>Fl_xqcYf*pOdA%z_zL65Io3&I@P%WkmLxc^GX9(Za&>JJwVO0G1 zfK1S(M1|Iv%zJ=+B+G&rfAMj!MUp7id4~f~nnV~XNworjkJGCsbvBK6ZD6UtOLwwh zB$WJj-9*h(Ivsf@?Z97VbfC0OA*`Te@0L;i-B0pyu3fq22nUSKi`D*r)xsK%ga3Sg ztvRzd@erlOgG{tyOGx$ZAjo$wCO0aoyZyjJN7#G`W7 z!YNN`mN~|Ic7N&~t)t@hlb)^AQasE5l?8UVBpemF$CH|$$P!Af1BD!{PPCK zH`fc#AS-GkO#!8RIz;|BxcJ1juN=lDe%4(TCHfn=JHHz6TJ(NFS(`MWEEQVRkcwM~0yV zL{dsQ_8D7^TBR8dYiC>caV1R6jp)OQvabNoJvSfRqIDXw=z3k6R*1}Wrc4oQI9F{U z>6U7_EStik6tYkTRiA4`!$;1o5DH4o1QlMS3UTMNe>+WkzpVxyFP&c(1Hv3&Qngv* z^-B*Z*{?^d%}r_V%2x z@r$bFMfGQKHenswvC*~#2CM#a2H4nxzE!*?Uvv6VxdMd}89h&< zz$UZV-XH8EM#cae(^_U0K@f{7x*L<>7B+wAU@_jLc^v)Bq{wgb(xgtM-<^s%xH@9n z^aR+f)Tvbz@i;8b)tz#3J=A3hbn}R@u>y;&M+B+H<`|`~(8h-G6s9692V{7nT%3uH{Lq_#$uw(b)6=OPCUfo>D#TIvmPx zv@%cq6VAwIJ9cLNJ*7v#PlD;}J=JL}#R`FHSlEJ5D`?>XgIECEg~973l2m9d7EyNJ zjjehRa){@?w2^JZ>Lp=ViCmctZNs4OLWduS44GIO+k}u9*O_En$qlsR#=+2ZNnFHj zvXbYZ$j%1@?w{VJLd%FhTmX;IPOl~?laCLl1WN$ly|XU`GXZ}!0s;WVJTHQc(NQk$ zr|Y+l&4{;A=lJLv-hIZJ=;>Q?#aYzHl`TlkAURhFzTtdX`!-=Le!PC#We<}3M8`_w z?c@Z!p!yathX++ft6?WTwypH$9?O^FMe=A$$iv8jdC>Fs%enncppu_UQr-H)@V zoU!8FJzjJm9{PVe&ez+@7T?oDRt&4=owN@eILoB-3oYV}ovo3x{b&Io;O>t%2W-9J zCE>Z*rcu#?50RiHF$i0fP>N%2?!-C7O$#i`seM^CX+hJvNH)@HSxF*?phM+yTFfe5 zT6G5va;ZP*#%}<9mj+z*suM#?@8$d(C2S3kVA6wWS)@cvA4mWGtp#DtKUgrJ=F417 z8`M{GZKmPK{&^v_s56gZgne33Vv=-8T*Pg%lINhv&IbhUpWdZH%ZNW*0FTg4uO=vy zj}O2ESIlX5vo8fR0e>q5N&tamAIgp8?eA!EU)iuh1B&!Tv(vrcf^-c0`%B`2@0Nj}y1=E=yGoWcj_yU9n|0_fo0S!;qaC9_UKy#q#0K;`ICHJE3b) zzXNG!af9P9kkvYF8WZL-{%G`4Q#T-`r!X5->OZ5ud|9$*M}L?$)%dI@Rk5k|f8a(s zY~#Z2yfDv@r@ut9p6b1nsYvKe%g3FE(n{f9Brzlis2(7=_-n@9iy|#29`Go};i37i zf`0qg!xX&)uZ=jb9Z^dfFtxj{)?kVE1=jK2Q8@dY5|p-!*sx8_v9__z=bfeGKHw|s zfR=dK3GM`Al79#Z5cZZ;4vrwk{7nd(WPHt*zl%J$->#pzkgVRv_(SX8#+oI`eR#&0 zFKLnDRt-v5DsEP4U0rF^6+{rakg%1KzB}-P#FZk6iG1a$n$UQq_>8q8$&$$I_FBBV z{vcQ|c{1*6o0{^u9H1gEN0St`i>1d6DbVXOGPPpdI)58-4DI#?AGGIb>Gu;OSbH(5 z+_E0e@FWf6IGNL1IzJx;wq1Q>co*uM{ZIMhT-=@+@;4hSFqgnQ%DN!wZ*5)I|3~7v zjB5)~Zu2t7z~T%2A%2EZ`=aB;HB5xDPVa2<&DbG~=hkAw@4BOKb7w%kw=V{6jCMs6 zLWM&xf`5JHx|O&}8KSUT-F4UcbZ|@<_&his?a<<$yYEbWN$PIWp_1GcsqKk~ggLk- zxZs?IVwnm0;xcGdDx1#dN#g%ZegOTA1$?H`KMXfIk%O$*N1vU5iS#`Tk^G(8F~>1} zK~Kge(3L==?bxnFYtD?9#Sf`lL4kg(8f5us@y{ zctc-*aGgFP^%UdaxL*-cly^hK)fkQyBzcqVQ9XF~+A(B~z!?xN7o%(mRpi0oze!Sl0RfY+1Plqd%|iR1|9JsHv%wZ10e=T=Q4VPRK;r0V z1Q7m`BKL_bclmA+1C$ydLn8?F8D=&9pZ^sBy8zaad1<`z(+sVUx@iHWMC_fUI>_OYoS>G(fE#n##YTRqX@gEQ!%zX(#qR5SV}5 zmXI%eX1wTaNemF(s=N_2BAZ`|l&8Z3794V;-Pm2_xn70M=B==Lag9a}0h)^*3O*Hp zn5vVo6fToa3lx*E1bdUP1V;%Y)PX;r|JVUNv%wZ10e@9%Gfl8wJ!;*Xb*#$^6g+NI z5XDv=d(#!_b(Pwpvh!4q^k>FccueJ3Q*t~7Pj!B)_rC#QJPF(8EX2>PEx7HO?#y~5G50xOZuoVaalivXb1VIt;ShKDfB>{hs0c-$!z_s0E z{NTGC*k+$ru#u<_!jK8>0W<*K{rs4d$Pxc@uI8a0F5EMEf&cU!9mRC3TDE5how|@q z4gBK;pJe^bdPIp$ZHpj-LMqC@Ugf3wk1(CiP$t95i{3!L`b%Px^xMUwA$mu|+(iQ| zQs#GlUt4z4I|Z*3rY%%qK``-Tw-NapPg!1C{U2_sX~WY-ubXI_tV9T*z~#jx2N?#U zsH&Qe6#Wxv!7mk#DUK*W$|6zaDyM6}Uqb6kon&Pfy__hd#iR7FmGX#~8(P(q5gK Date: Wed, 28 Oct 2026 13:58:10 +0100 Subject: [PATCH 19/75] docs(file): explain a checksum mismatch after a command without -b A command without -b changes the image but leaves the bmap next to it, so the next command on that image finds a bmap with old checksums and fails. The error and the README now name this case and the way out: use -b on each command, or pass the matching bmap with --input-bmap. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 2 +- src/file/bmap.rs | 5 +++-- tests/integration_tests.rs | 31 +++++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8062664f..1e2f8441 100644 --- a/README.md +++ b/README.md @@ -188,7 +188,7 @@ omnect-cli file copy-to-image --help Commands that change an image create a bmap file with `-b` (`--output-bmap`). It is written next to the image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic.xz`, and replaces a file with this name. -The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image, otherwise the command fails. Every range the input bmap maps stays mapped, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). The created bmap maps these ranges, the blocks `omnect-cli` changed, and all other blocks that the image file stores as data. For an uncompressed image file that is not sparse, this is the whole image. +The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image and sha256 range checksums that match it, otherwise the command fails. A command without `-b` changes the image but not the bmap next to it, so when you run several commands on one image, use `-b` on each of them, or pass the matching bmap with `--input-bmap`. Every range the input bmap maps stays mapped, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). The created bmap maps these ranges, the blocks `omnect-cli` changed, and all other blocks that the image file stores as data. For an uncompressed image file that is not sparse, this is the whole image. Without an input bmap, the created bmap maps the whole image, and a flash writes every block. diff --git a/src/file/bmap.rs b/src/file/bmap.rs index c7125a65..b3b09733 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -167,8 +167,9 @@ pub fn verify_checksums(image_file: &Path, bmap: &Bmap) -> Result<()> { let actual = hex(&hasher.finalize()); anyhow::ensure!( actual.eq_ignore_ascii_case(expected.trim()), - "verify_checksums: range {range:?} of the image does not match the bmap; \ - the bmap is for another image, pass the right one with --input-bmap" + "verify_checksums: range {range:?} of the image does not match the bmap. It is the \ + bmap of another image, or of this image before a command without -b changed it. \ + Pass the right bmap with --input-bmap." ); } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 422237d1..472948cb 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1145,6 +1145,37 @@ fn check_input_bmap_of_another_image_with_the_same_size() { assert_eq!(Testrunner::file_hash(&image_path), image_hash); } +#[test] +fn check_chained_commands_with_input_bmap() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_xz_path = tr.to_pathbuf("testfiles/image.wic.xz"); + let image_path = image_xz_path.with_extension(""); + tr.to_pathbuf("testfiles/image.wic.bmap"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + let copy_to_image = |image: &Path, output_bmap: bool| { + let mut cmd = Command::cargo_bin("omnect-cli").expect("omnect-cli binary"); + cmd.arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(image); + if output_bmap { + cmd.arg("-b"); + } + cmd.assert() + }; + + // without -b, the bmap next to the image still has the checksums of the original image + copy_to_image(&image_xz_path, false).success(); + copy_to_image(&image_path, true) + .failure() + .stderr(predicate::str::contains("without -b changed it")); + + copy_to_image(&image_xz_path, true).success(); + copy_to_image(&image_path, true).success(); +} + #[test] fn check_bmap_generation_wic() { let tr = Testrunner::new(function_name!().split("::").last().unwrap()); From 279f8a80505fb4187d6a8879a5fca8193ec081c8 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:16:57 +0100 Subject: [PATCH 20/75] feat(file): write the output bmap in Rust and keep the image sparse omnect-cli wrote the zero blocks of the input bmap ranges into the image, or every zero block without an input bmap, only so that bmaptool create mapped them. That fully allocated the image in /tmp and in the output, also without -b, and a sparse copy of the output lost the ranges again. The bmap is now written by omnect-cli: the ranges of the input bmap, or the whole image without one, plus every block the image file stores as data, with a sha256 per range. bmaptool copy writes a mapped hole as zeros, so the flash is the same, and the image stays sparse. The input bmap is only read with -b, --input-bmap requires -b, and -b works in the Docker image because bmaptool is not needed at runtime any more. The bmap is copied next to the image after the image was written back, so a failed write back does not leave a bmap for an image that is not there. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- Cargo.toml | 2 +- Dockerfile | 1 - README.md | 7 +- project-context.md | 5 +- src/cli.rs | 40 ++--- src/file/bmap.rs | 319 ++++++++++++++++++++----------------- src/file/sparse.rs | 21 +++ src/lib.rs | 248 ++++++++++++---------------- tests/integration_tests.rs | 62 +++++++ 9 files changed, 387 insertions(+), 318 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e9081bae..c1b0c5d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -82,5 +82,5 @@ tar = "0.4" # metadata for building with cargo-deb (https://crates.io/crates/cargo-deb) [package.metadata.deb] -depends = "bmap-tools, e2tools, keychain, libc6 (>= 2.39), libmagic1, libssl3 (>= 3.0.0), mtools" +depends = "e2tools, keychain, libc6 (>= 2.39), libmagic1, libssl3 (>= 3.0.0), mtools" revision = "" diff --git a/Dockerfile b/Dockerfile index 9300a868..9da78d24 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,6 @@ COPY ${debian_dir}/omnect-cli_${omnect_cli_version}_amd64.deb omnect-cli_${omnec RUN apt-get update && \ apt-get install -y --no-install-recommends \ - bmap-tools \ ca-certificates \ e2tools \ fdisk \ diff --git a/README.md b/README.md index 1e2f8441..eda55c06 100644 --- a/README.md +++ b/README.md @@ -48,9 +48,8 @@ docker run --rm -it \ omnect/omnect-cli:latest file copy-to-image --files /source/my-source-file,boot:/my-dest-file -i /source/my-image.wic ``` - **Note1**: `-b` (`--output-bmap`) option to create a bmap file is not supported by omnect-cli docker image.
- **Note2**: The ssh tunnel option requires some additional settings. See [here](Usage-with-docker) for more details.
- **Note3**: The docker inject command is not supported by omnect-cli docker image.
. + **Note1**: The ssh tunnel option requires some additional settings. See [here](Usage-with-docker) for more details.
+ **Note2**: The docker inject command is not supported by omnect-cli docker image.
. # Build from sources @@ -188,7 +187,7 @@ omnect-cli file copy-to-image --help Commands that change an image create a bmap file with `-b` (`--output-bmap`). It is written next to the image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic.xz`, and replaces a file with this name. -The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image and sha256 range checksums that match it, otherwise the command fails. A command without `-b` changes the image but not the bmap next to it, so when you run several commands on one image, use `-b` on each of them, or pass the matching bmap with `--input-bmap`. Every range the input bmap maps stays mapped, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). The created bmap maps these ranges, the blocks `omnect-cli` changed, and all other blocks that the image file stores as data. For an uncompressed image file that is not sparse, this is the whole image. +The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image and sha256 range checksums that match it, otherwise the command fails. A command without `-b` changes the image but not the bmap next to it, so when you run several commands on one image, use `-b` on each of them, or pass the matching bmap with `--input-bmap`. The created bmap maps every range of the input bmap, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). It also maps every block that the image file stores as data, which includes the blocks `omnect-cli` changed. The image file itself stays sparse. Without an input bmap, the created bmap maps the whole image, and a flash writes every block. diff --git a/project-context.md b/project-context.md index 554c6b73..3b8bbd1a 100644 --- a/project-context.md +++ b/project-context.md @@ -50,8 +50,9 @@ - `src/docker.rs` — `docker pull --platform` + `docker save` for multi-arch images - `src/image.rs` — firmware image architecture detection (ARM32/ARM64/x86_64) - `src/file/mod.rs` — high-level image operations: identity config, certs, hostname patching -- `src/file/functions.rs` — partition read via dd, modify via e2cp/mcopy, write back of changed blocks only -- `src/file/bmap.rs` — input bmap lookup; maps its ranges in the working image, or the whole image without one; bmap generation via `bmaptool create` +- `src/file/functions.rs` — sparse partition read, modify via e2cp/mcopy, write back of changed blocks only +- `src/file/bmap.rs` — input bmap lookup and checksum check; writes the output bmap: input ranges (or the whole image) plus data blocks +- `src/file/sparse.rs` — data/hole ranges via `SEEK_DATA`/`SEEK_HOLE`, sparse range copy - `src/file/partition.rs` — GPT/MBR partition table parsing - `src/file/compression.rs` — xz/bzip2/gzip compress/decompress with auto-detection - `src/validators/` — validation for identity config (TOML), device-update config (JSON), SSH keys diff --git a/src/cli.rs b/src/cli.rs index 8d7278a7..cece8d9a 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -34,11 +34,11 @@ pub enum Docker { /// destination path of the docker image in the firmware image (must end in ".tar.gz") #[clap(short = 'e', long = "dest")] dest: PathBuf, - /// optional: create a bmap of the output image (currently not working in docker image) + /// optional: create a bmap of the output image #[arg(short = 'b', long = "output-bmap")] output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap")] + #[arg(long = "input-bmap", requires = "output_bmap")] input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] @@ -58,11 +58,11 @@ pub enum File { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: create a bmap of the output image (currently not working in docker image) + /// optional: create a bmap of the output image #[arg(short = 'b', long = "output-bmap")] output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap")] + #[arg(long = "input-bmap", requires = "output_bmap")] input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] @@ -94,11 +94,11 @@ pub enum IdentityConfig { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: create a bmap of the output image (currently not working in docker image) + /// optional: create a bmap of the output image #[arg(short = 'b', long = "output-bmap")] output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap")] + #[arg(long = "input-bmap", requires = "output_bmap")] input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] @@ -121,11 +121,11 @@ pub enum IdentityConfig { /// path to device identity certificate key file #[arg(short = 'k', long = "device_identity_key")] device_identity_key: PathBuf, - /// optional: create a bmap of the output image (currently not working in docker image) + /// optional: create a bmap of the output image #[arg(short = 'b', long = "output-bmap")] output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap")] + #[arg(long = "input-bmap", requires = "output_bmap")] input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] @@ -142,11 +142,11 @@ pub enum IdentityConfig { /// path to root ca certificate file #[arg(short = 'r', long = "root_ca")] root_ca: PathBuf, - /// optional: create a bmap of the output image (currently not working in docker image) + /// optional: create a bmap of the output image #[arg(short = 'b', long = "output-bmap")] output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap")] + #[arg(long = "input-bmap", requires = "output_bmap")] input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] @@ -169,11 +169,11 @@ pub enum IdentityConfig { /// period of validity in days #[arg(short = 'D', long = "days")] days: u32, - /// optional: create a bmap of the output image (currently not working in docker image) + /// optional: create a bmap of the output image #[arg(short = 'b', long = "output-bmap")] output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap")] + #[arg(long = "input-bmap", requires = "output_bmap")] input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] @@ -190,11 +190,11 @@ pub enum IdentityConfig { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: create a bmap of the output image (currently not working in docker image) + /// optional: create a bmap of the output image #[arg(short = 'b', long = "output-bmap")] output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap")] + #[arg(long = "input-bmap", requires = "output_bmap")] input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] @@ -217,11 +217,11 @@ pub enum IdentityConfig { /// period of validity in days #[arg(short = 'D', long = "days")] days: u32, - /// optional: create a bmap of the output image (currently not working in docker image) + /// optional: create a bmap of the output image #[arg(short = 'b', long = "output-bmap")] output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap")] + #[arg(long = "input-bmap", requires = "output_bmap")] input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] @@ -241,11 +241,11 @@ pub enum IotHubDeviceUpdate { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: create a bmap of the output image (currently not working in docker image) + /// optional: create a bmap of the output image #[arg(short = 'b', long = "output-bmap")] output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap")] + #[arg(long = "input-bmap", requires = "output_bmap")] input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] @@ -363,11 +363,11 @@ pub enum SshConfig { /// path to public key of the ssh root ca #[arg(short = 'r', long = "root_ca")] root_ca: PathBuf, - /// optional: create a bmap of the output image (currently not working in docker image) + /// optional: create a bmap of the output image #[arg(short = 'b', long = "output-bmap")] output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap")] + #[arg(long = "input-bmap", requires = "output_bmap")] input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] diff --git a/src/file/bmap.rs b/src/file/bmap.rs index b3b09733..62a2cf80 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -1,17 +1,18 @@ -use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, is_zero}; +use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, sparse}; use anyhow::{Context, Result}; use sha2::{Digest, Sha256}; use std::{ fs, - ops::RangeInclusive, + ops::{Range, RangeInclusive}, os::unix::fs::FileExt, path::{Path, PathBuf}, - process::Command, }; -const BMAP_EXTENSION: &str = "bmap"; +pub const BMAP_EXTENSION: &str = "bmap"; const COMPRESSION_EXTENSIONS: [&str; 5] = ["xz", "gz", "gzip", "bz2", "bzip2"]; const SHA256_CHECKSUM_TYPE: &str = "sha256"; +const SHA256_HEX_LEN: usize = 64; +const BMAP_VERSION: &str = "2.0"; /// The block ranges of an image that a bmap file maps. #[derive(Debug)] @@ -74,21 +75,6 @@ impl Bmap { sha256, }) } - - pub fn whole_image(image_size: u64) -> Self { - let block_size = IMAGE_BLOCK_SIZE as u64; - let ranges = match image_size { - 0 => vec![], - size => vec![RangeInclusive::new(0, (size - 1) / block_size)], - }; - - Self { - block_size, - image_size, - ranges, - sha256: None, - } - } } fn parse_range(text: &str) -> Result> { @@ -153,18 +139,9 @@ pub fn verify_checksums(image_file: &Path, bmap: &Bmap) -> Result<()> { let mut buf = vec![0u8; COPY_CHUNK_SIZE]; for (range, expected) in bmap.ranges.iter().zip(sha256) { - let end = ((range.end() + 1) * bmap.block_size).min(image_len); - let mut pos = range.start() * bmap.block_size; - let mut hasher = Sha256::new(); - while pos < end { - let n = usize::try_from((end - pos).min(COPY_CHUNK_SIZE as u64))?; - image - .read_exact_at(&mut buf[..n], pos) - .context("verify_checksums: cannot read image")?; - hasher.update(&buf[..n]); - pos += n as u64; - } - let actual = hex(&hasher.finalize()); + let bytes = + range.start() * bmap.block_size..((range.end() + 1) * bmap.block_size).min(image_len); + let actual = range_sha256(&image, bytes, &mut buf)?; anyhow::ensure!( actual.eq_ignore_ascii_case(expected.trim()), "verify_checksums: range {range:?} of the image does not match the bmap. It is the \ @@ -176,6 +153,105 @@ pub fn verify_checksums(image_file: &Path, bmap: &Bmap) -> Result<()> { Ok(()) } +/// Writes a bmap of `image_file` to `bmap_file`. It maps the ranges of `input`, or the whole +/// image without one, and every block that the image file stores as data. A range of `input` +/// can be all zeros and a hole in the image, and a bmap flash still has to write it, else the +/// device keeps what an earlier installation left there. +pub fn write_bmap(image_file: &Path, input: Option<&Bmap>, bmap_file: &Path) -> Result<()> { + let image = fs::File::open(image_file) + .context(format!("write_bmap: cannot open {}", image_file.display()))?; + let image_len = image + .metadata() + .context("write_bmap: cannot get image size")? + .len(); + let block_size = IMAGE_BLOCK_SIZE as u64; + + let ranges = mapped_ranges(&image, image_len, input)?; + + let mut buf = vec![0u8; COPY_CHUNK_SIZE]; + let mut block_map = String::new(); + for range in &ranges { + let bytes = range.start() * block_size..((range.end() + 1) * block_size).min(image_len); + let sha256 = range_sha256(&image, bytes, &mut buf)?; + block_map.push_str(&format!( + " {}-{} \n", + range.start(), + range.end() + )); + } + let blocks = image_len.div_ceil(block_size); + let mapped: u64 = ranges.iter().map(|r| r.end() - r.start() + 1).sum(); + + // like bmaptool, the file checksum is taken with zeros in place of itself + let zeros = "0".repeat(SHA256_HEX_LEN); + let xml = format!( + "\n\ + \n \ + {image_len} \n \ + {block_size} \n \ + {blocks} \n \ + {mapped} \n \ + {SHA256_CHECKSUM_TYPE} \n \ + {zeros} \n \ + \n{block_map} \n\ + \n" + ); + let xml = xml.replacen(&zeros, &hex(&Sha256::digest(xml.as_bytes())), 1); + + fs::write(bmap_file, xml).context(format!("write_bmap: cannot write {}", bmap_file.display())) +} + +/// The blocks of the created bmap: the ranges of `input`, or the whole image without one, and +/// every block that the image file stores as data. +fn mapped_ranges( + image: &fs::File, + image_len: u64, + input: Option<&Bmap>, +) -> Result>> { + let block_size = IMAGE_BLOCK_SIZE as u64; + let mut ranges: Vec> = match input { + Some(bmap) => { + check_layout(bmap, image_len)?; + bmap.ranges + .iter() + .map(|r| r.start() * bmap.block_size..(r.end() + 1) * bmap.block_size) + .collect() + } + None => vec![Range { + start: 0, + end: image_len, + }], + }; + ranges.extend(sparse::data_ranges(image, 0, image_len)?); + Ok(sparse::merge( + ranges + .into_iter() + .map(|r| { + r.start / block_size * block_size..r.end.min(image_len).next_multiple_of(block_size) + }) + .filter(|r| !r.is_empty()) + .collect(), + ) + .into_iter() + .map(|r| r.start / block_size..=r.end / block_size - 1) + .collect()) +} + +fn range_sha256(image: &fs::File, bytes: Range, buf: &mut [u8]) -> Result { + let mut hasher = Sha256::new(); + let mut pos = bytes.start; + while pos < bytes.end { + let n = usize::try_from((bytes.end - pos).min(buf.len() as u64))?; + image + .read_exact_at(&mut buf[..n], pos) + .context("range_sha256: cannot read image")?; + hasher.update(&buf[..n]); + pos += n as u64; + } + + Ok(hex(&hasher.finalize())) +} + fn hex(bytes: &[u8]) -> String { bytes.iter().map(|b| format!("{b:02x}")).collect() } @@ -195,82 +271,6 @@ fn check_layout(bmap: &Bmap, image_len: u64) -> Result<()> { Ok(()) } -/// Writes the zero blocks in the ranges of `bmap` as data. A zero block can be a hole, and a -/// bmap created from the image would skip it, so a bmap-based flash would keep what an earlier -/// installation left there. -pub fn allocate_ranges(image_file: &Path, bmap: &Bmap) -> Result<()> { - let image = fs::OpenOptions::new() - .read(true) - .write(true) - .open(image_file) - .context(format!( - "allocate_ranges: cannot open {}", - image_file.display() - ))?; - let image_len = image - .metadata() - .context("allocate_ranges: cannot get image size")? - .len(); - check_layout(bmap, image_len)?; - - let image_block_size = IMAGE_BLOCK_SIZE as u64; - let mut buf = vec![0u8; COPY_CHUNK_SIZE]; - for range in &bmap.ranges { - // check whole image blocks, also when the bmap uses a smaller block size - let start = range.start() * bmap.block_size / image_block_size * image_block_size; - let end = ((range.end() + 1) * bmap.block_size) - .next_multiple_of(image_block_size) - .min(image_len); - - let mut pos = start; - while pos < end { - let n = usize::try_from((end - pos).min(COPY_CHUNK_SIZE as u64))?; - let chunk = &mut buf[..n]; - image - .read_exact_at(chunk, pos) - .context("allocate_ranges: cannot read image")?; - write_zero_blocks(&image, chunk, pos)?; - pos += n as u64; - } - } - - Ok(()) -} - -fn write_zero_blocks(image: &fs::File, chunk: &[u8], pos: u64) -> Result<()> { - let blocks: Vec<(u64, &[u8])> = if is_zero(chunk) { - vec![(pos, chunk)] - } else { - chunk - .chunks(IMAGE_BLOCK_SIZE) - .enumerate() - .filter(|(_, block)| is_zero(block)) - .map(|(i, block)| (pos + (i * IMAGE_BLOCK_SIZE) as u64, block)) - .collect() - }; - - blocks.into_iter().try_for_each(|(pos, block)| { - image - .write_all_at(block, pos) - .context("allocate_ranges: cannot write image") - }) -} - -pub fn generate_bmap_file(image_file: &str) -> Result<()> { - let mut bmaptool = Command::new("bmaptool"); - bmaptool - .arg("create") - .arg("-o") - .arg(format!("{image_file}.{BMAP_EXTENSION}")) - .arg(image_file); - let status = bmaptool - .status() - .context(format!("generate_bmap_file: cannot run {bmaptool:?}"))?; - anyhow::ensure!(status.success(), "generate_bmap_file: {bmaptool:?} failed"); - - Ok(()) -} - #[cfg(test)] mod tests { use crate::file::bmap::*; @@ -296,7 +296,7 @@ mod tests { ) } - fn write_bmap(path: &Path, image_size: u64, ranges: &[RangeInclusive]) { + fn write_input_bmap(path: &Path, image_size: u64, ranges: &[RangeInclusive]) { let block_map: String = ranges .iter() .map(|r| format!("{}-{}", r.start(), r.end())) @@ -317,10 +317,10 @@ mod tests { path } - fn created_bmap(image: &Path) -> Bmap { - let image = image.to_str().expect("image path"); - generate_bmap_file(image).expect("generate bmap"); - Bmap::read(Path::new(&format!("{image}.{BMAP_EXTENSION}"))).expect("read bmap") + fn created_bmap(image: &Path, input: Option<&Bmap>) -> Bmap { + let path = image.with_extension(BMAP_EXTENSION); + write_bmap(image, input, &path).expect("write bmap"); + Bmap::read(&path).expect("read bmap") } fn is_mapped(bmap: &Bmap, block: u64) -> bool { @@ -428,27 +428,29 @@ mod tests { } #[test] - fn allocate_ranges_maps_zero_ranges_of_the_input_bmap() { + fn write_bmap_maps_zero_ranges_of_the_input_bmap() { let dir = tempfile::tempdir().expect("create temp dir"); let image = sparse_image(dir.path()); let input_bmap = dir.path().join("input.bmap"); - write_bmap( - &input_bmap, - IMAGE_BLOCKS * BLOCK_SIZE, - &[DATA_BLOCK..=DATA_BLOCK, ZERO_BLOCKS], - ); + write_input_bmap(&input_bmap, IMAGE_BLOCKS * BLOCK_SIZE, &[ZERO_BLOCKS]); let input = Bmap::read(&input_bmap).expect("read bmap"); - let before = created_bmap(&image); - assert!(ZERO_BLOCKS.clone().all(|b| !is_mapped(&before, b))); - - allocate_ranges(&image, &input).expect("allocate ranges"); - let after = created_bmap(&image); - assert!(ZERO_BLOCKS.clone().all(|b| is_mapped(&after, b))); - assert!(!is_mapped(&after, UNMAPPED_BLOCK)); + let bmap = created_bmap(&image, Some(&input)); + assert!(ZERO_BLOCKS.clone().all(|b| is_mapped(&bmap, b))); + assert!( + is_mapped(&bmap, DATA_BLOCK), + "data outside the input ranges" + ); + assert!(!is_mapped(&bmap, UNMAPPED_BLOCK)); + let file = fs::File::open(&image).expect("open image"); + assert_eq!( + sparse::data_ranges(&file, 0, IMAGE_BLOCKS * BLOCK_SIZE).expect("data ranges"), + vec![DATA_BLOCK * BLOCK_SIZE..(DATA_BLOCK + 1) * BLOCK_SIZE], + "image is not sparse any more" + ); let device = dir.path().join("device"); - bmap_flash(&image, &after, &device); + bmap_flash(&image, &bmap, &device); let device = fs::read(&device).expect("read device"); let zero_start = usize::try_from(ZERO_BLOCKS.start() * BLOCK_SIZE).expect("start"); let zero_end = usize::try_from((ZERO_BLOCKS.end() + 1) * BLOCK_SIZE).expect("end"); @@ -456,17 +458,18 @@ mod tests { } #[test] - fn allocate_ranges_maps_whole_image() { + fn write_bmap_without_input_maps_whole_image() { let dir = tempfile::tempdir().expect("create temp dir"); let image = sparse_image(dir.path()); - allocate_ranges(&image, &Bmap::whole_image(IMAGE_BLOCKS * BLOCK_SIZE)) - .expect("allocate ranges"); - assert_eq!(created_bmap(&image).ranges, vec![0..=IMAGE_BLOCKS - 1]); + assert_eq!( + created_bmap(&image, None).ranges, + vec![0..=IMAGE_BLOCKS - 1] + ); } #[test] - fn allocate_ranges_maps_whole_blocks_for_a_smaller_bmap_block_size() { + fn write_bmap_maps_whole_blocks_for_a_smaller_bmap_block_size() { let dir = tempfile::tempdir().expect("create temp dir"); let image = sparse_image(dir.path()); // starts in DATA_BLOCK and ends in the hole after it @@ -478,8 +481,35 @@ mod tests { sha256: None, }; - allocate_ranges(&image, &input).expect("allocate ranges"); - assert!(is_mapped(&created_bmap(&image), DATA_BLOCK + 1)); + assert!(is_mapped( + &created_bmap(&image, Some(&input)), + DATA_BLOCK + 1 + )); + } + + #[test] + fn write_bmap_is_accepted_by_bmaptool() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + // a mapped last block that is shorter than the block size + fs::OpenOptions::new() + .write(true) + .open(&image) + .expect("open image") + .write_all_at(&[DATA_BYTE; PARTIAL_BLOCK_LEN], IMAGE_BLOCKS * BLOCK_SIZE) + .expect("write partial block"); + let bmap = image.with_extension(BMAP_EXTENSION); + write_bmap(&image, None, &bmap).expect("write bmap"); + + let status = std::process::Command::new("bmaptool") + .arg("copy") + .arg("--bmap") + .arg(&bmap) + .arg(&image) + .arg(dir.path().join("device")) + .status() + .expect("run bmaptool"); + assert!(status.success(), "bmaptool rejected the bmap"); } #[test] @@ -494,14 +524,14 @@ mod tests { .write_all_at(&[DATA_BYTE; PARTIAL_BLOCK_LEN], IMAGE_BLOCKS * BLOCK_SIZE) .expect("write partial block"); - verify_checksums(&image, &created_bmap(&image)).expect("verify checksums"); + verify_checksums(&image, &created_bmap(&image, None)).expect("verify checksums"); } #[test] fn verify_checksums_rejects_changed_image() { let dir = tempfile::tempdir().expect("create temp dir"); let image = sparse_image(dir.path()); - let bmap = created_bmap(&image); + let bmap = created_bmap(&image, None); fs::OpenOptions::new() .write(true) .open(&image) @@ -517,26 +547,29 @@ mod tests { fn verify_checksums_rejects_bmap_without_checksums() { let dir = tempfile::tempdir().expect("create temp dir"); let image = sparse_image(dir.path()); - let mut bmap = created_bmap(&image); + let mut bmap = created_bmap(&image, None); bmap.sha256 = None; assert!(verify_checksums(&image, &bmap).is_err()); } #[test] - fn allocate_ranges_rejects_bmap_of_another_image() { + fn bmap_of_another_image_is_rejected() { let dir = tempfile::tempdir().expect("create temp dir"); let image = sparse_image(dir.path()); - - let other_size = Bmap::whole_image((IMAGE_BLOCKS + 1) * BLOCK_SIZE); - assert!(allocate_ranges(&image, &other_size).is_err()); - + let bmap_file = dir.path().join("out.bmap"); + let mut other_size = created_bmap(&image, None); + other_size.image_size += BLOCK_SIZE; let outside = Bmap { block_size: BLOCK_SIZE, image_size: IMAGE_BLOCKS * BLOCK_SIZE, ranges: vec![RangeInclusive::new(IMAGE_BLOCKS, IMAGE_BLOCKS)], - sha256: None, + sha256: Some(vec![String::new()]), }; - assert!(allocate_ranges(&image, &outside).is_err()); + + for bmap in [&other_size, &outside] { + assert!(verify_checksums(&image, bmap).is_err()); + assert!(write_bmap(&image, Some(bmap), &bmap_file).is_err()); + } } } diff --git a/src/file/sparse.rs b/src/file/sparse.rs index 1c9d62bd..4193ddd6 100644 --- a/src/file/sparse.rs +++ b/src/file/sparse.rs @@ -31,6 +31,19 @@ pub fn data_ranges(file: &File, start: u64, end: u64) -> Result>> Ok(ranges) } +/// Sorts `ranges` and joins the ones that overlap or touch. +pub fn merge(mut ranges: Vec>) -> Vec> { + ranges.sort_by_key(|r| r.start); + let mut merged: Vec> = Vec::with_capacity(ranges.len()); + for range in ranges { + match merged.last_mut() { + Some(last) if range.start <= last.end => last.end = last.end.max(range.end), + _ => merged.push(range), + } + } + merged +} + /// Copies `len` bytes of `src` at `src_offset` to `dst` at `dst_offset`. Holes of `src` are /// skipped, so they stay holes in a new `dst`. pub fn copy_range( @@ -113,6 +126,14 @@ mod tests { ); } + #[test] + fn merge_joins_overlapping_and_touching_ranges() { + assert_eq!( + merge(vec![8..10, 0..2, 1..4, 4..5, 12..12]), + vec![0..5, 8..10, 12..12] + ); + } + #[test] fn copy_range_keeps_holes() { let dir = tempfile::tempdir().expect("create temp dir"); diff --git a/src/lib.rs b/src/lib.rs index d7dc96f3..257b8548 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,7 +20,11 @@ use cli::{ IotHubDeviceUpdate::{self, SetDeviceConfig as IotHubDeviceUpdateSet}, SshConfig::{SetCertificate, SetConnection}, }; -use file::{bmap::Bmap, compression::Compression, functions::FileCopyToParams}; +use file::{ + bmap::{BMAP_EXTENSION, Bmap}, + compression::Compression, + functions::FileCopyToParams, +}; use log::{error, info}; use std::{ fs, @@ -51,44 +55,28 @@ impl Drop for TempDirGuard { } } -/// What an image command does with the holes and mapped blocks of the image. -enum Layout { - /// The command does not change the image. - Unchanged, - /// Keep the ranges of this bmap, or of the one found next to the image, mapped. - Keep(Option), -} - fn run_image_command( image_file: PathBuf, - layout: Layout, output_bmap: bool, + input_bmap: Option, target_compression: Option, command: F, ) -> Result<()> where F: FnOnce(&PathBuf) -> Result<()>, { - if let Ok("true") | Ok("1") = std::env::var("CONTAINERIZED").as_deref() { - anyhow::ensure!( - !output_bmap, - "run_image_command: generating bmap file is not supported in containerized environments." - ); - } - anyhow::ensure!( image_file.try_exists().is_ok_and(|exists| exists), "run_image_command: image doesn't exist {}", image_file.to_str().context("cannot get image file path")? ); - let input_bmap = match layout { - Layout::Unchanged => None, - Layout::Keep(explicit) => Some( - file::bmap::find_input_bmap(&image_file, explicit.as_deref())? - .map(|path| Bmap::read(&path).map(|bmap| (path, bmap))) - .transpose()?, - ), + let input_bmap = if output_bmap { + file::bmap::find_input_bmap(&image_file, input_bmap.as_deref())? + .map(|path| Bmap::read(&path).map(|bmap| (path, bmap))) + .transpose()? + } else { + None }; let mut dest_image_file = image_file.clone(); @@ -119,48 +107,30 @@ where ))?; } - match input_bmap { - None => {} - Some(Some((path, bmap))) => { - info!("keeping the ranges of {} mapped", path.display()); - file::bmap::verify_checksums(&tmp_image_file, &bmap) - .and_then(|_| file::bmap::allocate_ranges(&tmp_image_file, &bmap)) - .context(format!("cannot use input bmap {}", path.display()))?; - } - Some(None) => { - info!("no input bmap found, so every block of the image is allocated"); - let image_size = fs::metadata(&tmp_image_file) - .context("run_image_command: cannot get image size")? - .len(); - file::bmap::allocate_ranges(&tmp_image_file, &Bmap::whole_image(image_size))?; - } + if let Some((path, bmap)) = &input_bmap { + file::bmap::verify_checksums(&tmp_image_file, bmap) + .context(format!("cannot use input bmap {}", path.display()))?; } // run command command(&tmp_image_file)?; - // create and copy back bmap file if one was created + let tmp_bmap = PathBuf::from(format!( + "{}.{BMAP_EXTENSION}", + tmp_image_file + .to_str() + .context("cannot get image file path")? + )); if output_bmap { - let mut target_bmap = image_file - .parent() - .context("cannot get parent dir of image path")? - .to_path_buf(); - let tmp_bmap = PathBuf::from(format!( - "{}.bmap", - tmp_image_file - .to_str() - .context("cannot get image file path")? - )); - file::bmap::generate_bmap_file( - tmp_image_file - .to_str() - .context("cannot get image file path")?, + match &input_bmap { + Some((path, _)) => info!("the bmap maps the ranges of {}", path.display()), + None => info!("no input bmap found, so the bmap maps the whole image"), + } + file::bmap::write_bmap( + &tmp_image_file, + input_bmap.as_ref().map(|(_, bmap)| bmap), + &tmp_bmap, )?; - target_bmap.push(tmp_bmap.file_name().context("cannot get bmap file name")?); - std::fs::copy(&tmp_bmap, &target_bmap).context(format!( - "error: std::fs::copy({:?}, {:?})", - tmp_bmap, target_bmap - ))?; } // if applicable compress image @@ -182,6 +152,18 @@ where ))?; } + // copy the bmap last, so it never describes an image that was not written back + if output_bmap { + let target_bmap = image_file + .parent() + .context("cannot get parent dir of image path")? + .join(tmp_bmap.file_name().context("cannot get bmap file name")?); + std::fs::copy(&tmp_bmap, &target_bmap).context(format!( + "error: std::fs::copy({:?}, {:?})", + tmp_bmap, target_bmap + ))?; + } + Ok(()) } @@ -240,8 +222,6 @@ fn create_image_cert(image: &Path, cert_opts: CertificateOptions) -> Result Result<()> { - use Layout::{Keep, Unchanged}; - match cli::from_args() { Command::Docker(Inject { docker_image, @@ -251,46 +231,40 @@ pub fn run() -> Result<()> { output_bmap, input_bmap, compress_image, - }) => run_image_command( - image, - Keep(input_bmap), - output_bmap, - compress_image, - |img| { - anyhow::ensure!( - dest.to_string_lossy().ends_with(".tar.gz"), - format!( - "invalid destination file path \"{}\". Must end in \".tar.gz\".", - dest.to_string_lossy(), - ), + }) => run_image_command(image, output_bmap, input_bmap, compress_image, |img| { + anyhow::ensure!( + dest.to_string_lossy().ends_with(".tar.gz"), + format!( + "invalid destination file path \"{}\". Must end in \".tar.gz\".", + dest.to_string_lossy(), + ), + ); + + let arch = image::image_arch(img)?; + + let docker_path = docker::pull_image(&docker_image, arch)?; + + let result = file::copy_to_image( + &[FileCopyToParams::new( + &docker_path, + partition.clone(), + &dest, + )], + img, + ); + std::fs::remove_file(docker_path)?; + + if result.is_ok() { + println!( + "Stored {} to {}:{}", + docker_image, + partition, + dest.to_string_lossy(), ); + } - let arch = image::image_arch(img)?; - - let docker_path = docker::pull_image(&docker_image, arch)?; - - let result = file::copy_to_image( - &[FileCopyToParams::new( - &docker_path, - partition.clone(), - &dest, - )], - img, - ); - std::fs::remove_file(docker_path)?; - - if result.is_ok() { - println!( - "Stored {} to {}:{}", - docker_image, - partition, - dest.to_string_lossy(), - ); - } - - result - }, - )?, + result + })?, Command::Identity(SetConfig { config, image, @@ -298,13 +272,9 @@ pub fn run() -> Result<()> { output_bmap, input_bmap, compress_image, - }) => run_image_command( - image, - Keep(input_bmap), - output_bmap, - compress_image, - |img| file::set_identity_config(&config, img, payload.as_deref()), - )?, + }) => run_image_command(image, output_bmap, input_bmap, compress_image, |img| { + file::set_identity_config(&config, img, payload.as_deref()) + })?, Command::Identity(SetDeviceCertificate { intermediate_full_chain_cert, intermediate_key, @@ -328,20 +298,14 @@ pub fn run() -> Result<()> { ) .context("set_device_certificate: could not create certificate")?; - run_image_command( - image, - Keep(input_bmap), - output_bmap, - compress_image, - |img| { - file::set_device_cert( - Some(&intermediate_full_chain_cert), - &cert_info.cert_path, - &cert_info.key_path, - img, - ) - }, - )? + run_image_command(image, output_bmap, input_bmap, compress_image, |img| { + file::set_device_cert( + Some(&intermediate_full_chain_cert), + &cert_info.cert_path, + &cert_info.key_path, + img, + ) + })? } Command::Identity(SetEdgeCaCertificate { intermediate_full_chain_cert, @@ -366,20 +330,14 @@ pub fn run() -> Result<()> { ) .context("set_edge_ca_certificate: could not create certificate")?; - run_image_command( - image, - Keep(input_bmap), - output_bmap, - compress_image, - |img| { - file::set_edge_ca_cert( - Some(&intermediate_full_chain_cert), - &cert_info.cert_path, - &cert_info.key_path, - img, - ) - }, - )? + run_image_command(image, output_bmap, input_bmap, compress_image, |img| { + file::set_edge_ca_cert( + Some(&intermediate_full_chain_cert), + &cert_info.cert_path, + &cert_info.key_path, + img, + ) + })? } Command::Identity(SetDeviceCertificateNoEst { device_cert: device_cert_pem, @@ -388,13 +346,9 @@ pub fn run() -> Result<()> { output_bmap, input_bmap, compress_image, - }) => run_image_command( - image, - Keep(input_bmap), - output_bmap, - compress_image, - |img| file::set_device_cert(None, &device_cert_pem, &device_key_pem, img), - )?, + }) => run_image_command(image, output_bmap, input_bmap, compress_image, |img| { + file::set_device_cert(None, &device_cert_pem, &device_key_pem, img) + })?, Command::Identity(SetIotedgeGatewayConfig { config, image, @@ -406,8 +360,8 @@ pub fn run() -> Result<()> { compress_image, }) => run_image_command( image, - Keep(input_bmap), output_bmap, + input_bmap, compress_image, |img: &PathBuf| { file::set_iotedge_gateway_config( @@ -428,8 +382,8 @@ pub fn run() -> Result<()> { compress_image, }) => run_image_command( image, - Keep(input_bmap), output_bmap, + input_bmap, compress_image, |img: &PathBuf| file::set_iot_leaf_sas_config(&config, img, &root_ca), )?, @@ -441,8 +395,8 @@ pub fn run() -> Result<()> { compress_image, }) => run_image_command( image, - Keep(input_bmap), output_bmap, + input_bmap, compress_image, |img: &PathBuf| file::set_ssh_tunnel_certificate(img, &root_ca), )?, @@ -454,8 +408,8 @@ pub fn run() -> Result<()> { compress_image, }) => run_image_command( image, - Keep(input_bmap), output_bmap, + input_bmap, compress_image, |img: &PathBuf| { file::set_iot_hub_device_update_config(&iot_hub_device_update_config, img) @@ -578,15 +532,15 @@ pub fn run() -> Result<()> { compress_image, }) => run_image_command( image, - Keep(input_bmap), output_bmap, + input_bmap, compress_image, |img: &PathBuf| file::copy_to_image(&file_copy_params, img), )?, Command::File(CopyFromImage { file_copy_params, image, - }) => run_image_command(image, Unchanged, false, None, |img: &PathBuf| { + }) => run_image_command(image, false, None, None, |img: &PathBuf| { file::copy_from_image(&file_copy_params, img) })?, } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 472948cb..b76a8648 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1098,6 +1098,7 @@ fn check_input_bmap_of_another_image() { .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) .arg("-i") .arg(&image_path) + .arg("-b") .assert() .failure() .stderr(predicate::str::contains("cannot use input bmap")); @@ -1116,6 +1117,66 @@ fn check_input_bmap_of_another_image() { .success(); } +#[test] +fn check_file_copy_without_output_bmap_keeps_image_sparse() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let sparse_path = image_path.with_file_name("sparse.wic"); + let bmap_path = PathBuf::from(format!("{}.bmap", sparse_path.to_string_lossy())); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + sparse_copy(&image_path, &sparse_path); + // without -b, a bmap next to the image is not used + fs::write(&bmap_path, OTHER_IMAGE_BMAP).expect("write bmap"); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},factory:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&sparse_path) + .assert() + .success(); + + let meta = sparse_path.metadata().expect("image metadata"); + assert!( + meta.blocks() * STAT_BLOCK_SIZE < meta.len(), + "image is not sparse" + ); + assert_eq!( + fs::read_to_string(&bmap_path).expect("read bmap"), + OTHER_IMAGE_BMAP + ); +} + +#[test] +fn check_no_output_bmap_when_image_write_back_fails() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let bmap_path = PathBuf::from(format!("{}.bmap", image_path.to_string_lossy())); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + let mut permissions = image_path.metadata().expect("image metadata").permissions(); + permissions.set_readonly(true); + fs::set_permissions(&image_path, permissions).expect("make image read-only"); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .arg("-b") + .assert() + .failure(); + assert!( + !bmap_path.exists(), + "bmap written for an image that was not written back" + ); +} + #[test] fn check_input_bmap_of_another_image_with_the_same_size() { let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); @@ -1138,6 +1199,7 @@ fn check_input_bmap_of_another_image_with_the_same_size() { .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) .arg("-i") .arg(&image_path) + .arg("-b") .assert() .failure() .stderr(predicate::str::contains("cannot use input bmap")) From 57a1580fff3a225e9dd24f0c5878a271ffe202f1 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:16:58 +0100 Subject: [PATCH 21/75] perf(file): compare only blocks that hold data in a partition write back write_partition read and compared the whole partition, also where both the partition file and the image are holes and no write can be needed. It now goes only over the data ranges of both files. A test pins that a zero block written into an unaligned partition is still mapped. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/functions.rs | 90 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 82 insertions(+), 8 deletions(-) diff --git a/src/file/functions.rs b/src/file/functions.rs index 22c3a15e..3c9429f9 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -459,11 +459,49 @@ fn write_partition( // image too, else a bmap flash keeps the old content there. let partition_data = sparse::data_ranges(&partition, 0, len)?; let image_data = sparse::data_ranges(&image, image_offset, image_offset + len)?; + // a block that is a hole in both files is zero in both, so it never needs a write + let block_size = IMAGE_BLOCK_SIZE as u64; + let spans = sparse::merge( + partition_data + .iter() + .cloned() + .chain( + image_data + .iter() + .map(|r| r.start - image_offset..r.end - image_offset), + ) + .map(|r| r.start / block_size * block_size..r.end.next_multiple_of(block_size).min(len)) + .collect(), + ); let mut src = vec![0u8; COPY_CHUNK_SIZE]; let mut dst = vec![0u8; COPY_CHUNK_SIZE]; - let mut pos = 0; - while pos < len { - let n = usize::try_from((len - pos).min(COPY_CHUNK_SIZE as u64))?; + for span in spans { + write_changed_blocks( + &partition, + &image, + image_offset, + span, + (&partition_data, &image_data), + (&mut src, &mut dst), + )?; + } + + image + .sync_all() + .context("write_partition: cannot sync image") +} + +fn write_changed_blocks( + partition: &fs::File, + image: &fs::File, + image_offset: u64, + span: Range, + (partition_data, image_data): (&[Range], &[Range]), + (src, dst): (&mut [u8], &mut [u8]), +) -> Result<()> { + let mut pos = span.start; + while pos < span.end { + let n = usize::try_from((span.end - pos).min(COPY_CHUNK_SIZE as u64))?; partition .read_exact_at(&mut src[..n], pos) .context("write_partition: cannot read partition file")?; @@ -478,8 +516,7 @@ fn write_partition( let block_pos = pos + (i * IMAGE_BLOCK_SIZE) as u64; let block = block_pos..block_pos + s.len() as u64; let image_block = image_offset + block.start..image_offset + block.end; - if s != d || (has_data(&partition_data, &block) && has_hole(&image_data, &image_block)) - { + if s != d || (has_data(partition_data, &block) && has_hole(image_data, &image_block)) { image .write_all_at(s, image_block.start) .context("write_partition: cannot write image")?; @@ -488,9 +525,7 @@ fn write_partition( pos += n as u64; } - image - .sync_all() - .context("write_partition: cannot sync image") + Ok(()) } fn has_data(data_ranges: &[Range], block: &Range) -> bool { @@ -515,6 +550,8 @@ mod tests { const PART_SECTORS: u64 = 64; const OLD_CONTENT_BYTE: u8 = 0xa5; const ZEROED_BLOCK: usize = 2; + // the first partition of an old MBR layout + const UNALIGNED_PART_START: u64 = 63; fn partition_info() -> PartitionInfo { PartitionInfo { @@ -574,6 +611,43 @@ mod tests { assert_eq!(allocated(), before, "unchanged zero blocks were written"); } + #[test] + fn write_partition_maps_zero_block_of_an_unaligned_partition() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + let partition = dir.path().join("partition.img"); + let info = PartitionInfo { + num: PART_NUM, + start: UNALIGNED_PART_START, + count: PART_SECTORS, + }; + let image_file = fs::File::create(&image).expect("create image"); + image_file + .set_len((UNALIGNED_PART_START + PART_SECTORS) * SECTOR_SIZE) + .expect("set image size"); + let partition_file = fs::File::create(&partition).expect("create partition"); + partition_file + .set_len(PART_SECTORS * SECTOR_SIZE) + .expect("set partition size"); + let block = (ZEROED_BLOCK * IMAGE_BLOCK_SIZE) as u64; + partition_file + .write_all_at(&[0; IMAGE_BLOCK_SIZE], block) + .expect("write zero block"); + + write_partition( + image.to_str().expect("image path"), + partition.to_str().expect("partition path"), + &info, + ) + .expect("write partition"); + + let block = UNALIGNED_PART_START * SECTOR_SIZE + block; + let block = block..block + IMAGE_BLOCK_SIZE as u64; + let image_data = + sparse::data_ranges(&image_file, block.start, block.end).expect("data ranges"); + assert_eq!(image_data, vec![block], "zero block is still a hole"); + } + #[test] fn write_partition_maps_zero_block_written_into_a_hole() { let dir = tempfile::tempdir().expect("create temp dir"); From b59788cff1b328cb9239e13507bb3df192cf01dd Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:25:15 +0100 Subject: [PATCH 22/75] test(file): check the bmaptool format only in the integration tests The unit tests run where bmaptool is not installed. The integration tests already flash the created bmap with bmaptool copy, and the range checksums of a short last block are checked by verify_checksums. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/bmap.rs | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/src/file/bmap.rs b/src/file/bmap.rs index 62a2cf80..b6ea230c 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -487,31 +487,6 @@ mod tests { )); } - #[test] - fn write_bmap_is_accepted_by_bmaptool() { - let dir = tempfile::tempdir().expect("create temp dir"); - let image = sparse_image(dir.path()); - // a mapped last block that is shorter than the block size - fs::OpenOptions::new() - .write(true) - .open(&image) - .expect("open image") - .write_all_at(&[DATA_BYTE; PARTIAL_BLOCK_LEN], IMAGE_BLOCKS * BLOCK_SIZE) - .expect("write partial block"); - let bmap = image.with_extension(BMAP_EXTENSION); - write_bmap(&image, None, &bmap).expect("write bmap"); - - let status = std::process::Command::new("bmaptool") - .arg("copy") - .arg("--bmap") - .arg(&bmap) - .arg(&image) - .arg(dir.path().join("device")) - .status() - .expect("run bmaptool"); - assert!(status.success(), "bmaptool rejected the bmap"); - } - #[test] fn verify_checksums_accepts_bmap_of_the_image() { let dir = tempfile::tempdir().expect("create temp dir"); From 53ebcd8c12e1d3cda13c1cb2c592d73d7aae03fb Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:11:27 +0100 Subject: [PATCH 23/75] chore(docker): drop dd and sync from the image No code calls them any more. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- Dockerfile | 2 -- project-context.md | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9da78d24..91ba58a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,13 +33,11 @@ RUN < Date: Wed, 28 Oct 2026 19:11:28 +0100 Subject: [PATCH 24/75] fix(file): decompress images named .gz and .bz2 The output path was the same as the source for these names, so creating it truncated the source before it was read. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/compression.rs | 82 +++++++++++++++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 12 deletions(-) diff --git a/src/file/compression.rs b/src/file/compression.rs index 2af9cefc..f34b73d4 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -2,12 +2,14 @@ use crate::file::{IMAGE_BLOCK_SIZE, is_zero}; use anyhow::{Context, Result}; use filemagic::Magic; use log::debug; -use std::env; -use std::fs::File; -use std::io::{Seek, SeekFrom, Write}; -use std::os::unix::fs::FileExt; -use std::path::PathBuf; -use std::str::FromStr; +use std::{ + env, + fs::File, + io::{Seek, SeekFrom, Write}, + os::unix::fs::FileExt, + path::{Path, PathBuf}, + str::FromStr, +}; use strum::IntoEnumIterator; use strum_macros::EnumIter; @@ -108,13 +110,24 @@ impl Compression { } fn extension(&self) -> &'static str { + self.extensions()[0] + } + + /// The first one is used for a compressed image that omnect-cli creates. + fn extensions(&self) -> &'static [&'static str] { match &self { - Compression::bzip2 => "bzip2", - Compression::gzip => "gzip", - Compression::xz { .. } => "xz", + Compression::bzip2 => &["bzip2", "bz2"], + Compression::gzip => &["gzip", "gz"], + Compression::xz { .. } => &["xz"], } } + pub fn has_extension(path: &Path) -> bool { + path.extension() + .and_then(|e| e.to_str()) + .is_some_and(|e| Compression::iter().any(|c| c.extensions().contains(&e))) + } + pub fn from_file(image_file_name: &PathBuf) -> Result> { let detector = Magic::open(Default::default()) .context("image::compression: failed to open libmagic")?; @@ -193,10 +206,18 @@ pub fn decompress(image_file_name: &PathBuf, compression: &Compression) -> Resul if new_image_file .extension() - .is_some_and(|ext| ext == compression.extension()) + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| compression.extensions().contains(&ext)) { new_image_file.set_extension(""); } + // creating the destination would truncate the source + anyhow::ensure!( + new_image_file != *image_file_name, + "decompress: {} has no {:?} file extension", + image_file_name.display(), + compression.extensions() + ); let mut destination = File::create(&new_image_file)?; let mut source = File::open(image_file_name)?; @@ -223,8 +244,7 @@ pub fn compress(image_file_name: &PathBuf, compression: &Compression) -> Result< #[cfg(test)] mod tests { use crate::file::compression::*; - use std::io::Read; - use std::os::unix::fs::MetadataExt; + use std::{fs, io::Read, os::unix::fs::MetadataExt}; // unit of `st_blocks` const STAT_BLOCK_SIZE: u64 = 512; @@ -339,4 +359,42 @@ mod tests { ); } } + + #[test] + fn decompress_file_strips_every_extension_of_the_compression() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = test_image(); + + for compression in all_compressions() { + for extension in compression.extensions() { + let path = dir.path().join(format!("image.wic.{extension}")); + let mut compressed = compress_image(&compression, &image); + fs::write(&path, read_all(&mut compressed)).expect("write compressed"); + + let decompressed = decompress(&path, &compression).expect("decompress"); + assert_eq!(decompressed, dir.path().join("image.wic")); + assert!( + fs::read(&decompressed).expect("read decompressed") == image, + "{compression:?}: content of .{extension} differs" + ); + assert!(Compression::has_extension(&path)); + } + } + } + + #[test] + fn decompress_file_rejects_unknown_extension() { + let dir = tempfile::tempdir().expect("create temp dir"); + let compression = Compression::gzip; + let path = dir.path().join("image.wic"); + let mut compressed = compress_image(&compression, &test_image()); + let content = read_all(&mut compressed); + fs::write(&path, &content).expect("write compressed"); + + assert!(decompress(&path, &compression).is_err()); + assert!( + fs::read(&path).expect("read compressed") == content, + "source was changed" + ); + } } From cdc299c44f12d16b0a01809decc9119aa3969064 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:11:29 +0100 Subject: [PATCH 25/75] refactor(cli): share the bmap and compression options of image commands Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/cli.rs | 125 ++++++++++------------------------- src/file/bmap.rs | 28 ++++---- src/file/functions.rs | 18 ++--- src/lib.rs | 148 ++++++++++++++---------------------------- 4 files changed, 105 insertions(+), 214 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index cece8d9a..954171f6 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -2,12 +2,25 @@ use crate::file::{ compression::Compression, functions::{FileCopyFromParams, FileCopyToParams, Partition}, }; -use clap::Parser; +use clap::{Args, Parser}; use std::path::PathBuf; use url::Url; const COPYRIGHT: &str = "Copyright © 2021 by conplement AG"; +#[derive(Args, Debug, Default)] +pub struct ImageOutput { + /// optional: create a bmap of the output image + #[arg(short = 'b', long = "output-bmap")] + pub output_bmap: bool, + /// optional: bmap of the input image; default: .bmap or .bmap next to the image + #[arg(long = "input-bmap", requires = "output_bmap")] + pub input_bmap: Option, + /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') + #[arg(short = 'p', long = "pack-image", value_enum)] + pub compress_image: Option, +} + // ToDo: command completion #[derive(Parser, Debug)] #[command(after_help = COPYRIGHT)] @@ -34,15 +47,8 @@ pub enum Docker { /// destination path of the docker image in the firmware image (must end in ".tar.gz") #[clap(short = 'e', long = "dest")] dest: PathBuf, - /// optional: create a bmap of the output image - #[arg(short = 'b', long = "output-bmap")] - output_bmap: bool, - /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap", requires = "output_bmap")] - input_bmap: Option, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, } @@ -58,15 +64,8 @@ pub enum File { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: create a bmap of the output image - #[arg(short = 'b', long = "output-bmap")] - output_bmap: bool, - /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap", requires = "output_bmap")] - input_bmap: Option, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// copy files from image CopyFromImage { @@ -94,15 +93,8 @@ pub enum IdentityConfig { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: create a bmap of the output image - #[arg(short = 'b', long = "output-bmap")] - output_bmap: bool, - /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap", requires = "output_bmap")] - input_bmap: Option, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// EXPERIMENTAL: set transparent gateway config.toml file and additional certificates and keys SetIotedgeGatewayConfig { @@ -121,15 +113,8 @@ pub enum IdentityConfig { /// path to device identity certificate key file #[arg(short = 'k', long = "device_identity_key")] device_identity_key: PathBuf, - /// optional: create a bmap of the output image - #[arg(short = 'b', long = "output-bmap")] - output_bmap: bool, - /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap", requires = "output_bmap")] - input_bmap: Option, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// EXPERIMENTAL: set leaf device config.toml file and additional certificate SetIotLeafSasConfig { @@ -142,15 +127,8 @@ pub enum IdentityConfig { /// path to root ca certificate file #[arg(short = 'r', long = "root_ca")] root_ca: PathBuf, - /// optional: create a bmap of the output image - #[arg(short = 'b', long = "output-bmap")] - output_bmap: bool, - /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap", requires = "output_bmap")] - input_bmap: Option, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// set certificates in order to support X.509 based DPS provisioning and certificate renewal via EST SetDeviceCertificate { @@ -169,15 +147,8 @@ pub enum IdentityConfig { /// period of validity in days #[arg(short = 'D', long = "days")] days: u32, - /// optional: create a bmap of the output image - #[arg(short = 'b', long = "output-bmap")] - output_bmap: bool, - /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap", requires = "output_bmap")] - input_bmap: Option, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// set certificates in order to support X.509 based DPS provisioning WITHOUT certificate renewal via EST SetDeviceCertificateNoEst { @@ -190,15 +161,8 @@ pub enum IdentityConfig { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: create a bmap of the output image - #[arg(short = 'b', long = "output-bmap")] - output_bmap: bool, - /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap", requires = "output_bmap")] - input_bmap: Option, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// generate and set bootstrap certificate for edge ca issuance/renewal. SetEdgeCaCertificate { @@ -217,15 +181,8 @@ pub enum IdentityConfig { /// period of validity in days #[arg(short = 'D', long = "days")] days: u32, - /// optional: create a bmap of the output image - #[arg(short = 'b', long = "output-bmap")] - output_bmap: bool, - /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap", requires = "output_bmap")] - input_bmap: Option, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, } @@ -241,15 +198,8 @@ pub enum IotHubDeviceUpdate { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: create a bmap of the output image - #[arg(short = 'b', long = "output-bmap")] - output_bmap: bool, - /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap", requires = "output_bmap")] - input_bmap: Option, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// import update to azure iot-hub ImportUpdate { @@ -363,15 +313,8 @@ pub enum SshConfig { /// path to public key of the ssh root ca #[arg(short = 'r', long = "root_ca")] root_ca: PathBuf, - /// optional: create a bmap of the output image - #[arg(short = 'b', long = "output-bmap")] - output_bmap: bool, - /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap", requires = "output_bmap")] - input_bmap: Option, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// set ssh connection parameters (see diff --git a/src/file/bmap.rs b/src/file/bmap.rs index b6ea230c..fbcdd295 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -1,4 +1,4 @@ -use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, sparse}; +use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, compression::Compression, sparse}; use anyhow::{Context, Result}; use sha2::{Digest, Sha256}; use std::{ @@ -8,8 +8,7 @@ use std::{ path::{Path, PathBuf}, }; -pub const BMAP_EXTENSION: &str = "bmap"; -const COMPRESSION_EXTENSIONS: [&str; 5] = ["xz", "gz", "gzip", "bz2", "bzip2"]; +const BMAP_EXTENSION: &str = "bmap"; const SHA256_CHECKSUM_TYPE: &str = "sha256"; const SHA256_HEX_LEN: usize = 64; const BMAP_VERSION: &str = "2.0"; @@ -103,23 +102,20 @@ pub fn find_input_bmap(image: &Path, explicit: Option<&Path>) -> Result PathBuf { + let mut path = image.as_os_str().to_owned(); + path.push(format!(".{BMAP_EXTENSION}")); + PathBuf::from(path) +} + /// Fails if a range of `image_file` does not match its checksum in `bmap`, so the bmap of /// another image with the same size is not used. pub fn verify_checksums(image_file: &Path, bmap: &Bmap) -> Result<()> { @@ -201,8 +197,6 @@ pub fn write_bmap(image_file: &Path, input: Option<&Bmap>, bmap_file: &Path) -> fs::write(bmap_file, xml).context(format!("write_bmap: cannot write {}", bmap_file.display())) } -/// The blocks of the created bmap: the ranges of `input`, or the whole image without one, and -/// every block that the image file stores as data. fn mapped_ranges( image: &fs::File, image_len: u64, diff --git a/src/file/functions.rs b/src/file/functions.rs index 3c9429f9..e40f4147 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -1,14 +1,16 @@ use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, partition::SECTOR_SIZE, sparse}; use anyhow::{Context, Result}; use log::{debug, warn}; -use std::collections::HashMap; -use std::fmt::{self, Display}; -use std::fs; -use std::ops::Range; -use std::os::unix::fs::FileExt; -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::str::FromStr; +use std::{ + collections::HashMap, + fmt::{self, Display}, + fs, + ops::Range, + os::unix::fs::FileExt, + path::{Path, PathBuf}, + process::Command, + str::FromStr, +}; use stdext::function_name; use uuid::Uuid; diff --git a/src/lib.rs b/src/lib.rs index 257b8548..2fd57ead 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,11 +17,12 @@ use cli::{ SetConfig, SetDeviceCertificate, SetDeviceCertificateNoEst, SetEdgeCaCertificate, SetIotLeafSasConfig, SetIotedgeGatewayConfig, }, + ImageOutput, IotHubDeviceUpdate::{self, SetDeviceConfig as IotHubDeviceUpdateSet}, SshConfig::{SetCertificate, SetConnection}, }; use file::{ - bmap::{BMAP_EXTENSION, Bmap}, + bmap::{self, Bmap}, compression::Compression, functions::FileCopyToParams, }; @@ -55,16 +56,16 @@ impl Drop for TempDirGuard { } } -fn run_image_command( - image_file: PathBuf, - output_bmap: bool, - input_bmap: Option, - target_compression: Option, - command: F, -) -> Result<()> +fn run_image_command(image_file: PathBuf, output: ImageOutput, command: F) -> Result<()> where F: FnOnce(&PathBuf) -> Result<()>, { + let ImageOutput { + output_bmap, + input_bmap, + compress_image: target_compression, + } = output; + anyhow::ensure!( image_file.try_exists().is_ok_and(|exists| exists), "run_image_command: image doesn't exist {}", @@ -72,7 +73,7 @@ where ); let input_bmap = if output_bmap { - file::bmap::find_input_bmap(&image_file, input_bmap.as_deref())? + bmap::find_input_bmap(&image_file, input_bmap.as_deref())? .map(|path| Bmap::read(&path).map(|bmap| (path, bmap))) .transpose()? } else { @@ -108,25 +109,20 @@ where } if let Some((path, bmap)) = &input_bmap { - file::bmap::verify_checksums(&tmp_image_file, bmap) + bmap::verify_checksums(&tmp_image_file, bmap) .context(format!("cannot use input bmap {}", path.display()))?; } // run command command(&tmp_image_file)?; - let tmp_bmap = PathBuf::from(format!( - "{}.{BMAP_EXTENSION}", - tmp_image_file - .to_str() - .context("cannot get image file path")? - )); + let tmp_bmap = bmap::bmap_path(&tmp_image_file); if output_bmap { match &input_bmap { Some((path, _)) => info!("the bmap maps the ranges of {}", path.display()), None => info!("no input bmap found, so the bmap maps the whole image"), } - file::bmap::write_bmap( + bmap::write_bmap( &tmp_image_file, input_bmap.as_ref().map(|(_, bmap)| bmap), &tmp_bmap, @@ -228,10 +224,8 @@ pub fn run() -> Result<()> { image, partition, dest, - output_bmap, - input_bmap, - compress_image, - }) => run_image_command(image, output_bmap, input_bmap, compress_image, |img| { + output, + }) => run_image_command(image, output, |img| { anyhow::ensure!( dest.to_string_lossy().ends_with(".tar.gz"), format!( @@ -269,10 +263,8 @@ pub fn run() -> Result<()> { config, image, payload, - output_bmap, - input_bmap, - compress_image, - }) => run_image_command(image, output_bmap, input_bmap, compress_image, |img| { + output, + }) => run_image_command(image, output, |img| { file::set_identity_config(&config, img, payload.as_deref()) })?, Command::Identity(SetDeviceCertificate { @@ -281,9 +273,7 @@ pub fn run() -> Result<()> { image, device_id, days, - output_bmap, - input_bmap, - compress_image, + output, }) => { let cert_info = create_image_cert( &image, @@ -298,7 +288,7 @@ pub fn run() -> Result<()> { ) .context("set_device_certificate: could not create certificate")?; - run_image_command(image, output_bmap, input_bmap, compress_image, |img| { + run_image_command(image, output, |img| { file::set_device_cert( Some(&intermediate_full_chain_cert), &cert_info.cert_path, @@ -313,9 +303,7 @@ pub fn run() -> Result<()> { image, device_id, days, - output_bmap, - input_bmap, - compress_image, + output, }) => { let cert_info = create_image_cert( &image, @@ -330,7 +318,7 @@ pub fn run() -> Result<()> { ) .context("set_edge_ca_certificate: could not create certificate")?; - run_image_command(image, output_bmap, input_bmap, compress_image, |img| { + run_image_command(image, output, |img| { file::set_edge_ca_cert( Some(&intermediate_full_chain_cert), &cert_info.cert_path, @@ -343,10 +331,8 @@ pub fn run() -> Result<()> { device_cert: device_cert_pem, device_key: device_key_pem, image, - output_bmap, - input_bmap, - compress_image, - }) => run_image_command(image, output_bmap, input_bmap, compress_image, |img| { + output, + }) => run_image_command(image, output, |img| { file::set_device_cert(None, &device_cert_pem, &device_key_pem, img) })?, Command::Identity(SetIotedgeGatewayConfig { @@ -355,66 +341,38 @@ pub fn run() -> Result<()> { root_ca, device_identity, device_identity_key, - output_bmap, - input_bmap, - compress_image, - }) => run_image_command( - image, - output_bmap, - input_bmap, - compress_image, - |img: &PathBuf| { - file::set_iotedge_gateway_config( - &config, - img, - &root_ca, - &device_identity, - &device_identity_key, - ) - }, - )?, + output, + }) => run_image_command(image, output, |img: &PathBuf| { + file::set_iotedge_gateway_config( + &config, + img, + &root_ca, + &device_identity, + &device_identity_key, + ) + })?, Command::Identity(SetIotLeafSasConfig { config, image, root_ca, - output_bmap, - input_bmap, - compress_image, - }) => run_image_command( - image, - output_bmap, - input_bmap, - compress_image, - |img: &PathBuf| file::set_iot_leaf_sas_config(&config, img, &root_ca), - )?, + output, + }) => run_image_command(image, output, |img: &PathBuf| { + file::set_iot_leaf_sas_config(&config, img, &root_ca) + })?, Command::Ssh(SetCertificate { image, root_ca, - output_bmap, - input_bmap, - compress_image, - }) => run_image_command( - image, - output_bmap, - input_bmap, - compress_image, - |img: &PathBuf| file::set_ssh_tunnel_certificate(img, &root_ca), - )?, + output, + }) => run_image_command(image, output, |img: &PathBuf| { + file::set_ssh_tunnel_certificate(img, &root_ca) + })?, Command::IotHubDeviceUpdate(IotHubDeviceUpdateSet { iot_hub_device_update_config, image, - output_bmap, - input_bmap, - compress_image, - }) => run_image_command( - image, - output_bmap, - input_bmap, - compress_image, - |img: &PathBuf| { - file::set_iot_hub_device_update_config(&iot_hub_device_update_config, img) - }, - )?, + output, + }) => run_image_command(image, output, |img: &PathBuf| { + file::set_iot_hub_device_update_config(&iot_hub_device_update_config, img) + })?, Command::IotHubDeviceUpdate(IotHubDeviceUpdate::ImportUpdate { import_manifest: import_manifest_path, storage_container_name, @@ -527,20 +485,14 @@ pub fn run() -> Result<()> { Command::File(CopyToImage { file_copy_params, image, - output_bmap, - input_bmap, - compress_image, - }) => run_image_command( - image, - output_bmap, - input_bmap, - compress_image, - |img: &PathBuf| file::copy_to_image(&file_copy_params, img), - )?, + output, + }) => run_image_command(image, output, |img: &PathBuf| { + file::copy_to_image(&file_copy_params, img) + })?, Command::File(CopyFromImage { file_copy_params, image, - }) => run_image_command(image, false, None, None, |img: &PathBuf| { + }) => run_image_command(image, ImageOutput::default(), |img: &PathBuf| { file::copy_from_image(&file_copy_params, img) })?, } From 92ce896a034aa14f3e53486c8f41f7758f5cec27 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:11:29 +0100 Subject: [PATCH 26/75] test(file): make the write back failure test work as root Also check that --input-bmap requires -b. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- tests/integration_tests.rs | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index b76a8648..4cf7ea55 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1153,12 +1153,12 @@ fn check_file_copy_without_output_bmap_keeps_image_sparse() { #[test] fn check_no_output_bmap_when_image_write_back_fails() { let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); - let image_path = tr.to_pathbuf("testfiles/image.wic"); - let bmap_path = PathBuf::from(format!("{}.bmap", image_path.to_string_lossy())); + let image_path = tr.to_pathbuf("testfiles/image.wic.xz"); + let written_image_path = image_path.with_extension(""); + let bmap_path = image_path.with_extension("bmap"); let in_file = tr.to_pathbuf("testfiles/boot.scr"); - let mut permissions = image_path.metadata().expect("image metadata").permissions(); - permissions.set_readonly(true); - fs::set_permissions(&image_path, permissions).expect("make image read-only"); + // a directory in place of the written image makes the write back fail, also for root + create_dir_all(&written_image_path).expect("create directory"); Command::cargo_bin("omnect-cli") .expect("omnect-cli binary") @@ -1177,6 +1177,29 @@ fn check_no_output_bmap_when_image_write_back_fails() { ); } +#[test] +fn check_input_bmap_requires_output_bmap() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let bmap_path = tr.to_pathbuf("testfiles/image.wic.bmap"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + let image_hash = Testrunner::file_hash(&image_path); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .arg("--input-bmap") + .arg(&bmap_path) + .assert() + .failure(); + assert_eq!(Testrunner::file_hash(&image_path), image_hash); +} + #[test] fn check_input_bmap_of_another_image_with_the_same_size() { let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); @@ -1292,8 +1315,7 @@ fn check_bmap_generation_wic_xz() { let image_path_wic_xz = tr.to_pathbuf("testfiles/image.wic.xz"); let image_path_wic = image_path_wic_xz.with_extension(""); let image_path_wic_copy = image_path_wic_xz.with_extension("copy"); - // a Yocto bmap of an image with the same layout; omnect-cli replaces it with the one it - // creates + // the Yocto bmap of the image; omnect-cli replaces it with the one it creates let image_path_bmap = tr.to_pathbuf("testfiles/image.wic.bmap"); let input_bmap = read_bmap(&image_path_bmap); let in_file = tr.to_pathbuf("testfiles/boot.scr"); From 04bb5f1bb216590d8bab6de7087d21fd46e849b7 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:20:23 +0100 Subject: [PATCH 27/75] fix(file): name the output bmap after the output image The bmap was named after the uncompressed image. When the output image is another file than the input image, e.g. with -p, it replaced the bmap of the input image, which stays unchanged. bmaptool finds .bmap first, also for a compressed image. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 2 +- src/file/bmap.rs | 6 +++--- src/lib.rs | 5 +---- tests/integration_tests.rs | 40 +++++++++++++++++++++++++++++++++++++- 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index eda55c06..9f30eb97 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ omnect-cli file copy-to-image --help ## bmap files -Commands that change an image create a bmap file with `-b` (`--output-bmap`). It is written next to the image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic.xz`, and replaces a file with this name. +Commands that change an image create a bmap file with `-b` (`--output-bmap`). It is written next to the output image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic` and `my-image.wic.xz.bmap` for `my-image.wic.xz`, and replaces a file with this name. `bmaptool` finds it there without `--bmap`. The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image and sha256 range checksums that match it, otherwise the command fails. A command without `-b` changes the image but not the bmap next to it, so when you run several commands on one image, use `-b` on each of them, or pass the matching bmap with `--input-bmap`. The created bmap maps every range of the input bmap, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). It also maps every block that the image file stores as data, which includes the blocks `omnect-cli` changed. The image file itself stays sparse. diff --git a/src/file/bmap.rs b/src/file/bmap.rs index fbcdd295..793a9fd5 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -140,9 +140,9 @@ pub fn verify_checksums(image_file: &Path, bmap: &Bmap) -> Result<()> { let actual = range_sha256(&image, bytes, &mut buf)?; anyhow::ensure!( actual.eq_ignore_ascii_case(expected.trim()), - "verify_checksums: range {range:?} of the image does not match the bmap. It is the \ - bmap of another image, or of this image before a command without -b changed it. \ - Pass the right bmap with --input-bmap." + "verify_checksums: range {range:?} of the image does not match the bmap, so it is the \ + bmap of another image or of an older version of this image. Pass the right bmap \ + with --input-bmap." ); } diff --git a/src/lib.rs b/src/lib.rs index 2fd57ead..263a4b4b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -150,10 +150,7 @@ where // copy the bmap last, so it never describes an image that was not written back if output_bmap { - let target_bmap = image_file - .parent() - .context("cannot get parent dir of image path")? - .join(tmp_bmap.file_name().context("cannot get bmap file name")?); + let target_bmap = bmap::bmap_path(&dest_image_file); std::fs::copy(&tmp_bmap, &target_bmap).context(format!( "error: std::fs::copy({:?}, {:?})", tmp_bmap, target_bmap diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 4cf7ea55..0f146f11 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1255,7 +1255,7 @@ fn check_chained_commands_with_input_bmap() { copy_to_image(&image_xz_path, false).success(); copy_to_image(&image_path, true) .failure() - .stderr(predicate::str::contains("without -b changed it")); + .stderr(predicate::str::contains("older version of this image")); copy_to_image(&image_xz_path, true).success(); copy_to_image(&image_path, true).success(); @@ -1370,6 +1370,44 @@ fn check_bmap_generation_wic_xz_without_input_bmap() { assert!(maps_whole_image(&read_bmap(&image_path_bmap))); } +#[test] +fn check_bmap_of_compressed_output_keeps_input_bmap() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let input_bmap_path = tr.to_pathbuf("testfiles/image.wic.bmap"); + let input_bmap_hash = Testrunner::file_hash(&input_bmap_path); + let image_path_xz = image_path.with_extension("wic.xz"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .arg("-b") + .arg("-p") + .arg("xz") + .assert() + .success(); + + assert_eq!( + Testrunner::file_hash(&input_bmap_path), + input_bmap_hash, + "bmap of the unchanged input image was replaced" + ); + // bmaptool finds .bmap without --bmap and checks the image against it + Command::new("bmaptool") + .arg("copy") + .arg(&image_path_xz) + .arg(image_path.with_extension("copy")) + .assert() + .success() + .stderr(predicate::str::contains("image.wic.xz.bmap")); +} + #[test] fn check_image_compression() { let tr = Testrunner::new(function_name!().split("::").last().unwrap()); From beafb45cd5329b9b2de891fca5fc1279f73c6e72 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:20:33 +0100 Subject: [PATCH 28/75] perf(file): do not sync the temp image after a partition write SEEK_DATA also sees data that is not flushed yet, and the partition files that e2cp and mcopy write were never synced either. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/functions.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/file/functions.rs b/src/file/functions.rs index e40f4147..09c4312f 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -488,9 +488,7 @@ fn write_partition( )?; } - image - .sync_all() - .context("write_partition: cannot sync image") + Ok(()) } fn write_changed_blocks( From d7ba2da5e32623f92d41dccb5b64a0a1a027214d Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:21:10 +0100 Subject: [PATCH 29/75] test(file): cover a failed compress trailer and a partition past the image end Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/compression.rs | 19 +++++++++++++++++++ src/file/functions.rs | 20 ++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/src/file/compression.rs b/src/file/compression.rs index f34b73d4..4c79da2f 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -397,4 +397,23 @@ mod tests { "source was changed" ); } + + #[test] + fn compress_fails_when_the_trailer_cannot_be_written() { + let dir = tempfile::tempdir().expect("create temp dir"); + // an empty source makes `finish` the first write to the destination + let source = dir.path().join("empty.wic"); + fs::write(&source, []).expect("write source"); + for compression in all_compressions() { + let mut source = fs::File::open(&source).expect("open source"); + let mut full = fs::OpenOptions::new() + .write(true) + .open("/dev/full") + .expect("open /dev/full"); + assert!( + compression.compress(&mut source, &mut full).is_err(), + "{compression:?}: write error was ignored" + ); + } + } } diff --git a/src/file/functions.rs b/src/file/functions.rs index 09c4312f..c1255e33 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -561,6 +561,26 @@ mod tests { } } + #[test] + fn read_partition_fails_for_partition_past_the_image_end() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + let partition = dir.path().join("partition.img"); + fs::File::create(&image) + .expect("create image") + .set_len((PART_START + PART_SECTORS - 1) * SECTOR_SIZE) + .expect("set image size"); + + assert!( + read_partition( + image.to_str().expect("image path"), + partition.to_str().expect("partition path"), + &partition_info(), + ) + .is_err() + ); + } + #[test] fn write_partition_writes_block_changed_to_zeros() { let dir = tempfile::tempdir().expect("create temp dir"); From 1ce9e1a31cce8066a4ba77b17f9c5233b09e410f Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:15:53 +0100 Subject: [PATCH 30/75] refactor(file): share the sparse copy into a new file Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/functions.rs | 15 ++++----------- src/file/sparse.rs | 27 ++++++++++++++++++--------- src/lib.rs | 5 +---- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/src/file/functions.rs b/src/file/functions.rs index c1255e33..9c9ffecc 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -419,19 +419,12 @@ fn read_partition( let image = fs::File::open(image_file).context(format!("read_partition: cannot open {image_file}"))?; - let partition = fs::File::create(partition_file) - .context(format!("read_partition: cannot create {partition_file}"))?; - let len = partition_info.count * SECTOR_SIZE; - partition - .set_len(len) - .context("read_partition: cannot set partition file size")?; - - sparse::copy_range( + + sparse::copy_to_new_file( &image, partition_info.start * SECTOR_SIZE, - &partition, - 0, - len, + Path::new(partition_file), + partition_info.count * SECTOR_SIZE, ) } diff --git a/src/file/sparse.rs b/src/file/sparse.rs index 4193ddd6..3f5a4005 100644 --- a/src/file/sparse.rs +++ b/src/file/sparse.rs @@ -7,6 +7,7 @@ use std::{ fs::File, io::{self, Read, Seek}, ops::Range, + path::Path, }; /// Returns the byte ranges of `file` between `start` and `end` that hold data. @@ -81,6 +82,17 @@ pub fn copy_range( Ok(()) } +/// Creates `dst` with `len` bytes and copies `len` bytes of `src` at `src_offset` into it. +/// Holes of `src` stay holes in `dst`. +pub fn copy_to_new_file(src: &File, src_offset: u64, dst: &Path, len: u64) -> Result<()> { + let dst = + File::create(dst).context(format!("copy_to_new_file: cannot create {}", dst.display()))?; + dst.set_len(len) + .context("copy_to_new_file: cannot set destination size")?; + + copy_range(src, src_offset, &dst, 0, len) +} + #[cfg(test)] mod tests { use crate::file::sparse::*; @@ -135,20 +147,17 @@ mod tests { } #[test] - fn copy_range_keeps_holes() { + fn copy_to_new_file_keeps_holes() { let dir = tempfile::tempdir().expect("create temp dir"); let src = sparse_file(dir.path()); - let dst = File::options() - .read(true) - .write(true) - .create_new(true) - .open(dir.path().join("dst")) - .expect("create file"); + let dst_path = dir.path().join("dst"); let len = (FILE_BLOCKS - DST_OFFSET_BLOCKS) * BLOCK_SIZE; - dst.set_len(len).expect("set size"); - copy_range(&src, DST_OFFSET_BLOCKS * BLOCK_SIZE, &dst, 0, len).expect("copy range"); + copy_to_new_file(&src, DST_OFFSET_BLOCKS * BLOCK_SIZE, &dst_path, len) + .expect("copy to new file"); + let dst = File::open(&dst_path).expect("open file"); + assert_eq!(dst.metadata().expect("metadata").len(), len); let data_start = (DATA_BLOCK - DST_OFFSET_BLOCKS) * BLOCK_SIZE; assert_eq!( data_ranges(&dst, 0, len).expect("data ranges"), diff --git a/src/lib.rs b/src/lib.rs index 263a4b4b..a6709365 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -167,11 +167,8 @@ fn copy_image(from: &Path, to: &Path) -> Result<()> { .metadata() .context("copy_image: cannot get source size")? .len(); - let dst = fs::File::create(to).context("copy_image: cannot create destination")?; - dst.set_len(len) - .context("copy_image: cannot set destination size")?; - file::sparse::copy_range(&src, 0, &dst, 0, len) + file::sparse::copy_to_new_file(&src, 0, to, len) } struct CertInfo { From c18fc17fb6c8a7e51790fb841733de9bb4961e22 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:16:59 +0100 Subject: [PATCH 31/75] fix(file): do not write the image back after copy-from-image copy-from-image only reads the image. Writing it back truncated the source first, so an error during the copy lost the source, and for a compressed image it left a decompressed copy next to it. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/cli.rs | 2 +- src/lib.rs | 98 ++++++++++++++++++++++++++------------ tests/integration_tests.rs | 23 +++++++++ 3 files changed, 92 insertions(+), 31 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 954171f6..25fb8505 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -8,7 +8,7 @@ use url::Url; const COPYRIGHT: &str = "Copyright © 2021 by conplement AG"; -#[derive(Args, Debug, Default)] +#[derive(Args, Debug)] pub struct ImageOutput { /// optional: create a bmap of the output image #[arg(short = 'b', long = "output-bmap")] diff --git a/src/lib.rs b/src/lib.rs index a6709365..18b66e7c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -36,8 +36,20 @@ use uuid::Uuid; use crate::file::compression; +/// A new directory `/tmp/{uuid}`, removed on drop. struct TempDirGuard(PathBuf); +impl TempDirGuard { + fn new() -> Result { + let tmp_dir = PathBuf::from(format!("/tmp/{}", Uuid::new_v4())); + fs::create_dir_all(&tmp_dir).context(format!( + "TempDirGuard::new: cannot create {}", + tmp_dir.display() + ))?; + Ok(Self(tmp_dir)) + } +} + impl Drop for TempDirGuard { fn drop(&mut self) { let Ok(rt) = tokio::runtime::Builder::new_current_thread() @@ -66,11 +78,7 @@ where compress_image: target_compression, } = output; - anyhow::ensure!( - image_file.try_exists().is_ok_and(|exists| exists), - "run_image_command: image doesn't exist {}", - image_file.to_str().context("cannot get image file path")? - ); + ensure_image_exists(&image_file)?; let input_bmap = if output_bmap { bmap::find_input_bmap(&image_file, input_bmap.as_deref())? @@ -81,32 +89,12 @@ where }; let mut dest_image_file = image_file.clone(); - // create /tmp/{uuid}/ and copy image into - let tmp_dir = PathBuf::from(format!("/tmp/{}", Uuid::new_v4())); - fs::create_dir_all(tmp_dir.clone()).context(format!( - "run_image_command: couldn't create destination path {}", - tmp_dir.to_str().context("cannot get tmp dir name")? - ))?; - - let _guard = TempDirGuard(tmp_dir.clone()); - - let mut tmp_image_file = tmp_dir.join( - image_file - .file_name() - .context("cannot get image file name")?, - ); - - // if applicable decompress image to *.wic - if let Some(source_compression) = Compression::from_file(&image_file)? { - std::fs::copy(&image_file, &tmp_image_file)?; - tmp_image_file = compression::decompress(&tmp_image_file, &source_compression)?; + let tmp_dir = TempDirGuard::new()?; + let source_compression = Compression::from_file(&image_file)?; + if source_compression.is_some() { dest_image_file.set_extension(""); - } else { - copy_image(&image_file, &tmp_image_file).context(format!( - "error: copy_image({:?}, {:?})", - image_file, tmp_image_file - ))?; } + let mut tmp_image_file = temp_image(&image_file, source_compression.as_ref(), &tmp_dir.0)?; if let Some((path, bmap)) = &input_bmap { bmap::verify_checksums(&tmp_image_file, bmap) @@ -160,6 +148,56 @@ where Ok(()) } +/// Runs `command` on a temp copy of the image and writes nothing back. +fn run_read_only_image_command(image_file: PathBuf, command: F) -> Result<()> +where + F: FnOnce(&PathBuf) -> Result<()>, +{ + ensure_image_exists(&image_file)?; + + let tmp_dir = TempDirGuard::new()?; + let source_compression = Compression::from_file(&image_file)?; + let tmp_image_file = temp_image(&image_file, source_compression.as_ref(), &tmp_dir.0)?; + + command(&tmp_image_file) +} + +fn ensure_image_exists(image_file: &Path) -> Result<()> { + anyhow::ensure!( + image_file.try_exists().is_ok_and(|exists| exists), + "image doesn't exist {}", + image_file.display() + ); + Ok(()) +} + +/// Copies `image_file` into `tmp_dir`, decompressed if it is compressed, and returns the copy. +fn temp_image( + image_file: &Path, + source_compression: Option<&Compression>, + tmp_dir: &Path, +) -> Result { + let tmp_image_file = tmp_dir.join( + image_file + .file_name() + .context("cannot get image file name")?, + ); + + match source_compression { + Some(c) => { + std::fs::copy(image_file, &tmp_image_file)?; + compression::decompress(&tmp_image_file, c) + } + None => { + copy_image(image_file, &tmp_image_file).context(format!( + "error: copy_image({:?}, {:?})", + image_file, tmp_image_file + ))?; + Ok(tmp_image_file) + } + } +} + /// Copies `from` to `to` and keeps the holes of a sparse `from`. fn copy_image(from: &Path, to: &Path) -> Result<()> { let src = fs::File::open(from).context("copy_image: cannot open source")?; @@ -486,7 +524,7 @@ pub fn run() -> Result<()> { Command::File(CopyFromImage { file_copy_params, image, - }) => run_image_command(image, ImageOutput::default(), |img: &PathBuf| { + }) => run_read_only_image_command(image, |img: &PathBuf| { file::copy_from_image(&file_copy_params, img) })?, } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 0f146f11..01e2b221 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1489,6 +1489,29 @@ fn check_image_decompression() { assert_eq!(image_path_wic_xz_hash1, image_path_wic_xz_hash2); } +#[test] +fn check_copy_from_image_writes_no_image() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path_wic_xz = tr.to_pathbuf("testfiles/image.wic.xz"); + let image_hash = Testrunner::file_hash(&image_path_wic_xz); + let out_file = tr.pathbuf().join("hosts"); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-from-image") + .arg("-f") + .arg(format!("rootA:/etc/hosts,{}", out_file.to_string_lossy())) + .arg("-i") + .arg(&image_path_wic_xz) + .assert() + .success(); + + assert!(out_file.exists()); + assert!(!image_path_wic_xz.with_extension("").exists()); + assert_eq!(Testrunner::file_hash(&image_path_wic_xz), image_hash); +} + #[tokio::test] async fn check_ssh_tunnel_setup() { let tr = Testrunner::new("check_ssh_tunnel_setup"); From c51066fd64a311aee1f04954965ceb808055b9d8 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:20:45 +0100 Subject: [PATCH 32/75] feat(file)!: fail on output name conflicts, add --output-image and --no-input-bmap omnect-cli derives the output image and bmap names from the input image. A command now fails instead of overwriting an existing file that is not the input image, replacing the input bmap of an image that stays (e.g. x.wic.bmap for x.wic.xz), or leaving a bmap next to the output image that no longer matches it. --output-image chooses the output path, and --no-input-bmap skips a bmap next to the image that does not match. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 10 +- src/cli.rs | 10 ++ src/file/bmap.rs | 5 +- src/file/compression.rs | 13 ++- src/file/mod.rs | 1 + src/file/output.rs | 182 +++++++++++++++++++++++++++++++++++++ src/lib.rs | 37 +++++--- tests/integration_tests.rs | 145 +++++++++++++++++++++++------ 8 files changed, 351 insertions(+), 52 deletions(-) create mode 100644 src/file/output.rs diff --git a/README.md b/README.md index 9f30eb97..c7e714c3 100644 --- a/README.md +++ b/README.md @@ -183,11 +183,17 @@ omnect-cli file copy-to-image --help - File permissions: inject `systemd-tmpfiles.d` - Wifi: inject `wpa_supplicant-wlan0.conf` +## Output image + +A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression, e.g. `my-image.wic.xz` for `my-image.wic`. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. + +The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. `copy-from-image` only reads the image and writes nothing. + ## bmap files -Commands that change an image create a bmap file with `-b` (`--output-bmap`). It is written next to the output image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic` and `my-image.wic.xz.bmap` for `my-image.wic.xz`, and replaces a file with this name. `bmaptool` finds it there without `--bmap`. +Commands that change an image create a bmap file with `-b` (`--output-bmap`). It is written next to the output image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic` and `my-image.wic.xz.bmap` for `my-image.wic.xz`, and replaces a file with this name. `bmaptool` finds it there without `--bmap`. The command fails if this file is the input bmap of an image that stays, e.g. `my-image.wic.bmap` when `-i my-image.wic.xz` writes `my-image.wic`. Choose another output with `-o` then. -The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image and sha256 range checksums that match it, otherwise the command fails. A command without `-b` changes the image but not the bmap next to it, so when you run several commands on one image, use `-b` on each of them, or pass the matching bmap with `--input-bmap`. The created bmap maps every range of the input bmap, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). It also maps every block that the image file stores as data, which includes the blocks `omnect-cli` changed. The image file itself stays sparse. +The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image and sha256 range checksums that match it, otherwise the command fails. A command without `-b` fails if a bmap is next to its output image, because the bmap would no longer match the image. So when you run several commands on one image, use `-b` on each of them. `--no-input-bmap` skips a bmap next to the image that does not match, and the created bmap then maps the whole image. The created bmap maps every range of the input bmap, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). It also maps every block that the image file stores as data, which includes the blocks `omnect-cli` changed. The image file itself stays sparse. Without an input bmap, the created bmap maps the whole image, and a flash writes every block. diff --git a/src/cli.rs b/src/cli.rs index 25fb8505..7b824c6f 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -16,6 +16,16 @@ pub struct ImageOutput { /// optional: bmap of the input image; default: .bmap or .bmap next to the image #[arg(long = "input-bmap", requires = "output_bmap")] pub input_bmap: Option, + /// optional: do not use a bmap next to the image, so the bmap maps the whole image + #[arg( + long = "no-input-bmap", + requires = "output_bmap", + conflicts_with = "input_bmap" + )] + pub no_input_bmap: bool, + /// optional: write the output image to this path; default: the input image, without the compression extension, plus the --pack-image extension + #[arg(short = 'o', long = "output-image")] + pub output_image: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] pub compress_image: Option, diff --git a/src/file/bmap.rs b/src/file/bmap.rs index 793a9fd5..96b13b5d 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -141,8 +141,7 @@ pub fn verify_checksums(image_file: &Path, bmap: &Bmap) -> Result<()> { anyhow::ensure!( actual.eq_ignore_ascii_case(expected.trim()), "verify_checksums: range {range:?} of the image does not match the bmap, so it is the \ - bmap of another image or of an older version of this image. Pass the right bmap \ - with --input-bmap." + bmap of another image or of an older version of this image" ); } @@ -509,7 +508,7 @@ mod tests { .expect("change data block"); let err = verify_checksums(&image, &bmap).expect_err("changed image accepted"); - assert!(err.to_string().contains("--input-bmap"), "{err}"); + assert!(err.to_string().contains("does not match the bmap"), "{err}"); } #[test] diff --git a/src/file/compression.rs b/src/file/compression.rs index 4c79da2f..85e8fc99 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -122,6 +122,13 @@ impl Compression { } } + /// `path` with the extension omnect-cli gives a compressed image appended. + pub fn compressed_path(&self, path: &Path) -> PathBuf { + let mut compressed = path.as_os_str().to_owned(); + compressed.push(format!(".{}", self.extension())); + PathBuf::from(compressed) + } + pub fn has_extension(path: &Path) -> bool { path.extension() .and_then(|e| e.to_str()) @@ -228,11 +235,7 @@ pub fn decompress(image_file_name: &PathBuf, compression: &Compression) -> Resul } pub fn compress(image_file_name: &PathBuf, compression: &Compression) -> Result { - let new_image_file = PathBuf::from(format!( - "{}.{}", - image_file_name.to_str().unwrap(), - compression.extension() - )); + let new_image_file = compression.compressed_path(image_file_name); let mut destination = File::create(&new_image_file)?; let mut source = File::open(image_file_name)?; debug!("compress {image_file_name:?} to {new_image_file:?}"); diff --git a/src/file/mod.rs b/src/file/mod.rs index f8999cef..92d9c677 100644 --- a/src/file/mod.rs +++ b/src/file/mod.rs @@ -1,6 +1,7 @@ pub mod bmap; pub mod compression; pub mod functions; +pub(crate) mod output; mod partition; pub(crate) mod sparse; diff --git a/src/file/output.rs b/src/file/output.rs new file mode 100644 index 00000000..4db32785 --- /dev/null +++ b/src/file/output.rs @@ -0,0 +1,182 @@ +use crate::file::{bmap, compression::Compression}; +use anyhow::{Context, Result}; +use std::{ + fs, + path::{Path, PathBuf}, +}; + +/// Returns `explicit`, else `image` without its last extension if it is compressed, with the +/// extension of `target` appended. +pub fn output_image( + image: &Path, + source_compressed: bool, + target: Option<&Compression>, + explicit: Option<&Path>, +) -> PathBuf { + if let Some(path) = explicit { + return path.to_path_buf(); + } + + let mut output = image.to_path_buf(); + if source_compressed { + output.set_extension(""); + } + match target { + Some(c) => c.compressed_path(&output), + None => output, + } +} + +/// Fails if writing `output_image` and its bmap would overwrite a file the user did not name, +/// replace the bmap of an image that stays, or leave a bmap that no longer matches the image. +pub fn check_conflicts( + input_image: &Path, + output_image: &Path, + explicit_output: bool, + output_bmap: bool, + input_bmap: Option<&Path>, +) -> Result<()> { + let in_place = same_file(input_image, output_image)?; + let target_bmap = bmap::bmap_path(output_image); + + anyhow::ensure!( + explicit_output || in_place || !output_image.exists(), + "{} exists; choose the output image with --output-image", + output_image.display() + ); + if output_bmap { + if let Some(input_bmap) = input_bmap { + anyhow::ensure!( + in_place || !same_file(input_bmap, &target_bmap)?, + "{} is the bmap of {} and would be replaced; choose the output image with \ + --output-image", + input_bmap.display(), + input_image.display() + ); + } + } else { + anyhow::ensure!( + !target_bmap.exists(), + "{} would no longer match {}; pass -b to update it, or remove it", + target_bmap.display(), + output_image.display() + ); + } + + Ok(()) +} + +fn same_file(a: &Path, b: &Path) -> Result { + Ok(resolve(a)? == resolve(b)?) +} + +/// The absolute path of `path` with symlinks resolved. A file that does not exist yet is +/// resolved through its directory. +fn resolve(path: &Path) -> Result { + if path.exists() { + return fs::canonicalize(path).context(format!("cannot resolve {}", path.display())); + } + let dir = match path.parent() { + Some(dir) if !dir.as_os_str().is_empty() => dir, + _ => Path::new("."), + }; + let name = path + .file_name() + .context(format!("{} has no file name", path.display()))?; + + Ok(fs::canonicalize(dir) + .context(format!("cannot resolve {}", dir.display()))? + .join(name)) +} + +#[cfg(test)] +mod tests { + use crate::file::output::*; + + const IMAGE: &str = "image.wic"; + const IMAGE_XZ: &str = "image.wic.xz"; + const OTHER_IMAGE: &str = "device.wic"; + const XZ_LEVEL: u32 = 9; + + fn touch(path: &Path) { + fs::write(path, []).expect("create file"); + } + + #[test] + fn output_image_follows_compression() { + let image = Path::new(IMAGE); + let image_xz = Path::new(IMAGE_XZ); + let xz = Compression::xz { + compression_level: XZ_LEVEL, + }; + + assert_eq!(output_image(image, false, None, None), image); + assert_eq!(output_image(image_xz, true, None, None), image); + assert_eq!(output_image(image, false, Some(&xz), None), image_xz); + assert_eq!(output_image(image_xz, true, Some(&xz), None), image_xz); + assert_eq!( + output_image(image_xz, true, Some(&xz), Some(Path::new(OTHER_IMAGE))), + Path::new(OTHER_IMAGE) + ); + } + + #[test] + fn check_conflicts_allows_in_place_with_bmap() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let image_bmap = bmap::bmap_path(&image); + touch(&image); + touch(&image_bmap); + + check_conflicts(&image, &image, false, true, Some(&image_bmap)).expect("no conflict"); + } + + #[test] + fn check_conflicts_rejects_existing_derived_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_xz = dir.path().join(IMAGE_XZ); + let image = dir.path().join(IMAGE); + touch(&image_xz); + touch(&image); + + assert!(check_conflicts(&image_xz, &image, false, false, None).is_err()); + check_conflicts(&image_xz, &image, true, false, None).expect("explicit output"); + } + + #[test] + fn check_conflicts_rejects_replacing_the_input_bmap() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_xz = dir.path().join(IMAGE_XZ); + let image = dir.path().join(IMAGE); + let image_bmap = bmap::bmap_path(&image); + touch(&image_xz); + touch(&image_bmap); + + assert!(check_conflicts(&image_xz, &image, false, true, Some(&image_bmap)).is_err()); + let other = dir.path().join(OTHER_IMAGE); + check_conflicts(&image_xz, &other, true, true, Some(&image_bmap)).expect("other output"); + } + + #[test] + fn check_conflicts_rejects_leaving_a_stale_bmap() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + touch(&image); + touch(&bmap::bmap_path(&image)); + + assert!(check_conflicts(&image, &image, false, false, None).is_err()); + } + + #[test] + fn same_file_resolves_relative_and_missing_paths() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + touch(&image); + let relative = dir.path().join(".").join(IMAGE); + let missing = dir.path().join(OTHER_IMAGE); + + assert!(same_file(&image, &relative).expect("same file")); + assert!(same_file(&missing, &missing).expect("same file")); + assert!(!same_file(&image, &missing).expect("same file")); + } +} diff --git a/src/lib.rs b/src/lib.rs index 18b66e7c..47ba4f31 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -75,30 +75,44 @@ where let ImageOutput { output_bmap, input_bmap, + no_input_bmap, + output_image, compress_image: target_compression, } = output; ensure_image_exists(&image_file)?; - let input_bmap = if output_bmap { + let input_bmap = if output_bmap && !no_input_bmap { bmap::find_input_bmap(&image_file, input_bmap.as_deref())? .map(|path| Bmap::read(&path).map(|bmap| (path, bmap))) .transpose()? } else { None }; - let mut dest_image_file = image_file.clone(); + let source_compression = Compression::from_file(&image_file)?; + let dest_image_file = file::output::output_image( + &image_file, + source_compression.is_some(), + target_compression.as_ref(), + output_image.as_deref(), + ); + file::output::check_conflicts( + &image_file, + &dest_image_file, + output_image.is_some(), + output_bmap, + input_bmap.as_ref().map(|(path, _)| path.as_path()), + )?; let tmp_dir = TempDirGuard::new()?; - let source_compression = Compression::from_file(&image_file)?; - if source_compression.is_some() { - dest_image_file.set_extension(""); - } let mut tmp_image_file = temp_image(&image_file, source_compression.as_ref(), &tmp_dir.0)?; if let Some((path, bmap)) = &input_bmap { - bmap::verify_checksums(&tmp_image_file, bmap) - .context(format!("cannot use input bmap {}", path.display()))?; + bmap::verify_checksums(&tmp_image_file, bmap).context(format!( + "cannot use input bmap {}; pass the right one with --input-bmap, or map the \ + whole image with --no-input-bmap", + path.display() + ))?; } // run command @@ -108,7 +122,7 @@ where if output_bmap { match &input_bmap { Some((path, _)) => info!("the bmap maps the ranges of {}", path.display()), - None => info!("no input bmap found, so the bmap maps the whole image"), + None => info!("no input bmap, so the bmap maps the whole image"), } bmap::write_bmap( &tmp_image_file, @@ -120,11 +134,6 @@ where // if applicable compress image if let Some(c) = target_compression { tmp_image_file = compression::compress(&tmp_image_file, &c)?; - dest_image_file.set_file_name( - tmp_image_file - .file_name() - .context("cannot get image file name")?, - ); std::fs::copy(&tmp_image_file, &dest_image_file).context(format!( "error: std::fs::copy({:?}, {:?})", tmp_image_file, dest_image_file diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 01e2b221..4510c54e 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -6,6 +6,7 @@ use httpmock::prelude::*; use omnect_cli::{file::bmap::Bmap, ssh}; use predicates::prelude::*; use std::{ + ffi::OsStr, fs::{self, create_dir_all}, os::unix::fs::MetadataExt, path::{Path, PathBuf}, @@ -909,6 +910,10 @@ const ZERO_BLOCKS_FILE_ZEROS: usize = 64 * 1024; const OTHER_IMAGE_BMAP: &str = "40964096\ 0"; +fn bmap_path_of(image: &Path) -> PathBuf { + PathBuf::from(format!("{}.bmap", image.to_string_lossy())) +} + fn read_bmap(path: &Path) -> Bmap { Bmap::read(path).expect("read bmap") } @@ -1122,11 +1127,8 @@ fn check_file_copy_without_output_bmap_keeps_image_sparse() { let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); let image_path = tr.to_pathbuf("testfiles/image.wic"); let sparse_path = image_path.with_file_name("sparse.wic"); - let bmap_path = PathBuf::from(format!("{}.bmap", sparse_path.to_string_lossy())); let in_file = tr.to_pathbuf("testfiles/boot.scr"); sparse_copy(&image_path, &sparse_path); - // without -b, a bmap next to the image is not used - fs::write(&bmap_path, OTHER_IMAGE_BMAP).expect("write bmap"); Command::cargo_bin("omnect-cli") .expect("omnect-cli binary") @@ -1144,10 +1146,6 @@ fn check_file_copy_without_output_bmap_keeps_image_sparse() { meta.blocks() * STAT_BLOCK_SIZE < meta.len(), "image is not sparse" ); - assert_eq!( - fs::read_to_string(&bmap_path).expect("read bmap"), - OTHER_IMAGE_BMAP - ); } #[test] @@ -1157,7 +1155,7 @@ fn check_no_output_bmap_when_image_write_back_fails() { let written_image_path = image_path.with_extension(""); let bmap_path = image_path.with_extension("bmap"); let in_file = tr.to_pathbuf("testfiles/boot.scr"); - // a directory in place of the written image makes the write back fail, also for root + // a directory as the output image makes the write back fail, also for root create_dir_all(&written_image_path).expect("create directory"); Command::cargo_bin("omnect-cli") @@ -1169,8 +1167,11 @@ fn check_no_output_bmap_when_image_write_back_fails() { .arg("-i") .arg(&image_path) .arg("-b") + .arg("-o") + .arg(&written_image_path) .assert() - .failure(); + .failure() + .stderr(predicate::str::contains("copy_image")); assert!( !bmap_path.exists(), "bmap written for an image that was not written back" @@ -1232,33 +1233,120 @@ fn check_input_bmap_of_another_image_with_the_same_size() { #[test] fn check_chained_commands_with_input_bmap() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_xz_path = tr.to_pathbuf("testfiles/image.wic.xz"); + let image_xz_hash = Testrunner::file_hash(&image_xz_path); + let input_bmap_path = tr.to_pathbuf("testfiles/image.wic.bmap"); + let input_bmap_hash = Testrunner::file_hash(&input_bmap_path); + let device_path = tr.pathbuf().join("device.wic"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + let copy_to_image = |image: &Path, args: &[&OsStr]| { + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(image) + .args(args) + .assert() + }; + + // image.wic would be written next to image.wic.bmap without updating it + copy_to_image(&image_xz_path, &[]) + .failure() + .stderr(predicate::str::contains("pass -b to update it")); + // the bmap of image.wic would replace the bmap of image.wic.xz + copy_to_image(&image_xz_path, &["-b".as_ref()]) + .failure() + .stderr(predicate::str::contains("--output-image")); + + copy_to_image( + &image_xz_path, + &["-b".as_ref(), "-o".as_ref(), device_path.as_os_str()], + ) + .success(); + copy_to_image(&device_path, &["-b".as_ref()]).success(); + copy_to_image(&device_path, &[]) + .failure() + .stderr(predicate::str::contains("pass -b to update it")); + + assert_eq!(Testrunner::file_hash(&image_xz_path), image_xz_hash); + assert_eq!(Testrunner::file_hash(&input_bmap_path), input_bmap_hash); + assert!(maps_all_ranges_of( + &read_bmap(&bmap_path_of(&device_path)), + &read_bmap(&input_bmap_path) + )); +} + +#[test] +fn check_derived_output_image_exists() { let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); let image_xz_path = tr.to_pathbuf("testfiles/image.wic.xz"); let image_path = image_xz_path.with_extension(""); - tr.to_pathbuf("testfiles/image.wic.bmap"); let in_file = tr.to_pathbuf("testfiles/boot.scr"); - let copy_to_image = |image: &Path, output_bmap: bool| { - let mut cmd = Command::cargo_bin("omnect-cli").expect("omnect-cli binary"); - cmd.arg("file") + fs::write(&image_path, "an older image").expect("write image"); + let copy_to_image = |args: &[&OsStr]| { + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_xz_path) + .args(args) + .assert() + }; + + copy_to_image(&[]) + .failure() + .stderr(predicate::str::contains("--output-image")); + // a path given with -o is overwritten + copy_to_image(&["-o".as_ref(), image_path.as_os_str()]).success(); + assert_ne!( + fs::read(&image_path).expect("read image"), + b"an older image" + ); +} + +#[test] +fn check_no_input_bmap() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let bmap_path = bmap_path_of(&image_path); + let other_bmap_path = tr.pathbuf().join("other.bmap"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + fs::write(&bmap_path, OTHER_IMAGE_BMAP).expect("write bmap"); + fs::write(&other_bmap_path, OTHER_IMAGE_BMAP).expect("write bmap"); + let copy_to_image = |args: &[&OsStr]| { + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") .arg("copy-to-image") .arg("-f") .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) .arg("-i") - .arg(image); - if output_bmap { - cmd.arg("-b"); - } - cmd.assert() + .arg(&image_path) + .args(args) + .assert() }; - // without -b, the bmap next to the image still has the checksums of the original image - copy_to_image(&image_xz_path, false).success(); - copy_to_image(&image_path, true) + copy_to_image(&["--no-input-bmap".as_ref()]).failure(); + copy_to_image(&[ + "-b".as_ref(), + "--no-input-bmap".as_ref(), + "--input-bmap".as_ref(), + other_bmap_path.as_os_str(), + ]) + .failure(); + copy_to_image(&["-b".as_ref()]) .failure() - .stderr(predicate::str::contains("older version of this image")); + .stderr(predicate::str::contains("--no-input-bmap")); - copy_to_image(&image_xz_path, true).success(); - copy_to_image(&image_path, true).success(); + copy_to_image(&["-b".as_ref(), "--no-input-bmap".as_ref()]).success(); + assert!(maps_whole_image(&read_bmap(&bmap_path))); } #[test] @@ -1313,11 +1401,10 @@ fn check_bmap_generation_wic() { fn check_bmap_generation_wic_xz() { let tr = Testrunner::new(function_name!().split("::").last().unwrap()); let image_path_wic_xz = tr.to_pathbuf("testfiles/image.wic.xz"); - let image_path_wic = image_path_wic_xz.with_extension(""); + let image_path_wic = tr.pathbuf().join("device.wic"); let image_path_wic_copy = image_path_wic_xz.with_extension("copy"); - // the Yocto bmap of the image; omnect-cli replaces it with the one it creates - let image_path_bmap = tr.to_pathbuf("testfiles/image.wic.bmap"); - let input_bmap = read_bmap(&image_path_bmap); + let image_path_bmap = bmap_path_of(&image_path_wic); + let input_bmap = read_bmap(&tr.to_pathbuf("testfiles/image.wic.bmap")); let in_file = tr.to_pathbuf("testfiles/boot.scr"); let in_file = in_file.to_str().unwrap(); @@ -1330,6 +1417,8 @@ fn check_bmap_generation_wic_xz() { .arg("-i") .arg(&image_path_wic_xz) .arg("-b") + .arg("-o") + .arg(&image_path_wic) .assert(); assert.success(); From f0ead2173ab39b4b0175fe40fe6ca8062172014c Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:21:14 +0100 Subject: [PATCH 33/75] perf(file): check the input bmap of an uncompressed image before the copy Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/lib.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 47ba4f31..515a5fdf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -104,15 +104,25 @@ where input_bmap.as_ref().map(|(path, _)| path.as_path()), )?; - let tmp_dir = TempDirGuard::new()?; - let mut tmp_image_file = temp_image(&image_file, source_compression.as_ref(), &tmp_dir.0)?; - - if let Some((path, bmap)) = &input_bmap { - bmap::verify_checksums(&tmp_image_file, bmap).context(format!( + let verify_input_bmap = |image: &Path| -> Result<()> { + let Some((path, bmap)) = &input_bmap else { + return Ok(()); + }; + bmap::verify_checksums(image, bmap).context(format!( "cannot use input bmap {}; pass the right one with --input-bmap, or map the \ whole image with --no-input-bmap", path.display() - ))?; + )) + }; + + // a wrong bmap of an uncompressed image fails before the copy + if source_compression.is_none() { + verify_input_bmap(&image_file)?; + } + let tmp_dir = TempDirGuard::new()?; + let mut tmp_image_file = temp_image(&image_file, source_compression.as_ref(), &tmp_dir.0)?; + if source_compression.is_some() { + verify_input_bmap(&tmp_image_file)?; } // run command From be84b3386916cd9c31b9e930512dc9a05c3708ec Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:57:04 +0100 Subject: [PATCH 34/75] fix(file): read concatenated compressed streams Decompress with the multi-stream decoders, so an image made of several streams (e.g. written by pbzip2) is read completely. Decompress straight from the input image into the temp dir, without a copy of the compressed file first. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/compression.rs | 100 ++++++++++++++++++++++++++-------------- src/lib.rs | 16 +++---- 2 files changed, 71 insertions(+), 45 deletions(-) diff --git a/src/file/compression.rs b/src/file/compression.rs index 85e8fc99..261506ac 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -1,11 +1,11 @@ -use crate::file::{IMAGE_BLOCK_SIZE, is_zero}; +use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, is_zero}; use anyhow::{Context, Result}; use filemagic::Magic; use log::debug; use std::{ env, fs::File, - io::{Seek, SeekFrom, Write}, + io::{BufWriter, IntoInnerError, Read, Seek, SeekFrom, Write}, os::unix::fs::FileExt, path::{Path, PathBuf}, str::FromStr, @@ -81,24 +81,19 @@ impl Compression { source: &mut std::fs::File, destination: &mut std::fs::File, ) -> std::io::Result { - let sparse = SparseWriter::new(destination)?; - // only `finish` detects a truncated or corrupt stream - let (bytes_read, sparse) = match &self { - Compression::bzip2 => { - let mut dec = bzip2::write::BzDecoder::new(sparse); - (std::io::copy(source, &mut dec)?, dec.finish()?) - } - Compression::gzip => { - let mut dec = flate2::write::GzDecoder::new(sparse); - (std::io::copy(source, &mut dec)?, dec.finish()?) - } - Compression::xz { .. } => { - let mut dec = xz2::write::XzDecoder::new(sparse); - (std::io::copy(source, &mut dec)?, dec.finish()?) - } + // the multi decoders also read concatenated streams, as pbzip2 writes them + let mut decoder: Box = match &self { + Compression::bzip2 => Box::new(bzip2::read::MultiBzDecoder::new(source)), + Compression::gzip => Box::new(flate2::read::MultiGzDecoder::new(source)), + Compression::xz { .. } => Box::new(xz2::read::XzDecoder::new_multi_decoder(source)), }; - sparse.finish()?; - Ok(bytes_read) + let mut sparse = BufWriter::with_capacity(COPY_CHUNK_SIZE, SparseWriter::new(destination)?); + let bytes_written = std::io::copy(&mut decoder, &mut sparse)?; + sparse + .into_inner() + .map_err(IntoInnerError::into_error)? + .finish()?; + Ok(bytes_written) } fn marker(&self) -> &'static str { @@ -208,26 +203,35 @@ impl Write for SparseWriter<'_> { } } -pub fn decompress(image_file_name: &PathBuf, compression: &Compression) -> Result { - let mut new_image_file = PathBuf::from(image_file_name); - - if new_image_file - .extension() - .and_then(|ext| ext.to_str()) - .is_some_and(|ext| compression.extensions().contains(&ext)) - { - new_image_file.set_extension(""); - } - // creating the destination would truncate the source +/// Decompresses `image_file_name` into `dest_dir`, named without the compression extension. +pub fn decompress( + image_file_name: &Path, + compression: &Compression, + dest_dir: &Path, +) -> Result { anyhow::ensure!( - new_image_file != *image_file_name, + image_file_name + .extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| compression.extensions().contains(&ext)), "decompress: {} has no {:?} file extension", image_file_name.display(), compression.extensions() ); + let new_image_file = dest_dir.join( + image_file_name + .file_stem() + .context("decompress: cannot get image file name")?, + ); - let mut destination = File::create(&new_image_file)?; - let mut source = File::open(image_file_name)?; + let mut source = File::open(image_file_name).context(format!( + "decompress: cannot open {}", + image_file_name.display() + ))?; + let mut destination = File::create(&new_image_file).context(format!( + "decompress: cannot create {}", + new_image_file.display() + ))?; debug!("decompress {image_file_name:?} to {new_image_file:?}"); let bytes_written = compression.decompress(&mut source, &mut destination)?; debug!("image::decompress: copied {} bytes.", bytes_written); @@ -363,6 +367,32 @@ mod tests { } } + #[test] + fn decompress_reads_concatenated_streams() { + let image = test_image(); + let (first, second) = image.split_at(DATA_OFFSET); + + for compression in all_compressions() { + let mut concatenated = tempfile::tempfile().expect("create concatenated"); + for part in [first, second] { + let mut compressed = compress_image(&compression, part); + concatenated + .write_all(&read_all(&mut compressed)) + .expect("write stream"); + } + concatenated.rewind().expect("rewind concatenated"); + + let mut decompressed = tempfile::tempfile().expect("create decompressed"); + compression + .decompress(&mut concatenated, &mut decompressed) + .expect("decompress"); + assert!( + read_all(&mut decompressed) == image, + "{compression:?}: content differs" + ); + } + } + #[test] fn decompress_file_strips_every_extension_of_the_compression() { let dir = tempfile::tempdir().expect("create temp dir"); @@ -374,7 +404,7 @@ mod tests { let mut compressed = compress_image(&compression, &image); fs::write(&path, read_all(&mut compressed)).expect("write compressed"); - let decompressed = decompress(&path, &compression).expect("decompress"); + let decompressed = decompress(&path, &compression, dir.path()).expect("decompress"); assert_eq!(decompressed, dir.path().join("image.wic")); assert!( fs::read(&decompressed).expect("read decompressed") == image, @@ -394,7 +424,7 @@ mod tests { let content = read_all(&mut compressed); fs::write(&path, &content).expect("write compressed"); - assert!(decompress(&path, &compression).is_err()); + assert!(decompress(&path, &compression, dir.path()).is_err()); assert!( fs::read(&path).expect("read compressed") == content, "source was changed" diff --git a/src/lib.rs b/src/lib.rs index 515a5fdf..a2ce84d7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -196,18 +196,14 @@ fn temp_image( source_compression: Option<&Compression>, tmp_dir: &Path, ) -> Result { - let tmp_image_file = tmp_dir.join( - image_file - .file_name() - .context("cannot get image file name")?, - ); - match source_compression { - Some(c) => { - std::fs::copy(image_file, &tmp_image_file)?; - compression::decompress(&tmp_image_file, c) - } + Some(c) => compression::decompress(image_file, c, tmp_dir), None => { + let tmp_image_file = tmp_dir.join( + image_file + .file_name() + .context("cannot get image file name")?, + ); copy_image(image_file, &tmp_image_file).context(format!( "error: copy_image({:?}, {:?})", image_file, tmp_image_file From 453b1245e6aec59c6b979aeca2697b9ec749c81f Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:57:04 +0100 Subject: [PATCH 35/75] fix(file): check every bmap that bmaptool would use Without -b, also fail when the bmap of the uncompressed image is next to a compressed output image, because bmaptool falls back to it. With -b and an output that is not the input image, also refuse to replace the bmap next to the input image when --input-bmap or --no-input-bmap is given. An input bmap that cannot be read gets the same hint as one that does not match. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 4 +-- src/file/output.rs | 62 +++++++++++++++++++++++++++++--------- src/lib.rs | 42 +++++++++++++++----------- tests/integration_tests.rs | 11 +++++-- 4 files changed, 83 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index c7e714c3..e914d157 100644 --- a/README.md +++ b/README.md @@ -191,9 +191,9 @@ The command fails instead of overwriting an existing file that is not the input ## bmap files -Commands that change an image create a bmap file with `-b` (`--output-bmap`). It is written next to the output image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic` and `my-image.wic.xz.bmap` for `my-image.wic.xz`, and replaces a file with this name. `bmaptool` finds it there without `--bmap`. The command fails if this file is the input bmap of an image that stays, e.g. `my-image.wic.bmap` when `-i my-image.wic.xz` writes `my-image.wic`. Choose another output with `-o` then. +Commands that change an image create a bmap file with `-b` (`--output-bmap`). It is written next to the output image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic` and `my-image.wic.xz.bmap` for `my-image.wic.xz`, and replaces a file with this name. `bmaptool` finds it there without `--bmap`. The command fails if this file is the bmap of an image that stays, e.g. `my-image.wic.bmap` when `-i my-image.wic.xz` writes `my-image.wic`, also with `--input-bmap` or `--no-input-bmap`. Choose another output with `-o` then. -The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image and sha256 range checksums that match it, otherwise the command fails. A command without `-b` fails if a bmap is next to its output image, because the bmap would no longer match the image. So when you run several commands on one image, use `-b` on each of them. `--no-input-bmap` skips a bmap next to the image that does not match, and the created bmap then maps the whole image. The created bmap maps every range of the input bmap, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). It also maps every block that the image file stores as data, which includes the blocks `omnect-cli` changed. The image file itself stays sparse. +The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image and sha256 range checksums that match it, otherwise the command fails. A command without `-b` fails if `.bmap` exists, or for a compressed output image the bmap of the uncompressed image (e.g. `my-image.wic.bmap` for `my-image.wic.xz`), because `bmaptool` would use it and it would not match the image. So when you run several commands on one image, use `-b` on each of them. `--no-input-bmap` skips a bmap next to the image that does not match, and the created bmap then maps the whole image. The created bmap maps every range of the input bmap, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). It also maps every block that the image file stores as data, which includes the blocks `omnect-cli` changed. The image file itself stays sparse. Without an input bmap, the created bmap maps the whole image, and a flash writes every block. diff --git a/src/file/output.rs b/src/file/output.rs index 4db32785..ea7ffae0 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -28,7 +28,9 @@ pub fn output_image( } /// Fails if writing `output_image` and its bmap would overwrite a file the user did not name, -/// replace the bmap of an image that stays, or leave a bmap that no longer matches the image. +/// replace the bmap of an image that stays, or leave a bmap next to the output image that does +/// not match it: `.bmap`, or for a compressed output image the bmap of the +/// uncompressed image, which `bmaptool` also uses. pub fn check_conflicts( input_image: &Path, output_image: &Path, @@ -37,7 +39,6 @@ pub fn check_conflicts( input_bmap: Option<&Path>, ) -> Result<()> { let in_place = same_file(input_image, output_image)?; - let target_bmap = bmap::bmap_path(output_image); anyhow::ensure!( explicit_output || in_place || !output_image.exists(), @@ -45,20 +46,23 @@ pub fn check_conflicts( output_image.display() ); if output_bmap { - if let Some(input_bmap) = input_bmap { - anyhow::ensure!( - in_place || !same_file(input_bmap, &target_bmap)?, - "{} is the bmap of {} and would be replaced; choose the output image with \ - --output-image", - input_bmap.display(), - input_image.display() - ); + let target_bmap = bmap::bmap_path(output_image); + if !in_place { + let found_bmap = bmap::find_input_bmap(input_image, None)?; + for bmap in input_bmap.into_iter().chain(found_bmap.as_deref()) { + anyhow::ensure!( + !same_file(bmap, &target_bmap)?, + "{} is the bmap of {} and would be replaced; choose another output image \ + with --output-image", + bmap.display(), + input_image.display() + ); + } } - } else { - anyhow::ensure!( - !target_bmap.exists(), - "{} would no longer match {}; pass -b to update it, or remove it", - target_bmap.display(), + } else if let Some(stale_bmap) = bmap::find_input_bmap(output_image, None)? { + anyhow::bail!( + "{} would not match {}; pass -b to write a matching bmap", + stale_bmap.display(), output_image.display() ); } @@ -167,6 +171,34 @@ mod tests { assert!(check_conflicts(&image, &image, false, false, None).is_err()); } + #[test] + fn check_conflicts_rejects_a_stale_bmap_of_the_uncompressed_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_xz = dir.path().join(IMAGE_XZ); + let image_bmap = bmap::bmap_path(&dir.path().join(IMAGE)); + touch(&image_xz); + touch(&image_bmap); + + // bmaptool uses image.wic.bmap for image.wic.xz + assert!(check_conflicts(&image_xz, &image_xz, false, false, None).is_err()); + check_conflicts(&image_xz, &image_xz, false, true, Some(&image_bmap)).expect("with -b"); + } + + #[test] + fn check_conflicts_rejects_replacing_the_bmap_next_to_the_input_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_xz = dir.path().join(IMAGE_XZ); + let image = dir.path().join(IMAGE); + let other_bmap = dir.path().join("other.bmap"); + touch(&image_xz); + touch(&bmap::bmap_path(&image)); + touch(&other_bmap); + + // --input-bmap and --no-input-bmap + assert!(check_conflicts(&image_xz, &image, false, true, Some(&other_bmap)).is_err()); + assert!(check_conflicts(&image_xz, &image, false, true, None).is_err()); + } + #[test] fn same_file_resolves_relative_and_missing_paths() { let dir = tempfile::tempdir().expect("create temp dir"); diff --git a/src/lib.rs b/src/lib.rs index a2ce84d7..5711540c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,8 +23,10 @@ use cli::{ }; use file::{ bmap::{self, Bmap}, - compression::Compression, + compression::{self, Compression}, functions::FileCopyToParams, + output::{check_conflicts, output_image}, + sparse, }; use log::{error, info}; use std::{ @@ -34,8 +36,6 @@ use std::{ use tokio::fs::remove_dir_all; use uuid::Uuid; -use crate::file::compression; - /// A new directory `/tmp/{uuid}`, removed on drop. struct TempDirGuard(PathBuf); @@ -76,7 +76,7 @@ where output_bmap, input_bmap, no_input_bmap, - output_image, + output_image: explicit_output, compress_image: target_compression, } = output; @@ -84,22 +84,26 @@ where let input_bmap = if output_bmap && !no_input_bmap { bmap::find_input_bmap(&image_file, input_bmap.as_deref())? - .map(|path| Bmap::read(&path).map(|bmap| (path, bmap))) + .map(|path| { + Bmap::read(&path) + .with_context(|| input_bmap_error(&path)) + .map(|bmap| (path, bmap)) + }) .transpose()? } else { None }; let source_compression = Compression::from_file(&image_file)?; - let dest_image_file = file::output::output_image( + let dest_image_file = output_image( &image_file, source_compression.is_some(), target_compression.as_ref(), - output_image.as_deref(), + explicit_output.as_deref(), ); - file::output::check_conflicts( + check_conflicts( &image_file, &dest_image_file, - output_image.is_some(), + explicit_output.is_some(), output_bmap, input_bmap.as_ref().map(|(path, _)| path.as_path()), )?; @@ -108,11 +112,7 @@ where let Some((path, bmap)) = &input_bmap else { return Ok(()); }; - bmap::verify_checksums(image, bmap).context(format!( - "cannot use input bmap {}; pass the right one with --input-bmap, or map the \ - whole image with --no-input-bmap", - path.display() - )) + bmap::verify_checksums(image, bmap).with_context(|| input_bmap_error(path)) }; // a wrong bmap of an uncompressed image fails before the copy @@ -144,7 +144,7 @@ where // if applicable compress image if let Some(c) = target_compression { tmp_image_file = compression::compress(&tmp_image_file, &c)?; - std::fs::copy(&tmp_image_file, &dest_image_file).context(format!( + fs::copy(&tmp_image_file, &dest_image_file).context(format!( "error: std::fs::copy({:?}, {:?})", tmp_image_file, dest_image_file ))?; @@ -158,7 +158,7 @@ where // copy the bmap last, so it never describes an image that was not written back if output_bmap { let target_bmap = bmap::bmap_path(&dest_image_file); - std::fs::copy(&tmp_bmap, &target_bmap).context(format!( + fs::copy(&tmp_bmap, &target_bmap).context(format!( "error: std::fs::copy({:?}, {:?})", tmp_bmap, target_bmap ))?; @@ -167,6 +167,14 @@ where Ok(()) } +fn input_bmap_error(path: &Path) -> String { + format!( + "cannot use input bmap {}; pass the right one with --input-bmap, or map the whole \ + image with --no-input-bmap", + path.display() + ) +} + /// Runs `command` on a temp copy of the image and writes nothing back. fn run_read_only_image_command(image_file: PathBuf, command: F) -> Result<()> where @@ -221,7 +229,7 @@ fn copy_image(from: &Path, to: &Path) -> Result<()> { .context("copy_image: cannot get source size")? .len(); - file::sparse::copy_to_new_file(&src, 0, to, len) + sparse::copy_to_new_file(&src, 0, to, len) } struct CertInfo { diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 4510c54e..2802a1ae 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1256,11 +1256,18 @@ fn check_chained_commands_with_input_bmap() { // image.wic would be written next to image.wic.bmap without updating it copy_to_image(&image_xz_path, &[]) .failure() - .stderr(predicate::str::contains("pass -b to update it")); + .stderr(predicate::str::contains("pass -b to write a matching bmap")); + // bmaptool would use image.wic.bmap for the changed image.wic.xz + copy_to_image(&image_xz_path, &["-p".as_ref(), "xz".as_ref()]) + .failure() + .stderr(predicate::str::contains("pass -b to write a matching bmap")); // the bmap of image.wic would replace the bmap of image.wic.xz copy_to_image(&image_xz_path, &["-b".as_ref()]) .failure() .stderr(predicate::str::contains("--output-image")); + copy_to_image(&image_xz_path, &["-b".as_ref(), "--no-input-bmap".as_ref()]) + .failure() + .stderr(predicate::str::contains("--output-image")); copy_to_image( &image_xz_path, @@ -1270,7 +1277,7 @@ fn check_chained_commands_with_input_bmap() { copy_to_image(&device_path, &["-b".as_ref()]).success(); copy_to_image(&device_path, &[]) .failure() - .stderr(predicate::str::contains("pass -b to update it")); + .stderr(predicate::str::contains("pass -b to write a matching bmap")); assert_eq!(Testrunner::file_hash(&image_xz_path), image_xz_hash); assert_eq!(Testrunner::file_hash(&input_bmap_path), input_bmap_hash); From d0800f83bcaab653af587f1e6e1172ecdf809faa Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:00:08 +0100 Subject: [PATCH 36/75] fix(file)!: name a bzip2 image .bz2 bmaptool does not read the .bzip2 extension, so it could not flash an image written with -p bzip2. BREAKING CHANGE: -p bzip2 writes .bz2 instead of .bzip2. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 2 +- src/file/compression.rs | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e914d157..62103058 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ omnect-cli file copy-to-image --help ## Output image -A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression, e.g. `my-image.wic.xz` for `my-image.wic`. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. +A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gzip`), e.g. `my-image.wic.xz` for `my-image.wic`. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. `copy-from-image` only reads the image and writes nothing. diff --git a/src/file/compression.rs b/src/file/compression.rs index 261506ac..5f37efcb 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -111,7 +111,7 @@ impl Compression { /// The first one is used for a compressed image that omnect-cli creates. fn extensions(&self) -> &'static [&'static str] { match &self { - Compression::bzip2 => &["bzip2", "bz2"], + Compression::bzip2 => &["bz2", "bzip2"], Compression::gzip => &["gzip", "gz"], Compression::xz { .. } => &["xz"], } @@ -415,6 +415,16 @@ mod tests { } } + #[test] + fn compressed_path_has_an_extension_bmaptool_reads() { + let image = Path::new("image.wic"); + let expected = ["image.wic.bz2", "image.wic.gzip", "image.wic.xz"]; + + for (compression, expected) in all_compressions().iter().zip(expected) { + assert_eq!(compression.compressed_path(image), Path::new(expected)); + } + } + #[test] fn decompress_file_rejects_unknown_extension() { let dir = tempfile::tempdir().expect("create temp dir"); From c477f9a2d0c4c9a83f2133c69519018caab883ed Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:01:11 +0100 Subject: [PATCH 37/75] fix(file): check the extension of the --output-image name The name given with -o must have the extension of the -p compression, or no compression extension without -p, so the name matches the content. For a compressed input, strip only the extension of its compression from the derived output name. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 2 +- src/file/compression.rs | 13 +++-- src/file/output.rs | 110 ++++++++++++++++++++++++++++++++----- src/lib.rs | 4 +- tests/integration_tests.rs | 10 ++++ 5 files changed, 117 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 62103058..e8a87336 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ omnect-cli file copy-to-image --help ## Output image -A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gzip`), e.g. `my-image.wic.xz` for `my-image.wic`. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. +A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gzip`), e.g. `my-image.wic.xz` for `my-image.wic`. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. Its name must have the extension of the `-p` compression, or no compression extension without `-p`. The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. `copy-from-image` only reads the image and writes nothing. diff --git a/src/file/compression.rs b/src/file/compression.rs index 5f37efcb..cf3f173d 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -124,10 +124,14 @@ impl Compression { PathBuf::from(compressed) } - pub fn has_extension(path: &Path) -> bool { + pub fn matches_extension(&self, path: &Path) -> bool { path.extension() .and_then(|e| e.to_str()) - .is_some_and(|e| Compression::iter().any(|c| c.extensions().contains(&e))) + .is_some_and(|e| self.extensions().contains(&e)) + } + + pub fn has_extension(path: &Path) -> bool { + Compression::iter().any(|c| c.matches_extension(path)) } pub fn from_file(image_file_name: &PathBuf) -> Result> { @@ -210,10 +214,7 @@ pub fn decompress( dest_dir: &Path, ) -> Result { anyhow::ensure!( - image_file_name - .extension() - .and_then(|ext| ext.to_str()) - .is_some_and(|ext| compression.extensions().contains(&ext)), + compression.matches_extension(image_file_name), "decompress: {} has no {:?} file extension", image_file_name.display(), compression.extensions() diff --git a/src/file/output.rs b/src/file/output.rs index ea7ffae0..953173d0 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -5,26 +5,40 @@ use std::{ path::{Path, PathBuf}, }; -/// Returns `explicit`, else `image` without its last extension if it is compressed, with the -/// extension of `target` appended. +/// Returns `explicit`, else `image` without the extension of its `source` compression, with the +/// extension of `target` appended. Fails if `explicit` has no extension of `target`, or a +/// compression extension while `target` is none. pub fn output_image( image: &Path, - source_compressed: bool, + source: Option<&Compression>, target: Option<&Compression>, explicit: Option<&Path>, -) -> PathBuf { +) -> Result { if let Some(path) = explicit { - return path.to_path_buf(); + match target { + Some(c) => anyhow::ensure!( + c.matches_extension(path), + "{} has no extension of the -p compression; name it e.g. {}", + path.display(), + c.compressed_path(path).display() + ), + None => anyhow::ensure!( + !Compression::has_extension(path), + "{} has a compression extension, but the image is written uncompressed; pass -p", + path.display() + ), + } + return Ok(path.to_path_buf()); } let mut output = image.to_path_buf(); - if source_compressed { + if source.is_some_and(|c| c.matches_extension(image)) { output.set_extension(""); } - match target { + Ok(match target { Some(c) => c.compressed_path(&output), None => output, - } + }) } /// Fails if writing `output_image` and its bmap would overwrite a file the user did not name, @@ -113,15 +127,85 @@ mod tests { let xz = Compression::xz { compression_level: XZ_LEVEL, }; + let output = |image, source, target| { + output_image(image, source, target, None).expect("output image") + }; + + assert_eq!(output(image, None, None), image); + assert_eq!(output(image_xz, Some(&xz), None), image); + assert_eq!(output(image, None, Some(&xz)), image_xz); + assert_eq!(output(image_xz, Some(&xz), Some(&xz)), image_xz); + } + + #[test] + fn output_image_strips_only_the_source_compression_extension() { + let image = Path::new("image.bin"); + let xz = Compression::xz { + compression_level: XZ_LEVEL, + }; - assert_eq!(output_image(image, false, None, None), image); - assert_eq!(output_image(image_xz, true, None, None), image); - assert_eq!(output_image(image, false, Some(&xz), None), image_xz); - assert_eq!(output_image(image_xz, true, Some(&xz), None), image_xz); assert_eq!( - output_image(image_xz, true, Some(&xz), Some(Path::new(OTHER_IMAGE))), + output_image(image, Some(&xz), None, None).expect("output image"), + image + ); + } + + #[test] + fn output_image_takes_explicit_path_with_matching_extension() { + let xz = Compression::xz { + compression_level: XZ_LEVEL, + }; + let other_xz = Path::new("device.wic.xz"); + + assert_eq!( + output_image( + Path::new(IMAGE_XZ), + Some(&xz), + None, + Some(Path::new(OTHER_IMAGE)) + ) + .expect("uncompressed"), Path::new(OTHER_IMAGE) ); + assert_eq!( + output_image(Path::new(IMAGE), None, Some(&xz), Some(other_xz)).expect("compressed"), + other_xz + ); + } + + #[test] + fn output_image_rejects_explicit_path_with_wrong_extension() { + let xz = Compression::xz { + compression_level: XZ_LEVEL, + }; + + assert!( + output_image( + Path::new(IMAGE), + None, + Some(&xz), + Some(Path::new(OTHER_IMAGE)) + ) + .is_err() + ); + assert!( + output_image( + Path::new(IMAGE), + None, + Some(&xz), + Some(Path::new("device.wic.gz")) + ) + .is_err() + ); + assert!( + output_image( + Path::new(IMAGE_XZ), + Some(&xz), + None, + Some(Path::new("device.wic.xz")) + ) + .is_err() + ); } #[test] diff --git a/src/lib.rs b/src/lib.rs index 5711540c..f4eb0168 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -96,10 +96,10 @@ where let source_compression = Compression::from_file(&image_file)?; let dest_image_file = output_image( &image_file, - source_compression.is_some(), + source_compression.as_ref(), target_compression.as_ref(), explicit_output.as_deref(), - ); + )?; check_conflicts( &image_file, &dest_image_file, diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 2802a1ae..9aeb30a9 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1310,6 +1310,16 @@ fn check_derived_output_image_exists() { copy_to_image(&[]) .failure() .stderr(predicate::str::contains("--output-image")); + copy_to_image(&[ + "-o".as_ref(), + image_path.as_os_str(), + "-p".as_ref(), + "xz".as_ref(), + ]) + .failure() + .stderr(predicate::str::contains( + "has no extension of the -p compression", + )); // a path given with -o is overwritten copy_to_image(&["-o".as_ref(), image_path.as_os_str()]).success(); assert_ne!( From 7ab73f8ddf7a13ef3255196e16f3fef244e04966 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:02:09 +0100 Subject: [PATCH 38/75] fix(file): write the output image through a rename The write-back truncated the output image first, so a failed copy, e.g. on a full disk, destroyed an image that was changed in place. Write the image and the bmap to a new file in the same directory and rename it over the old one. A symlink is followed, and the old file keeps its permissions. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 2 +- src/file/output.rs | 76 ++++++++++++++++++++++++++++++++++++++ src/lib.rs | 29 ++++++++------- tests/integration_tests.rs | 2 +- 4 files changed, 93 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index e8a87336..bc366430 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,7 @@ omnect-cli file copy-to-image --help A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gzip`), e.g. `my-image.wic.xz` for `my-image.wic`. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. Its name must have the extension of the `-p` compression, or no compression extension without `-p`. -The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. `copy-from-image` only reads the image and writes nothing. +The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. The output image is written to a new file in its directory and then renamed, so the directory needs space for one more image. `copy-from-image` only reads the image and writes nothing. ## bmap files diff --git a/src/file/output.rs b/src/file/output.rs index 953173d0..f1349b7c 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -5,6 +5,8 @@ use std::{ path::{Path, PathBuf}, }; +const TMP_DIR_PREFIX: &str = ".omnect-cli-"; + /// Returns `explicit`, else `image` without the extension of its `source` compression, with the /// extension of `target` appended. Fails if `explicit` has no extension of `target`, or a /// compression extension while `target` is none. @@ -84,6 +86,34 @@ pub fn check_conflicts( Ok(()) } +/// Writes `dest` as a new file in its directory and renames it, so `dest` stays unchanged if +/// `write` fails. A symlink is followed, and an existing `dest` keeps its permissions. +pub fn replace(dest: &Path, write: impl FnOnce(&Path) -> Result<()>) -> Result<()> { + let dest = resolve(dest)?; + let dir = dest + .parent() + .context(format!("{} has no directory", dest.display()))?; + let tmp_dir = tempfile::Builder::new() + .prefix(TMP_DIR_PREFIX) + .tempdir_in(dir) + .context(format!("cannot create a temp dir in {}", dir.display()))?; + let tmp_file = tmp_dir.path().join( + dest.file_name() + .context(format!("{} has no file name", dest.display()))?, + ); + + write(&tmp_file)?; + if let Ok(metadata) = fs::metadata(&dest) { + fs::set_permissions(&tmp_file, metadata.permissions()) + .context(format!("cannot set permissions of {}", tmp_file.display()))?; + } + fs::rename(&tmp_file, &dest).context(format!( + "cannot rename {} to {}", + tmp_file.display(), + dest.display() + )) +} + fn same_file(a: &Path, b: &Path) -> Result { Ok(resolve(a)? == resolve(b)?) } @@ -110,11 +140,16 @@ fn resolve(path: &Path) -> Result { #[cfg(test)] mod tests { use crate::file::output::*; + use std::os::unix::fs::{PermissionsExt, symlink}; const IMAGE: &str = "image.wic"; const IMAGE_XZ: &str = "image.wic.xz"; const OTHER_IMAGE: &str = "device.wic"; const XZ_LEVEL: u32 = 9; + const OLD_CONTENT: &[u8] = b"old image"; + const NEW_CONTENT: &[u8] = b"new image"; + const IMAGE_MODE: u32 = 0o644; + const MODE_MASK: u32 = 0o777; fn touch(path: &Path) { fs::write(path, []).expect("create file"); @@ -283,6 +318,47 @@ mod tests { assert!(check_conflicts(&image_xz, &image, false, true, None).is_err()); } + #[test] + fn replace_keeps_dest_when_write_fails() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + fs::write(&image, OLD_CONTENT).expect("write image"); + + assert!( + replace(&image, |tmp| { + fs::write(tmp, NEW_CONTENT).expect("write tmp"); + anyhow::bail!("write failed") + }) + .is_err() + ); + assert_eq!(fs::read(&image).expect("read image"), OLD_CONTENT); + assert_eq!( + fs::read_dir(dir.path()).expect("read dir").count(), + 1, + "temp dir was left" + ); + } + + #[test] + fn replace_keeps_permissions_and_symlink() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let link = dir.path().join(OTHER_IMAGE); + fs::write(&image, OLD_CONTENT).expect("write image"); + fs::set_permissions(&image, fs::Permissions::from_mode(IMAGE_MODE)) + .expect("set permissions"); + symlink(&image, &link).expect("create symlink"); + + replace(&link, |tmp| Ok(fs::write(tmp, NEW_CONTENT)?)).expect("replace"); + + assert!(link.is_symlink()); + assert_eq!(fs::read(&image).expect("read image"), NEW_CONTENT); + assert_eq!( + fs::metadata(&image).expect("metadata").permissions().mode() & MODE_MASK, + IMAGE_MODE + ); + } + #[test] fn same_file_resolves_relative_and_missing_paths() { let dir = tempfile::tempdir().expect("create temp dir"); diff --git a/src/lib.rs b/src/lib.rs index f4eb0168..3f32e2f9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,7 +25,7 @@ use file::{ bmap::{self, Bmap}, compression::{self, Compression}, functions::FileCopyToParams, - output::{check_conflicts, output_image}, + output::{check_conflicts, output_image, replace}, sparse, }; use log::{error, info}; @@ -144,24 +144,25 @@ where // if applicable compress image if let Some(c) = target_compression { tmp_image_file = compression::compress(&tmp_image_file, &c)?; - fs::copy(&tmp_image_file, &dest_image_file).context(format!( - "error: std::fs::copy({:?}, {:?})", - tmp_image_file, dest_image_file - ))?; + replace(&dest_image_file, |dest| { + fs::copy(&tmp_image_file, dest) + .map(drop) + .context(format!("cannot copy {tmp_image_file:?} to {dest:?}")) + })?; } else { - copy_image(&tmp_image_file, &dest_image_file).context(format!( - "error: copy_image({:?}, {:?})", - tmp_image_file, dest_image_file - ))?; + replace(&dest_image_file, |dest| { + copy_image(&tmp_image_file, dest) + .context(format!("cannot copy {tmp_image_file:?} to {dest:?}")) + })?; } // copy the bmap last, so it never describes an image that was not written back if output_bmap { - let target_bmap = bmap::bmap_path(&dest_image_file); - fs::copy(&tmp_bmap, &target_bmap).context(format!( - "error: std::fs::copy({:?}, {:?})", - tmp_bmap, target_bmap - ))?; + replace(&bmap::bmap_path(&dest_image_file), |dest| { + fs::copy(&tmp_bmap, dest) + .map(drop) + .context(format!("cannot copy {tmp_bmap:?} to {dest:?}")) + })?; } Ok(()) diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 9aeb30a9..32616a39 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1171,7 +1171,7 @@ fn check_no_output_bmap_when_image_write_back_fails() { .arg(&written_image_path) .assert() .failure() - .stderr(predicate::str::contains("copy_image")); + .stderr(predicate::str::contains("cannot rename")); assert!( !bmap_path.exists(), "bmap written for an image that was not written back" From 20c114b2692b5214374b1b6ed821d6f31288219a Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:02:49 +0100 Subject: [PATCH 39/75] refactor(file): use tempfile::TempDir for the temp image The temp dir now follows TMPDIR and no longer needs its own tokio runtime to be removed. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/lib.rs | 44 +++++--------------------------------------- 1 file changed, 5 insertions(+), 39 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 3f32e2f9..85ab18d9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,45 +28,11 @@ use file::{ output::{check_conflicts, output_image, replace}, sparse, }; -use log::{error, info}; +use log::info; use std::{ fs, path::{Path, PathBuf}, }; -use tokio::fs::remove_dir_all; -use uuid::Uuid; - -/// A new directory `/tmp/{uuid}`, removed on drop. -struct TempDirGuard(PathBuf); - -impl TempDirGuard { - fn new() -> Result { - let tmp_dir = PathBuf::from(format!("/tmp/{}", Uuid::new_v4())); - fs::create_dir_all(&tmp_dir).context(format!( - "TempDirGuard::new: cannot create {}", - tmp_dir.display() - ))?; - Ok(Self(tmp_dir)) - } -} - -impl Drop for TempDirGuard { - fn drop(&mut self) { - let Ok(rt) = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - else { - error!("cannot create tokio runtime"); - return; - }; - - rt.block_on(async { - if let Err(e) = remove_dir_all(self.0.clone()).await { - error!("cannot remove tmp dir: {e}") - } - }) - } -} fn run_image_command(image_file: PathBuf, output: ImageOutput, command: F) -> Result<()> where @@ -119,8 +85,8 @@ where if source_compression.is_none() { verify_input_bmap(&image_file)?; } - let tmp_dir = TempDirGuard::new()?; - let mut tmp_image_file = temp_image(&image_file, source_compression.as_ref(), &tmp_dir.0)?; + let tmp_dir = tempfile::tempdir().context("cannot create a temp dir")?; + let mut tmp_image_file = temp_image(&image_file, source_compression.as_ref(), tmp_dir.path())?; if source_compression.is_some() { verify_input_bmap(&tmp_image_file)?; } @@ -183,9 +149,9 @@ where { ensure_image_exists(&image_file)?; - let tmp_dir = TempDirGuard::new()?; + let tmp_dir = tempfile::tempdir().context("cannot create a temp dir")?; let source_compression = Compression::from_file(&image_file)?; - let tmp_image_file = temp_image(&image_file, source_compression.as_ref(), &tmp_dir.0)?; + let tmp_image_file = temp_image(&image_file, source_compression.as_ref(), tmp_dir.path())?; command(&tmp_image_file) } From 342bea1bb1c5cdcaff9cdc16a4c19c1683489ba6 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:03:57 +0100 Subject: [PATCH 40/75] fix(file): keep the partition files in their own temp dir The partition files were named .img in the directory of the image, so an image named e.g. 1.img was overwritten by its own partition 1. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/functions.rs | 22 ++++++----------- tests/integration_tests.rs | 49 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/src/file/functions.rs b/src/file/functions.rs index 9c9ffecc..5e4a2794 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -189,12 +189,8 @@ macro_rules! try_exec_cmd { } pub fn copy_to_image(file_copy_params: &[FileCopyToParams], image_file: &Path) -> Result<()> { - // we use the folder the image is located in - // the caller is responsible to create a /tmp/ directory if needed - let working_dir = image_file - .parent() - .context("copy_to_image: cannot get directory of image")? - .to_path_buf(); + // the partition files get their own dir, so they never collide with the image + let working_dir = tempfile::tempdir().context("copy_to_image: cannot create a temp dir")?; let image_file = image_file.to_str().unwrap(); let mut partition_map: HashMap<&Partition, Vec<(&PathBuf, &PathBuf)>> = HashMap::new(); @@ -209,7 +205,7 @@ pub fn copy_to_image(file_copy_params: &[FileCopyToParams], image_file: &Path) - // 1. for each involved partition for partition in partition_map.keys() { - let mut partition_file = working_dir.clone(); + let mut partition_file = working_dir.path().to_path_buf(); let partition_info = get_partition_info(image_file, partition)?; partition_file.push(Path::new(&format!("{}.img", partition_info.num))); @@ -271,16 +267,12 @@ pub fn copy_to_image(file_copy_params: &[FileCopyToParams], image_file: &Path) - } pub fn copy_from_image(file_copy_params: &[FileCopyFromParams], image_file: &Path) -> Result<()> { - // we use the folder the image is located in - // the caller is responsible to create a /tmp/ directory if needed - let working_dir = image_file - .parent() - .context("copy_to_image: cannot get directory of image")? - .to_path_buf(); + // the partition files get their own dir, so they never collide with the image + let working_dir = tempfile::tempdir().context("copy_from_image: cannot create a temp dir")?; let image_file = image_file.to_str().unwrap(); for param in file_copy_params.iter() { - let mut partition_file = working_dir.clone(); + let mut partition_file = working_dir.path().to_path_buf(); let partition_info = get_partition_info(image_file, ¶m.partition)?; let in_file = param.in_file.to_str().unwrap(); @@ -302,7 +294,7 @@ pub fn copy_from_image(file_copy_params: &[FileCopyFromParams], image_file: &Pat // copy if param.partition == Partition::boot { - let mut tmp_out_file = working_dir.clone(); + let mut tmp_out_file = working_dir.path().to_path_buf(); // mcopy deadlocks when target file is not residing in workingdir so we copy to a temp file tmp_out_file.push(format!( "{}-{}", diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 32616a39..d0b358e5 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1618,6 +1618,55 @@ fn check_copy_from_image_writes_no_image() { assert_eq!(Testrunner::file_hash(&image_path_wic_xz), image_hash); } +#[test] +fn check_partition_files_stay_out_of_the_image_dir() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + // the name of the file that holds partition 1 + let image_path = tr.pathbuf().join("1.img"); + fs::rename(tr.to_pathbuf("testfiles/image.wic"), &image_path).expect("rename image"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + let out_file = tr.pathbuf().join("my-file"); + let file_image = |command: &str, files: String| { + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg(command) + .arg("-f") + .arg(files) + .arg("-i") + .arg(&image_path) + .assert() + .success(); + }; + let dir_entries = || { + let mut entries = fs::read_dir(tr.pathbuf()) + .expect("read test dir") + .map(|entry| entry.expect("dir entry").file_name()) + .collect::>(); + entries.sort(); + entries + }; + + file_image( + "copy-to-image", + format!("{},boot:/my-file", in_file.to_string_lossy()), + ); + let entries = dir_entries(); + file_image( + "copy-from-image", + format!("boot:/my-file,{}", out_file.to_string_lossy()), + ); + + assert_eq!( + fs::read(&out_file).expect("read out file"), + fs::read(&in_file).expect("read in file") + ); + let mut expected = entries; + expected.push(out_file.file_name().expect("file name").to_owned()); + expected.sort(); + assert_eq!(dir_entries(), expected); +} + #[tokio::test] async fn check_ssh_tunnel_setup() { let tr = Testrunner::new("check_ssh_tunnel_setup"); From 53672ec37f3da9c5a9b09de792ba9e03d2821075 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:03:57 +0100 Subject: [PATCH 41/75] perf(file): run copy-from-image on an uncompressed image directly copy-from-image only reads the image, and the partition files no longer go next to it, so a temp copy is needed only to decompress the image. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/lib.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 85ab18d9..a6b6eb27 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -142,18 +142,20 @@ fn input_bmap_error(path: &Path) -> String { ) } -/// Runs `command` on a temp copy of the image and writes nothing back. +/// Runs `command` on the image, or on a decompressed temp copy of a compressed image. fn run_read_only_image_command(image_file: PathBuf, command: F) -> Result<()> where F: FnOnce(&PathBuf) -> Result<()>, { ensure_image_exists(&image_file)?; - let tmp_dir = tempfile::tempdir().context("cannot create a temp dir")?; - let source_compression = Compression::from_file(&image_file)?; - let tmp_image_file = temp_image(&image_file, source_compression.as_ref(), tmp_dir.path())?; - - command(&tmp_image_file) + match Compression::from_file(&image_file)? { + Some(c) => { + let tmp_dir = tempfile::tempdir().context("cannot create a temp dir")?; + command(&compression::decompress(&image_file, &c, tmp_dir.path())?) + } + None => command(&image_file), + } } fn ensure_image_exists(image_file: &Path) -> Result<()> { From b328b451d7b5e5c788f10b47e1bbe22beed9fc99 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:04:39 +0100 Subject: [PATCH 42/75] fix(file): read partitions of a GPT with 4096-byte sectors gptman finds a GPT with 512 or 4096-byte sectors, but the partition offsets were always counted in 512-byte sectors. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/partition.rs | 43 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/src/file/partition.rs b/src/file/partition.rs index 7a617fcd..747c8f4f 100644 --- a/src/file/partition.rs +++ b/src/file/partition.rs @@ -31,10 +31,12 @@ pub fn get_partition_data>(path: P, partition_num: u32) -> Result entry.ending_lba >= entry.starting_lba, "GPT partition {partition_num} has invalid LBA range (ending_lba < starting_lba)" ); + // the callers count in SECTOR_SIZE units + let scale = gpt.sector_size / SECTOR_SIZE; return Ok(PartitionData { num: partition_num, - start: entry.starting_lba, - count: entry.ending_lba - entry.starting_lba + 1, + start: entry.starting_lba * scale, + count: (entry.ending_lba - entry.starting_lba + 1) * scale, }); } Err(e) => e, @@ -61,3 +63,40 @@ pub fn is_gpt>(path: P) -> Result { let mut file = File::open(path.as_ref()).context("is_gpt: failed to open image")?; Ok(gptman::GPT::find_from(&mut file).is_ok()) } + +#[cfg(test)] +mod tests { + use crate::file::partition::*; + use gptman::{GPT, GPTPartitionEntry}; + + const GPT_SECTOR_SIZE: u64 = 4096; + const DISK_SECTORS: u64 = 64; + const PART_NUM: u32 = 1; + const PART_SECTORS: u64 = 8; + const GUID: [u8; 16] = [0xff; 16]; + + #[test] + fn get_partition_data_converts_4k_gpt_sectors() { + let dir = tempfile::tempdir().expect("create temp dir"); + let path = dir.path().join("image.wic"); + let mut file = File::create(&path).expect("create image"); + file.set_len(DISK_SECTORS * GPT_SECTOR_SIZE) + .expect("set image size"); + let mut gpt = GPT::new_from(&mut file, GPT_SECTOR_SIZE, GUID).expect("create gpt"); + let start = gpt.header.first_usable_lba; + gpt[PART_NUM] = GPTPartitionEntry { + partition_type_guid: GUID, + unique_partition_guid: GUID, + starting_lba: start, + ending_lba: start + PART_SECTORS - 1, + attribute_bits: 0, + partition_name: "part".into(), + }; + gpt.write_into(&mut file).expect("write gpt"); + + let data = get_partition_data(&path, PART_NUM).expect("partition data"); + let scale = GPT_SECTOR_SIZE / SECTOR_SIZE; + assert_eq!(data.start, start * scale); + assert_eq!(data.count, PART_SECTORS * scale); + } +} From ee181b42b2a605aa7ac72337aa22b1c9860c0423 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:07:35 +0100 Subject: [PATCH 43/75] docs(file): describe the --output-image extension rule and the rename Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 2 +- src/cli.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bc366430..61afc38d 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,7 @@ omnect-cli file copy-to-image --help A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gzip`), e.g. `my-image.wic.xz` for `my-image.wic`. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. Its name must have the extension of the `-p` compression, or no compression extension without `-p`. -The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. The output image is written to a new file in its directory and then renamed, so the directory needs space for one more image. `copy-from-image` only reads the image and writes nothing. +The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. The output image is written to a new file in its directory and then renamed, so the directory needs write access and space for one more image. The new file belongs to the user who runs `omnect-cli`. `copy-from-image` only reads the image and writes nothing. ## bmap files diff --git a/src/cli.rs b/src/cli.rs index 7b824c6f..3c2e77de 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -23,7 +23,7 @@ pub struct ImageOutput { conflicts_with = "input_bmap" )] pub no_input_bmap: bool, - /// optional: write the output image to this path; default: the input image, without the compression extension, plus the --pack-image extension + /// optional: write the output image to this path, with the --pack-image extension or no compression extension; default: the input image, without the compression extension, plus the --pack-image extension #[arg(short = 'o', long = "output-image")] pub output_image: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') From 961f95b16bc50f4ce0017aea77cfbdc81f43adb2 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:42:21 +0100 Subject: [PATCH 44/75] fix(file): keep the name of an image with any extension of the -p compression `-i x.wic.gz -p gzip` wrote `x.wic.gzip`, and `-i x.wic.bzip2 -p bzip2` wrote `x.wic.bz2`, so a second run failed. gzip images now get `.gz`. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 2 +- src/file/compression.rs | 4 ++-- src/file/output.rs | 26 +++++++++++++++++++++++--- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 61afc38d..6d7ef853 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ omnect-cli file copy-to-image --help ## Output image -A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gzip`), e.g. `my-image.wic.xz` for `my-image.wic`. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. Its name must have the extension of the `-p` compression, or no compression extension without `-p`. +A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gz`), e.g. `my-image.wic.xz` for `my-image.wic`. An image that already has an extension of that compression keeps its name. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. Its name must have the extension of the `-p` compression, or no compression extension without `-p`. The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. The output image is written to a new file in its directory and then renamed, so the directory needs write access and space for one more image. The new file belongs to the user who runs `omnect-cli`. `copy-from-image` only reads the image and writes nothing. diff --git a/src/file/compression.rs b/src/file/compression.rs index cf3f173d..6d9b5ed4 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -112,7 +112,7 @@ impl Compression { fn extensions(&self) -> &'static [&'static str] { match &self { Compression::bzip2 => &["bz2", "bzip2"], - Compression::gzip => &["gzip", "gz"], + Compression::gzip => &["gz", "gzip"], Compression::xz { .. } => &["xz"], } } @@ -419,7 +419,7 @@ mod tests { #[test] fn compressed_path_has_an_extension_bmaptool_reads() { let image = Path::new("image.wic"); - let expected = ["image.wic.bz2", "image.wic.gzip", "image.wic.xz"]; + let expected = ["image.wic.bz2", "image.wic.gz", "image.wic.xz"]; for (compression, expected) in all_compressions().iter().zip(expected) { assert_eq!(compression.compressed_path(image), Path::new(expected)); diff --git a/src/file/output.rs b/src/file/output.rs index f1349b7c..389b3601 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -7,9 +7,9 @@ use std::{ const TMP_DIR_PREFIX: &str = ".omnect-cli-"; -/// Returns `explicit`, else `image` without the extension of its `source` compression, with the -/// extension of `target` appended. Fails if `explicit` has no extension of `target`, or a -/// compression extension while `target` is none. +/// Returns `explicit`, else `image` if it has an extension of `target`, else `image` without the +/// extension of its `source` compression, with the extension of `target` appended. Fails if +/// `explicit` has no extension of `target`, or a compression extension while `target` is none. pub fn output_image( image: &Path, source: Option<&Compression>, @@ -32,6 +32,9 @@ pub fn output_image( } return Ok(path.to_path_buf()); } + if target.is_some_and(|c| c.matches_extension(image)) { + return Ok(image.to_path_buf()); + } let mut output = image.to_path_buf(); if source.is_some_and(|c| c.matches_extension(image)) { @@ -172,6 +175,23 @@ mod tests { assert_eq!(output(image_xz, Some(&xz), Some(&xz)), image_xz); } + #[test] + fn output_image_keeps_every_extension_of_the_same_compression() { + for (image, compression) in [ + ("image.wic.gz", Compression::gzip), + ("image.wic.gzip", Compression::gzip), + ("image.wic.bz2", Compression::bzip2), + ("image.wic.bzip2", Compression::bzip2), + ] { + let image = Path::new(image); + assert_eq!( + output_image(image, Some(&compression), Some(&compression), None) + .expect("output image"), + image + ); + } + } + #[test] fn output_image_strips_only_the_source_compression_extension() { let image = Path::new("image.bin"); From 3504b5a9e50eace9faaa86e0a1b71f81051f8417 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:43:26 +0100 Subject: [PATCH 45/75] fix(file): find the input bmap the way bmaptool does bmaptool removes one extension of the image at a time until it finds a bmap, so it also uses e.g. `x.bmap` for `x.wic`. The input bmap and the stale bmap check now follow the same rule. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 2 +- src/cli.rs | 2 +- src/file/bmap.rs | 35 +++++++++++++++++++++++++++-------- src/file/output.rs | 15 ++++++++++++--- 4 files changed, 41 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 6d7ef853..a6ebc073 100644 --- a/README.md +++ b/README.md @@ -193,7 +193,7 @@ The command fails instead of overwriting an existing file that is not the input Commands that change an image create a bmap file with `-b` (`--output-bmap`). It is written next to the output image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic` and `my-image.wic.xz.bmap` for `my-image.wic.xz`, and replaces a file with this name. `bmaptool` finds it there without `--bmap`. The command fails if this file is the bmap of an image that stays, e.g. `my-image.wic.bmap` when `-i my-image.wic.xz` writes `my-image.wic`, also with `--input-bmap` or `--no-input-bmap`. Choose another output with `-o` then. -The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image and sha256 range checksums that match it, otherwise the command fails. A command without `-b` fails if `.bmap` exists, or for a compressed output image the bmap of the uncompressed image (e.g. `my-image.wic.bmap` for `my-image.wic.xz`), because `bmaptool` would use it and it would not match the image. So when you run several commands on one image, use `-b` on each of them. `--no-input-bmap` skips a bmap next to the image that does not match, and the created bmap then maps the whole image. The created bmap maps every range of the input bmap, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). It also maps every block that the image file stores as data, which includes the blocks `omnect-cli` changed. The image file itself stays sparse. +The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else the bmap next to the image that `bmaptool` would use: `.bmap`, else the same with one extension of the image removed at a time, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image and sha256 range checksums that match it, otherwise the command fails. A command without `-b` fails if such a bmap exists for the output image (e.g. `my-image.wic.bmap` for `my-image.wic.xz`), because `bmaptool` would use it and it would not match the image. So when you run several commands on one image, use `-b` on each of them. `--no-input-bmap` skips a bmap next to the image that does not match, and the created bmap then maps the whole image. The created bmap maps every range of the input bmap, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). It also maps every block that the image file stores as data, which includes the blocks `omnect-cli` changed. The image file itself stays sparse. Without an input bmap, the created bmap maps the whole image, and a flash writes every block. diff --git a/src/cli.rs b/src/cli.rs index 3c2e77de..dd20908c 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -13,7 +13,7 @@ pub struct ImageOutput { /// optional: create a bmap of the output image #[arg(short = 'b', long = "output-bmap")] pub output_bmap: bool, - /// optional: bmap of the input image; default: .bmap or .bmap next to the image + /// optional: bmap of the input image; default: the bmap next to the image that bmaptool would use #[arg(long = "input-bmap", requires = "output_bmap")] pub input_bmap: Option, /// optional: do not use a bmap next to the image, so the bmap maps the whole image diff --git a/src/file/bmap.rs b/src/file/bmap.rs index 96b13b5d..aacc6dfc 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -1,4 +1,4 @@ -use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, compression::Compression, sparse}; +use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, sparse}; use anyhow::{Context, Result}; use sha2::{Digest, Sha256}; use std::{ @@ -90,8 +90,8 @@ fn parse_range(text: &str) -> Result> { Ok(first..=last) } -/// Returns `explicit`, else `.bmap`, else `.bmap`, -/// whichever exists first. +/// Returns `explicit`, else the bmap `bmaptool` uses for `image`: `.bmap`, else the same +/// with one extension of `image` removed at a time, whichever exists first. pub fn find_input_bmap(image: &Path, explicit: Option<&Path>) -> Result> { if let Some(path) = explicit { anyhow::ensure!( @@ -102,12 +102,17 @@ pub fn find_input_bmap(image: &Path, explicit: Option<&Path>) -> Result PathBuf { @@ -420,6 +425,20 @@ mod tests { ); } + #[test] + fn find_input_bmap_strips_extensions_like_bmaptool() { + let dir = tempfile::tempdir().expect("create temp dir"); + let bmap = dir.path().join("image.bmap"); + fs::write(&bmap, "").expect("write bmap"); + + for image in ["image.wic", "image.wic.xz"] { + assert_eq!( + find_input_bmap(&dir.path().join(image), None).expect("find bmap"), + Some(bmap.clone()) + ); + } + } + #[test] fn write_bmap_maps_zero_ranges_of_the_input_bmap() { let dir = tempfile::tempdir().expect("create temp dir"); diff --git a/src/file/output.rs b/src/file/output.rs index 389b3601..fc213713 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -47,9 +47,8 @@ pub fn output_image( } /// Fails if writing `output_image` and its bmap would overwrite a file the user did not name, -/// replace the bmap of an image that stays, or leave a bmap next to the output image that does -/// not match it: `.bmap`, or for a compressed output image the bmap of the -/// uncompressed image, which `bmaptool` also uses. +/// replace the bmap of an image that stays, or leave a bmap that `bmaptool` would use for the +/// output image and that does not match it. pub fn check_conflicts( input_image: &Path, output_image: &Path, @@ -310,6 +309,16 @@ mod tests { assert!(check_conflicts(&image, &image, false, false, None).is_err()); } + #[test] + fn check_conflicts_rejects_a_stale_bmap_bmaptool_finds() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + touch(&image); + touch(&dir.path().join("image.bmap")); + + assert!(check_conflicts(&image, &image, false, false, None).is_err()); + } + #[test] fn check_conflicts_rejects_a_stale_bmap_of_the_uncompressed_image() { let dir = tempfile::tempdir().expect("create temp dir"); From 3da5a4d43dc48d03aa5b02b82a5a5251b4947070 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:46:08 +0100 Subject: [PATCH 46/75] fix(file): sync the output image and remove the old bmap before the rename The output image and its bmap are now staged first: written, synced on the handle that wrote them, and given their permissions. Then the old bmap is removed, the image and the bmap are renamed, and their directories are synced. So a power loss cannot leave an empty or partial image, and a failed bmap rename leaves no bmap that does not match the new image. A compressed image is written into its staged file directly, without a copy in TMPDIR. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/compression.rs | 19 +++-- src/file/output.rs | 167 ++++++++++++++++++++++++++++++++-------- src/file/sparse.rs | 11 ++- src/lib.rs | 56 ++++++-------- 4 files changed, 182 insertions(+), 71 deletions(-) diff --git a/src/file/compression.rs b/src/file/compression.rs index 6d9b5ed4..d15c1c30 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -239,14 +239,19 @@ pub fn decompress( Ok(new_image_file) } -pub fn compress(image_file_name: &PathBuf, compression: &Compression) -> Result { - let new_image_file = compression.compressed_path(image_file_name); - let mut destination = File::create(&new_image_file)?; - let mut source = File::open(image_file_name)?; - debug!("compress {image_file_name:?} to {new_image_file:?}"); - let bytes_written = compression.compress(&mut source, &mut destination)?; +pub fn compress( + image_file_name: &Path, + compression: &Compression, + destination: &mut File, +) -> Result<()> { + let mut source = File::open(image_file_name).context(format!( + "compress: cannot open {}", + image_file_name.display() + ))?; + debug!("compress {image_file_name:?}"); + let bytes_written = compression.compress(&mut source, destination)?; debug!("image::compress: copied {} bytes.", bytes_written); - Ok(new_image_file) + Ok(()) } #[cfg(test)] diff --git a/src/file/output.rs b/src/file/output.rs index fc213713..d52dd02a 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -1,9 +1,14 @@ use crate::file::{bmap, compression::Compression}; use anyhow::{Context, Result}; +use log::warn; use std::{ - fs, + collections::BTreeSet, + fs::{self, File}, + io::ErrorKind, + iter, path::{Path, PathBuf}, }; +use tempfile::TempDir; const TMP_DIR_PREFIX: &str = ".omnect-cli-"; @@ -88,32 +93,89 @@ pub fn check_conflicts( Ok(()) } -/// Writes `dest` as a new file in its directory and renames it, so `dest` stays unchanged if -/// `write` fails. A symlink is followed, and an existing `dest` keeps its permissions. -pub fn replace(dest: &Path, write: impl FnOnce(&Path) -> Result<()>) -> Result<()> { - let dest = resolve(dest)?; - let dir = dest - .parent() - .context(format!("{} has no directory", dest.display()))?; - let tmp_dir = tempfile::Builder::new() - .prefix(TMP_DIR_PREFIX) - .tempdir_in(dir) - .context(format!("cannot create a temp dir in {}", dir.display()))?; - let tmp_file = tmp_dir.path().join( - dest.file_name() - .context(format!("{} has no file name", dest.display()))?, - ); +/// A new file in the directory of its destination, which `commit` renames to the destination. +/// Until then the destination stays unchanged. +pub struct Staged { + _tmp_dir: TempDir, + file: PathBuf, + dest: PathBuf, +} + +impl Staged { + /// Writes and syncs the new file for `dest`. A symlink is followed, and an existing `dest` + /// keeps its permissions. + pub fn write(dest: &Path, write: impl FnOnce(&mut File) -> Result<()>) -> Result { + let dest = resolve(dest)?; + let dir = dest + .parent() + .context(format!("{} has no directory", dest.display()))?; + let tmp_dir = tempfile::Builder::new() + .prefix(TMP_DIR_PREFIX) + .tempdir_in(dir) + .context(format!("cannot create a temp dir in {}", dir.display()))?; + let file = tmp_dir.path().join( + dest.file_name() + .context(format!("{} has no file name", dest.display()))?, + ); + + let mut handle = + File::create(&file).context(format!("cannot create {}", file.display()))?; + write(&mut handle)?; + if let Ok(metadata) = fs::metadata(&dest) { + handle + .set_permissions(metadata.permissions()) + .context(format!("cannot set permissions of {}", file.display()))?; + } + // the handle that wrote the data also gets the write errors + handle + .sync_all() + .context(format!("cannot sync {}", file.display()))?; + + Ok(Self { + _tmp_dir: tmp_dir, + file, + dest, + }) + } + + fn rename(&self) -> Result<()> { + fs::rename(&self.file, &self.dest).context(format!( + "cannot rename {} to {}", + self.file.display(), + self.dest.display() + )) + } +} - write(&tmp_file)?; - if let Ok(metadata) = fs::metadata(&dest) { - fs::set_permissions(&tmp_file, metadata.permissions()) - .context(format!("cannot set permissions of {}", tmp_file.display()))?; +/// Renames the staged `image` and its `bmap`, then syncs their directories, so the renames are +/// not lost on a power loss. The old bmap is removed before the image is renamed, so if renaming +/// the bmap fails, no bmap is left that does not match the new image. +pub fn commit(image: Staged, bmap: Option) -> Result<()> { + if let Some(bmap) = &bmap { + match fs::remove_file(&bmap.dest) { + Err(e) if e.kind() != ErrorKind::NotFound => { + return Err(e).context(format!("cannot remove {}", bmap.dest.display())); + } + _ => {} + } + } + image.rename()?; + if let Some(bmap) = &bmap { + bmap.rename()?; + } + + let dirs: BTreeSet<_> = iter::once(&image) + .chain(&bmap) + .filter_map(|staged| staged.dest.parent()) + .collect(); + for dir in dirs { + // both files are complete already, only a power loss could undo the rename + if let Err(e) = File::open(dir).and_then(|dir| dir.sync_all()) { + warn!("cannot sync {}: {e}", dir.display()); + } } - fs::rename(&tmp_file, &dest).context(format!( - "cannot rename {} to {}", - tmp_file.display(), - dest.display() - )) + + Ok(()) } fn same_file(a: &Path, b: &Path) -> Result { @@ -142,7 +204,10 @@ fn resolve(path: &Path) -> Result { #[cfg(test)] mod tests { use crate::file::output::*; - use std::os::unix::fs::{PermissionsExt, symlink}; + use std::{ + io::Write, + os::unix::fs::{PermissionsExt, symlink}, + }; const IMAGE: &str = "image.wic"; const IMAGE_XZ: &str = "image.wic.xz"; @@ -347,15 +412,19 @@ mod tests { assert!(check_conflicts(&image_xz, &image, false, true, None).is_err()); } + fn stage(dest: &Path, content: &'static [u8]) -> Staged { + Staged::write(dest, |file| Ok(file.write_all(content)?)).expect("stage") + } + #[test] - fn replace_keeps_dest_when_write_fails() { + fn staged_write_keeps_dest_when_write_fails() { let dir = tempfile::tempdir().expect("create temp dir"); let image = dir.path().join(IMAGE); fs::write(&image, OLD_CONTENT).expect("write image"); assert!( - replace(&image, |tmp| { - fs::write(tmp, NEW_CONTENT).expect("write tmp"); + Staged::write(&image, |file| { + file.write_all(NEW_CONTENT).expect("write file"); anyhow::bail!("write failed") }) .is_err() @@ -369,7 +438,7 @@ mod tests { } #[test] - fn replace_keeps_permissions_and_symlink() { + fn staged_commit_keeps_permissions_and_symlink() { let dir = tempfile::tempdir().expect("create temp dir"); let image = dir.path().join(IMAGE); let link = dir.path().join(OTHER_IMAGE); @@ -378,7 +447,7 @@ mod tests { .expect("set permissions"); symlink(&image, &link).expect("create symlink"); - replace(&link, |tmp| Ok(fs::write(tmp, NEW_CONTENT)?)).expect("replace"); + commit(stage(&link, NEW_CONTENT), None).expect("commit"); assert!(link.is_symlink()); assert_eq!(fs::read(&image).expect("read image"), NEW_CONTENT); @@ -388,6 +457,42 @@ mod tests { ); } + #[test] + fn commit_leaves_no_old_bmap_when_the_bmap_rename_fails() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let image_bmap = bmap::bmap_path(&image); + fs::write(&image, OLD_CONTENT).expect("write image"); + fs::write(&image_bmap, OLD_CONTENT).expect("write bmap"); + + let staged_bmap = stage(&image_bmap, NEW_CONTENT); + fs::remove_file(&staged_bmap.file).expect("remove staged bmap"); + + assert!(commit(stage(&image, NEW_CONTENT), Some(staged_bmap)).is_err()); + assert_eq!(fs::read(&image).expect("read image"), NEW_CONTENT); + assert!(!image_bmap.exists()); + } + + #[test] + fn commit_replaces_the_target_of_a_symlinked_bmap() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let image_bmap = bmap::bmap_path(&image); + let target_bmap = dir.path().join("target.bmap"); + fs::write(&image, OLD_CONTENT).expect("write image"); + fs::write(&target_bmap, OLD_CONTENT).expect("write bmap"); + symlink(&target_bmap, &image_bmap).expect("create symlink"); + + commit( + stage(&image, NEW_CONTENT), + Some(stage(&image_bmap, NEW_CONTENT)), + ) + .expect("commit"); + + assert!(image_bmap.is_symlink()); + assert_eq!(fs::read(&target_bmap).expect("read bmap"), NEW_CONTENT); + } + #[test] fn same_file_resolves_relative_and_missing_paths() { let dir = tempfile::tempdir().expect("create temp dir"); diff --git a/src/file/sparse.rs b/src/file/sparse.rs index 3f5a4005..7953b8e1 100644 --- a/src/file/sparse.rs +++ b/src/file/sparse.rs @@ -87,10 +87,17 @@ pub fn copy_range( pub fn copy_to_new_file(src: &File, src_offset: u64, dst: &Path, len: u64) -> Result<()> { let dst = File::create(dst).context(format!("copy_to_new_file: cannot create {}", dst.display()))?; + + copy_to_empty_file(src, src_offset, &dst, len) +} + +/// Sets the size of the empty `dst` to `len` and copies `len` bytes of `src` at `src_offset` +/// into it. Holes of `src` stay holes in `dst`. +pub fn copy_to_empty_file(src: &File, src_offset: u64, dst: &File, len: u64) -> Result<()> { dst.set_len(len) - .context("copy_to_new_file: cannot set destination size")?; + .context("copy_to_empty_file: cannot set destination size")?; - copy_range(src, src_offset, &dst, 0, len) + copy_range(src, src_offset, dst, 0, len) } #[cfg(test)] diff --git a/src/lib.rs b/src/lib.rs index a6b6eb27..95ba6e53 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,12 +25,12 @@ use file::{ bmap::{self, Bmap}, compression::{self, Compression}, functions::FileCopyToParams, - output::{check_conflicts, output_image, replace}, + output::{self, Staged, check_conflicts, output_image}, sparse, }; use log::info; use std::{ - fs, + fs, io, path::{Path, PathBuf}, }; @@ -86,7 +86,7 @@ where verify_input_bmap(&image_file)?; } let tmp_dir = tempfile::tempdir().context("cannot create a temp dir")?; - let mut tmp_image_file = temp_image(&image_file, source_compression.as_ref(), tmp_dir.path())?; + let tmp_image_file = temp_image(&image_file, source_compression.as_ref(), tmp_dir.path())?; if source_compression.is_some() { verify_input_bmap(&tmp_image_file)?; } @@ -107,31 +107,23 @@ where )?; } - // if applicable compress image - if let Some(c) = target_compression { - tmp_image_file = compression::compress(&tmp_image_file, &c)?; - replace(&dest_image_file, |dest| { - fs::copy(&tmp_image_file, dest) - .map(drop) - .context(format!("cannot copy {tmp_image_file:?} to {dest:?}")) - })?; - } else { - replace(&dest_image_file, |dest| { - copy_image(&tmp_image_file, dest) - .context(format!("cannot copy {tmp_image_file:?} to {dest:?}")) - })?; - } - - // copy the bmap last, so it never describes an image that was not written back - if output_bmap { - replace(&bmap::bmap_path(&dest_image_file), |dest| { - fs::copy(&tmp_bmap, dest) - .map(drop) - .context(format!("cannot copy {tmp_bmap:?} to {dest:?}")) - })?; - } + let image = Staged::write(&dest_image_file, |dest| match &target_compression { + Some(c) => compression::compress(&tmp_image_file, c, dest), + None => copy_image(&tmp_image_file, dest), + })?; + let bmap = output_bmap + .then(|| { + Staged::write(&bmap::bmap_path(&dest_image_file), |dest| { + let mut src = fs::File::open(&tmp_bmap) + .context(format!("cannot open {}", tmp_bmap.display()))?; + io::copy(&mut src, dest) + .map(drop) + .context(format!("cannot copy {}", tmp_bmap.display())) + }) + }) + .transpose()?; - Ok(()) + output::commit(image, bmap) } fn input_bmap_error(path: &Path) -> String { @@ -181,7 +173,9 @@ fn temp_image( .file_name() .context("cannot get image file name")?, ); - copy_image(image_file, &tmp_image_file).context(format!( + let tmp_image = fs::File::create(&tmp_image_file) + .context(format!("cannot create {}", tmp_image_file.display()))?; + copy_image(image_file, &tmp_image).context(format!( "error: copy_image({:?}, {:?})", image_file, tmp_image_file ))?; @@ -190,15 +184,15 @@ fn temp_image( } } -/// Copies `from` to `to` and keeps the holes of a sparse `from`. -fn copy_image(from: &Path, to: &Path) -> Result<()> { +/// Copies `from` to the empty `to` and keeps the holes of a sparse `from`. +fn copy_image(from: &Path, to: &fs::File) -> Result<()> { let src = fs::File::open(from).context("copy_image: cannot open source")?; let len = src .metadata() .context("copy_image: cannot get source size")? .len(); - sparse::copy_to_new_file(&src, 0, to, len) + sparse::copy_to_empty_file(&src, 0, to, len) } struct CertInfo { From 039b8617b5e7f8179609933fe0a85af9b49592d2 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:40:01 +0100 Subject: [PATCH 47/75] fix(file): copy into an output image that is a mount point rename(2) to a mount point fails with EBUSY, so an in-place command on an image file mounted into a container failed. If the kernel reports the image as a mount root, its new content is staged in TMPDIR, because the directory may not be writable then, and copied into the image. A rename that fails with EBUSY or EXDEV also falls back to the copy. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 2 +- src/file/output.rs | 105 ++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 95 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index a6ebc073..7bc99000 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,7 @@ omnect-cli file copy-to-image --help A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gz`), e.g. `my-image.wic.xz` for `my-image.wic`. An image that already has an extension of that compression keeps its name. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. Its name must have the extension of the `-p` compression, or no compression extension without `-p`. -The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. The output image is written to a new file in its directory and then renamed, so the directory needs write access and space for one more image. The new file belongs to the user who runs `omnect-cli`. `copy-from-image` only reads the image and writes nothing. +The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. The output image is written to a new file in its directory and then renamed, so the directory needs write access and space for one more image. An output image that is a mount point, e.g. with `docker run -v "$(pwd)/my-image.wic":/source/my-image.wic`, cannot be renamed to, so the new image is copied into it. Mount the directory instead of the image if you use `-b`, because the bmap is written next to the image. The new file belongs to the user who runs `omnect-cli`. `copy-from-image` only reads the image and writes nothing. ## bmap files diff --git a/src/file/output.rs b/src/file/output.rs index d52dd02a..5e188226 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -1,9 +1,11 @@ -use crate::file::{bmap, compression::Compression}; +use crate::file::{bmap, compression::Compression, sparse}; use anyhow::{Context, Result}; -use log::warn; +use log::{info, warn}; +use rustix::fs::{AtFlags, CWD, StatxAttributes, StatxFlags}; use std::{ collections::BTreeSet, - fs::{self, File}, + env, + fs::{self, File, OpenOptions}, io::ErrorKind, iter, path::{Path, PathBuf}, @@ -94,7 +96,9 @@ pub fn check_conflicts( } /// A new file in the directory of its destination, which `commit` renames to the destination. -/// Until then the destination stays unchanged. +/// Until then the destination stays unchanged. A destination that is a mount point, e.g. an image +/// file mounted into a container, cannot be renamed to, so its new file is staged in TMPDIR and +/// copied into it. pub struct Staged { _tmp_dir: TempDir, file: PathBuf, @@ -109,10 +113,18 @@ impl Staged { let dir = dest .parent() .context(format!("{} has no directory", dest.display()))?; + let tmp_parent = if is_mount_root(&dest) { + env::temp_dir() + } else { + dir.to_path_buf() + }; let tmp_dir = tempfile::Builder::new() .prefix(TMP_DIR_PREFIX) - .tempdir_in(dir) - .context(format!("cannot create a temp dir in {}", dir.display()))?; + .tempdir_in(&tmp_parent) + .context(format!( + "cannot create a temp dir in {}", + tmp_parent.display() + ))?; let file = tmp_dir.path().join( dest.file_name() .context(format!("{} has no file name", dest.display()))?, @@ -139,14 +151,62 @@ impl Staged { } fn rename(&self) -> Result<()> { - fs::rename(&self.file, &self.dest).context(format!( - "cannot rename {} to {}", - self.file.display(), - self.dest.display() - )) + match fs::rename(&self.file, &self.dest) { + Err(e) + if matches!( + e.kind(), + ErrorKind::ResourceBusy | ErrorKind::CrossesDevices + ) => + { + info!( + "cannot rename to {} ({e}), so copy into it", + self.dest.display() + ); + self.copy_into_dest() + } + result => result.context(format!( + "cannot rename {} to {}", + self.file.display(), + self.dest.display() + )), + } + } + + fn copy_into_dest(&self) -> Result<()> { + let src = File::open(&self.file).context(format!("cannot open {}", self.file.display()))?; + let len = src + .metadata() + .context(format!("cannot get the size of {}", self.file.display()))? + .len(); + let dest = OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&self.dest) + .context(format!("cannot open {}", self.dest.display()))?; + + sparse::copy_to_empty_file(&src, 0, &dest, len)?; + dest.sync_all() + .context(format!("cannot sync {}", self.dest.display())) } } +/// `false` if the kernel cannot tell. +fn is_mount_root(path: &Path) -> bool { + rustix::fs::statx( + CWD, + path, + AtFlags::SYMLINK_NOFOLLOW | AtFlags::NO_AUTOMOUNT, + StatxFlags::empty(), + ) + .is_ok_and(|statx| { + statx + .stx_attributes_mask + .contains(StatxAttributes::MOUNT_ROOT) + && statx.stx_attributes.contains(StatxAttributes::MOUNT_ROOT) + }) +} + /// Renames the staged `image` and its `bmap`, then syncs their directories, so the renames are /// not lost on a power loss. The old bmap is removed before the image is renamed, so if renaming /// the bmap fails, no bmap is left that does not match the new image. @@ -473,6 +533,29 @@ mod tests { assert!(!image_bmap.exists()); } + #[test] + fn copy_into_dest_replaces_the_content() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + fs::write(&image, b"old image with more bytes").expect("write image"); + + stage(&image, NEW_CONTENT) + .copy_into_dest() + .expect("copy into dest"); + + assert_eq!(fs::read(&image).expect("read image"), NEW_CONTENT); + } + + #[test] + fn is_mount_root_detects_the_root_dir() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + touch(&image); + + assert!(is_mount_root(Path::new("/"))); + assert!(!is_mount_root(&image)); + } + #[test] fn commit_replaces_the_target_of_a_symlinked_bmap() { let dir = tempfile::tempdir().expect("create temp dir"); From 6c599be9046904930ef546a2a7762c4e05c2d358 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:40:40 +0100 Subject: [PATCH 48/75] fix(file): keep a read-only output image unchanged A rename needs write access only to the directory, so a read-only image was replaced without an error. The command now fails before it starts if the output image or its bmap exists and is not writable. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 2 +- src/file/output.rs | 42 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7bc99000..1e4d03a3 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,7 @@ omnect-cli file copy-to-image --help A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gz`), e.g. `my-image.wic.xz` for `my-image.wic`. An image that already has an extension of that compression keeps its name. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. Its name must have the extension of the `-p` compression, or no compression extension without `-p`. -The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. The output image is written to a new file in its directory and then renamed, so the directory needs write access and space for one more image. An output image that is a mount point, e.g. with `docker run -v "$(pwd)/my-image.wic":/source/my-image.wic`, cannot be renamed to, so the new image is copied into it. Mount the directory instead of the image if you use `-b`, because the bmap is written next to the image. The new file belongs to the user who runs `omnect-cli`. `copy-from-image` only reads the image and writes nothing. +The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. It also fails if the output image or its bmap exists and is not writable for the user. The output image is written to a new file in its directory and then renamed, so the directory needs write access and space for one more image. An output image that is a mount point, e.g. with `docker run -v "$(pwd)/my-image.wic":/source/my-image.wic`, cannot be renamed to, so the new image is copied into it. Mount the directory instead of the image if you use `-b`, because the bmap is written next to the image. The new file belongs to the user who runs `omnect-cli`. `copy-from-image` only reads the image and writes nothing. ## bmap files diff --git a/src/file/output.rs b/src/file/output.rs index 5e188226..0e1c58ea 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -1,7 +1,10 @@ use crate::file::{bmap, compression::Compression, sparse}; use anyhow::{Context, Result}; use log::{info, warn}; -use rustix::fs::{AtFlags, CWD, StatxAttributes, StatxFlags}; +use rustix::{ + fs::{Access, AtFlags, CWD, StatxAttributes, StatxFlags}, + io::Errno, +}; use std::{ collections::BTreeSet, env, @@ -64,6 +67,7 @@ pub fn check_conflicts( input_bmap: Option<&Path>, ) -> Result<()> { let in_place = same_file(input_image, output_image)?; + ensure_writable(output_image)?; anyhow::ensure!( explicit_output || in_place || !output_image.exists(), @@ -72,6 +76,7 @@ pub fn check_conflicts( ); if output_bmap { let target_bmap = bmap::bmap_path(output_image); + ensure_writable(&target_bmap)?; if !in_place { let found_bmap = bmap::find_input_bmap(input_image, None)?; for bmap in input_bmap.into_iter().chain(found_bmap.as_deref()) { @@ -95,6 +100,19 @@ pub fn check_conflicts( Ok(()) } +/// Fails if `path` exists and the user cannot write it. Replacing it by a rename would not need +/// that. +fn ensure_writable(path: &Path) -> Result<()> { + match rustix::fs::access(path, Access::WRITE_OK) { + Err(Errno::NOENT) => Ok(()), + result => result.context(format!( + "cannot write {}; make it writable or choose another output image with \ + --output-image", + path.display() + )), + } +} + /// A new file in the directory of its destination, which `commit` renames to the destination. /// Until then the destination stays unchanged. A destination that is a mount point, e.g. an image /// file mounted into a container, cannot be renamed to, so its new file is staged in TMPDIR and @@ -276,6 +294,7 @@ mod tests { const OLD_CONTENT: &[u8] = b"old image"; const NEW_CONTENT: &[u8] = b"new image"; const IMAGE_MODE: u32 = 0o644; + const READ_ONLY_MODE: u32 = 0o444; const MODE_MASK: u32 = 0o777; fn touch(path: &Path) { @@ -398,6 +417,27 @@ mod tests { check_conflicts(&image, &image, false, true, Some(&image_bmap)).expect("no conflict"); } + #[test] + fn check_conflicts_rejects_a_read_only_output() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let image_bmap = bmap::bmap_path(&image); + touch(&image); + touch(&image_bmap); + let read_only = fs::Permissions::from_mode(READ_ONLY_MODE); + fs::set_permissions(&image_bmap, read_only.clone()).expect("set permissions"); + // root can write a read-only file + if OpenOptions::new().write(true).open(&image_bmap).is_ok() { + return; + } + + assert!(check_conflicts(&image, &image, false, true, Some(&image_bmap)).is_err()); + fs::remove_file(&image_bmap).expect("remove bmap"); + check_conflicts(&image, &image, false, false, None).expect("writable image"); + fs::set_permissions(&image, read_only).expect("set permissions"); + assert!(check_conflicts(&image, &image, false, false, None).is_err()); + } + #[test] fn check_conflicts_rejects_existing_derived_image() { let dir = tempfile::tempdir().expect("create temp dir"); From 8d674335745006018383ed7098f46a2fb8a40551 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:41:14 +0100 Subject: [PATCH 49/75] refactor(file): write the output bmap into its staged file directly write_bmap writes into any Write, so the bmap no longer goes through a temp file, and nothing is done for it without -b. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/bmap.rs | 14 ++++++++------ src/lib.rs | 29 ++++++++++------------------- 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/src/file/bmap.rs b/src/file/bmap.rs index aacc6dfc..c4f1c4fa 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -3,6 +3,7 @@ use anyhow::{Context, Result}; use sha2::{Digest, Sha256}; use std::{ fs, + io::Write, ops::{Range, RangeInclusive}, os::unix::fs::FileExt, path::{Path, PathBuf}, @@ -153,11 +154,11 @@ pub fn verify_checksums(image_file: &Path, bmap: &Bmap) -> Result<()> { Ok(()) } -/// Writes a bmap of `image_file` to `bmap_file`. It maps the ranges of `input`, or the whole +/// Writes a bmap of `image_file` to `out`. It maps the ranges of `input`, or the whole /// image without one, and every block that the image file stores as data. A range of `input` /// can be all zeros and a hole in the image, and a bmap flash still has to write it, else the /// device keeps what an earlier installation left there. -pub fn write_bmap(image_file: &Path, input: Option<&Bmap>, bmap_file: &Path) -> Result<()> { +pub fn write_bmap(image_file: &Path, input: Option<&Bmap>, out: &mut impl Write) -> Result<()> { let image = fs::File::open(image_file) .context(format!("write_bmap: cannot open {}", image_file.display()))?; let image_len = image @@ -198,7 +199,8 @@ pub fn write_bmap(image_file: &Path, input: Option<&Bmap>, bmap_file: &Path) -> ); let xml = xml.replacen(&zeros, &hex(&Sha256::digest(xml.as_bytes())), 1); - fs::write(bmap_file, xml).context(format!("write_bmap: cannot write {}", bmap_file.display())) + out.write_all(xml.as_bytes()) + .context("write_bmap: cannot write the bmap") } fn mapped_ranges( @@ -317,7 +319,8 @@ mod tests { fn created_bmap(image: &Path, input: Option<&Bmap>) -> Bmap { let path = image.with_extension(BMAP_EXTENSION); - write_bmap(image, input, &path).expect("write bmap"); + let mut file = fs::File::create(&path).expect("create bmap"); + write_bmap(image, input, &mut file).expect("write bmap"); Bmap::read(&path).expect("read bmap") } @@ -544,7 +547,6 @@ mod tests { fn bmap_of_another_image_is_rejected() { let dir = tempfile::tempdir().expect("create temp dir"); let image = sparse_image(dir.path()); - let bmap_file = dir.path().join("out.bmap"); let mut other_size = created_bmap(&image, None); other_size.image_size += BLOCK_SIZE; let outside = Bmap { @@ -556,7 +558,7 @@ mod tests { for bmap in [&other_size, &outside] { assert!(verify_checksums(&image, bmap).is_err()); - assert!(write_bmap(&image, Some(bmap), &bmap_file).is_err()); + assert!(write_bmap(&image, Some(bmap), &mut Vec::new()).is_err()); } } } diff --git a/src/lib.rs b/src/lib.rs index 95ba6e53..22da82ce 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -30,7 +30,7 @@ use file::{ }; use log::info; use std::{ - fs, io, + fs, path::{Path, PathBuf}, }; @@ -94,31 +94,22 @@ where // run command command(&tmp_image_file)?; - let tmp_bmap = bmap::bmap_path(&tmp_image_file); - if output_bmap { - match &input_bmap { - Some((path, _)) => info!("the bmap maps the ranges of {}", path.display()), - None => info!("no input bmap, so the bmap maps the whole image"), - } - bmap::write_bmap( - &tmp_image_file, - input_bmap.as_ref().map(|(_, bmap)| bmap), - &tmp_bmap, - )?; - } - let image = Staged::write(&dest_image_file, |dest| match &target_compression { Some(c) => compression::compress(&tmp_image_file, c, dest), None => copy_image(&tmp_image_file, dest), })?; let bmap = output_bmap .then(|| { + match &input_bmap { + Some((path, _)) => info!("the bmap maps the ranges of {}", path.display()), + None => info!("no input bmap, so the bmap maps the whole image"), + } Staged::write(&bmap::bmap_path(&dest_image_file), |dest| { - let mut src = fs::File::open(&tmp_bmap) - .context(format!("cannot open {}", tmp_bmap.display()))?; - io::copy(&mut src, dest) - .map(drop) - .context(format!("cannot copy {}", tmp_bmap.display())) + bmap::write_bmap( + &tmp_image_file, + input_bmap.as_ref().map(|(_, bmap)| bmap), + dest, + ) }) }) .transpose()?; From 9126d556c8673c50de42e321334efc0e60091850 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:41:31 +0100 Subject: [PATCH 50/75] refactor(file): use Path::with_added_extension for bmap and compressed paths Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/bmap.rs | 4 +--- src/file/compression.rs | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/file/bmap.rs b/src/file/bmap.rs index c4f1c4fa..83841d1a 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -117,9 +117,7 @@ pub fn find_input_bmap(image: &Path, explicit: Option<&Path>) -> Result PathBuf { - let mut path = image.as_os_str().to_owned(); - path.push(format!(".{BMAP_EXTENSION}")); - PathBuf::from(path) + image.with_added_extension(BMAP_EXTENSION) } /// Fails if a range of `image_file` does not match its checksum in `bmap`, so the bmap of diff --git a/src/file/compression.rs b/src/file/compression.rs index d15c1c30..8efac234 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -119,9 +119,7 @@ impl Compression { /// `path` with the extension omnect-cli gives a compressed image appended. pub fn compressed_path(&self, path: &Path) -> PathBuf { - let mut compressed = path.as_os_str().to_owned(); - compressed.push(format!(".{}", self.extension())); - PathBuf::from(compressed) + path.with_added_extension(self.extension()) } pub fn matches_extension(&self, path: &Path) -> bool { From 8f087356c753032e25b5854aa412100ffe232b66 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:42:03 +0100 Subject: [PATCH 51/75] fix(file): refuse a bmap with another root element or a newer major version Like bmaptool, a bmap needs a version, and a major version above 2 is refused. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/bmap.rs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/file/bmap.rs b/src/file/bmap.rs index 83841d1a..52a501c0 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -13,6 +13,8 @@ const BMAP_EXTENSION: &str = "bmap"; const SHA256_CHECKSUM_TYPE: &str = "sha256"; const SHA256_HEX_LEN: usize = 64; const BMAP_VERSION: &str = "2.0"; +const BMAP_MAJOR_VERSION: u32 = 2; +const BMAP_ROOT: &str = "bmap"; /// The block ranges of an image that a bmap file maps. #[derive(Debug)] @@ -31,6 +33,25 @@ impl Bmap { let doc = roxmltree::Document::parse(&text) .context(format!("Bmap::read: invalid xml in {}", path.display()))?; let root = doc.root_element(); + anyhow::ensure!( + root.has_tag_name(BMAP_ROOT), + "Bmap::read: {} is not a bmap", + path.display() + ); + let major_version = root + .attribute("version") + .and_then(|v| v.split('.').next()?.parse::().ok()) + .context(format!( + "Bmap::read: no valid version in {}", + path.display() + ))?; + // like bmaptool, refuse a bmap of a newer major version + anyhow::ensure!( + major_version <= BMAP_MAJOR_VERSION, + "Bmap::read: {} has version {major_version}.x, only up to {BMAP_MAJOR_VERSION}.x is \ + supported", + path.display() + ); let child = |name: &str| { root.children() .find(|n| n.has_tag_name(name)) @@ -384,6 +405,11 @@ mod tests { bmap_xml(BLOCK_SIZE, "7-5"), bmap_xml(BLOCK_SIZE, "x"), "".to_string(), + bmap_xml(BLOCK_SIZE, "") + .replace("", ""), + bmap_xml(BLOCK_SIZE, "").replace("version=\"2.0\"", "version=\"3.0\""), + bmap_xml(BLOCK_SIZE, "").replace(" version=\"2.0\"", ""), ]; for xml in invalid { From 19bb6be4ad21d8678c133f438c46d2352f410477 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:42:18 +0100 Subject: [PATCH 52/75] docs(file): describe the space a command needs in TMPDIR Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 1e4d03a3..ed6507e7 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,8 @@ A command that changes an image writes the result to the input image. For a comp The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. It also fails if the output image or its bmap exists and is not writable for the user. The output image is written to a new file in its directory and then renamed, so the directory needs write access and space for one more image. An output image that is a mount point, e.g. with `docker run -v "$(pwd)/my-image.wic":/source/my-image.wic`, cannot be renamed to, so the new image is copied into it. Mount the directory instead of the image if you use `-b`, because the bmap is written next to the image. The new file belongs to the user who runs `omnect-cli`. `copy-from-image` only reads the image and writes nothing. +A command that changes an image, or reads a compressed one, works on a copy of the image in `TMPDIR` (default `/tmp`). It needs space there for the uncompressed image and for a copy of each partition the command reads or changes, and for an output image that is a mount point also for the new image. If `/tmp` is a small tmpfs, set `TMPDIR` to a directory with more space. + ## bmap files Commands that change an image create a bmap file with `-b` (`--output-bmap`). It is written next to the output image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic` and `my-image.wic.xz.bmap` for `my-image.wic.xz`, and replaces a file with this name. `bmaptool` finds it there without `--bmap`. The command fails if this file is the bmap of an image that stays, e.g. `my-image.wic.bmap` when `-i my-image.wic.xz` writes `my-image.wic`, also with `--input-bmap` or `--no-input-bmap`. Choose another output with `-o` then. From a75f17f1c8c9a1339bfe91e05f14a917560360bd Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:45:08 +0100 Subject: [PATCH 53/75] fix(file): replace an output bmap that is a mount point A mount point cannot be removed, so with an image and its bmap both mounted into a container, -b failed before the image was written. The copy into the bmap replaces its content anyway. Also clearer wording for the mount point case and the TMPDIR space. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 4 ++-- src/file/output.rs | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index ed6507e7..1018bfe0 100644 --- a/README.md +++ b/README.md @@ -187,9 +187,9 @@ omnect-cli file copy-to-image --help A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gz`), e.g. `my-image.wic.xz` for `my-image.wic`. An image that already has an extension of that compression keeps its name. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. Its name must have the extension of the `-p` compression, or no compression extension without `-p`. -The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. It also fails if the output image or its bmap exists and is not writable for the user. The output image is written to a new file in its directory and then renamed, so the directory needs write access and space for one more image. An output image that is a mount point, e.g. with `docker run -v "$(pwd)/my-image.wic":/source/my-image.wic`, cannot be renamed to, so the new image is copied into it. Mount the directory instead of the image if you use `-b`, because the bmap is written next to the image. The new file belongs to the user who runs `omnect-cli`. `copy-from-image` only reads the image and writes nothing. +The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. It also fails if the output image or its bmap exists and is not writable for the user. The output image is written to a new file in its directory and then renamed, so the directory needs write access and space for one more image. A rename cannot replace a mount point, e.g. an image mounted with `docker run -v "$(pwd)/my-image.wic":/source/my-image.wic`, so the new image is copied into it. Mount the directory instead of the image if you use `-b`, because the bmap is written next to the image. The new file belongs to the user who runs `omnect-cli`. `copy-from-image` only reads the image and writes nothing. -A command that changes an image, or reads a compressed one, works on a copy of the image in `TMPDIR` (default `/tmp`). It needs space there for the uncompressed image and for a copy of each partition the command reads or changes, and for an output image that is a mount point also for the new image. If `/tmp` is a small tmpfs, set `TMPDIR` to a directory with more space. +A command copies each partition it reads or changes to `TMPDIR` (default `/tmp`). A command that changes an image, or reads a compressed one, also copies the uncompressed image there, and for an output image that is a mount point also the new image. So `TMPDIR` needs space for all of them. If `/tmp` is a small tmpfs, set `TMPDIR` to a directory with more space. ## bmap files diff --git a/src/file/output.rs b/src/file/output.rs index 0e1c58ea..56e4deb9 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -100,8 +100,7 @@ pub fn check_conflicts( Ok(()) } -/// Fails if `path` exists and the user cannot write it. Replacing it by a rename would not need -/// that. +/// Fails if `path` exists and the user cannot write it, so a read-only file is not replaced. fn ensure_writable(path: &Path) -> Result<()> { match rustix::fs::access(path, Access::WRITE_OK) { Err(Errno::NOENT) => Ok(()), @@ -114,9 +113,9 @@ fn ensure_writable(path: &Path) -> Result<()> { } /// A new file in the directory of its destination, which `commit` renames to the destination. -/// Until then the destination stays unchanged. A destination that is a mount point, e.g. an image -/// file mounted into a container, cannot be renamed to, so its new file is staged in TMPDIR and -/// copied into it. +/// Until then the destination stays unchanged. A rename cannot replace a mount point, e.g. an +/// image file mounted into a container, so for such a destination the new file is staged in +/// TMPDIR and copied into it. pub struct Staged { _tmp_dir: TempDir, file: PathBuf, @@ -177,7 +176,7 @@ impl Staged { ) => { info!( - "cannot rename to {} ({e}), so copy into it", + "cannot rename to {}: {e}; copying into it instead", self.dest.display() ); self.copy_into_dest() @@ -230,8 +229,9 @@ fn is_mount_root(path: &Path) -> bool { /// the bmap fails, no bmap is left that does not match the new image. pub fn commit(image: Staged, bmap: Option) -> Result<()> { if let Some(bmap) = &bmap { + // a bmap that is a mount point cannot be removed, the copy into it replaces it match fs::remove_file(&bmap.dest) { - Err(e) if e.kind() != ErrorKind::NotFound => { + Err(e) if !matches!(e.kind(), ErrorKind::NotFound | ErrorKind::ResourceBusy) => { return Err(e).context(format!("cannot remove {}", bmap.dest.display())); } _ => {} From 6e9c9e16c8c095795750086fe6acecb5337380ce Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:51:43 +0100 Subject: [PATCH 54/75] fix(file): copy into a detected mount point without a rename rename(2) and unlink(2) need write access to the directory, which a container often does not have when only the image is mounted. So a detected mount point is not renamed to or removed, its new content is copied into it. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/output.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/file/output.rs b/src/file/output.rs index 56e4deb9..db6c047b 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -120,6 +120,7 @@ pub struct Staged { _tmp_dir: TempDir, file: PathBuf, dest: PathBuf, + mount_root: bool, } impl Staged { @@ -130,7 +131,8 @@ impl Staged { let dir = dest .parent() .context(format!("{} has no directory", dest.display()))?; - let tmp_parent = if is_mount_root(&dest) { + let mount_root = is_mount_root(&dest); + let tmp_parent = if mount_root { env::temp_dir() } else { dir.to_path_buf() @@ -164,10 +166,14 @@ impl Staged { _tmp_dir: tmp_dir, file, dest, + mount_root, }) } fn rename(&self) -> Result<()> { + if self.mount_root { + return self.copy_into_dest(); + } match fs::rename(&self.file, &self.dest) { Err(e) if matches!( @@ -228,8 +234,8 @@ fn is_mount_root(path: &Path) -> bool { /// not lost on a power loss. The old bmap is removed before the image is renamed, so if renaming /// the bmap fails, no bmap is left that does not match the new image. pub fn commit(image: Staged, bmap: Option) -> Result<()> { - if let Some(bmap) = &bmap { - // a bmap that is a mount point cannot be removed, the copy into it replaces it + if let Some(bmap) = bmap.as_ref().filter(|bmap| !bmap.mount_root) { + // a mount point that was not detected cannot be removed, the copy into it replaces it match fs::remove_file(&bmap.dest) { Err(e) if !matches!(e.kind(), ErrorKind::NotFound | ErrorKind::ResourceBusy) => { return Err(e).context(format!("cannot remove {}", bmap.dest.display())); From 32c4a0b0c01144cd5b27ac49cb5815ff4d335c21 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:49:13 +0100 Subject: [PATCH 55/75] refactor(file): share the sparse file copy in sparse::copy_file The copy into a mounted destination, the copy of an uncompressed input image and the uncompressed output image used the same open, size and copy steps. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/output.rs | 7 +------ src/file/sparse.rs | 35 +++++++++++++++++++++++++++++++++++ src/lib.rs | 18 ++---------------- 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/src/file/output.rs b/src/file/output.rs index db6c047b..b3575ff9 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -196,11 +196,6 @@ impl Staged { } fn copy_into_dest(&self) -> Result<()> { - let src = File::open(&self.file).context(format!("cannot open {}", self.file.display()))?; - let len = src - .metadata() - .context(format!("cannot get the size of {}", self.file.display()))? - .len(); let dest = OpenOptions::new() .write(true) .create(true) @@ -208,7 +203,7 @@ impl Staged { .open(&self.dest) .context(format!("cannot open {}", self.dest.display()))?; - sparse::copy_to_empty_file(&src, 0, &dest, len)?; + sparse::copy_file(&self.file, &dest)?; dest.sync_all() .context(format!("cannot sync {}", self.dest.display())) } diff --git a/src/file/sparse.rs b/src/file/sparse.rs index 7953b8e1..22acbfbd 100644 --- a/src/file/sparse.rs +++ b/src/file/sparse.rs @@ -100,6 +100,20 @@ pub fn copy_to_empty_file(src: &File, src_offset: u64, dst: &File, len: u64) -> copy_range(src, src_offset, dst, 0, len) } +/// Copies the file `src` into the empty `dst`. Holes of `src` stay holes in `dst`. +pub fn copy_file(src: &Path, dst: &File) -> Result<()> { + let src_file = File::open(src).context(format!("copy_file: cannot open {}", src.display()))?; + let len = src_file + .metadata() + .context(format!( + "copy_file: cannot get the size of {}", + src.display() + ))? + .len(); + + copy_to_empty_file(&src_file, 0, dst, len) +} + #[cfg(test)] mod tests { use crate::file::sparse::*; @@ -176,6 +190,27 @@ mod tests { assert!(block.iter().all(|b| *b == DATA_BYTE)); } + #[test] + fn copy_file_keeps_holes() { + let dir = tempfile::tempdir().expect("create temp dir"); + sparse_file(dir.path()); + let dst = File::options() + .read(true) + .write(true) + .create_new(true) + .open(dir.path().join("dst")) + .expect("create file"); + + copy_file(&dir.path().join("src"), &dst).expect("copy file"); + + let len = FILE_BLOCKS * BLOCK_SIZE; + assert_eq!(dst.metadata().expect("metadata").len(), len); + assert_eq!( + data_ranges(&dst, 0, len).expect("data ranges"), + vec![DATA_BLOCK * BLOCK_SIZE..(DATA_BLOCK + 1) * BLOCK_SIZE] + ); + } + #[test] fn copy_range_rejects_range_after_source_end() { let dir = tempfile::tempdir().expect("create temp dir"); diff --git a/src/lib.rs b/src/lib.rs index 22da82ce..03ea7214 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -96,7 +96,7 @@ where let image = Staged::write(&dest_image_file, |dest| match &target_compression { Some(c) => compression::compress(&tmp_image_file, c, dest), - None => copy_image(&tmp_image_file, dest), + None => sparse::copy_file(&tmp_image_file, dest), })?; let bmap = output_bmap .then(|| { @@ -166,26 +166,12 @@ fn temp_image( ); let tmp_image = fs::File::create(&tmp_image_file) .context(format!("cannot create {}", tmp_image_file.display()))?; - copy_image(image_file, &tmp_image).context(format!( - "error: copy_image({:?}, {:?})", - image_file, tmp_image_file - ))?; + sparse::copy_file(image_file, &tmp_image)?; Ok(tmp_image_file) } } } -/// Copies `from` to the empty `to` and keeps the holes of a sparse `from`. -fn copy_image(from: &Path, to: &fs::File) -> Result<()> { - let src = fs::File::open(from).context("copy_image: cannot open source")?; - let len = src - .metadata() - .context("copy_image: cannot get source size")? - .len(); - - sparse::copy_to_empty_file(&src, 0, to, len) -} - struct CertInfo { cert_path: PathBuf, key_path: PathBuf, From 44e456f8e05b57c1fab21da4dead5496ecbbd268 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:50:00 +0100 Subject: [PATCH 56/75] fix(file): keep the staged file when the copy into the destination fails The copy into a mounted destination truncates it first. If the copy or the sync fails, the staged file was removed with its temp dir, so neither the old nor the new content was left. Now its temp dir stays, and the error names the file. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/output.rs | 44 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/src/file/output.rs b/src/file/output.rs index b3575ff9..3571d8a9 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -117,7 +117,7 @@ fn ensure_writable(path: &Path) -> Result<()> { /// image file mounted into a container, so for such a destination the new file is staged in /// TMPDIR and copied into it. pub struct Staged { - _tmp_dir: TempDir, + tmp_dir: TempDir, file: PathBuf, dest: PathBuf, mount_root: bool, @@ -163,14 +163,14 @@ impl Staged { .context(format!("cannot sync {}", file.display()))?; Ok(Self { - _tmp_dir: tmp_dir, + tmp_dir, file, dest, mount_root, }) } - fn rename(&self) -> Result<()> { + fn rename(&mut self) -> Result<()> { if self.mount_root { return self.copy_into_dest(); } @@ -195,7 +195,8 @@ impl Staged { } } - fn copy_into_dest(&self) -> Result<()> { + /// Keeps the staged file if the copy fails, because `dest` is truncated already. + fn copy_into_dest(&mut self) -> Result<()> { let dest = OpenOptions::new() .write(true) .create(true) @@ -203,9 +204,20 @@ impl Staged { .open(&self.dest) .context(format!("cannot open {}", self.dest.display()))?; - sparse::copy_file(&self.file, &dest)?; - dest.sync_all() - .context(format!("cannot sync {}", self.dest.display())) + let result = sparse::copy_file(&self.file, &dest).and_then(|()| { + dest.sync_all() + .context(format!("cannot sync {}", self.dest.display())) + }); + if result.is_err() { + self.tmp_dir.disable_cleanup(true); + } + result.with_context(|| { + format!( + "cannot copy into {}; the new file is kept in {}", + self.dest.display(), + self.file.display() + ) + }) } } @@ -228,7 +240,7 @@ fn is_mount_root(path: &Path) -> bool { /// Renames the staged `image` and its `bmap`, then syncs their directories, so the renames are /// not lost on a power loss. The old bmap is removed before the image is renamed, so if renaming /// the bmap fails, no bmap is left that does not match the new image. -pub fn commit(image: Staged, bmap: Option) -> Result<()> { +pub fn commit(mut image: Staged, mut bmap: Option) -> Result<()> { if let Some(bmap) = bmap.as_ref().filter(|bmap| !bmap.mount_root) { // a mount point that was not detected cannot be removed, the copy into it replaces it match fs::remove_file(&bmap.dest) { @@ -239,7 +251,7 @@ pub fn commit(image: Staged, bmap: Option) -> Result<()> { } } image.rename()?; - if let Some(bmap) = &bmap { + if let Some(bmap) = &mut bmap { bmap.rename()?; } @@ -297,6 +309,7 @@ mod tests { const IMAGE_MODE: u32 = 0o644; const READ_ONLY_MODE: u32 = 0o444; const MODE_MASK: u32 = 0o777; + const UNRESIZABLE_FILE: &str = "/dev/full"; fn touch(path: &Path) { fs::write(path, []).expect("create file"); @@ -587,6 +600,19 @@ mod tests { assert_eq!(fs::read(&image).expect("read image"), NEW_CONTENT); } + #[test] + fn copy_into_dest_keeps_the_staged_file_when_the_copy_fails() { + let dir = tempfile::tempdir().expect("create temp dir"); + let mut staged = stage(&dir.path().join(IMAGE), NEW_CONTENT); + // a character device cannot be resized, so the copy fails after the open + staged.dest = PathBuf::from(UNRESIZABLE_FILE); + + assert!(staged.copy_into_dest().is_err()); + let file = staged.file.clone(); + drop(staged); + assert_eq!(fs::read(&file).expect("read staged file"), NEW_CONTENT); + } + #[test] fn is_mount_root_detects_the_root_dir() { let dir = tempfile::tempdir().expect("create temp dir"); From 5e772f98ac6a83ac52ea839156d0346827c8666f Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:50:00 +0100 Subject: [PATCH 57/75] fix(file): skip the sync of a staged file that is only copied For a mounted destination the staged file in TMPDIR is copied into the destination and then removed, and the copy is synced. Syncing the staged file too wrote the whole image to disk once more. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/output.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/file/output.rs b/src/file/output.rs index 3571d8a9..2c469a52 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -157,10 +157,13 @@ impl Staged { .set_permissions(metadata.permissions()) .context(format!("cannot set permissions of {}", file.display()))?; } - // the handle that wrote the data also gets the write errors - handle - .sync_all() - .context(format!("cannot sync {}", file.display()))?; + // a file in TMPDIR is only copied into `dest`, and the copy is synced + if !mount_root { + // the handle that wrote the data also gets the write errors + handle + .sync_all() + .context(format!("cannot sync {}", file.display()))?; + } Ok(Self { tmp_dir, From bac7b430208466c7f8108b390cc3d3e619cef736 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:50:21 +0100 Subject: [PATCH 58/75] test(file): check / as mount root only where the kernel reports it Kernels before 5.8 do not report mount roots, and in a chroot `/` is no mount root. The test now checks `/` only if statx supports the attribute and `/` is a mount point in mountinfo. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/output.rs | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/file/output.rs b/src/file/output.rs index 2c469a52..3801ffe4 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -313,6 +313,8 @@ mod tests { const READ_ONLY_MODE: u32 = 0o444; const MODE_MASK: u32 = 0o777; const UNRESIZABLE_FILE: &str = "/dev/full"; + const MOUNTINFO: &str = "/proc/self/mountinfo"; + const MOUNTINFO_MOUNT_POINT: usize = 4; fn touch(path: &Path) { fs::write(path, []).expect("create file"); @@ -622,8 +624,24 @@ mod tests { let image = dir.path().join(IMAGE); touch(&image); - assert!(is_mount_root(Path::new("/"))); assert!(!is_mount_root(&image)); + + // kernels before 5.8 do not report mount roots, and `/` of a chroot is no mount root + let root = Path::new("/"); + let reported = rustix::fs::statx(CWD, root, AtFlags::empty(), StatxFlags::empty()) + .is_ok_and(|statx| { + statx + .stx_attributes_mask + .contains(StatxAttributes::MOUNT_ROOT) + }); + let mounted = fs::read_to_string(MOUNTINFO).is_ok_and(|mountinfo| { + mountinfo + .lines() + .any(|line| line.split(' ').nth(MOUNTINFO_MOUNT_POINT) == Some("/")) + }); + if reported && mounted { + assert!(is_mount_root(root)); + } } #[test] From 483adce36f233f7abc55aa4bd6667c5450aca77d Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:51:08 +0100 Subject: [PATCH 59/75] test(file): make the bmap of another image valid except for its size The test bmap had no version and no checksums, so `Bmap::read` rejected it and the tests never reached the image size check. It is now a valid bmap of a one-block image, and the tests check for the size error. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- tests/integration_tests.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index d0b358e5..be3f7918 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -907,8 +907,12 @@ const OLD_CONTENT_BYTE: u8 = 0xa5; // a file with a zero range that spans whole image blocks const ZERO_BLOCKS_FILE_EDGE: usize = 4096; const ZERO_BLOCKS_FILE_ZEROS: usize = 64 * 1024; -const OTHER_IMAGE_BMAP: &str = "40964096\ - 0"; +// a valid bmap of a one-block image, so only the image size check rejects it +const OTHER_IMAGE_BMAP: &str = "4096\ + 4096sha256\ + 0\ + "; +const OTHER_IMAGE_SIZE_ERROR: &str = "bmap is for an image of 4096 bytes"; fn bmap_path_of(image: &Path) -> PathBuf { PathBuf::from(format!("{}.bmap", image.to_string_lossy())) @@ -1106,7 +1110,8 @@ fn check_input_bmap_of_another_image() { .arg("-b") .assert() .failure() - .stderr(predicate::str::contains("cannot use input bmap")); + .stderr(predicate::str::contains("cannot use input bmap")) + .stderr(predicate::str::contains(OTHER_IMAGE_SIZE_ERROR)); assert_eq!(Testrunner::file_hash(&image_path), image_hash); // reading from the image does not use the bmap @@ -1360,7 +1365,8 @@ fn check_no_input_bmap() { .failure(); copy_to_image(&["-b".as_ref()]) .failure() - .stderr(predicate::str::contains("--no-input-bmap")); + .stderr(predicate::str::contains("--no-input-bmap")) + .stderr(predicate::str::contains(OTHER_IMAGE_SIZE_ERROR)); copy_to_image(&["-b".as_ref(), "--no-input-bmap".as_ref()]).success(); assert!(maps_whole_image(&read_bmap(&bmap_path))); From f91820426ebd7415501bb7152df36449eac6450f Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:51:09 +0100 Subject: [PATCH 60/75] test(file): build bmap paths with bmap::bmap_path The tests had their own copy of the naming rule. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- tests/integration_tests.rs | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index be3f7918..9978d782 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -3,7 +3,10 @@ use assert_cmd::Command; use assert_json_diff::assert_json_eq; use common::Testrunner; use httpmock::prelude::*; -use omnect_cli::{file::bmap::Bmap, ssh}; +use omnect_cli::{ + file::bmap::{Bmap, bmap_path}, + ssh, +}; use predicates::prelude::*; use std::{ ffi::OsStr, @@ -914,10 +917,6 @@ const OTHER_IMAGE_BMAP: &str = "4096"; const OTHER_IMAGE_SIZE_ERROR: &str = "bmap is for an image of 4096 bytes"; -fn bmap_path_of(image: &Path) -> PathBuf { - PathBuf::from(format!("{}.bmap", image.to_string_lossy())) -} - fn read_bmap(path: &Path) -> Bmap { Bmap::read(path).expect("read bmap") } @@ -965,7 +964,7 @@ fn sparse_copy(from: &Path, to: &Path) { fn check_file_copy_without_input_bmap_maps_whole_image() { let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); let image_path = tr.to_pathbuf("testfiles/image.wic"); - let bmap_path = PathBuf::from(format!("{}.bmap", image_path.to_string_lossy())); + let bmap_path = bmap_path(&image_path); let in_file = tr.to_pathbuf("testfiles/boot.scr"); Command::cargo_bin("omnect-cli") @@ -989,7 +988,7 @@ fn check_file_copy_keeps_holes() { let image_path = tr.to_pathbuf("testfiles/image.wic"); let input_bmap_path = tr.to_pathbuf("testfiles/image.wic.bmap"); let sparse_path = image_path.with_file_name("sparse.wic"); - let bmap_path = PathBuf::from(format!("{}.bmap", sparse_path.to_string_lossy())); + let bmap_path = bmap_path(&sparse_path); let in_file = tr.to_pathbuf("testfiles/boot.scr"); sparse_copy(&image_path, &sparse_path); @@ -1036,7 +1035,7 @@ fn check_file_copy_maps_zero_blocks_of_copied_files() { let image_path = tr.to_pathbuf("testfiles/image.wic"); let input_bmap_path = tr.to_pathbuf("testfiles/image.wic.bmap"); let sparse_path = image_path.with_file_name("sparse.wic"); - let bmap_path = PathBuf::from(format!("{}.bmap", sparse_path.to_string_lossy())); + let bmap_path = bmap_path(&sparse_path); let device_path = tr.pathbuf().join("device.wic"); let in_file = tr.pathbuf().join("zero-blocks.bin"); let in_content = [ @@ -1093,7 +1092,7 @@ fn check_file_copy_maps_zero_blocks_of_copied_files() { fn check_input_bmap_of_another_image() { let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); let image_path = tr.to_pathbuf("testfiles/image.wic"); - let bmap_path = PathBuf::from(format!("{}.bmap", image_path.to_string_lossy())); + let bmap_path = bmap_path(&image_path); let in_file = tr.to_pathbuf("testfiles/boot.scr"); let out_file = tr.pathbuf().join("hosts"); fs::write(&bmap_path, OTHER_IMAGE_BMAP).expect("write bmap"); @@ -1158,7 +1157,7 @@ fn check_no_output_bmap_when_image_write_back_fails() { let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); let image_path = tr.to_pathbuf("testfiles/image.wic.xz"); let written_image_path = image_path.with_extension(""); - let bmap_path = image_path.with_extension("bmap"); + let bmap_path = bmap_path(&written_image_path); let in_file = tr.to_pathbuf("testfiles/boot.scr"); // a directory as the output image makes the write back fail, also for root create_dir_all(&written_image_path).expect("create directory"); @@ -1287,7 +1286,7 @@ fn check_chained_commands_with_input_bmap() { assert_eq!(Testrunner::file_hash(&image_xz_path), image_xz_hash); assert_eq!(Testrunner::file_hash(&input_bmap_path), input_bmap_hash); assert!(maps_all_ranges_of( - &read_bmap(&bmap_path_of(&device_path)), + &read_bmap(&bmap_path(&device_path)), &read_bmap(&input_bmap_path) )); } @@ -1337,7 +1336,7 @@ fn check_derived_output_image_exists() { fn check_no_input_bmap() { let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); let image_path = tr.to_pathbuf("testfiles/image.wic"); - let bmap_path = bmap_path_of(&image_path); + let bmap_path = bmap_path(&image_path); let other_bmap_path = tr.pathbuf().join("other.bmap"); let in_file = tr.to_pathbuf("testfiles/boot.scr"); fs::write(&bmap_path, OTHER_IMAGE_BMAP).expect("write bmap"); @@ -1377,7 +1376,7 @@ fn check_bmap_generation_wic() { let tr = Testrunner::new(function_name!().split("::").last().unwrap()); let image_path = tr.to_pathbuf("testfiles/image.wic"); let image_path_copy = PathBuf::from(format!("{}.copy", image_path.to_str().unwrap())); - let bmap_path = PathBuf::from(format!("{}.bmap", image_path.to_str().unwrap())); + let bmap_path = bmap_path(&image_path); let in_file = tr.to_pathbuf("testfiles/boot.scr"); let in_file = in_file.to_str().unwrap(); @@ -1426,7 +1425,7 @@ fn check_bmap_generation_wic_xz() { let image_path_wic_xz = tr.to_pathbuf("testfiles/image.wic.xz"); let image_path_wic = tr.pathbuf().join("device.wic"); let image_path_wic_copy = image_path_wic_xz.with_extension("copy"); - let image_path_bmap = bmap_path_of(&image_path_wic); + let image_path_bmap = bmap_path(&image_path_wic); let input_bmap = read_bmap(&tr.to_pathbuf("testfiles/image.wic.bmap")); let in_file = tr.to_pathbuf("testfiles/boot.scr"); let in_file = in_file.to_str().unwrap(); From ddb60e95f7df3325bffd286916bf1317573021be Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:51:47 +0100 Subject: [PATCH 61/75] fix(identity): write the new device certificate and key into a temp dir `set-device-certificate` and `set-edge-ca-certificate` wrote them under fixed names next to the input image and did not remove them. Two runs on the same base image could use each other's key, a read-only image directory failed, and the private key stayed next to the image. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/lib.rs | 54 +++++++++++++++++++------------------- tests/integration_tests.rs | 16 +++++++++++ 2 files changed, 43 insertions(+), 27 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 03ea7214..3a7eb08d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,6 +33,7 @@ use std::{ fs, path::{Path, PathBuf}, }; +use tempfile::TempDir; fn run_image_command(image_file: PathBuf, output: ImageOutput, command: F) -> Result<()> where @@ -172,7 +173,10 @@ fn temp_image( } } +/// A new device certificate and key in a temp dir, so parallel runs do not share them and no +/// private key is left behind. struct CertInfo { + _dir: TempDir, cert_path: PathBuf, key_path: PathBuf, } @@ -186,7 +190,7 @@ struct CertificateOptions<'a> { validity_days: u32, } -fn create_image_cert(image: &Path, cert_opts: CertificateOptions) -> Result { +fn create_image_cert(cert_opts: CertificateOptions) -> Result { let intermediate_full_chain_cert_str = std::fs::read_to_string(cert_opts.intermediate_full_chain_cert) .context("create_and_set_image_cert: couldn't read intermediate fullchain cert")?; @@ -200,15 +204,17 @@ fn create_image_cert(image: &Path, cert_opts: CertificateOptions) -> Result Result<()> { days, output, }) => { - let cert_info = create_image_cert( - &image, - CertificateOptions { - intermediate_full_chain_cert: &intermediate_full_chain_cert, - intermediate_key: &intermediate_key, - target_cert: "device_cert_path.pem", - target_key: "device_key_path.key.pem", - subject: &device_id, - validity_days: days, - }, - ) + let cert_info = create_image_cert(CertificateOptions { + intermediate_full_chain_cert: &intermediate_full_chain_cert, + intermediate_key: &intermediate_key, + target_cert: "device_cert_path.pem", + target_key: "device_key_path.key.pem", + subject: &device_id, + validity_days: days, + }) .context("set_device_certificate: could not create certificate")?; run_image_command(image, output, |img| { @@ -300,17 +303,14 @@ pub fn run() -> Result<()> { days, output, }) => { - let cert_info = create_image_cert( - &image, - CertificateOptions { - intermediate_full_chain_cert: &intermediate_full_chain_cert, - intermediate_key: &intermediate_key, - target_cert: "edge_ca_cert_path.pem", - target_key: "edge_ca_key_path.key.pem", - subject: &device_id, - validity_days: days, - }, - ) + let cert_info = create_image_cert(CertificateOptions { + intermediate_full_chain_cert: &intermediate_full_chain_cert, + intermediate_key: &intermediate_key, + target_cert: "edge_ca_cert_path.pem", + target_key: "edge_ca_key_path.key.pem", + subject: &device_id, + validity_days: days, + }) .context("set_edge_ca_certificate: could not create certificate")?; run_image_command(image, output, |img| { diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 9978d782..93f7b6bf 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -484,6 +484,14 @@ fn check_set_identity_config_tpm_template() { ); } +/// Fails if a file with one of `names` is in the directory of `image`. +fn assert_no_files_next_to(image: &Path, names: &[&str]) { + for name in names { + let path = image.with_file_name(name); + assert!(!path.exists(), "{} was left behind", path.display()); + } +} + #[test] fn check_set_device_cert_est() { let tr = Testrunner::new(function_name!().split("::").last().unwrap()); @@ -507,6 +515,10 @@ fn check_set_device_cert_est() { .arg("1") .assert(); assert.success(); + assert_no_files_next_to( + &image_path, + &["device_cert_path.pem", "device_key_path.key.pem"], + ); let mut device_id_cert_out_path = tr.pathbuf(); device_id_cert_out_path.push("dir1"); @@ -638,6 +650,10 @@ fn check_set_edge_ca_cert() { .arg("1") .assert(); assert.success(); + assert_no_files_next_to( + &image_path, + &["edge_ca_cert_path.pem", "edge_ca_key_path.key.pem"], + ); let mut edge_ca_cert_out_path = tr.pathbuf(); edge_ca_cert_out_path.push("dir1"); From 7b6161841fab1202102c15fd416e9dc7dd98ba97 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:53:29 +0100 Subject: [PATCH 62/75] fix(file): remove the staging dir when a stop signal ends the command The staged output image is in a hidden dir next to the output image, and a dir in TMPDIR for a mount point. SIGINT, SIGTERM or SIGHUP ended the process without destructors, so a dir with a whole image stayed there. A signal-hook thread now removes the registered dirs and then stops the process with the same signal. The README names the dir for the cases no handler can catch. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- Cargo.lock | 11 ++++++ Cargo.toml | 1 + README.md | 2 +- src/file/mod.rs | 1 + src/file/output.rs | 48 ++++++++++++++++++++------- src/file/signal.rs | 83 ++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 133 insertions(+), 13 deletions(-) create mode 100644 src/file/signal.rs diff --git a/Cargo.lock b/Cargo.lock index 38a63979..f4a1a849 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1783,6 +1783,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "sha2 0.11.1", + "signal-hook", "stdext", "strum", "strum_macros", @@ -2446,6 +2447,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f7447afd772ca79ad17ae0a3972a6b0df0cb4b80990fd82707593e9e012a74" +dependencies = [ + "libc", + "signal-hook-registry", +] + [[package]] name = "signal-hook-registry" version = "1.4.8" diff --git a/Cargo.toml b/Cargo.toml index c1b0c5d2..64ca8ef6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,6 +49,7 @@ serde = { version = "1.0", default-features = false, features = ["derive"] } serde_json = { version = "1.0", default-features = false } serde_path_to_error = { version = "0.1", default-features = false } sha2 = { version = "0.11", default-features = false } +signal-hook = { version = "0.4", default-features = false, features = ["iterator"] } stdext = { version = "0.3", default-features = false } strum = { version = "0.28", default-features = false } strum_macros = { version = "0.28", default-features = false } diff --git a/README.md b/README.md index 1018bfe0..60d54070 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,7 @@ omnect-cli file copy-to-image --help A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gz`), e.g. `my-image.wic.xz` for `my-image.wic`. An image that already has an extension of that compression keeps its name. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. Its name must have the extension of the `-p` compression, or no compression extension without `-p`. -The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. It also fails if the output image or its bmap exists and is not writable for the user. The output image is written to a new file in its directory and then renamed, so the directory needs write access and space for one more image. A rename cannot replace a mount point, e.g. an image mounted with `docker run -v "$(pwd)/my-image.wic":/source/my-image.wic`, so the new image is copied into it. Mount the directory instead of the image if you use `-b`, because the bmap is written next to the image. The new file belongs to the user who runs `omnect-cli`. `copy-from-image` only reads the image and writes nothing. +The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. It also fails if the output image or its bmap exists and is not writable for the user. The output image is written to a new file in its directory and then renamed, so the directory needs write access and space for one more image. Until the rename, the new file is in a hidden `.omnect-cli-*` directory. The command removes it when it ends or is stopped with Ctrl-C, `SIGTERM` or `SIGHUP`. It stays after a `SIGKILL` or a power loss, and you can delete it. A rename cannot replace a mount point, e.g. an image mounted with `docker run -v "$(pwd)/my-image.wic":/source/my-image.wic`, so the new image is copied into it. Mount the directory instead of the image if you use `-b`, because the bmap is written next to the image. The new file belongs to the user who runs `omnect-cli`. `copy-from-image` only reads the image and writes nothing. A command copies each partition it reads or changes to `TMPDIR` (default `/tmp`). A command that changes an image, or reads a compressed one, also copies the uncompressed image there, and for an output image that is a mount point also the new image. So `TMPDIR` needs space for all of them. If `/tmp` is a small tmpfs, set `TMPDIR` to a directory with more space. diff --git a/src/file/mod.rs b/src/file/mod.rs index 92d9c677..43e15dd8 100644 --- a/src/file/mod.rs +++ b/src/file/mod.rs @@ -3,6 +3,7 @@ pub mod compression; pub mod functions; pub(crate) mod output; mod partition; +mod signal; pub(crate) mod sparse; use super::validators::{ diff --git a/src/file/output.rs b/src/file/output.rs index 3801ffe4..c73a6240 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -1,4 +1,4 @@ -use crate::file::{bmap, compression::Compression, sparse}; +use crate::file::{bmap, compression::Compression, signal, sparse}; use anyhow::{Context, Result}; use log::{info, warn}; use rustix::{ @@ -148,29 +148,35 @@ impl Staged { dest.file_name() .context(format!("{} has no file name", dest.display()))?, ); + signal::register(tmp_dir.path()); + // from here on, drop unregisters the temp dir + let staged = Self { + tmp_dir, + file, + dest, + mount_root, + }; - let mut handle = - File::create(&file).context(format!("cannot create {}", file.display()))?; + let mut handle = File::create(&staged.file) + .context(format!("cannot create {}", staged.file.display()))?; write(&mut handle)?; - if let Ok(metadata) = fs::metadata(&dest) { + if let Ok(metadata) = fs::metadata(&staged.dest) { handle .set_permissions(metadata.permissions()) - .context(format!("cannot set permissions of {}", file.display()))?; + .context(format!( + "cannot set permissions of {}", + staged.file.display() + ))?; } // a file in TMPDIR is only copied into `dest`, and the copy is synced if !mount_root { // the handle that wrote the data also gets the write errors handle .sync_all() - .context(format!("cannot sync {}", file.display()))?; + .context(format!("cannot sync {}", staged.file.display()))?; } - Ok(Self { - tmp_dir, - file, - dest, - mount_root, - }) + Ok(staged) } fn rename(&mut self) -> Result<()> { @@ -212,6 +218,7 @@ impl Staged { .context(format!("cannot sync {}", self.dest.display())) }); if result.is_err() { + signal::unregister(self.tmp_dir.path()); self.tmp_dir.disable_cleanup(true); } result.with_context(|| { @@ -224,6 +231,12 @@ impl Staged { } } +impl Drop for Staged { + fn drop(&mut self) { + signal::unregister(self.tmp_dir.path()); + } +} + /// `false` if the kernel cannot tell. fn is_mount_root(path: &Path) -> bool { rustix::fs::statx( @@ -556,6 +569,17 @@ mod tests { ); } + #[test] + fn staged_registers_its_temp_dir_until_drop() { + let dir = tempfile::tempdir().expect("create temp dir"); + let staged = stage(&dir.path().join(IMAGE), NEW_CONTENT); + let tmp_dir = staged.tmp_dir.path().to_path_buf(); + + assert!(signal::is_registered(&tmp_dir)); + drop(staged); + assert!(!signal::is_registered(&tmp_dir)); + } + #[test] fn staged_commit_keeps_permissions_and_symlink() { let dir = tempfile::tempdir().expect("create temp dir"); diff --git a/src/file/signal.rs b/src/file/signal.rs new file mode 100644 index 00000000..b579ea16 --- /dev/null +++ b/src/file/signal.rs @@ -0,0 +1,83 @@ +use log::warn; +use signal_hook::{ + consts::{SIGHUP, SIGINT, SIGTERM}, + iterator::Signals, + low_level, +}; +use std::{ + collections::BTreeSet, + ffi::c_int, + fs, + path::{Path, PathBuf}, + process, + sync::{Mutex, MutexGuard, Once, PoisonError}, + thread, +}; + +const STOP_SIGNALS: [c_int; 3] = [SIGHUP, SIGINT, SIGTERM]; +// exit code of a process that a signal stopped, plus the signal number +const SIGNAL_EXIT_CODE_BASE: c_int = 128; + +static DIRS: Mutex> = Mutex::new(BTreeSet::new()); +static HANDLER: Once = Once::new(); + +/// Removes `dir` if a stop signal ends the process before `unregister`. A dir next to the +/// output image, which can hold a whole image, is not left there by e.g. Ctrl-C. +pub fn register(dir: &Path) { + HANDLER.call_once(start_handler); + dirs().insert(dir.to_path_buf()); +} + +pub fn unregister(dir: &Path) { + dirs().remove(dir); +} + +#[cfg(test)] +pub fn is_registered(dir: &Path) -> bool { + dirs().contains(dir) +} + +fn dirs() -> MutexGuard<'static, BTreeSet> { + DIRS.lock().unwrap_or_else(PoisonError::into_inner) +} + +fn start_handler() { + let mut signals = match Signals::new(STOP_SIGNALS) { + Ok(signals) => signals, + Err(e) => { + warn!("cannot handle stop signals, so a stop leaves temp dirs: {e}"); + return; + } + }; + thread::spawn(move || { + if let Some(signal) = signals.forever().next() { + // the lock is held until the process ends, so no new dir is registered + let dirs = dirs(); + for dir in dirs.iter() { + if let Err(e) = fs::remove_dir_all(dir) { + warn!("cannot remove {}: {e}", dir.display()); + } + } + // the caller sees the signal as the cause of the stop + if let Err(e) = low_level::emulate_default_handler(signal) { + warn!("cannot stop with signal {signal}: {e}"); + } + process::exit(SIGNAL_EXIT_CODE_BASE + signal); + } + }); +} + +#[cfg(test)] +mod tests { + use crate::file::signal::*; + + #[test] + fn register_and_unregister() { + let dir = tempfile::tempdir().expect("create temp dir"); + + register(dir.path()); + assert!(is_registered(dir.path())); + unregister(dir.path()); + assert!(!is_registered(dir.path())); + } +} From 334d81d08dec6e5db705389b19815fdd8ad6e801 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:53:36 +0100 Subject: [PATCH 63/75] docs: list the output and signal modules in the project context Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- project-context.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/project-context.md b/project-context.md index 7d8ff673..373a1f5a 100644 --- a/project-context.md +++ b/project-context.md @@ -52,7 +52,9 @@ - `src/file/mod.rs` — high-level image operations: identity config, certs, hostname patching - `src/file/functions.rs` — sparse partition read, modify via e2cp/mcopy, write back of changed blocks only - `src/file/bmap.rs` — input bmap lookup and checksum check; writes the output bmap: input ranges (or the whole image) plus data blocks +- `src/file/output.rs` — output image name, conflict checks, staged write and commit of the output image and bmap - `src/file/sparse.rs` — data/hole ranges via `SEEK_DATA`/`SEEK_HOLE`, sparse range copy +- `src/file/signal.rs` — removes staging dirs on SIGINT/SIGTERM/SIGHUP - `src/file/partition.rs` — GPT/MBR partition table parsing - `src/file/compression.rs` — xz/bzip2/gzip compress/decompress with auto-detection - `src/validators/` — validation for identity config (TOML), device-update config (JSON), SSH keys From 224b7bf17d9d25bfc9044254088627f1dc1012e5 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:55:25 +0100 Subject: [PATCH 64/75] fix(file): keep the staged file when a signal stops the copy into a mount point The copy truncates the destination first. A stop signal during the copy removed the staged file with its registered dir, so neither the old nor the new image was left. The dir is now unregistered before the copy, and the log names the staged file. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 2 +- src/file/output.rs | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 60d54070..3bc02c3e 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,7 @@ omnect-cli file copy-to-image --help A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gz`), e.g. `my-image.wic.xz` for `my-image.wic`. An image that already has an extension of that compression keeps its name. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. Its name must have the extension of the `-p` compression, or no compression extension without `-p`. -The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. It also fails if the output image or its bmap exists and is not writable for the user. The output image is written to a new file in its directory and then renamed, so the directory needs write access and space for one more image. Until the rename, the new file is in a hidden `.omnect-cli-*` directory. The command removes it when it ends or is stopped with Ctrl-C, `SIGTERM` or `SIGHUP`. It stays after a `SIGKILL` or a power loss, and you can delete it. A rename cannot replace a mount point, e.g. an image mounted with `docker run -v "$(pwd)/my-image.wic":/source/my-image.wic`, so the new image is copied into it. Mount the directory instead of the image if you use `-b`, because the bmap is written next to the image. The new file belongs to the user who runs `omnect-cli`. `copy-from-image` only reads the image and writes nothing. +The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. It also fails if the output image or its bmap exists and is not writable for the user. The output image is written to a new file in its directory and then renamed, so the directory needs write access and space for one more image. Until the rename, the new file is in a hidden `.omnect-cli-*` directory. The command removes it when it ends or is stopped with Ctrl-C, `SIGTERM` or `SIGHUP`. It stays after a `SIGKILL` or a power loss, and you can delete it. A rename cannot replace a mount point, e.g. an image mounted with `docker run -v "$(pwd)/my-image.wic":/source/my-image.wic`, so the new image is staged in `TMPDIR` and copied into it. If this copy fails or is stopped, the new image stays in `TMPDIR`, and the log names it. Mount the directory instead of the image if you use `-b`, because the bmap is written next to the image. The new file belongs to the user who runs `omnect-cli`. `copy-from-image` only reads the image and writes nothing. A command copies each partition it reads or changes to `TMPDIR` (default `/tmp`). A command that changes an image, or reads a compressed one, also copies the uncompressed image there, and for an output image that is a mount point also the new image. So `TMPDIR` needs space for all of them. If `/tmp` is a small tmpfs, set `TMPDIR` to a directory with more space. diff --git a/src/file/output.rs b/src/file/output.rs index c73a6240..e54c23f1 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -204,8 +204,15 @@ impl Staged { } } - /// Keeps the staged file if the copy fails, because `dest` is truncated already. + /// Keeps the staged file if the copy fails or a signal stops it, because `dest` is truncated + /// already. fn copy_into_dest(&mut self) -> Result<()> { + signal::unregister(self.tmp_dir.path()); + info!( + "copying {} into {}; it stays if the copy fails", + self.file.display(), + self.dest.display() + ); let dest = OpenOptions::new() .write(true) .create(true) @@ -218,7 +225,6 @@ impl Staged { .context(format!("cannot sync {}", self.dest.display())) }); if result.is_err() { - signal::unregister(self.tmp_dir.path()); self.tmp_dir.disable_cleanup(true); } result.with_context(|| { @@ -637,6 +643,7 @@ mod tests { staged.dest = PathBuf::from(UNRESIZABLE_FILE); assert!(staged.copy_into_dest().is_err()); + assert!(!signal::is_registered(staged.tmp_dir.path())); let file = staged.file.clone(); drop(staged); assert_eq!(fs::read(&file).expect("read staged file"), NEW_CONTENT); From 47c05a5ea94b2373d926d2ad31b9bea90bb7c206 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:56:07 +0100 Subject: [PATCH 65/75] fix(identity): remove the new device key when a stop signal ends the command The temp dir with the new device certificate and key is registered with the stop signal handler, so Ctrl-C or SIGTERM during the image command does not leave the private key in TMPDIR. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- project-context.md | 2 +- src/file/mod.rs | 2 +- src/file/signal.rs | 3 +-- src/lib.rs | 32 ++++++++++++++++++++------------ 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/project-context.md b/project-context.md index 373a1f5a..89974387 100644 --- a/project-context.md +++ b/project-context.md @@ -54,7 +54,7 @@ - `src/file/bmap.rs` — input bmap lookup and checksum check; writes the output bmap: input ranges (or the whole image) plus data blocks - `src/file/output.rs` — output image name, conflict checks, staged write and commit of the output image and bmap - `src/file/sparse.rs` — data/hole ranges via `SEEK_DATA`/`SEEK_HOLE`, sparse range copy -- `src/file/signal.rs` — removes staging dirs on SIGINT/SIGTERM/SIGHUP +- `src/file/signal.rs` — removes registered temp dirs (staged output, device key) on SIGINT/SIGTERM/SIGHUP - `src/file/partition.rs` — GPT/MBR partition table parsing - `src/file/compression.rs` — xz/bzip2/gzip compress/decompress with auto-detection - `src/validators/` — validation for identity config (TOML), device-update config (JSON), SSH keys diff --git a/src/file/mod.rs b/src/file/mod.rs index 43e15dd8..3d02cd3b 100644 --- a/src/file/mod.rs +++ b/src/file/mod.rs @@ -3,7 +3,7 @@ pub mod compression; pub mod functions; pub(crate) mod output; mod partition; -mod signal; +pub(crate) mod signal; pub(crate) mod sparse; use super::validators::{ diff --git a/src/file/signal.rs b/src/file/signal.rs index b579ea16..7bc574e1 100644 --- a/src/file/signal.rs +++ b/src/file/signal.rs @@ -21,8 +21,7 @@ const SIGNAL_EXIT_CODE_BASE: c_int = 128; static DIRS: Mutex> = Mutex::new(BTreeSet::new()); static HANDLER: Once = Once::new(); -/// Removes `dir` if a stop signal ends the process before `unregister`. A dir next to the -/// output image, which can hold a whole image, is not left there by e.g. Ctrl-C. +/// Removes `dir` if a stop signal ends the process before `unregister`. pub fn register(dir: &Path) { HANDLER.call_once(start_handler); dirs().insert(dir.to_path_buf()); diff --git a/src/lib.rs b/src/lib.rs index 3a7eb08d..ae558452 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,7 +26,7 @@ use file::{ compression::{self, Compression}, functions::FileCopyToParams, output::{self, Staged, check_conflicts, output_image}, - sparse, + signal, sparse, }; use log::info; use std::{ @@ -174,13 +174,19 @@ fn temp_image( } /// A new device certificate and key in a temp dir, so parallel runs do not share them and no -/// private key is left behind. +/// private key is left behind, also not after a stop signal. struct CertInfo { - _dir: TempDir, + dir: TempDir, cert_path: PathBuf, key_path: PathBuf, } +impl Drop for CertInfo { + fn drop(&mut self) { + signal::unregister(self.dir.path()); + } +} + struct CertificateOptions<'a> { intermediate_full_chain_cert: &'a Path, intermediate_key: &'a Path, @@ -205,17 +211,19 @@ fn create_image_cert(cert_opts: CertificateOptions) -> Result { .context("create_and_set_image_cert: couldn't create device cert and key")?; let dir = tempfile::tempdir().context("create_and_set_image_cert: cannot create a temp dir")?; - let cert_path = dir.path().join(cert_opts.target_cert); - let key_path = dir.path().join(cert_opts.target_key); + signal::register(dir.path()); + let cert_info = CertInfo { + cert_path: dir.path().join(cert_opts.target_cert), + key_path: dir.path().join(cert_opts.target_key), + dir, + }; - fs::write(&cert_path, cert_pem).context("create_and_set_image_cert: write device_cert_path")?; - fs::write(&key_path, key_pem).context("create_and_set_image_cert: write device_key_path")?; + fs::write(&cert_info.cert_path, cert_pem) + .context("create_and_set_image_cert: write device_cert_path")?; + fs::write(&cert_info.key_path, key_pem) + .context("create_and_set_image_cert: write device_key_path")?; - Ok(CertInfo { - _dir: dir, - cert_path, - key_path, - }) + Ok(cert_info) } pub fn run() -> Result<()> { From acf2cb351df690ec22a13daefe4a237c1c26f69d Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:41:10 +0100 Subject: [PATCH 66/75] fix(file): do not handle a stop signal the process ignores signal-hook runs its handler also for a signal set to SIG_IGN. So `nohup` and background jobs of a shell, which ignore SIGHUP or SIGINT, were stopped and lost the staged image. The handler now skips signals that are ignored when it starts. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- Cargo.lock | 1 + Cargo.toml | 1 + src/file/signal.rs | 27 +++++++++++++++++++++++++-- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f4a1a849..f9b65470 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1767,6 +1767,7 @@ dependencies = [ "hmac", "httpmock", "keyring-core", + "libc", "log", "mbrman", "num_cpus", diff --git a/Cargo.toml b/Cargo.toml index 64ca8ef6..d9077bc7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ hmac = { version = "0.13", default-features = false } mbrman = { version = "0.6", default-features = false } omnect-crypto = { git = "https://github.com/omnect/omnect-crypto.git", tag = "0.5.0" } keyring-core = { version = "1.0", default-features = false } +libc = { version = "0.2", default-features = false } log = { version = "0.4", default-features = false } num_cpus = { version = "1.17", default-features = false } oauth2 = { version = "5.0", default-features = false, features = [ diff --git a/src/file/signal.rs b/src/file/signal.rs index 7bc574e1..6fd65099 100644 --- a/src/file/signal.rs +++ b/src/file/signal.rs @@ -8,8 +8,9 @@ use std::{ collections::BTreeSet, ffi::c_int, fs, + mem::MaybeUninit, path::{Path, PathBuf}, - process, + process, ptr, sync::{Mutex, MutexGuard, Once, PoisonError}, thread, }; @@ -41,7 +42,12 @@ fn dirs() -> MutexGuard<'static, BTreeSet> { } fn start_handler() { - let mut signals = match Signals::new(STOP_SIGNALS) { + // e.g. `nohup` ignores SIGHUP, and a shell ignores SIGINT for a background job + let signals: Vec<_> = STOP_SIGNALS + .into_iter() + .filter(|signal| !is_ignored(*signal)) + .collect(); + let mut signals = match Signals::new(signals) { Ok(signals) => signals, Err(e) => { warn!("cannot handle stop signals, so a stop leaves temp dirs: {e}"); @@ -66,6 +72,14 @@ fn start_handler() { }); } +fn is_ignored(signal: c_int) -> bool { + let mut action = MaybeUninit::::zeroed(); + // SAFETY: without a new action, sigaction only writes the current one into `action` + let result = unsafe { libc::sigaction(signal, ptr::null(), action.as_mut_ptr()) }; + // SAFETY: the action was zeroed, and sigaction fills it on success + result == 0 && unsafe { action.assume_init() }.sa_sigaction == libc::SIG_IGN +} + #[cfg(test)] mod tests { use crate::file::signal::*; @@ -79,4 +93,13 @@ mod tests { unregister(dir.path()); assert!(!is_registered(dir.path())); } + + #[test] + fn is_ignored_reads_the_disposition() { + // SAFETY: SIGUSR2 is not used by the tests + unsafe { libc::signal(libc::SIGUSR2, libc::SIG_IGN) }; + + assert!(is_ignored(libc::SIGUSR2)); + assert!(!is_ignored(libc::SIGUSR1)); + } } From 13dfac79543bb7d2255625fc5858ed72181a8eb7 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:41:39 +0100 Subject: [PATCH 67/75] fix(file): replace the image before the bmap `commit` removed the old bmap before it replaced the image. If the image rename or copy failed, or a stop signal came in between, the old image stayed without its bmap, and the next `-b` run mapped the whole image. Now the image is replaced first. If the bmap then fails, the old bmap is removed, because it does not match the new image. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/output.rs | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/src/file/output.rs b/src/file/output.rs index e54c23f1..3fefa7dc 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -259,22 +259,21 @@ fn is_mount_root(path: &Path) -> bool { }) } -/// Renames the staged `image` and its `bmap`, then syncs their directories, so the renames are -/// not lost on a power loss. The old bmap is removed before the image is renamed, so if renaming -/// the bmap fails, no bmap is left that does not match the new image. +/// Renames the staged `image` and then its `bmap`, then syncs their directories, so the renames +/// are not lost on a power loss. If the image fails, the old image keeps its bmap. If the bmap +/// fails, the old bmap is removed, because it does not match the new image. pub fn commit(mut image: Staged, mut bmap: Option) -> Result<()> { - if let Some(bmap) = bmap.as_ref().filter(|bmap| !bmap.mount_root) { - // a mount point that was not detected cannot be removed, the copy into it replaces it + image.rename()?; + if let Some(bmap) = &mut bmap + && let Err(e) = bmap.rename() + { match fs::remove_file(&bmap.dest) { - Err(e) if !matches!(e.kind(), ErrorKind::NotFound | ErrorKind::ResourceBusy) => { - return Err(e).context(format!("cannot remove {}", bmap.dest.display())); + Err(remove) if remove.kind() != ErrorKind::NotFound => { + warn!("cannot remove the old {}: {remove}", bmap.dest.display()) } _ => {} } - } - image.rename()?; - if let Some(bmap) = &mut bmap { - bmap.rename()?; + return Err(e); } let dirs: BTreeSet<_> = iter::once(&image) @@ -622,6 +621,22 @@ mod tests { assert!(!image_bmap.exists()); } + #[test] + fn commit_keeps_the_old_bmap_when_the_image_rename_fails() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let image_bmap = bmap::bmap_path(&image); + fs::write(&image, OLD_CONTENT).expect("write image"); + fs::write(&image_bmap, OLD_CONTENT).expect("write bmap"); + + let staged_image = stage(&image, NEW_CONTENT); + fs::remove_file(&staged_image.file).expect("remove staged image"); + + assert!(commit(staged_image, Some(stage(&image_bmap, NEW_CONTENT))).is_err()); + assert_eq!(fs::read(&image).expect("read image"), OLD_CONTENT); + assert_eq!(fs::read(&image_bmap).expect("read bmap"), OLD_CONTENT); + } + #[test] fn copy_into_dest_replaces_the_content() { let dir = tempfile::tempdir().expect("create temp dir"); From 95c091818a2d52ea968b431296212c5bca281f5c Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:43:41 +0100 Subject: [PATCH 68/75] fix(file): refuse a partition that does not fit into the image The partition offsets were computed without overflow checks since dd no longer does this math. A corrupt GPT with valid CRCs and a huge LBA could wrap the byte offset in a release build, and `write_partition` would write at the wrapped offset. Now the sector numbers are computed with checked math, and the partition must end inside the image. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/partition.rs | 91 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 83 insertions(+), 8 deletions(-) diff --git a/src/file/partition.rs b/src/file/partition.rs index 747c8f4f..4ff7abe3 100644 --- a/src/file/partition.rs +++ b/src/file/partition.rs @@ -6,20 +6,40 @@ use std::path::Path; pub const SECTOR_SIZE: u64 = 512; +#[derive(Debug)] pub struct PartitionData { pub num: u32, pub start: u64, pub count: u64, } +/// Fails if the partition does not fit into the image, so a corrupt partition table cannot make +/// a read or write go to another offset. pub fn get_partition_data>(path: P, partition_num: u32) -> Result { let path = path.as_ref(); let mut file = File::open(path).with_context(|| format!("failed to open image: {}", path.display()))?; + let data = find_partition(&mut file, partition_num)?; + let image_len = file + .metadata() + .context("failed to get the image size")? + .len(); + let end = data + .start + .checked_add(data.count) + .and_then(|end| end.checked_mul(SECTOR_SIZE)); + anyhow::ensure!( + end.is_some_and(|end| end <= image_len), + "partition {partition_num} ends after the end of the image" + ); + Ok(data) +} + +fn find_partition(file: &mut File, partition_num: u32) -> Result { // Try GPT first (validates CRC32 — more robust than signature check). // Capture any error so we can attach it as context if MBR also fails. - let gpt_err = match gptman::GPT::find_from(&mut file) { + let gpt_err = match gptman::GPT::find_from(file) { Ok(gpt) => { let entry = gpt .iter() @@ -33,10 +53,14 @@ pub fn get_partition_data>(path: P, partition_num: u32) -> Result ); // the callers count in SECTOR_SIZE units let scale = gpt.sector_size / SECTOR_SIZE; + let sectors = |lba: u64| { + lba.checked_mul(scale) + .with_context(|| format!("GPT partition {partition_num} is too large")) + }; return Ok(PartitionData { num: partition_num, - start: entry.starting_lba * scale, - count: (entry.ending_lba - entry.starting_lba + 1) * scale, + start: sectors(entry.starting_lba)?, + count: sectors(entry.ending_lba - entry.starting_lba + 1)?, }); } Err(e) => e, @@ -45,7 +69,7 @@ pub fn get_partition_data>(path: P, partition_num: u32) -> Result // Try MBR — iter() includes logical partitions (5, 6, ...) file.seek(SeekFrom::Start(0)) .context("failed to seek to start of image")?; - let mbr = mbrman::MBR::read_from(&mut file, SECTOR_SIZE as u32) + let mbr = mbrman::MBR::read_from(file, SECTOR_SIZE as u32) .with_context(|| format!("image is neither valid GPT nor MBR (GPT error: {gpt_err})"))?; for (i, p) in mbr.iter() { if u32::try_from(i).ok() == Some(partition_num) && p.is_used() { @@ -68,17 +92,19 @@ pub fn is_gpt>(path: P) -> Result { mod tests { use crate::file::partition::*; use gptman::{GPT, GPTPartitionEntry}; + use std::path::PathBuf; const GPT_SECTOR_SIZE: u64 = 4096; const DISK_SECTORS: u64 = 64; const PART_NUM: u32 = 1; const PART_SECTORS: u64 = 8; const GUID: [u8; 16] = [0xff; 16]; + // times 8 for 4K sectors and times 512 bytes is more than u64::MAX + const OVERFLOWING_LBA: u64 = 1 << 55; - #[test] - fn get_partition_data_converts_4k_gpt_sectors() { - let dir = tempfile::tempdir().expect("create temp dir"); - let path = dir.path().join("image.wic"); + /// A 4K-sector GPT image with partition `PART_NUM` at `first_usable_lba`. + fn gpt_image(dir: &Path) -> (PathBuf, u64) { + let path = dir.join("image.wic"); let mut file = File::create(&path).expect("create image"); file.set_len(DISK_SECTORS * GPT_SECTOR_SIZE) .expect("set image size"); @@ -93,10 +119,59 @@ mod tests { partition_name: "part".into(), }; gpt.write_into(&mut file).expect("write gpt"); + (path, start) + } + + #[test] + fn get_partition_data_converts_4k_gpt_sectors() { + let dir = tempfile::tempdir().expect("create temp dir"); + let (path, start) = gpt_image(dir.path()); let data = get_partition_data(&path, PART_NUM).expect("partition data"); let scale = GPT_SECTOR_SIZE / SECTOR_SIZE; assert_eq!(data.start, start * scale); assert_eq!(data.count, PART_SECTORS * scale); } + + #[test] + fn get_partition_data_rejects_a_partition_after_the_image_end() { + let dir = tempfile::tempdir().expect("create temp dir"); + let (path, _) = gpt_image(dir.path()); + let file = File::options().write(true).open(&path).expect("open image"); + file.set_len(PART_SECTORS * GPT_SECTOR_SIZE) + .expect("set image size"); + + let error = get_partition_data(&path, PART_NUM).expect_err("partition after the end"); + assert!( + error + .to_string() + .contains("ends after the end of the image") + ); + } + + #[test] + fn get_partition_data_rejects_an_overflowing_offset() { + let dir = tempfile::tempdir().expect("create temp dir"); + let (path, _) = gpt_image(dir.path()); + // `GPT::write_into` refuses such an entry, the primary header and entries do not check it + let mut file = File::options() + .read(true) + .write(true) + .open(&path) + .expect("open image"); + let mut gpt = GPT::find_from(&mut file).expect("read gpt"); + gpt[PART_NUM].starting_lba = OVERFLOWING_LBA; + gpt[PART_NUM].ending_lba = OVERFLOWING_LBA + PART_SECTORS - 1; + let partitions: Vec<_> = gpt.iter().map(|(_, entry)| entry.clone()).collect(); + gpt.header + .write_into(&mut file, GPT_SECTOR_SIZE, &partitions) + .expect("write gpt header"); + + let error = get_partition_data(&path, PART_NUM).expect_err("overflowing offset"); + assert!( + error + .to_string() + .contains("ends after the end of the image") + ); + } } From 626c64f6703d5dffb9915e02ad5a1086d2f2f301 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:44:04 +0100 Subject: [PATCH 69/75] fix(file): follow a symlink whose target does not exist yet `resolve` used the link itself when its target was missing, so the staged image replaced the link with a regular file and the target was never created. It now follows the link, also through a chain, up to the Linux limit of 40 links. Such a link also counts as an existing derived output in the conflict check. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/output.rs | 56 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/src/file/output.rs b/src/file/output.rs index 3fefa7dc..50903e80 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -16,6 +16,8 @@ use std::{ use tempfile::TempDir; const TMP_DIR_PREFIX: &str = ".omnect-cli-"; +// the limit of Linux path resolution +const MAX_SYMLINKS: usize = 40; /// Returns `explicit`, else `image` if it has an extension of `target`, else `image` without the /// extension of its `source` compression, with the extension of `target` appended. Fails if @@ -70,7 +72,8 @@ pub fn check_conflicts( ensure_writable(output_image)?; anyhow::ensure!( - explicit_output || in_place || !output_image.exists(), + // a symlink counts also if its target does not exist yet + explicit_output || in_place || output_image.symlink_metadata().is_err(), "{} exists; choose the output image with --output-image", output_image.display() ); @@ -294,9 +297,22 @@ fn same_file(a: &Path, b: &Path) -> Result { Ok(resolve(a)? == resolve(b)?) } -/// The absolute path of `path` with symlinks resolved. A file that does not exist yet is -/// resolved through its directory. +/// The absolute path of `path` with symlinks resolved. A file that does not exist yet, also as +/// the target of a symlink, is resolved through its directory. fn resolve(path: &Path) -> Result { + let mut path = path.to_path_buf(); + for _ in 0..MAX_SYMLINKS { + if !path.is_symlink() || path.exists() { + return resolve_existing_dir(&path); + } + let target = fs::read_link(&path).context(format!("cannot read {}", path.display()))?; + // a relative target is relative to the directory of the link + path = path.parent().unwrap_or(Path::new("")).join(target); + } + anyhow::bail!("too many symlinks in {}", path.display()) +} + +fn resolve_existing_dir(path: &Path) -> Result { if path.exists() { return fs::canonicalize(path).context(format!("cannot resolve {}", path.display())); } @@ -710,6 +726,40 @@ mod tests { assert_eq!(fs::read(&target_bmap).expect("read bmap"), NEW_CONTENT); } + #[test] + fn staged_commit_creates_the_target_of_a_dangling_symlink() { + let dir = tempfile::tempdir().expect("create temp dir"); + let target = dir.path().join("target").join(IMAGE); + let link = dir.path().join(IMAGE); + fs::create_dir(target.parent().expect("target dir")).expect("create dir"); + symlink(Path::new("target").join(IMAGE), &link).expect("create symlink"); + + commit(stage(&link, NEW_CONTENT), None).expect("commit"); + + assert!(link.is_symlink()); + assert_eq!(fs::read(&target).expect("read target"), NEW_CONTENT); + } + + #[test] + fn check_conflicts_rejects_a_dangling_derived_output() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_xz = dir.path().join(IMAGE_XZ); + let image = dir.path().join(IMAGE); + touch(&image_xz); + symlink(dir.path().join("missing.wic"), &image).expect("create symlink"); + + assert!(check_conflicts(&image_xz, &image, false, false, None).is_err()); + } + + #[test] + fn resolve_stops_at_a_symlink_loop() { + let dir = tempfile::tempdir().expect("create temp dir"); + let link = dir.path().join(IMAGE); + symlink(&link, &link).expect("create symlink"); + + assert!(resolve(&link).is_err()); + } + #[test] fn same_file_resolves_relative_and_missing_paths() { let dir = tempfile::tempdir().expect("create temp dir"); From 7447910bb541b397dae349774f826cd154fbe05e Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:44:34 +0100 Subject: [PATCH 70/75] fix(file): check the staging directory before the command runs `check_conflicts` checked only the output file. The directory of the output image, or TMPDIR for a mount point, was found not writable only when the staged file was created, after the decompress and the command. Now the check fails up front. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/output.rs | 52 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/src/file/output.rs b/src/file/output.rs index 50903e80..ba3304ee 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -70,6 +70,7 @@ pub fn check_conflicts( ) -> Result<()> { let in_place = same_file(input_image, output_image)?; ensure_writable(output_image)?; + ensure_staging_dir_writable(output_image)?; anyhow::ensure!( // a symlink counts also if its target does not exist yet @@ -80,6 +81,7 @@ pub fn check_conflicts( if output_bmap { let target_bmap = bmap::bmap_path(output_image); ensure_writable(&target_bmap)?; + ensure_staging_dir_writable(&target_bmap)?; if !in_place { let found_bmap = bmap::find_input_bmap(input_image, None)?; for bmap in input_bmap.into_iter().chain(found_bmap.as_deref()) { @@ -115,6 +117,28 @@ fn ensure_writable(path: &Path) -> Result<()> { } } +/// Fails before the command runs if `Staged::write` could not create its temp dir for `dest`. +fn ensure_staging_dir_writable(dest: &Path) -> Result<()> { + let (dir, _) = staging_dir(&resolve(dest)?)?; + rustix::fs::access(&dir, Access::WRITE_OK).context(format!( + "cannot write to {}, where the new {} is staged", + dir.display(), + dest.display() + )) +} + +/// The directory for the staged file of the resolved `dest`, and whether `dest` is a mount +/// point. +fn staging_dir(dest: &Path) -> Result<(PathBuf, bool)> { + if is_mount_root(dest) { + return Ok((env::temp_dir(), true)); + } + let dir = dest + .parent() + .context(format!("{} has no directory", dest.display()))?; + Ok((dir.to_path_buf(), false)) +} + /// A new file in the directory of its destination, which `commit` renames to the destination. /// Until then the destination stays unchanged. A rename cannot replace a mount point, e.g. an /// image file mounted into a container, so for such a destination the new file is staged in @@ -131,15 +155,7 @@ impl Staged { /// keeps its permissions. pub fn write(dest: &Path, write: impl FnOnce(&mut File) -> Result<()>) -> Result { let dest = resolve(dest)?; - let dir = dest - .parent() - .context(format!("{} has no directory", dest.display()))?; - let mount_root = is_mount_root(&dest); - let tmp_parent = if mount_root { - env::temp_dir() - } else { - dir.to_path_buf() - }; + let (tmp_parent, mount_root) = staging_dir(&dest)?; let tmp_dir = tempfile::Builder::new() .prefix(TMP_DIR_PREFIX) .tempdir_in(&tmp_parent) @@ -346,6 +362,8 @@ mod tests { const IMAGE_MODE: u32 = 0o644; const READ_ONLY_MODE: u32 = 0o444; const MODE_MASK: u32 = 0o777; + const DIR_MODE: u32 = 0o755; + const READ_ONLY_DIR_MODE: u32 = 0o555; const UNRESIZABLE_FILE: &str = "/dev/full"; const MOUNTINFO: &str = "/proc/self/mountinfo"; const MOUNTINFO_MOUNT_POINT: usize = 4; @@ -491,6 +509,22 @@ mod tests { assert!(check_conflicts(&image, &image, false, false, None).is_err()); } + #[test] + fn check_conflicts_rejects_a_read_only_output_dir() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + touch(&image); + fs::set_permissions(dir.path(), fs::Permissions::from_mode(READ_ONLY_DIR_MODE)) + .expect("set permissions"); + // root can write a read-only directory + let writable = rustix::fs::access(dir.path(), Access::WRITE_OK).is_ok(); + + let result = check_conflicts(&image, &image, false, false, None); + fs::set_permissions(dir.path(), fs::Permissions::from_mode(DIR_MODE)) + .expect("set permissions"); + assert_eq!(result.is_ok(), writable); + } + #[test] fn check_conflicts_rejects_existing_derived_image() { let dir = tempfile::tempdir().expect("create temp dir"); From dd1f171c7e5a46ce1c3f47515a481ca238eb0955 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:45:00 +0100 Subject: [PATCH 71/75] fix(file): write a compressed image only with the extension bmaptool reads `-p gzip` and `-p bzip2` kept an input named `.gzip` or `.bzip2` and accepted such a name with `-o`, but `bmaptool` reads only `.gz` and `.bz2`. Now `-o` needs the short name, and such an input is written with it, e.g. `x.wic.gzip` to `x.wic.gz`. The long names are still read as input. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 2 +- src/file/compression.rs | 5 +++++ src/file/output.rs | 44 +++++++++++++++++++++++++++-------------- 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 3bc02c3e..4adb5d27 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ omnect-cli file copy-to-image --help ## Output image -A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gz`), e.g. `my-image.wic.xz` for `my-image.wic`. An image that already has an extension of that compression keeps its name. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. Its name must have the extension of the `-p` compression, or no compression extension without `-p`. +A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gz`), e.g. `my-image.wic.xz` for `my-image.wic`. An image that already has this extension keeps its name, and `.gzip` and `.bzip2` become `.gz` and `.bz2`, because `bmaptool` does not read the long names. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. Its name must have the extension of the `-p` compression, or no compression extension without `-p`. The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. It also fails if the output image or its bmap exists and is not writable for the user. The output image is written to a new file in its directory and then renamed, so the directory needs write access and space for one more image. Until the rename, the new file is in a hidden `.omnect-cli-*` directory. The command removes it when it ends or is stopped with Ctrl-C, `SIGTERM` or `SIGHUP`. It stays after a `SIGKILL` or a power loss, and you can delete it. A rename cannot replace a mount point, e.g. an image mounted with `docker run -v "$(pwd)/my-image.wic":/source/my-image.wic`, so the new image is staged in `TMPDIR` and copied into it. If this copy fails or is stopped, the new image stays in `TMPDIR`, and the log names it. Mount the directory instead of the image if you use `-b`, because the bmap is written next to the image. The new file belongs to the user who runs `omnect-cli`. `copy-from-image` only reads the image and writes nothing. diff --git a/src/file/compression.rs b/src/file/compression.rs index 8efac234..08ae665d 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -128,6 +128,11 @@ impl Compression { .is_some_and(|e| self.extensions().contains(&e)) } + /// Whether `path` has the extension of `compressed_path`, which `bmaptool` reads. + pub fn matches_output_extension(&self, path: &Path) -> bool { + path.extension().is_some_and(|e| e == self.extension()) + } + pub fn has_extension(path: &Path) -> bool { Compression::iter().any(|c| c.matches_extension(path)) } diff --git a/src/file/output.rs b/src/file/output.rs index ba3304ee..ede31ba1 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -19,9 +19,10 @@ const TMP_DIR_PREFIX: &str = ".omnect-cli-"; // the limit of Linux path resolution const MAX_SYMLINKS: usize = 40; -/// Returns `explicit`, else `image` if it has an extension of `target`, else `image` without the -/// extension of its `source` compression, with the extension of `target` appended. Fails if -/// `explicit` has no extension of `target`, or a compression extension while `target` is none. +/// Returns `explicit`, else `image` if it has the extension `target` writes, else `image` without +/// the extension of its `source` compression, with the extension of `target` appended. Fails if +/// `explicit` has not the extension `target` writes, or a compression extension while `target` is +/// none. pub fn output_image( image: &Path, source: Option<&Compression>, @@ -31,7 +32,7 @@ pub fn output_image( if let Some(path) = explicit { match target { Some(c) => anyhow::ensure!( - c.matches_extension(path), + c.matches_output_extension(path), "{} has no extension of the -p compression; name it e.g. {}", path.display(), c.compressed_path(path).display() @@ -44,7 +45,7 @@ pub fn output_image( } return Ok(path.to_path_buf()); } - if target.is_some_and(|c| c.matches_extension(image)) { + if target.is_some_and(|c| c.matches_output_extension(image)) { return Ok(image.to_path_buf()); } @@ -390,20 +391,33 @@ mod tests { } #[test] - fn output_image_keeps_every_extension_of_the_same_compression() { - for (image, compression) in [ - ("image.wic.gz", Compression::gzip), - ("image.wic.gzip", Compression::gzip), - ("image.wic.bz2", Compression::bzip2), - ("image.wic.bzip2", Compression::bzip2), + fn output_image_uses_the_extension_bmaptool_reads() { + for (image, compression, output) in [ + ("image.wic.gz", Compression::gzip, "image.wic.gz"), + ("image.wic.gzip", Compression::gzip, "image.wic.gz"), + ("image.wic.bz2", Compression::bzip2, "image.wic.bz2"), + ("image.wic.bzip2", Compression::bzip2, "image.wic.bz2"), ] { - let image = Path::new(image); assert_eq!( - output_image(image, Some(&compression), Some(&compression), None) - .expect("output image"), - image + output_image( + Path::new(image), + Some(&compression), + Some(&compression), + None + ) + .expect("output image"), + Path::new(output) ); } + assert!( + output_image( + Path::new(IMAGE), + None, + Some(&Compression::gzip), + Some(Path::new("device.wic.gzip")) + ) + .is_err() + ); } #[test] From 6d80f925d24fbcda94c67f89084723e2a622e55a Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:46:20 +0100 Subject: [PATCH 72/75] fix(file): remove every temp dir of a command on a stop signal Only the staged files and the new device key were removed on a stop signal. The decompressed or copied image in TMPDIR and the partition files stayed, also after a handler was installed. `RegisteredTempDir` registers a temp dir on create and unregisters it on drop, and all temp dirs of a command use it. It replaces the two hand-written Drop impls. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/functions.rs | 12 ++++++-- src/file/output.rs | 35 ++++++++++------------- src/file/signal.rs | 64 ++++++++++++++++++++++++++++++++++++------- src/lib.rs | 24 +++++++--------- 4 files changed, 87 insertions(+), 48 deletions(-) diff --git a/src/file/functions.rs b/src/file/functions.rs index 5e4a2794..546651c7 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -1,4 +1,6 @@ -use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, partition::SECTOR_SIZE, sparse}; +use crate::file::{ + COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, partition::SECTOR_SIZE, signal::RegisteredTempDir, sparse, +}; use anyhow::{Context, Result}; use log::{debug, warn}; use std::{ @@ -190,7 +192,9 @@ macro_rules! try_exec_cmd { pub fn copy_to_image(file_copy_params: &[FileCopyToParams], image_file: &Path) -> Result<()> { // the partition files get their own dir, so they never collide with the image - let working_dir = tempfile::tempdir().context("copy_to_image: cannot create a temp dir")?; + let working_dir = RegisteredTempDir::new( + tempfile::tempdir().context("copy_to_image: cannot create a temp dir")?, + ); let image_file = image_file.to_str().unwrap(); let mut partition_map: HashMap<&Partition, Vec<(&PathBuf, &PathBuf)>> = HashMap::new(); @@ -268,7 +272,9 @@ pub fn copy_to_image(file_copy_params: &[FileCopyToParams], image_file: &Path) - pub fn copy_from_image(file_copy_params: &[FileCopyFromParams], image_file: &Path) -> Result<()> { // the partition files get their own dir, so they never collide with the image - let working_dir = tempfile::tempdir().context("copy_from_image: cannot create a temp dir")?; + let working_dir = RegisteredTempDir::new( + tempfile::tempdir().context("copy_from_image: cannot create a temp dir")?, + ); let image_file = image_file.to_str().unwrap(); for param in file_copy_params.iter() { diff --git a/src/file/output.rs b/src/file/output.rs index ede31ba1..5ed834f2 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -1,4 +1,4 @@ -use crate::file::{bmap, compression::Compression, signal, sparse}; +use crate::file::{bmap, compression::Compression, signal::RegisteredTempDir, sparse}; use anyhow::{Context, Result}; use log::{info, warn}; use rustix::{ @@ -13,7 +13,6 @@ use std::{ iter, path::{Path, PathBuf}, }; -use tempfile::TempDir; const TMP_DIR_PREFIX: &str = ".omnect-cli-"; // the limit of Linux path resolution @@ -145,7 +144,7 @@ fn staging_dir(dest: &Path) -> Result<(PathBuf, bool)> { /// image file mounted into a container, so for such a destination the new file is staged in /// TMPDIR and copied into it. pub struct Staged { - tmp_dir: TempDir, + tmp_dir: RegisteredTempDir, file: PathBuf, dest: PathBuf, mount_root: bool, @@ -157,19 +156,19 @@ impl Staged { pub fn write(dest: &Path, write: impl FnOnce(&mut File) -> Result<()>) -> Result { let dest = resolve(dest)?; let (tmp_parent, mount_root) = staging_dir(&dest)?; - let tmp_dir = tempfile::Builder::new() - .prefix(TMP_DIR_PREFIX) - .tempdir_in(&tmp_parent) - .context(format!( - "cannot create a temp dir in {}", - tmp_parent.display() - ))?; + let tmp_dir = RegisteredTempDir::new( + tempfile::Builder::new() + .prefix(TMP_DIR_PREFIX) + .tempdir_in(&tmp_parent) + .context(format!( + "cannot create a temp dir in {}", + tmp_parent.display() + ))?, + ); let file = tmp_dir.path().join( dest.file_name() .context(format!("{} has no file name", dest.display()))?, ); - signal::register(tmp_dir.path()); - // from here on, drop unregisters the temp dir let staged = Self { tmp_dir, file, @@ -227,7 +226,7 @@ impl Staged { /// Keeps the staged file if the copy fails or a signal stops it, because `dest` is truncated /// already. fn copy_into_dest(&mut self) -> Result<()> { - signal::unregister(self.tmp_dir.path()); + self.tmp_dir.unregister(); info!( "copying {} into {}; it stays if the copy fails", self.file.display(), @@ -245,7 +244,7 @@ impl Staged { .context(format!("cannot sync {}", self.dest.display())) }); if result.is_err() { - self.tmp_dir.disable_cleanup(true); + self.tmp_dir.keep(); } result.with_context(|| { format!( @@ -257,12 +256,6 @@ impl Staged { } } -impl Drop for Staged { - fn drop(&mut self) { - signal::unregister(self.tmp_dir.path()); - } -} - /// `false` if the kernel cannot tell. fn is_mount_root(path: &Path) -> bool { rustix::fs::statx( @@ -348,7 +341,7 @@ fn resolve_existing_dir(path: &Path) -> Result { #[cfg(test)] mod tests { - use crate::file::output::*; + use crate::file::{output::*, signal}; use std::{ io::Write, os::unix::fs::{PermissionsExt, symlink}, diff --git a/src/file/signal.rs b/src/file/signal.rs index 6fd65099..a5eee9aa 100644 --- a/src/file/signal.rs +++ b/src/file/signal.rs @@ -14,6 +14,7 @@ use std::{ sync::{Mutex, MutexGuard, Once, PoisonError}, thread, }; +use tempfile::TempDir; const STOP_SIGNALS: [c_int; 3] = [SIGHUP, SIGINT, SIGTERM]; // exit code of a process that a signal stopped, plus the signal number @@ -22,18 +23,48 @@ const SIGNAL_EXIT_CODE_BASE: c_int = 128; static DIRS: Mutex> = Mutex::new(BTreeSet::new()); static HANDLER: Once = Once::new(); -/// Removes `dir` if a stop signal ends the process before `unregister`. -pub fn register(dir: &Path) { +/// A temp dir that is also removed if a stop signal ends the process. +pub struct RegisteredTempDir(TempDir); + +impl RegisteredTempDir { + pub fn new(dir: TempDir) -> Self { + register(dir.path()); + Self(dir) + } + + pub fn path(&self) -> &Path { + self.0.path() + } + + /// Keeps the dir if a stop signal ends the process. Drop still removes it. + pub fn unregister(&self) { + unregister(self.path()); + } + + /// Keeps the dir, also on drop. + pub fn keep(&mut self) { + self.unregister(); + self.0.disable_cleanup(true); + } +} + +impl Drop for RegisteredTempDir { + fn drop(&mut self) { + self.unregister(); + } +} + +fn register(dir: &Path) { HANDLER.call_once(start_handler); dirs().insert(dir.to_path_buf()); } -pub fn unregister(dir: &Path) { +fn unregister(dir: &Path) { dirs().remove(dir); } #[cfg(test)] -pub fn is_registered(dir: &Path) -> bool { +pub(crate) fn is_registered(dir: &Path) -> bool { dirs().contains(dir) } @@ -85,13 +116,26 @@ mod tests { use crate::file::signal::*; #[test] - fn register_and_unregister() { - let dir = tempfile::tempdir().expect("create temp dir"); + fn registered_temp_dir_is_registered_until_drop() { + let dir = RegisteredTempDir::new(tempfile::tempdir().expect("create temp dir")); + let path = dir.path().to_path_buf(); - register(dir.path()); - assert!(is_registered(dir.path())); - unregister(dir.path()); - assert!(!is_registered(dir.path())); + assert!(is_registered(&path)); + drop(dir); + assert!(!is_registered(&path)); + assert!(!path.exists()); + } + + #[test] + fn registered_temp_dir_keep_unregisters_and_keeps_the_dir() { + let mut dir = RegisteredTempDir::new(tempfile::tempdir().expect("create temp dir")); + let path = dir.path().to_path_buf(); + + dir.keep(); + assert!(!is_registered(&path)); + drop(dir); + assert!(path.exists()); + fs::remove_dir(&path).expect("remove dir"); } #[test] diff --git a/src/lib.rs b/src/lib.rs index ae558452..d1ac84ab 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,14 +26,14 @@ use file::{ compression::{self, Compression}, functions::FileCopyToParams, output::{self, Staged, check_conflicts, output_image}, - signal, sparse, + signal::RegisteredTempDir, + sparse, }; use log::info; use std::{ fs, path::{Path, PathBuf}, }; -use tempfile::TempDir; fn run_image_command(image_file: PathBuf, output: ImageOutput, command: F) -> Result<()> where @@ -86,7 +86,7 @@ where if source_compression.is_none() { verify_input_bmap(&image_file)?; } - let tmp_dir = tempfile::tempdir().context("cannot create a temp dir")?; + let tmp_dir = RegisteredTempDir::new(tempfile::tempdir().context("cannot create a temp dir")?); let tmp_image_file = temp_image(&image_file, source_compression.as_ref(), tmp_dir.path())?; if source_compression.is_some() { verify_input_bmap(&tmp_image_file)?; @@ -135,7 +135,8 @@ where match Compression::from_file(&image_file)? { Some(c) => { - let tmp_dir = tempfile::tempdir().context("cannot create a temp dir")?; + let tmp_dir = + RegisteredTempDir::new(tempfile::tempdir().context("cannot create a temp dir")?); command(&compression::decompress(&image_file, &c, tmp_dir.path())?) } None => command(&image_file), @@ -176,17 +177,11 @@ fn temp_image( /// A new device certificate and key in a temp dir, so parallel runs do not share them and no /// private key is left behind, also not after a stop signal. struct CertInfo { - dir: TempDir, + _dir: RegisteredTempDir, cert_path: PathBuf, key_path: PathBuf, } -impl Drop for CertInfo { - fn drop(&mut self) { - signal::unregister(self.dir.path()); - } -} - struct CertificateOptions<'a> { intermediate_full_chain_cert: &'a Path, intermediate_key: &'a Path, @@ -210,12 +205,13 @@ fn create_image_cert(cert_opts: CertificateOptions) -> Result { .create_cert_and_key(cert_opts.subject, cert_opts.validity_days) .context("create_and_set_image_cert: couldn't create device cert and key")?; - let dir = tempfile::tempdir().context("create_and_set_image_cert: cannot create a temp dir")?; - signal::register(dir.path()); + let dir = RegisteredTempDir::new( + tempfile::tempdir().context("create_and_set_image_cert: cannot create a temp dir")?, + ); let cert_info = CertInfo { cert_path: dir.path().join(cert_opts.target_cert), key_path: dir.path().join(cert_opts.target_key), - dir, + _dir: dir, }; fs::write(&cert_info.cert_path, cert_pem) From 59a6cf9952a155e411af2f5837de146f19236a93 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:46:30 +0100 Subject: [PATCH 73/75] docs: describe where a failed copy keeps the image and the stop signal cleanup A copy into a mount point that the EBUSY/EXDEV fallback found keeps the new image next to the image, not in TMPDIR, so the README now points to the log. It also names the up-front directory check and the cleanup of the TMPDIR copies. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 4 ++-- project-context.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4adb5d27..1a7be601 100644 --- a/README.md +++ b/README.md @@ -187,9 +187,9 @@ omnect-cli file copy-to-image --help A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gz`), e.g. `my-image.wic.xz` for `my-image.wic`. An image that already has this extension keeps its name, and `.gzip` and `.bzip2` become `.gz` and `.bz2`, because `bmaptool` does not read the long names. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. Its name must have the extension of the `-p` compression, or no compression extension without `-p`. -The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. It also fails if the output image or its bmap exists and is not writable for the user. The output image is written to a new file in its directory and then renamed, so the directory needs write access and space for one more image. Until the rename, the new file is in a hidden `.omnect-cli-*` directory. The command removes it when it ends or is stopped with Ctrl-C, `SIGTERM` or `SIGHUP`. It stays after a `SIGKILL` or a power loss, and you can delete it. A rename cannot replace a mount point, e.g. an image mounted with `docker run -v "$(pwd)/my-image.wic":/source/my-image.wic`, so the new image is staged in `TMPDIR` and copied into it. If this copy fails or is stopped, the new image stays in `TMPDIR`, and the log names it. Mount the directory instead of the image if you use `-b`, because the bmap is written next to the image. The new file belongs to the user who runs `omnect-cli`. `copy-from-image` only reads the image and writes nothing. +The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. It also fails before it starts if the output image or its bmap exists and is not writable for the user, or if the directory of the new file is not writable. The output image is written to a new file in its directory and then renamed, so the directory needs write access and space for one more image. Until the rename, the new file is in a hidden `.omnect-cli-*` directory. The command removes it when it ends or is stopped with Ctrl-C, `SIGTERM` or `SIGHUP`. It stays after a `SIGKILL` or a power loss, and you can delete it. A rename cannot replace a mount point, e.g. an image mounted with `docker run -v "$(pwd)/my-image.wic":/source/my-image.wic`, so the new image is staged in `TMPDIR` and copied into it. If this copy fails or is stopped, the new image stays, and the log names it. Mount the directory instead of the image if you use `-b`, because the bmap is written next to the image. The new file belongs to the user who runs `omnect-cli`. `copy-from-image` only reads the image and writes nothing. -A command copies each partition it reads or changes to `TMPDIR` (default `/tmp`). A command that changes an image, or reads a compressed one, also copies the uncompressed image there, and for an output image that is a mount point also the new image. So `TMPDIR` needs space for all of them. If `/tmp` is a small tmpfs, set `TMPDIR` to a directory with more space. +A command copies each partition it reads or changes to `TMPDIR` (default `/tmp`). A command that changes an image, or reads a compressed one, also copies the uncompressed image there, and for an output image that is a mount point also the new image. So `TMPDIR` needs space for all of them. Ctrl-C, `SIGTERM` or `SIGHUP` removes them too. If `/tmp` is a small tmpfs, set `TMPDIR` to a directory with more space. ## bmap files diff --git a/project-context.md b/project-context.md index 89974387..b6831767 100644 --- a/project-context.md +++ b/project-context.md @@ -54,7 +54,7 @@ - `src/file/bmap.rs` — input bmap lookup and checksum check; writes the output bmap: input ranges (or the whole image) plus data blocks - `src/file/output.rs` — output image name, conflict checks, staged write and commit of the output image and bmap - `src/file/sparse.rs` — data/hole ranges via `SEEK_DATA`/`SEEK_HOLE`, sparse range copy -- `src/file/signal.rs` — removes registered temp dirs (staged output, device key) on SIGINT/SIGTERM/SIGHUP +- `src/file/signal.rs` — `RegisteredTempDir`: a temp dir that SIGINT/SIGTERM/SIGHUP (unless ignored) also removes - `src/file/partition.rs` — GPT/MBR partition table parsing - `src/file/compression.rs` — xz/bzip2/gzip compress/decompress with auto-detection - `src/validators/` — validation for identity config (TOML), device-update config (JSON), SSH keys From aaa1c8a599d8ad43f72dfd51e6281609d8a21a2d Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:46:41 +0100 Subject: [PATCH 74/75] test(file): build the last inline bmap path with bmap::bmap_path Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- tests/integration_tests.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 93f7b6bf..2599055d 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1479,7 +1479,8 @@ fn check_bmap_generation_wic_xz() { fn check_bmap_generation_wic_xz_without_input_bmap() { let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); let image_path_wic_xz = tr.to_pathbuf("testfiles/image.wic.xz"); - let image_path_bmap = image_path_wic_xz.with_extension("bmap"); + // the output image is the uncompressed image next to the input + let image_path_bmap = bmap_path(&image_path_wic_xz.with_extension("")); let in_file = tr.to_pathbuf("testfiles/boot.scr"); Command::cargo_bin("omnect-cli") From 32d03c9e4b5af08dd5650e72fbc6b9fa401035ae Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:50:16 +0100 Subject: [PATCH 75/75] fix(file): suggest the right -o name for the -p compression Since `-o` needs `.gz` or `.bz2`, `-o x.wic.gzip -p gzip` failed with "has no extension of the -p compression" and suggested `x.wic.gzip.gz`. The error now names the expected extension and suggests the name with the compression extension replaced. The README and the `-o` help list the extensions, and the staging directory error names `--output-image` like the file check. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 2 +- src/cli.rs | 2 +- src/file/compression.rs | 2 +- src/file/output.rs | 38 ++++++++++++++++++++++++-------------- tests/integration_tests.rs | 2 +- 5 files changed, 28 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 1a7be601..835eb1c6 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ omnect-cli file copy-to-image --help ## Output image -A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gz`), e.g. `my-image.wic.xz` for `my-image.wic`. An image that already has this extension keeps its name, and `.gzip` and `.bzip2` become `.gz` and `.bz2`, because `bmaptool` does not read the long names. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. Its name must have the extension of the `-p` compression, or no compression extension without `-p`. +A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gz`), e.g. `my-image.wic.xz` for `my-image.wic`. An image that already has this extension keeps its name, and `.gzip` and `.bzip2` become `.gz` and `.bz2`, because `bmaptool` does not read the long names. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. With `-p` its name must end in the extension that `-p` writes (`.xz`, `.bz2` or `.gz`), and without `-p` it must not have a compression extension. The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. It also fails before it starts if the output image or its bmap exists and is not writable for the user, or if the directory of the new file is not writable. The output image is written to a new file in its directory and then renamed, so the directory needs write access and space for one more image. Until the rename, the new file is in a hidden `.omnect-cli-*` directory. The command removes it when it ends or is stopped with Ctrl-C, `SIGTERM` or `SIGHUP`. It stays after a `SIGKILL` or a power loss, and you can delete it. A rename cannot replace a mount point, e.g. an image mounted with `docker run -v "$(pwd)/my-image.wic":/source/my-image.wic`, so the new image is staged in `TMPDIR` and copied into it. If this copy fails or is stopped, the new image stays, and the log names it. Mount the directory instead of the image if you use `-b`, because the bmap is written next to the image. The new file belongs to the user who runs `omnect-cli`. `copy-from-image` only reads the image and writes nothing. diff --git a/src/cli.rs b/src/cli.rs index dd20908c..d0514e95 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -23,7 +23,7 @@ pub struct ImageOutput { conflicts_with = "input_bmap" )] pub no_input_bmap: bool, - /// optional: write the output image to this path, with the --pack-image extension or no compression extension; default: the input image, without the compression extension, plus the --pack-image extension + /// optional: write the output image to this path, ending in .xz, .bz2 or .gz with --pack-image, else without a compression extension; default: the input image, without the compression extension, plus the --pack-image extension #[arg(short = 'o', long = "output-image")] pub output_image: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') diff --git a/src/file/compression.rs b/src/file/compression.rs index 08ae665d..15af05f4 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -104,7 +104,7 @@ impl Compression { } } - fn extension(&self) -> &'static str { + pub fn extension(&self) -> &'static str { self.extensions()[0] } diff --git a/src/file/output.rs b/src/file/output.rs index 5ed834f2..d68ad10f 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -20,8 +20,8 @@ const MAX_SYMLINKS: usize = 40; /// Returns `explicit`, else `image` if it has the extension `target` writes, else `image` without /// the extension of its `source` compression, with the extension of `target` appended. Fails if -/// `explicit` has not the extension `target` writes, or a compression extension while `target` is -/// none. +/// `explicit` does not have the extension `target` writes, or has a compression extension while +/// `target` is none. pub fn output_image( image: &Path, source: Option<&Compression>, @@ -32,9 +32,11 @@ pub fn output_image( match target { Some(c) => anyhow::ensure!( c.matches_output_extension(path), - "{} has no extension of the -p compression; name it e.g. {}", + "{} does not end in .{}, the extension of the -p compression; name it e.g. {}", path.display(), - c.compressed_path(path).display() + c.extension(), + c.compressed_path(&without_compression_extension(path)) + .display() ), None => anyhow::ensure!( !Compression::has_extension(path), @@ -58,6 +60,14 @@ pub fn output_image( }) } +fn without_compression_extension(path: &Path) -> PathBuf { + let mut path = path.to_path_buf(); + if Compression::has_extension(&path) { + path.set_extension(""); + } + path +} + /// Fails if writing `output_image` and its bmap would overwrite a file the user did not name, /// replace the bmap of an image that stays, or leave a bmap that `bmaptool` would use for the /// output image and that does not match it. @@ -121,7 +131,8 @@ fn ensure_writable(path: &Path) -> Result<()> { fn ensure_staging_dir_writable(dest: &Path) -> Result<()> { let (dir, _) = staging_dir(&resolve(dest)?)?; rustix::fs::access(&dir, Access::WRITE_OK).context(format!( - "cannot write to {}, where the new {} is staged", + "cannot write to {}, where the new {} is staged; make it writable or choose another \ + output image with --output-image", dir.display(), dest.display() )) @@ -402,15 +413,14 @@ mod tests { Path::new(output) ); } - assert!( - output_image( - Path::new(IMAGE), - None, - Some(&Compression::gzip), - Some(Path::new("device.wic.gzip")) - ) - .is_err() - ); + let error = output_image( + Path::new(IMAGE), + None, + Some(&Compression::gzip), + Some(Path::new("device.wic.gzip")), + ) + .expect_err("long name with -o"); + assert!(error.to_string().contains("name it e.g. device.wic.gz")); } #[test] diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 2599055d..dae88385 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1338,7 +1338,7 @@ fn check_derived_output_image_exists() { ]) .failure() .stderr(predicate::str::contains( - "has no extension of the -p compression", + "does not end in .xz, the extension of the -p compression; name it e.g. ", )); // a path given with -o is overwritten copy_to_image(&["-o".as_ref(), image_path.as_os_str()]).success();