diff --git a/Cargo.lock b/Cargo.lock index 8faa9e11..81e44d6a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -798,7 +798,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1329,7 +1329,7 @@ checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] name = "isb" -version = "4.1.2" +version = "4.2.0" dependencies = [ "age", "clap", @@ -1340,7 +1340,7 @@ dependencies = [ "isb-server", "isb-tui", "libc", - "rustix 1.1.5", + "rustix 1.1.6", "rustls", "serde", "serde_json", @@ -1352,7 +1352,7 @@ dependencies = [ [[package]] name = "isb-apps" -version = "4.1.2" +version = "4.2.0" dependencies = [ "age", "base64 0.23.1", @@ -1376,7 +1376,7 @@ dependencies = [ [[package]] name = "isb-core" -version = "4.1.2" +version = "4.2.0" dependencies = [ "age", "base64 0.23.1", @@ -1385,7 +1385,7 @@ dependencies = [ "rcgen", "ring", "rusqlite", - "rustix 1.1.5", + "rustix 1.1.6", "rustls", "schemars", "serde", @@ -1403,7 +1403,7 @@ dependencies = [ [[package]] name = "isb-daemon" -version = "4.1.2" +version = "4.2.0" dependencies = [ "age", "httparse", @@ -1413,7 +1413,7 @@ dependencies = [ "isb-server", "libc", "ring", - "rustix 1.1.5", + "rustix 1.1.6", "rustls", "serde", "serde_json", @@ -1423,7 +1423,7 @@ dependencies = [ [[package]] name = "isb-egress" -version = "4.1.2" +version = "4.2.0" dependencies = [ "base64 0.23.1", "httparse", @@ -1436,7 +1436,7 @@ dependencies = [ [[package]] name = "isb-server" -version = "4.1.2" +version = "4.2.0" dependencies = [ "argon2", "base64 0.23.1", @@ -1445,7 +1445,7 @@ dependencies = [ "libc", "ring", "rusqlite", - "rustix 1.1.5", + "rustix 1.1.6", "rustls", "serde", "serde_json", @@ -1459,7 +1459,7 @@ dependencies = [ [[package]] name = "isb-tui" -version = "4.1.2" +version = "4.2.0" dependencies = [ "isb-core", "libc", @@ -2224,15 +2224,15 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2689,8 +2689,8 @@ dependencies = [ "fastrand", "getrandom 0.4.3", "once_cell", - "rustix 1.1.5", - "windows-sys 0.61.2", + "rustix 1.1.6", + "windows-sys 0.52.0", ] [[package]] @@ -3086,7 +3086,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index a1f19db2..1804fe1a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,7 +55,7 @@ members = ["crates/*"] default-members = [".", "crates/*"] [workspace.package] -version = "4.1.2" +version = "4.2.0" edition = "2024" rust-version = "1.89" license = "LicenseRef-EXA-SL-1.0" @@ -63,12 +63,12 @@ repository = "https://github.com/execution-associates/isb" homepage = "https://github.com/execution-associates/isb" [workspace.dependencies] -isb-core = { path = "crates/isb-core", version = "=4.1.2" } -isb-server = { path = "crates/isb-server", version = "=4.1.2" } -isb-apps = { path = "crates/isb-apps", version = "=4.1.2" } -isb-daemon = { path = "crates/isb-daemon", version = "=4.1.2" } -isb-tui = { path = "crates/isb-tui", version = "=4.1.2" } -isb-egress = { path = "crates/isb-egress", version = "=4.1.2" } +isb-core = { path = "crates/isb-core", version = "=4.2.0" } +isb-server = { path = "crates/isb-server", version = "=4.2.0" } +isb-apps = { path = "crates/isb-apps", version = "=4.2.0" } +isb-daemon = { path = "crates/isb-daemon", version = "=4.2.0" } +isb-tui = { path = "crates/isb-tui", version = "=4.2.0" } +isb-egress = { path = "crates/isb-egress", version = "=4.2.0" } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/SKILL.md b/SKILL.md index 6fa04686..ad3ef014 100644 --- a/SKILL.md +++ b/SKILL.md @@ -129,6 +129,7 @@ isb ps # status of the file's sandboxes isb port get NAME DEV # the port a published range picked isb config # the file with every ${VAR} filled in isb down # delete them (--volumes also deletes named volumes) +isb volume prune # list volumes whose sandbox is gone (-y deletes) ``` `isb up` blocks until every `command` exits (forever if none has one), Ctrl-C, @@ -195,7 +196,10 @@ cannot exec or read secrets. `sandbox_remove`. Remote callers' specs are refused privileged mode, raw config, host bind mounts outside the bind roots and non-loopback ports. `sandbox_settings` reads the org's defaults (`sandbox_expiry`, - `sandbox_idle`); org admins change them. + `sandbox_idle`, `sandbox_volume_ttl`); org admins change them. A volume a + sandbox created is deleted once that sandbox has been gone, with nothing + using the volume, for `sandbox_volume_ttl` (7d); `keep: true` on the + volume's definition exempts it. - **Apps:** `project_create`, `app_create` (an image, or a git repository and a builder), `app_env_set` (`.env` text; `KEY=${{secret.NAME}}` for secrets), `app_deploy` (`wait: true`), `app_deployment_log`, diff --git a/crates/isb-apps/src/volume_backup/incus.rs b/crates/isb-apps/src/volume_backup/incus.rs index f68beede..a45d66a8 100644 --- a/crates/isb-apps/src/volume_backup/incus.rs +++ b/crates/isb-apps/src/volume_backup/incus.rs @@ -289,6 +289,7 @@ mod tests { "/1.0/instances/ws".into(), "/1.0/profiles/default".into(), ], + created_at: String::new(), }; assert_eq!(instances_using(&v), ["web-1", "ws"]); } diff --git a/crates/isb-core/src/plan.rs b/crates/isb-core/src/plan.rs index 37cf4e5c..e02b61d4 100644 --- a/crates/isb-core/src/plan.rs +++ b/crates/isb-core/src/plan.rs @@ -131,13 +131,14 @@ impl HostFacts { } } -/// A named volume that must exist before the instance. +/// A named volume that must exist before the instance. `keep` is its definition's: never marked as a sandbox's. #[derive(Debug, Clone, PartialEq, Serialize)] pub struct EnsureVolume { pub pool: String, pub name: String, pub config: Props, pub external: bool, + pub keep: bool, } /// chown (and root-owned parents in the owner's home) and/or chmod a mount point. @@ -751,6 +752,7 @@ pub fn resolve( name: n.clone(), config, external: v.external || def.is_some_and(|d| d.external), + keep: def.is_some_and(|d| d.keep), }; if !volumes .iter() diff --git a/crates/isb-core/src/rpc.rs b/crates/isb-core/src/rpc.rs index 9fdb5516..ccc10ffa 100644 --- a/crates/isb-core/src/rpc.rs +++ b/crates/isb-core/src/rpc.rs @@ -184,6 +184,7 @@ impl Server { let a: SpecParams = params(p)?; let c = self.client(&a.project); let opts = EnsureOptions { + own_volumes: true, wait_ready: a.wait_ready, default_expiry: self.default_expiry(&c), ..Default::default() @@ -196,6 +197,7 @@ impl Server { let a: SpecParams = params(p)?; let c = self.client(&a.project); let opts = EnsureOptions { + own_volumes: true, wait_ready: a.wait_ready, diff: DiffOptions { prune_devices: a.prune_devices, @@ -370,6 +372,7 @@ impl Server { let proj = compose::load(&a.load())?; let c = self.client(&a.project); let opts = EnsureOptions { + own_volumes: true, wait_ready: a.wait_ready, diff: DiffOptions { prune_devices: a.prune_devices, @@ -552,6 +555,7 @@ fn create_desired( return Err(Error::AlreadyExists(d.name.clone())); } let plan = sandbox::plan_desired(c, d, opts.diff)?; + let plan = sandbox::own_volumes(c, d, plan, opts.own_volumes); let stamped = sandbox::expiry::stamped(d, &plan, opts.default_expiry); sandbox::apply(c, &stamped, &plan, report)?; sandbox::expiry::report_created(&stamped, report); diff --git a/crates/isb-core/src/sandbox.rs b/crates/isb-core/src/sandbox.rs index fa1a2a32..808b92ae 100644 --- a/crates/isb-core/src/sandbox.rs +++ b/crates/isb-core/src/sandbox.rs @@ -14,6 +14,7 @@ use crate::idmap::SubIds; use crate::lock::NameLock; pub mod expiry; pub mod images; +mod options; mod prune; mod update; @@ -22,6 +23,7 @@ use crate::plan::{ VolumeDefs, split_addr, }; use crate::spec::{ExecDefaults, PortSpec, ReadyCheck, SandboxSpec}; +pub use options::{EnsureOptions, own_volumes}; pub use prune::{PruneItem, prune_missing_path}; #[doc(hidden)] pub use update::update_instance; @@ -127,30 +129,6 @@ pub struct ApplyReport { pub restart_needed: Vec, } -/// Options for ensure / up. -#[derive(Debug, Clone, Copy)] -pub struct EnsureOptions { - pub diff: DiffOptions, - /// Run readiness checks after applying. - pub wait_ready: bool, - /// How long to wait for another isb holding this sandbox's lock. - pub lock_wait: Duration, - /// The lifetime of a sandbox whose spec gives no `expires` ([`expiry`]): - /// set by the CLI and `isb rpc`; `None` gives it no deadline. - pub default_expiry: Option, -} - -impl Default for EnsureOptions { - fn default() -> Self { - EnsureOptions { - diff: DiffOptions::default(), - wait_ready: true, - lock_wait: Duration::from_secs(900), - default_expiry: None, - } - } -} - fn inst_path(name: &str) -> String { format!("/1.0/instances/{}", encode_segment(name)) } @@ -861,6 +839,7 @@ pub fn ensure( }, )?; let plan = plan_desired(client, desired, opts.diff)?; + let plan = own_volumes(client, desired, plan, opts.own_volumes); let stamped = expiry::stamped(desired, &plan, opts.default_expiry); let mut out = apply(client, &stamped, &plan, report)?; if out.created { @@ -969,6 +948,7 @@ impl Sandbox { return Err(Error::AlreadyExists(d.name.clone())); } let plan = plan_desired(client, &d, opts.diff)?; + let plan = own_volumes(client, &d, plan, opts.own_volumes); let stamped = expiry::stamped(&d, &plan, opts.default_expiry); apply(client, &stamped, &plan, report)?; expiry::report_created(&stamped, report); diff --git a/crates/isb-core/src/sandbox/options.rs b/crates/isb-core/src/sandbox/options.rs new file mode 100644 index 00000000..38588485 --- /dev/null +++ b/crates/isb-core/src/sandbox/options.rs @@ -0,0 +1,70 @@ +//! The options of an ensure, and the mark on the volumes a plain sandbox +//! creates ([`crate::volume::KEY_SANDBOX`]). + +use std::time::Duration; + +use crate::client::Client; +use crate::plan::{Action, Desired, DiffOptions, SandboxPlan}; + +/// Options for ensure / up. +#[derive(Debug, Clone, Copy)] +pub struct EnsureOptions { + pub diff: DiffOptions, + /// Run readiness checks after applying. + pub wait_ready: bool, + /// How long to wait for another isb holding this sandbox's lock. + pub lock_wait: Duration, + /// The lifetime of a sandbox whose spec gives no `expires` ([`super::expiry`]): + /// set by the CLI and `isb rpc`; `None` gives it no deadline. + pub default_expiry: Option, + /// Mark the volumes this call creates as this sandbox's + /// ([`crate::volume::KEY_SANDBOX`]), so they are cleaned up once it is + /// gone. For plain sandboxes only: a stack's or an app's volumes hold + /// data that outlives every instance. + pub own_volumes: bool, +} + +impl Default for EnsureOptions { + fn default() -> Self { + EnsureOptions { + diff: DiffOptions::default(), + wait_ready: true, + lock_wait: Duration::from_secs(900), + default_expiry: None, + own_volumes: false, + } + } +} + +/// `plan`, with each volume it creates marked as `desired`'s when `own` +/// ([`EnsureOptions::own_volumes`]). A `keep` volume is never marked, and an +/// external one is never created. +pub fn own_volumes( + client: &Client, + desired: &Desired, + mut plan: SandboxPlan, + own: bool, +) -> SandboxPlan { + if !own { + return plan; + } + let owner = format!("{}/{}", client.project_name(), desired.name); + for a in &mut plan.actions { + let Action::CreateVolume { + pool, + volume, + config, + } = a + else { + continue; + }; + let keep = desired + .volumes + .iter() + .any(|v| (&v.pool, &v.name) == (&*pool, &*volume) && v.keep); + if !keep { + config.insert(crate::volume::KEY_SANDBOX.into(), owner.clone()); + } + } + plan +} diff --git a/crates/isb-core/src/spec.rs b/crates/isb-core/src/spec.rs index 42b9f3b1..d51e3d69 100644 --- a/crates/isb-core/src/spec.rs +++ b/crates/isb-core/src/spec.rs @@ -22,6 +22,8 @@ pub use mount::{MountType, VolumeOptions, VolumeSpec}; pub use restart::{BootSpec, RestartMode}; mod ready; pub use ready::ReadyCheck; +mod volume; +pub use volume::NamedVolumeSpec; /// A compose file: named volumes plus any number of services, each one /// sandbox. Mirrors docker compose wherever incus allows. @@ -53,39 +55,6 @@ pub struct ComposeFile { pub secrets: BTreeMap, } -/// A named custom storage volume. -#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] -#[serde(deny_unknown_fields)] -pub struct NamedVolumeSpec { - /// The incus volume name. Default: `_`, or the key itself - /// for an `external` volume. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub name: Option, - - /// Storage pool. `auto` (default) means the same pool the sandbox uses - /// (`storage`), resolved the same way. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub pool: Option, - - /// Volume config keys (e.g. `size: 10GiB`), applied only at creation. - #[serde( - default, - deserialize_with = "flex::string_map", - skip_serializing_if = "BTreeMap::is_empty" - )] - #[schemars(with = "BTreeMap")] - pub config: BTreeMap, - - /// The volume must already exist; isb never creates it. - #[serde( - default, - deserialize_with = "flex::bool", - skip_serializing_if = "std::ops::Not::not" - )] - #[schemars(with = "flex::BoolOrString")] - pub external: bool, -} - /// Instance type. #[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] diff --git a/crates/isb-core/src/spec/volume.rs b/crates/isb-core/src/spec/volume.rs new file mode 100644 index 00000000..23ee3df7 --- /dev/null +++ b/crates/isb-core/src/spec/volume.rs @@ -0,0 +1,52 @@ +//! A named volume's definition (`volumes:` at the top of a compose file). + +use std::collections::BTreeMap; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::flex; + +/// A named custom storage volume. +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct NamedVolumeSpec { + /// The incus volume name. Default: `_`, or the key itself + /// for an `external` volume. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + + /// Storage pool. `auto` (default) means the same pool the sandbox uses + /// (`storage`), resolved the same way. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pool: Option, + + /// Volume config keys (e.g. `size: 10GiB`), applied only at creation. + #[serde( + default, + deserialize_with = "flex::string_map", + skip_serializing_if = "BTreeMap::is_empty" + )] + #[schemars(with = "BTreeMap")] + pub config: BTreeMap, + + /// The volume must already exist; isb never creates it. + #[serde( + default, + deserialize_with = "flex::bool", + skip_serializing_if = "std::ops::Not::not" + )] + #[schemars(with = "flex::BoolOrString")] + pub external: bool, + + /// Never deleted by isb's cleanup of volumes whose sandbox is gone + /// (`isb volume prune`, `isb serve`): for a cache shared between + /// sandboxes, or data that must outlive them. + #[serde( + default, + deserialize_with = "flex::bool", + skip_serializing_if = "std::ops::Not::not" + )] + #[schemars(with = "flex::BoolOrString")] + pub keep: bool, +} diff --git a/crates/isb-core/src/volume.rs b/crates/isb-core/src/volume.rs index bac7648c..ebb6555a 100644 --- a/crates/isb-core/src/volume.rs +++ b/crates/isb-core/src/volume.rs @@ -1,5 +1,7 @@ //! Named custom storage volumes. +use std::time::Duration; + use serde::Serialize; use serde_json::{Value, json}; @@ -7,6 +9,14 @@ use crate::client::{Client, encode_segment}; use crate::error::{Error, Result}; use crate::plan::Props; +/// On a volume a plain sandbox created: that sandbox, as +/// `/`. Once the sandbox is gone and nothing has the +/// volume attached, [`sweep`] deletes it. A stack's, an app's and a `keep` +/// volume never carry it. +pub const KEY_SANDBOX: &str = "user.isb.sandbox"; +/// When [`sweep`] first found the volume unused (epoch seconds). +pub const KEY_UNUSED_SINCE: &str = "user.isb.unused_since"; + /// A custom volume as incus reports it. #[derive(Debug, Clone, Serialize)] pub struct VolumeInfo { @@ -16,6 +26,8 @@ pub struct VolumeInfo { pub config: Props, /// Instances (and profiles) using it, as incus URLs. pub used_by: Vec, + /// RFC 3339, as incus gives it; empty when it does not. + pub created_at: String, } impl VolumeInfo { @@ -53,8 +65,23 @@ impl VolumeInfo { .collect() }) .unwrap_or_default(), + created_at: v + .get("created_at") + .and_then(Value::as_str) + .unwrap_or("") + .into(), } } + + /// The sandbox that created it ([`KEY_SANDBOX`]): (incus project, name). + pub fn sandbox(&self) -> Option<(&str, &str)> { + self.config.get(KEY_SANDBOX)?.split_once('/') + } + + /// [`KEY_UNUSED_SINCE`], when set and a number. + pub fn unused_since(&self) -> Option { + self.config.get(KEY_UNUSED_SINCE)?.parse().ok() + } } fn vol_path(pool: &str, name: &str) -> String { @@ -137,3 +164,357 @@ pub fn remove(client: &Client, pool: &str, name: &str) -> Result<()> { ) .map(|_| ()) } + +/// What [`sweep`] makes of one volume. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Verdict { + /// Not a sandbox's volume ([`KEY_SANDBOX`] absent): never touched. + NotOwned, + /// Attached somewhere, or its sandbox still exists. + InUse, + /// Unused, seen so for the first time: the clock starts now. + Unused, + /// Unused, with this long left before it is deleted. + Waiting(Duration), + /// Unused for the whole of `ttl`. + Due, +} + +/// `sandbox_alive`: whether the sandbox that created `v` still exists. +pub fn verdict(v: &VolumeInfo, sandbox_alive: bool, now: u64, ttl: Duration) -> Verdict { + if v.sandbox().is_none() { + return Verdict::NotOwned; + } + if !v.used_by.is_empty() || sandbox_alive { + return Verdict::InUse; + } + match v.unused_since() { + None => Verdict::Unused, + Some(t) => { + let quiet = Duration::from_secs(now.saturating_sub(t)); + match ttl.checked_sub(quiet) { + Some(left) if !left.is_zero() => Verdict::Waiting(left), + _ => Verdict::Due, + } + } + } +} + +fn sandbox_alive(client: &Client, v: &VolumeInfo) -> Result { + let Some((project, name)) = v.sandbox() else { + return Ok(false); + }; + Ok(client + .clone() + .project(project) + .get_opt(&format!("/1.0/instances/{}", encode_segment(name)))? + .is_some()) +} + +/// Set (`Some`) or remove (`None`) one config key, leaving the rest as it is. +fn set_key(client: &Client, v: &VolumeInfo, key: &str, value: Option<&str>) -> Result<()> { + let path = vol_path(&v.pool, &v.name); + let Some(cur) = client.get_opt(&path)? else { + return Ok(()); + }; + let mut config = cur.get("config").cloned().unwrap_or_else(|| json!({})); + if let Some(m) = config.as_object_mut() { + match value { + Some(x) => m.insert(key.into(), json!(x)), + None => m.remove(key), + }; + } + let body = json!({ + "config": config, + "description": cur.get("description").cloned().unwrap_or_else(|| json!("")), + }); + client + .mutate( + "PUT", + &path, + Some(&body), + &format!("update volume {}", v.name), + client.timeouts.other, + ) + .map(|_| ()) +} + +/// A pool's storage driver (`dir`, `zfs`, `btrfs`, ...). +pub fn pool_driver(client: &Client, pool: &str) -> Result> { + Ok(client + .get_opt(&format!("/1.0/storage-pools/{}", encode_segment(pool)))? + .and_then(|p| p.get("driver").and_then(Value::as_str).map(String::from))) +} + +/// Whether `name` matches `pattern`, where `*` stands for any run of +/// characters (none included). +pub fn name_matches(pattern: &str, name: &str) -> bool { + let mut parts = pattern.split('*'); + let first = parts.next().unwrap_or(""); + let Some(mut rest) = name.strip_prefix(first) else { + return false; + }; + let mut parts: Vec<&str> = parts.collect(); + let Some(last) = parts.pop() else { + return rest.is_empty(); + }; + for p in parts { + match rest.find(p) { + Some(i) => rest = &rest[i + p.len()..], + None => return false, + } + } + rest.ends_with(last) +} + +/// The names of the storage pools, in server order. +pub fn pools(client: &Client) -> Result> { + Ok(client + .get("/1.0/storage-pools")? + .as_array() + .map(|a| { + a.iter() + .filter_map(Value::as_str) + .filter_map(|u| u.rsplit('/').next()) + .map(|s| s.split('?').next().unwrap_or(s).to_string()) + .collect() + }) + .unwrap_or_default()) +} + +/// Volumes nothing has attached: those a sandbox that is gone created +/// ([`KEY_SANDBOX`]), or with `unattached` every one, whoever made it. The +/// second kind includes data on purpose left unattached (a stack scaled to +/// zero), so it is for a person to read, never to delete wholesale. +pub fn orphans(client: &Client, pool: &str, unattached: bool) -> Result> { + let mut out = Vec::new(); + for v in list(client, pool)? { + if !v.used_by.is_empty() { + continue; + } + if unattached || (v.sandbox().is_some() && !sandbox_alive(client, &v)?) { + out.push(v); + } + } + Ok(out) +} + +/// Delete the volumes in `pool` that plain sandboxes created and left +/// behind, once each has been unused for `ttl`: the first pass that finds +/// one unused notes the time on it ([`KEY_UNUSED_SINCE`]), and a volume in +/// use again loses the note. Returns the names deleted. One volume failing +/// is reported and does not stop the rest. +pub fn sweep( + client: &Client, + pool: &str, + now: u64, + ttl: Duration, + report: &mut dyn FnMut(&str), +) -> Result> { + let mut deleted = Vec::new(); + for v in list(client, pool)? { + if v.sandbox().is_none() { + continue; + } + let step = sandbox_alive(client, &v).and_then(|alive| { + match verdict(&v, alive, now, ttl) { + Verdict::InUse if v.unused_since().is_some() => { + set_key(client, &v, KEY_UNUSED_SINCE, None) + } + Verdict::Unused => set_key(client, &v, KEY_UNUSED_SINCE, Some(&now.to_string())), + Verdict::Due => match remove(client, &v.pool, &v.name) { + Ok(()) => { + deleted.push(v.name.clone()); + Ok(()) + } + // Gone already, or attached since the listing: not ours to take. + Err(e) if e.is_not_found() || matches!(e, Error::Invalid(_)) => Ok(()), + Err(e) => Err(e), + }, + _ => Ok(()), + } + }); + if let Err(e) = step { + report(&format!("volume {}: {e}", v.name)); + } + } + Ok(deleted) +} + +#[cfg(test)] +mod tests { + use std::path::Path; + + use super::*; + use crate::plan::{Action, DiffOptions, HostFacts, SandboxPlan, VolumeDefs, diff, resolve}; + use crate::spec::{NamedVolumeSpec, SandboxSpec, Volume}; + + #[test] + fn a_plain_sandbox_marks_the_volumes_it_creates_as_its_own() { + let spec = SandboxSpec::new("web-dev", "dev-base") + .volume("/m", Volume::named("web-dev-modules")) + .volume("/c", Volume::named("shared-cache")); + let host = HostFacts { + pools: vec!["default".into()], + project: "default".into(), + ..Default::default() + }; + let mut defs = VolumeDefs::new(); + defs.insert( + "shared-cache".into(), + NamedVolumeSpec { + keep: true, + ..Default::default() + }, + ); + let d = resolve(&spec, &defs, &host, Path::new("/")).unwrap(); + let missing: Vec<(String, String)> = d + .volumes + .iter() + .map(|v| (v.pool.clone(), v.name.clone())) + .collect(); + let plan = diff(&d, None, &missing, DiffOptions::default()).unwrap(); + let owners = |p: &SandboxPlan| -> Vec<(String, Option)> { + p.actions + .iter() + .filter_map(|a| match a { + Action::CreateVolume { volume, config, .. } => Some(( + volume.clone(), + config.get(crate::volume::KEY_SANDBOX).cloned(), + )), + _ => None, + }) + .collect() + }; + let c = crate::Client::new().project("isb-acme"); + // A stack, an app, a job: nothing is marked, so nothing is ever swept. + let unowned = crate::sandbox::own_volumes(&c, &d, plan.clone(), false); + assert!(owners(&unowned).iter().all(|(_, o)| o.is_none())); + // A plain sandbox: its own volume, but never one the file says to keep. + let owned = crate::sandbox::own_volumes(&c, &d, plan, true); + assert_eq!( + owners(&owned), + [ + ( + "web-dev-modules".to_string(), + Some("isb-acme/web-dev".to_string()) + ), + ("shared-cache".to_string(), None), + ] + ); + // A volume that exists already is not in the plan, so never re-marked. + let again = diff(&d, None, &[], DiffOptions::default()).unwrap(); + assert!(owners(&crate::sandbox::own_volumes(&c, &d, again, true)).is_empty()); + } + + fn vol(owner: Option<&str>, used: bool, since: Option) -> VolumeInfo { + let mut config = Props::new(); + if let Some(o) = owner { + config.insert(KEY_SANDBOX.into(), o.into()); + } + if let Some(t) = since { + config.insert(KEY_UNUSED_SINCE.into(), t.to_string()); + } + VolumeInfo { + name: "v".into(), + pool: "default".into(), + content_type: "filesystem".into(), + config, + used_by: if used { + vec!["/1.0/instances/a".into()] + } else { + vec![] + }, + created_at: String::new(), + } + } + + #[test] + fn only_a_gone_sandboxs_unused_volume_is_ever_due() { + let day = Duration::from_secs(86_400); + let now = 1_000_000; + // A stack's, an app's or a hand-made volume carries no owner. + assert_eq!( + verdict(&vol(None, false, Some(0)), false, now, day), + Verdict::NotOwned + ); + // Attached, or its sandbox is still there (stopped, say). + assert_eq!( + verdict(&vol(Some("default/a"), true, Some(0)), false, now, day), + Verdict::InUse + ); + assert_eq!( + verdict(&vol(Some("default/a"), false, Some(0)), true, now, day), + Verdict::InUse + ); + // The clock starts at the first sighting, not at creation. + assert_eq!( + verdict(&vol(Some("default/a"), false, None), false, now, day), + Verdict::Unused + ); + assert_eq!( + verdict( + &vol(Some("default/a"), false, Some(now - 3600)), + false, + now, + day + ), + Verdict::Waiting(Duration::from_secs(86_400 - 3600)) + ); + assert_eq!( + verdict( + &vol(Some("default/a"), false, Some(now - 86_400)), + false, + now, + day + ), + Verdict::Due + ); + // A clock set forward and back again never makes a volume due early. + assert_eq!( + verdict( + &vol(Some("default/a"), false, Some(now + 50)), + false, + now, + day + ), + Verdict::Waiting(day) + ); + assert_eq!( + verdict( + &vol(Some("default/a"), false, Some(now)), + false, + now, + Duration::ZERO + ), + Verdict::Due + ); + } + + #[test] + fn names_match_with_stars() { + for (p, n, want) in [ + ("web-dev-modules", "web-dev-modules", true), + ("web-dev-modules", "web-dev-modules2", false), + ("*-dev-modules", "web-dev-modules", true), + ("*-dev-modules", "web-dev-mise", false), + ("x-*", "x-cache-ct", true), + ("x-*", "ax-cache", false), + ("*", "anything", true), + ("a*b*c", "a1b2c", true), + ("a*b*c", "a1c2b", false), + ("a*a", "a", false), + ] { + assert_eq!(name_matches(p, n), want, "{p} ~ {n}"); + } + } + + #[test] + fn the_owner_is_a_project_and_a_name() { + assert_eq!( + vol(Some("isb-exa/web-dev"), false, None).sandbox(), + Some(("isb-exa", "web-dev")) + ); + assert_eq!(vol(Some("nonsense"), false, None).sandbox(), None); + } +} diff --git a/crates/isb-core/src/x/mod.rs b/crates/isb-core/src/x/mod.rs index 610bc84c..ed32edac 100644 --- a/crates/isb-core/src/x/mod.rs +++ b/crates/isb-core/src/x/mod.rs @@ -156,6 +156,45 @@ pub fn last_used(instance: &str) -> Option { .ok() } +/// The sandbox a VM run's own cache volume (`x-run--cache`, +/// [`project::RunSpec::cache_volume`]) belongs to, if `volume` is one. +fn cache_owner(volume: &str) -> Option<&str> { + volume + .strip_suffix("-cache") + .filter(|i| i.starts_with(&format!("{PREFIX}run-"))) +} + +/// Delete the cache volumes of VM run sandboxes that are gone: nothing +/// else ever would. The containers' shared cache stays. Returns the names +/// (with `dry_run`, those it would delete). +pub fn sweep_caches( + client: &crate::Client, + dry_run: bool, + report: &mut dyn FnMut(&str), +) -> crate::Result> { + let mut out = Vec::new(); + for pool in crate::volume::pools(client)? { + for v in crate::volume::list(client, &pool)? { + let Some(owner) = cache_owner(&v.name) else { + continue; + }; + if !v.used_by.is_empty() || crate::Sandbox::get(client, owner).is_ok() { + continue; + } + if dry_run { + report(&format!("would delete volume {}", v.name)); + } else if let Err(e) = crate::volume::remove(client, &pool, &v.name) { + report(&format!("volume {}: {e}", v.name)); + continue; + } else { + report(&format!("deleted volume {}", v.name)); + } + out.push(v.name); + } + } + Ok(out) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/isb-daemon/src/daemon/guide.rs b/crates/isb-daemon/src/daemon/guide.rs index 6733cf8e..48e06e8b 100644 --- a/crates/isb-daemon/src/daemon/guide.rs +++ b/crates/isb-daemon/src/daemon/guide.rs @@ -107,7 +107,12 @@ mod tests { let families: std::collections::BTreeSet<&str> = tools.iter().filter_map(|t| t.split('_').next()).collect(); // Arguments and readiness checks that share a family's prefix. - let not_tools = ["user_exists", "sandbox_expiry", "sandbox_idle"]; + let not_tools = [ + "user_exists", + "sandbox_expiry", + "sandbox_idle", + "sandbox_volume_ttl", + ]; let texts = TOPICS.iter().map(|(_, t)| *t).chain([INSTRUCTIONS]); for text in texts { for word in text.split(|c: char| !(c.is_ascii_lowercase() || c == '_' || c == '*')) { diff --git a/crates/isb-daemon/src/daemon/guide/sandboxes.md b/crates/isb-daemon/src/daemon/guide/sandboxes.md index 861a0555..8115a7ad 100644 --- a/crates/isb-daemon/src/daemon/guide/sandboxes.md +++ b/crates/isb-daemon/src/daemon/guide/sandboxes.md @@ -16,8 +16,13 @@ Linux machine, starts in seconds) or a VM. - `sandbox_list`, `sandbox_extend` (push out the expiry before it passes, or the sandbox is deleted), `sandbox_remove`. - `sandbox_settings`: the org's defaults for new sandboxes (`sandbox_expiry`, - e.g. `24h`, at most `30d`; `sandbox_idle`, e.g. `2h`, or `none`). Without - changes it reads; org admins and above change them. + e.g. `24h`, at most `30d`; `sandbox_idle`, e.g. `2h`, or `none`; + `sandbox_volume_ttl`, e.g. `7d`, or `none`). Without changes it reads; org + admins and above change them. +- A named volume a sandbox creates is that sandbox's: once the sandbox is + gone and nothing mounts the volume, it is deleted after + `sandbox_volume_ttl`. Keep data that must outlive a sandbox in a volume + made with `volume_create`, or in a stack or an app. - `sandbox_start`, `sandbox_stop`, `instance_restart`: a stopped sandbox keeps its state. - `sandbox_logs`: output of a long-running `restart: always` service, or an diff --git a/crates/isb-daemon/src/daemon/mod.rs b/crates/isb-daemon/src/daemon/mod.rs index c60dd94e..44b1bf15 100644 --- a/crates/isb-daemon/src/daemon/mod.rs +++ b/crates/isb-daemon/src/daemon/mod.rs @@ -879,6 +879,7 @@ fn sandbox_create(d: &Daemon, a: Value, c: &Caller) -> Result { } let opts = EnsureOptions { wait_ready: a.wait_ready.unwrap_or(true), + own_volumes: true, ..Default::default() }; let mut log: Vec = Vec::new(); diff --git a/crates/isb-daemon/src/daemon/sandboxes.rs b/crates/isb-daemon/src/daemon/sandboxes.rs index e62538f9..aa283169 100644 --- a/crates/isb-daemon/src/daemon/sandboxes.rs +++ b/crates/isb-daemon/src/daemon/sandboxes.rs @@ -23,6 +23,9 @@ use crate::server::{Caller, Registry, Tool}; /// How often the reaper runs. const REAP_EVERY: Duration = Duration::from_secs(60); +/// Volumes left behind are looked for on every this-many-th reaper pass: +/// listing them costs incusd more than the sandboxes' own check does. +const SWEEP_EVERY: u32 = 15; /// A sandbox using this much of a core is active. const ACTIVE_CPU_PCT: f32 = 2.0; /// How long an isbx sandbox stays running after its last run, unless its @@ -163,11 +166,16 @@ impl Sandboxes { .name("isb-sandboxes".into()) .spawn(move || { let mut warned = HashSet::new(); + let mut pass = 0u32; loop { std::thread::sleep(REAP_EVERY); me.warn_unmanaged(&ctl, &mut warned); me.reap(&ctl); me.stop_idle_isbx(&ctl); + pass = pass.wrapping_add(1); + if pass % SWEEP_EVERY == 1 { + me.sweep_volumes(); + } } }); } @@ -237,6 +245,76 @@ impl Sandboxes { } } + /// Delete the volumes sandboxes created and left behind + /// ([`crate::volume::sweep`]), in every org's project and in incus' + /// `default` (plain sandboxes, outside every org: the default org's + /// `sandbox_volume_ttl` governs those too). + fn sweep_volumes(&self) { + let mut places: Vec<(Option, String)> = vec![(None, "default".into())]; + match crate::org::list(&self.client) { + Ok(orgs) => places.extend( + orgs.into_iter() + .map(|o| (Some(o.name.clone()), o.name.incus_project())), + ), + Err(e) => eprintln!("isb serve: volume sweep: could not list orgs: {e}"), + } + let pools = match crate::volume::pools(&self.client) { + Ok(p) => p, + Err(e) => { + eprintln!("isb serve: volume sweep: could not list pools: {e}"); + return; + } + }; + for (org, project) in places { + let settings_of = org.clone().unwrap_or_else(OrgId::default_org); + let ttl = match self + .settings(&settings_of) + .and_then(|s| life::volume_ttl(&s.sandbox_volume_ttl)) + { + Ok(Some(t)) => t, + Ok(None) => continue, + Err(e) => { + eprintln!("isb serve: volume sweep in {project}: {e}"); + continue; + } + }; + let c = self.client.clone().project(&project); + for pool in &pools { + let mut say = |m: &str| eprintln!("isb serve: volume sweep in {project}: {m}"); + let deleted = match crate::volume::sweep(&c, pool, now(), ttl, &mut say) { + Ok(d) => d, + Err(e) => { + eprintln!("isb serve: volume sweep in {project} ({pool}): {e}"); + continue; + } + }; + for v in deleted { + eprintln!( + "isb serve: deleted volume {project}/{v}: its sandbox is gone and nothing used it for {}", + life::human(ttl.as_secs()) + ); + self.recorder.record(crate::history::NewRecord { + source: "controller".into(), + org: org.as_ref().map(|o| o.as_str().to_string()), + project: Some(project.clone()), + kind: "volume.swept".into(), + object_type: Some("volume".into()), + object: Some(v.clone()), + objects: vec![v.clone()], + actor: Some("isb".into()), + level: Some("info".into()), + message: Some(format!( + "volume {v} deleted: its sandbox is gone and nothing used it for {}", + life::human(ttl.as_secs()) + )), + details: json!({"pool": pool, "ttl_seconds": ttl.as_secs()}), + ..Default::default() + }); + } + } + } + } + /// Delete sandboxes past their expiry or idle timeout. Idempotent: an /// instance already gone is not an error. fn reap(&self, ctl: &crate::stack::Controller) { @@ -427,13 +505,15 @@ struct SettingsArgs { sandbox_expiry: Option, #[serde(default)] sandbox_idle: Option, + #[serde(default)] + sandbox_volume_ttl: Option, } fn sandbox_settings(d: &Daemon, a: Value, c: &Caller) -> Result { let org = super::arg_org(&a)?; let a: SettingsArgs = args(a)?; let mut s = d.sandboxes.settings(&org)?; - if a.sandbox_expiry.is_some() || a.sandbox_idle.is_some() { + if a.sandbox_expiry.is_some() || a.sandbox_idle.is_some() || a.sandbox_volume_ttl.is_some() { super::volumes::require_admin(c, &org, "changing the sandbox defaults")?; if let Some(e) = a.sandbox_expiry { s.sandbox_expiry = e.trim().to_string(); @@ -441,6 +521,9 @@ fn sandbox_settings(d: &Daemon, a: Value, c: &Caller) -> Result { if let Some(i) = a.sandbox_idle { s.sandbox_idle = i.trim().to_string(); } + if let Some(t) = a.sandbox_volume_ttl { + s.sandbox_volume_ttl = t.trim().to_string(); + } life::put_settings(&d.state_dir, &org, &s)?; } Ok(json!({"org": org.as_str(), "settings": s})) @@ -450,11 +533,12 @@ pub(super) fn register(r: &mut Registry, d: Arc) -> Result<()> { r.register( Tool::new( "sandbox_settings", - "The org's defaults for new sandboxes: sandbox_expiry (24h, at most 30d), how long one lives, and sandbox_idle (2h, or none), how long it may sit idle before the reaper deletes it. sandbox_create's expires and idle_timeout override them per sandbox. Without changes it reads them; org admins and above change them.", + "The org's defaults for new sandboxes: sandbox_expiry (24h, at most 30d), how long one lives, and sandbox_idle (2h, or none), how long it may sit idle before the reaper deletes it. sandbox_create's expires and idle_timeout override them per sandbox. sandbox_volume_ttl (7d, or none) is how long a volume a sandbox created is kept after that sandbox is gone with nothing using it; a stack's, an app's or a `keep: true` volume is never deleted this way. Without changes it reads them; org admins and above change them.", obj( json!({ "sandbox_expiry": {"type": "string", "description": "e.g. 24h, 7d; at most 30d."}, - "sandbox_idle": {"type": "string", "description": "e.g. 2h, or none."} + "sandbox_idle": {"type": "string", "description": "e.g. 2h, or none."}, + "sandbox_volume_ttl": {"type": "string", "description": "e.g. 7d, 24h, or none."} }), &[], ), diff --git a/crates/isb-daemon/src/sandbox_life.rs b/crates/isb-daemon/src/sandbox_life.rs index 549db0dd..d5445cf6 100644 --- a/crates/isb-daemon/src/sandbox_life.rs +++ b/crates/isb-daemon/src/sandbox_life.rs @@ -33,6 +33,11 @@ pub struct Settings { /// `none`. #[serde(default = "default_idle")] pub sandbox_idle: String, + /// How long a volume a sandbox created is kept once that sandbox is + /// gone and nothing has it attached (`7d`), or `none` to keep it for + /// good. Never applies to a stack's, an app's or a `keep` volume. + #[serde(default = "default_volume_ttl")] + pub sandbox_volume_ttl: String, } fn default_expiry() -> String { @@ -43,11 +48,16 @@ fn default_idle() -> String { "2h".into() } +fn default_volume_ttl() -> String { + "7d".into() +} + impl Default for Settings { fn default() -> Self { Settings { sandbox_expiry: default_expiry(), sandbox_idle: default_idle(), + sandbox_volume_ttl: default_volume_ttl(), } } } @@ -57,6 +67,7 @@ impl Settings { pub fn validate(&self) -> Result<()> { lifetime(&self.sandbox_expiry)?; idle(&self.sandbox_idle)?; + volume_ttl(&self.sandbox_volume_ttl)?; Ok(()) } } @@ -121,6 +132,23 @@ pub fn lifetime(s: &str) -> Result { Ok(d) } +/// How long a gone sandbox's volume is kept: a duration of at least a +/// minute, or `none` (never deleted). +pub fn volume_ttl(s: &str) -> Result> { + let s = s.trim(); + if matches!(s, "none" | "never" | "off") { + return Ok(None); + } + let d = crate::flex::parse_duration(s) + .map_err(|e| Error::invalid(format!("volume ttl {s:?}: {e}")))?; + if d < Duration::from_secs(60) { + return Err(Error::invalid(format!( + "volume ttl {s:?}: at least a minute, or none" + ))); + } + Ok(Some(d)) +} + /// An idle timeout: a duration of at least a minute, or `none`. pub fn idle(s: &str) -> Result> { let s = s.trim(); @@ -241,6 +269,13 @@ mod tests { b.sandbox_expiry = "7d".into(); b.sandbox_idle = "none".into(); b.validate().unwrap(); + assert_eq!(s.sandbox_volume_ttl, "7d"); + assert_eq!(volume_ttl("24h").unwrap(), Some(Duration::from_secs(86400))); + assert_eq!(volume_ttl("none").unwrap(), None); + b.sandbox_volume_ttl = "10s".into(); + assert!(b.validate().is_err()); + b.sandbox_volume_ttl = "none".into(); + b.validate().unwrap(); let empty: Settings = serde_json::from_str("{}").unwrap(); assert_eq!(empty, Settings::default()); } diff --git a/docs/concepts/sandboxes.md b/docs/concepts/sandboxes.md index 03ca3df3..bbfd8072 100644 --- a/docs/concepts/sandboxes.md +++ b/docs/concepts/sandboxes.md @@ -203,9 +203,10 @@ in an org, or a plain sandbox in incus' `default` project. `never`, or `-` for none). `isb expiry NAME --in 24h` sets it from now, `--by 24h` pushes it out, `--never` clears it. Through the daemon, `sandbox_list` and `sandbox_extend`. -- **What goes.** The instance and its root disk. Named volumes stay, as they - do after `isb rm` and `isb down`: they are shared by name and outlive their - sandboxes by design. `isb down --volumes` or `isb volume rm` removes them. +- **What goes.** The instance and its root disk. Named volumes stay at + first, as they do after `isb rm` and `isb down`, so the next sandbox finds + its caches; [the ones the sandbox created go later](#what-happens-to-a-sandboxs-volumes). + `isb down --volumes` or `isb volume rm` removes them at once. - **Sandboxes without a deadline keep none.** The default applies when isb creates a sandbox. One made before isb had expiries, or by hand, has no `user.isb.expires_at`, is never reaped, and an `isb up` does not give it @@ -216,6 +217,30 @@ in an org, or a plain sandbox in incus' `default` project. - Nothing is reaped while `isb serve` is not running; the reaper takes what has expired when it starts. +## What happens to a sandbox's volumes + +A named volume that a plain sandbox creates (`isb up`, `isb create`, +`sandbox_create`, the SDKs) is marked as that sandbox's, with +`user.isb.sandbox: /` on the volume. Once the sandbox +is gone and nothing has the volume attached, `isb serve` deletes it after +the `sandbox_volume_ttl` (default 7 days; `isb sandbox settings +--volume-ttl 24h`, or `none` to keep such volumes for good). The clock +starts when isb first sees the volume unused (`user.isb.unused_since`), and +stops if the sandbox comes back or anything mounts the volume again. Each +deletion is in the history as `volume.swept`. + +A volume is never deleted this way when: + +- its definition says `keep: true`; +- it is `external`, or it existed before the sandbox that mounts it; +- a stack, an app, a database, a job or `isb volume create` made it: those + hold data, and a stack scaled to zero leaves its volumes unattached on + purpose. + +Plain sandboxes outside every org (incus' `default` project) follow the +default org's setting. `isb volume prune` lists the volumes whose sandbox is +gone, and with `-y` deletes them without waiting. + ## Sandboxes are short-lived Every sandbox made through `isb serve` (`sandbox_create`) gets deadlines, diff --git a/docs/guides/dev-environments.md b/docs/guides/dev-environments.md index 49e1920c..04b33d73 100644 --- a/docs/guides/dev-environments.md +++ b/docs/guides/dev-environments.md @@ -22,6 +22,7 @@ name: "shop-${BRANCH:-main}" volumes: bun-cache: name: bun-cache # pinned: shared by every worktree + keep: true # never cleaned up with a sandbox services: web: image: dev-base # or images:ubuntu/24.04 @@ -124,6 +125,17 @@ isb prune --label worktree --missing-path -y # deletes them `prune` never touches a sandbox without the label, or one whose path still exists. +A volume named after its sandbox (`name: "${BRANCH}-modules"`) is that +sandbox's: `isb down` keeps it, so the next `isb up` is fast, and once the +sandbox has been gone for a week with nothing using the volume, `isb serve` +deletes it ([volume lifetime](../concepts/sandboxes.md#what-happens-to-a-sandboxs-volumes)). +A cache every worktree shares takes `keep: true`. To clear the leftovers now: + +```sh +isb volume prune # lists the volumes whose sandbox is gone +isb volume prune -y # deletes them +``` + ## Safety rules These are what make the sandbox worth having. They are the same rules the diff --git a/docs/guides/isbx.md b/docs/guides/isbx.md index 8bc8b32c..454e28ed 100644 --- a/docs/guides/isbx.md +++ b/docs/guides/isbx.md @@ -314,7 +314,8 @@ Container sandboxes share one cache volume, `x-cache-ct`, at `/var/cache/isbx`, and point npm, pip, uv, go and bun at it (`npm_config_cache`, `PIP_CACHE_DIR`, `UV_CACHE_DIR`, `GOMODCACHE`, `BUN_INSTALL_CACHE_DIR`), so the second project's install is fast. A VM, -which is for code not trusted to share, gets a cache of its own. +which is for code not trusted to share, gets a cache of its own, deleted +with that sandbox (`isb x rm`, `isb x gc --missing -y`). `--no-cache` uses none. ## Finding and cleaning up @@ -325,7 +326,7 @@ isb x stop # stop them all (the next command starts one in ~ isb x gc --idle 30m # stop those unused for 30 minutes isb x gc --missing -y # delete those whose project directory is gone isb x rm x-run-1a2b3c4d5e # delete one (its list starts closed again) -isb x doctor # isb serve, incus, the shims' place on PATH, every tool +isb x doctor # isb serve, incus, the shims' place on PATH, every tool, the storage driver isb x update-shell # put the shims first on PATH in your shell's startup file ``` @@ -334,6 +335,11 @@ isb x update-shell # put the shims first on PATH in your shell's sta time), and never one a command is running in; it never deletes one. The next command starts it again in about a second. +Each sandbox and each tool is an instance of its own. On a copy-on-write +storage pool (zfs, btrfs) they share the image they were made from and cost +only what they add to it; on a `dir` pool each is a full copy, which +`isb x doctor` points out. + Everything isbx makes is named `x-…` and labelled `isbx.kind` (`run` or `tool`), with `isbx.root` (the project) or `isbx.tool` (the tool), so `isb ls -l isbx.kind` finds it too. diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 1ae824d3..8a14250d 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -209,6 +209,7 @@ isb volume create NAME [--pool P] [-c KEY=VALUE]... no-op if it exists isb volume ls [--pool P] [--json] isb volume inspect NAME [--pool P] [--json] JSON either way isb volume rm NAME [--pool P] refused while in use +isb volume prune [NAME...] [--unattached] [--pool P] [-y] [--json] isb port add NAME SPEC [--name DEV] [--search N] prints the listen address in use isb port get NAME DEV [KEY] [--json] one property (default listen); --json: every property @@ -219,6 +220,16 @@ isb device ls NAME [--json] instance-local devices isb device rm NAME DEV... (root cannot be removed) ``` +`volume prune` lists the volumes that sandboxes created and left behind (the +sandbox is gone and nothing has the volume attached), and deletes them with +`-y`; `isb serve` does the same by itself after the org's +`sandbox_volume_ttl` ([volume lifetime](../concepts/sandboxes.md#what-happens-to-a-sandboxs-volumes)). +`NAME` narrows it, with `*` as a wildcard. `--unattached` lists every volume +nothing has attached, whoever made it. That includes data left unattached on +purpose, such as a stack scaled to zero, so with `-y` it needs names: +`isb volume prune --unattached -y '*-dev-modules'`. + + These talk to incus directly. `port add` leaves a correct device alone; `--search N` steps past up to N taken host ports. Through `isb serve` (MCP and the HTTP API) an org's sandboxes have the same as tools: `sandbox_port_list`, @@ -316,7 +327,7 @@ apps, stacks and sandboxes first. See [Orgs](../concepts/orgs.md) and ```text isb sandbox ls [--json] isb sandbox extend NAME [--by 24h] [--idle-timeout 4h|none] -isb sandbox settings [--expiry 24h] [--idle 2h|none] +isb sandbox settings [--expiry 24h] [--idle 2h|none] [--volume-ttl 7d|none] ``` The org's sandboxes made through `isb serve` (`sandbox_create`, the web UI), diff --git a/docs/reference/compose.md b/docs/reference/compose.md index 8e7b3414..f762b00d 100644 --- a/docs/reference/compose.md +++ b/docs/reference/compose.md @@ -187,6 +187,7 @@ service mounts is never created. | `pool` | string | `auto` | Storage pool. `auto` means the root pool of the sandbox that mounts it. A named pool must exist. | | `config` | map of string | `{}` | Volume config keys (`size: 10GiB`). Applied only when isb creates the volume; never compared or changed later. | | `external` | bool | `false` | The volume must already exist. isb never creates it; `plan` and `up` fail if it is missing. | +| `keep` | bool | `false` | Never deleted by the cleanup of volumes whose sandbox is gone ([volume lifetime](../concepts/sandboxes.md#what-happens-to-a-sandboxs-volumes)). For a cache shared between sandboxes, or data that must outlive them. | A volume is created before the sandbox that mounts it. If two sandboxes resolve the same volume to different pools, each pool gets its own volume. diff --git a/docs/reference/mcp-tools.md b/docs/reference/mcp-tools.md index 4898b7e1..bb5aad2d 100644 --- a/docs/reference/mcp-tools.md +++ b/docs/reference/mcp-tools.md @@ -149,7 +149,7 @@ There is no `port-forward` tool: run `curl` with `app_exec`, or reach an instanc | `sandbox_list` | viewer | Instances, filtered by labels and `kind` (`sandbox`, `replica`, `build`): creator, age, expiry, idle timeout, last activity, limits and use, and `mine`. Remote callers see only instances isb manages. | | `sandbox_exec` | member | Run argv in a sandbox: exit code, stdout, stderr (each capped at 256 KiB, keeping the end), optional `stdin` text, `user`, `cwd`, `env`, `timeout` (default 10m). | | `sandbox_extend` | the sandbox's creator, or admin | Push the expiry out (`by`, default 24h, at most 30 days from now) or change `idle_timeout`. | -| `sandbox_settings` | member reads, admin changes | The org's defaults for new sandboxes: `sandbox_expiry` (e.g. `24h`, `7d`; at most `30d`) and `sandbox_idle` (e.g. `2h`, or `none`). Without changes it reads; returns `{org, settings: {sandbox_expiry, sandbox_idle}}`. Sandboxes that exist keep their deadlines. | +| `sandbox_settings` | member reads, admin changes | The org's defaults for new sandboxes: `sandbox_expiry` (e.g. `24h`, `7d`; at most `30d`) and `sandbox_idle` (e.g. `2h`, or `none`), and `sandbox_volume_ttl` (e.g. `7d`, or `none`): how long a volume a sandbox created is kept once that sandbox is gone and nothing uses it. Without changes it reads; returns `{org, settings: {sandbox_expiry, sandbox_idle, sandbox_volume_ttl}}`. Sandboxes that exist keep their deadlines. | | `sandbox_remove` | member | Delete a sandbox (not a stack replica). | | `sandbox_start` | member | Start a stopped sandbox and wait until it runs. Not a stack replica. | | `sandbox_stop` | member | Stop a sandbox and keep it: a clean shutdown within `timeout` (default 30s, at most 10m), or `force`. Not a stack replica. | diff --git a/docs/reference/rpc.md b/docs/reference/rpc.md index 2c148e01..f0afba2f 100644 --- a/docs/reference/rpc.md +++ b/docs/reference/rpc.md @@ -19,7 +19,7 @@ it; relative paths resolve against that directory. When it starts, the server writes one hello line: ```json -{"isb": "4.1.2", "protocol": 1} +{"isb": "4.2.0", "protocol": 1} ``` A client checks `protocol` and refuses versions it does not know. @@ -205,7 +205,7 @@ All take `files` (default `./isb.yaml` plus `./isb.override.yaml` if present), ## Example session ```text -<- {"isb":"4.1.2","protocol":1} +<- {"isb":"4.2.0","protocol":1} -> {"id":1,"method":"sandbox.ensure","params":{"spec":{"name":"web","image":"dev-base","cpus":2}}} <- {"id":1,"event":"progress","data":"web: creating from dev-base"} <- {"id":1,"event":"progress","data":"web: starting"} diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 36f20f59..423a56af 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "isb-sdk" -version = "4.1.2" +version = "4.2.0" description = "Python SDK for isb: declarative incus sandboxes (containers and VMs)" readme = "README.md" license = "MIT AND LicenseRef-EXA-SL-1.0" diff --git a/sdk/python/src/isb/__init__.py b/sdk/python/src/isb/__init__.py index 99b106ac..7cb68bfa 100644 --- a/sdk/python/src/isb/__init__.py +++ b/sdk/python/src/isb/__init__.py @@ -62,7 +62,7 @@ async def main(): from ._util import Duration from .volumes import Volumes -__version__ = "4.1.2" +__version__ = "4.2.0" __all__ = [ "PROTOCOL", diff --git a/sdk/python/src/isb/_spec.py b/sdk/python/src/isb/_spec.py index 9c539811..4f3e3bae 100644 --- a/sdk/python/src/isb/_spec.py +++ b/sdk/python/src/isb/_spec.py @@ -186,6 +186,10 @@ class NamedVolumeSpec(TypedDict, total=False): config: Mapping[str, Scalar] #: The volume must already exist; isb never creates it. external: BoolOrString + #: Never deleted by isb's cleanup of volumes whose sandbox is gone + #: (`isb volume prune`, `isb serve`): for a cache shared between + #: sandboxes, or data that must outlive them. + keep: BoolOrString #: The incus volume name. Default: `_`, or the key itself #: for an `external` volume. name: Optional[str] diff --git a/sdk/python/tests/test_unit.py b/sdk/python/tests/test_unit.py index 0e59cb14..f928079f 100644 --- a/sdk/python/tests/test_unit.py +++ b/sdk/python/tests/test_unit.py @@ -222,7 +222,7 @@ async def test_death_fails_pending_requests(self) -> None: path = fake_server( self.tmp.name, """\ - echo '{"isb":"4.1.2","protocol":1}' + echo '{"isb":"4.2.0","protocol":1}' read line echo "boom" >&2 exit 3 @@ -243,7 +243,7 @@ async def test_close_fails_pending(self) -> None: path = fake_server( self.tmp.name, """\ - echo '{"isb":"4.1.2","protocol":1}' + echo '{"isb":"4.2.0","protocol":1}' cat >/dev/null """, ) @@ -261,7 +261,7 @@ async def test_events_and_out_of_order_replies(self) -> None: path = fake_server( self.tmp.name, """\ - echo '{"isb":"4.1.2","protocol":1}' + echo '{"isb":"4.2.0","protocol":1}' read a read b echo 'not json' diff --git a/sdk/python/uv.lock b/sdk/python/uv.lock index 16e1ee1a..797840e0 100644 --- a/sdk/python/uv.lock +++ b/sdk/python/uv.lock @@ -72,7 +72,7 @@ wheels = [ [[package]] name = "isb-sdk" -version = "4.1.2" +version = "4.2.0" source = { editable = "." } [package.dev-dependencies] diff --git a/sdk/typescript/bun.lock b/sdk/typescript/bun.lock index 2695ebc3..7a55e593 100644 --- a/sdk/typescript/bun.lock +++ b/sdk/typescript/bun.lock @@ -12,8 +12,8 @@ "typescript": "^5.9.0", }, "optionalDependencies": { - "@execution-associates/isb-linux-arm64": "4.1.2", - "@execution-associates/isb-linux-x64": "4.1.2", + "@execution-associates/isb-linux-arm64": "4.2.0", + "@execution-associates/isb-linux-x64": "4.2.0", }, }, }, diff --git a/sdk/typescript/npm/linux-arm64/package.json b/sdk/typescript/npm/linux-arm64/package.json index 2acd2a95..619fb985 100644 --- a/sdk/typescript/npm/linux-arm64/package.json +++ b/sdk/typescript/npm/linux-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@execution-associates/isb-linux-arm64", - "version": "4.1.2", + "version": "4.2.0", "description": "The static isb binary for Linux arm64 (aarch64-unknown-linux-musl), used by @execution-associates/isb", "license": "SEE LICENSE IN LICENSE", "repository": { diff --git a/sdk/typescript/npm/linux-x64/package.json b/sdk/typescript/npm/linux-x64/package.json index 00cd7403..7f44d5a2 100644 --- a/sdk/typescript/npm/linux-x64/package.json +++ b/sdk/typescript/npm/linux-x64/package.json @@ -1,6 +1,6 @@ { "name": "@execution-associates/isb-linux-x64", - "version": "4.1.2", + "version": "4.2.0", "description": "The static isb binary for Linux x64 (x86_64-unknown-linux-musl), used by @execution-associates/isb", "license": "SEE LICENSE IN LICENSE", "repository": { diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json index 81b6a201..9d33bcff 100644 --- a/sdk/typescript/package.json +++ b/sdk/typescript/package.json @@ -1,6 +1,6 @@ { "name": "@execution-associates/isb", - "version": "4.1.2", + "version": "4.2.0", "description": "TypeScript SDK for isb: declarative incus sandboxes (containers and VMs)", "license": "MIT", "type": "module", @@ -51,8 +51,8 @@ "prepack": "bun run build" }, "optionalDependencies": { - "@execution-associates/isb-linux-arm64": "4.1.2", - "@execution-associates/isb-linux-x64": "4.1.2" + "@execution-associates/isb-linux-arm64": "4.2.0", + "@execution-associates/isb-linux-x64": "4.2.0" }, "devDependencies": { "@biomejs/biome": "^2.2.0", diff --git a/sdk/typescript/src/spec.ts b/sdk/typescript/src/spec.ts index e815c9f8..a1a15318 100644 --- a/sdk/typescript/src/spec.ts +++ b/sdk/typescript/src/spec.ts @@ -897,6 +897,7 @@ export interface NamedVolumeSpec { [k: string]: Scalar; }; external?: BoolOrString; + keep?: BoolOrString; /** * The incus volume name. Default: `_`, or the key itself * for an `external` volume. diff --git a/src/bin/isb.rs b/src/bin/isb.rs index e544863e..07a242e4 100644 --- a/src/bin/isb.rs +++ b/src/bin/isb.rs @@ -749,11 +749,107 @@ fn volume_local(ctx: &Ctx, v: VolumeCmd) -> Result { let pool = pool(p)?; isb::volume::remove(&c, &pool, &name)?; } + VolumeCmd::Prune { + names, + unattached, + pool: p, + yes, + json, + } => return volume_prune(&c, &names, unattached, p, yes, json), _ => unreachable!("isb serve's volume commands are handled in volume()"), } Ok(0) } +/// `isb volume prune`. +fn volume_prune( + c: &isb::Client, + names: &[String], + unattached: bool, + pool: Option, + yes: bool, + json: bool, +) -> Result { + if unattached && yes && names.is_empty() { + return Err(Error::Invalid( + "--unattached lists volumes isb cannot tell from data kept on purpose: with -y, name what to delete (`*` is a wildcard), e.g. isb volume prune --unattached -y '*-dev-modules'".into(), + )); + } + let pools = match pool { + Some(p) => vec![p], + None => isb::volume::pools(c)?, + }; + let mut found = Vec::new(); + for p in pools { + found.extend(isb::volume::orphans(c, &p, unattached)?); + } + found.retain(|v| { + names.is_empty() || names.iter().any(|n| isb::volume::name_matches(n, &v.name)) + }); + let mut failed = false; + let mut rows = Vec::new(); + for v in &found { + let state = if !yes { + "would delete".to_string() + } else { + match isb::volume::remove(c, &v.pool, &v.name) { + Ok(()) => "deleted".into(), + Err(e) => { + failed = true; + format!("failed: {e}") + } + } + }; + rows.push((v, state)); + } + if json { + print_json( + &rows + .iter() + .map(|(v, state)| { + serde_json::json!({ + "name": v.name, + "pool": v.pool, + "sandbox": v.config.get(isb::volume::KEY_SANDBOX), + "created_at": v.created_at, + "result": state, + }) + }) + .collect::>(), + ); + } else if rows.is_empty() { + println!("nothing to prune"); + } else { + let mut t = vec![vec![ + "NAME".into(), + "POOL".into(), + "SANDBOX".into(), + "CREATED".into(), + "RESULT".into(), + ]]; + for (v, state) in &rows { + t.push(vec![ + v.name.clone(), + v.pool.clone(), + v.config + .get(isb::volume::KEY_SANDBOX) + .cloned() + .unwrap_or_else(|| "-".into()), + v.created_at.chars().take(10).collect(), + state.clone(), + ]); + } + table(t); + if !yes { + println!( + "dry run; re-run with -y to delete ({} volume(s))", + rows.len() + ); + } + } + Ok(u8::from(failed)) +} + /// isb serve where there is no daemon socket (an org's instance, another /// machine): its URL (`$ISB_URL`, such as the org API listener on the org's /// bridge) with an API token (`$ISB_TOKEN`), as `isb ssh-config` and `isb diff --git a/src/bin/isb/instances.rs b/src/bin/isb/instances.rs index 9e9bc6af..f9c21a5e 100644 --- a/src/bin/isb/instances.rs +++ b/src/bin/isb/instances.rs @@ -153,6 +153,25 @@ pub(crate) enum VolumeCmd { #[arg(long)] pool: Option, }, + /// Delete volumes that sandboxes created and left behind: the sandbox + /// is gone and nothing has the volume attached. A dry run unless -y. + Prune { + /// Only volumes whose name matches (`*` is a wildcard), e.g. + /// '*-dev-modules'. + names: Vec, + /// Consider every volume nothing has attached, whoever made it. + /// This includes data left unattached on purpose (a stack scaled + /// to zero), so -y then needs NAMES. + #[arg(long)] + unattached: bool, + #[arg(long)] + pool: Option, + /// Actually delete (default is a dry run). + #[arg(short = 'y', long)] + yes: bool, + #[arg(long)] + json: bool, + }, /// Snapshots of a named volume in an org, now or on a schedule (isb serve). #[command(subcommand)] Snapshot(volumes::SnapshotCmd), @@ -277,6 +296,7 @@ pub(crate) fn create(ctx: &Ctx, a: CreateArgs) -> Result { let opts = EnsureOptions { wait_ready: !a.no_ready, default_expiry: super::expiry::default_expiry(&c), + own_volumes: true, ..Default::default() }; if a.ensure { diff --git a/src/bin/isb/project.rs b/src/bin/isb/project.rs index 9b265f30..d5994805 100644 --- a/src/bin/isb/project.rs +++ b/src/bin/isb/project.rs @@ -79,6 +79,7 @@ pub(crate) fn up(ctx: &Ctx, services: Vec, flags: UpFlags) -> Result }, wait_ready: !flags.no_ready, default_expiry: super::expiry::default_expiry(&compose::client_for(&c, &p)), + own_volumes: true, ..Default::default() }; let mut rep = ctx.report(); diff --git a/src/bin/isb/sandboxes.rs b/src/bin/isb/sandboxes.rs index 040bc5ac..be8b315b 100644 --- a/src/bin/isb/sandboxes.rs +++ b/src/bin/isb/sandboxes.rs @@ -37,6 +37,10 @@ pub enum SandboxCmd { /// or none. #[arg(long)] idle: Option, + /// How long a volume a sandbox created is kept once the sandbox is + /// gone and nothing uses it, e.g. 7d, or none. + #[arg(long)] + volume_ttl: Option, }, } @@ -96,7 +100,11 @@ pub fn sandbox(org: &Option, cmd: SandboxCmd) -> Result { println!("{}", v["message"].as_str().unwrap_or("extended")); Ok(0) } - SandboxCmd::Settings { expiry, idle } => { + SandboxCmd::Settings { + expiry, + idle, + volume_ttl, + } => { let mut a = json!({}); if let Some(e) = expiry { a["sandbox_expiry"] = json!(e); @@ -104,6 +112,9 @@ pub fn sandbox(org: &Option, cmd: SandboxCmd) -> Result { if let Some(i) = idle { a["sandbox_idle"] = json!(i); } + if let Some(t) = volume_ttl { + a["sandbox_volume_ttl"] = json!(t); + } print_json(&c("sandbox_settings", a)?["settings"]); Ok(0) } diff --git a/src/bin/isb/x.rs b/src/bin/isb/x.rs index c4f811f0..a37137af 100644 --- a/src/bin/isb/x.rs +++ b/src/bin/isb/x.rs @@ -311,6 +311,9 @@ fn run(ctx: &Ctx, a: RunArgs) -> Result { if let Err(e) = Sandbox::remove(&client, sb.name(), true) { eprintln!("isbx: could not delete {}: {e}", sb.name()); } + if a.vm { + let _ = x::sweep_caches(&client, false, &mut |_| {}); + } } else { x::touch_used(sb.name()); } @@ -500,6 +503,8 @@ fn gc(ctx: &Ctx, idle: Option, missing: bool, yes: bool) -> Result } if missing { prune_missing_path(&client, x::LABEL_ROOT, !yes, &mut ctx.report())?; + // The cache volumes of VM sandboxes that are gone, these included. + x::sweep_caches(&client, !yes, &mut ctx.report())?; } Ok(0) } diff --git a/src/bin/isb/x_tools.rs b/src/bin/isb/x_tools.rs index def9432c..401351ad 100644 --- a/src/bin/isb/x_tools.rs +++ b/src/bin/isb/x_tools.rs @@ -288,6 +288,10 @@ pub(crate) fn rm(ctx: &Ctx, names: Vec) -> Result { } Sandbox::remove(&client, &n, true)?; report(&format!("deleted {n}")); + // A VM run's own cache volume goes with it. + if let Err(e) = x::sweep_caches(&client, false, &mut report) { + report(&format!("cache volumes not checked: {e}")); + } } else { return Err(Error::Invalid(format!( "{n}: no such tool, and not an x-… sandbox (`isb x ls`)" @@ -383,6 +387,17 @@ pub(crate) fn doctor(ctx: &Ctx) -> Result { ), ); } + // Not a failure, but the reason a host fills up: without copy-on-write + // every sandbox holds a whole copy of its image. + if let Ok(pool) = isb::sandbox::host_facts(&client).and_then(|h| h.pick_pool(None)) { + if let Ok(Some(driver)) = isb::volume::pool_driver(&client, &pool) { + if driver == "dir" { + println!( + "note storage pool {pool} uses the `dir` driver: every sandbox is a full copy of its image. A zfs or btrfs pool shares the image between them (a pool named incus-zfs is preferred when it exists)" + ); + } + } + } let tools = Tool::all(); let mut names: Vec = tools.iter().flat_map(|t| t.bins.keys().cloned()).collect(); names.extend(shim::list().into_iter().map(|(n, _)| n)); diff --git a/tests/volume_owner.rs b/tests/volume_owner.rs index e6e067d1..f964c25b 100644 --- a/tests/volume_owner.rs +++ b/tests/volume_owner.rs @@ -1,8 +1,11 @@ //! A named volume's owner and mode on shell-less OCI images, and a //! multi-line `sh -c` script as an OCI command line, against a real incusd. //! +//! Also: the volumes a plain sandbox creates are its own, and are swept once +//! it is gone, where a `keep` volume and one nobody owns stay. +//! //! Gated by `ISB_INTEGRATION=1`. Everything happens in a throwaway incus -//! project of its own (`isbtest-own-`; not `isb-*`, which a running +//! project of its own (`isbtest--`; not `isb-*`, which a running //! daemon would take for an org), deleted afterwards with all it holds; no //! org's project is touched. @@ -45,8 +48,13 @@ impl Drop for Project { } fn project() -> Project { + project_named("own") +} + +/// A throwaway project of its own per test: the tests run side by side. +fn project_named(tag: &str) -> Project { let base = Client::new(); - let name = format!("isbtest-own-{}", std::process::id()); + let name = format!("isbtest-{tag}-{}", std::process::id()); let body = serde_json::json!({ "name": name, "description": "isb volume_owner integration test (throwaway)", @@ -201,3 +209,110 @@ fn owner_and_mode_without_a_shell_and_multi_line_commands() { } assert_eq!(stat(&c, sb.name(), "/srv").uid, 0); } + +#[test] +fn a_gone_sandboxs_volume_is_swept_and_nothing_else_is() { + if !enabled() { + return; + } + use isb::volume::{KEY_SANDBOX, KEY_UNUSED_SINCE}; + let mut p = project_named("sweep"); + let c = p.client.clone(); + let pid = std::process::id(); + let (plain, stack) = ( + format!("isb-test-{pid}-plain"), + format!("isb-test-{pid}-stack"), + ); + p.instances.extend([plain.clone(), stack.clone()]); + let (own, kept, data) = ( + format!("isb-test-{pid}-own"), + format!("isb-test-{pid}-kept"), + format!("isb-test-{pid}-data"), + ); + p.volumes.extend([own.clone(), kept.clone(), data.clone()]); + let idle = |name: &str, mounts| { + let mut s = spec(name, "docker:busybox", mounts); + s.command = Some(["sleep", "600"].map(String::from).to_vec()); + s + }; + // A plain sandbox (`isb up`, `isb create`): one volume of its own, one + // the file says to keep. + let mut defs = isb::plan::VolumeDefs::new(); + defs.insert( + kept.clone(), + isb::spec::NamedVolumeSpec { + keep: true, + ..Default::default() + }, + ); + let opts = isb::EnsureOptions { + own_volumes: true, + ..Default::default() + }; + Sandbox::connect_or_create_with( + &c, + &idle( + &plain, + vec![ + ("/own", Volume::named(&own).device("own")), + ("/kept", Volume::named(&kept).device("kept")), + ], + ), + &defs, + opts, + &mut |_| {}, + ) + .expect("create the plain sandbox"); + // What a stack or an app does: no owner, so its data is never swept. + Sandbox::connect_or_create( + &c, + &idle(&stack, vec![("/data", Volume::named(&data).device("data"))]), + ) + .expect("create the unowned one"); + let get = |v: &str| isb::volume::get(&c, "default", v).unwrap(); + assert_eq!( + get(&own).unwrap().config.get(KEY_SANDBOX), + Some(&format!("{}/{plain}", p.name)) + ); + assert!(!get(&kept).unwrap().config.contains_key(KEY_SANDBOX)); + assert!(!get(&data).unwrap().config.contains_key(KEY_SANDBOX)); + + let sweep = |now: u64| { + isb::volume::sweep(&c, "default", now, Duration::from_secs(3600), &mut |m| { + panic!("sweep: {m}") + }) + .unwrap() + }; + // In use: untouched, and no clock started. + assert!(sweep(1_000).is_empty()); + assert!(!get(&own).unwrap().config.contains_key(KEY_UNUSED_SINCE)); + // Both sandboxes go. The first pass only starts the clock. + Sandbox::remove(&c, &plain, true).unwrap(); + Sandbox::remove(&c, &stack, true).unwrap(); + assert_eq!( + isb::volume::orphans(&c, "default", false) + .unwrap() + .iter() + .map(|v| v.name.clone()) + .collect::>(), + std::slice::from_ref(&own) + ); + assert!(sweep(1_000).is_empty()); + assert_eq!(get(&own).unwrap().unused_since(), Some(1_000)); + // Still inside the hour. + assert!(sweep(1_000 + 3_599).is_empty()); + // Past it: the sandbox's own volume goes, and only that one. + assert_eq!(sweep(1_000 + 3_600), std::slice::from_ref(&own)); + assert!(get(&own).is_none()); + assert!(get(&kept).is_some() && get(&data).is_some()); + // Every unattached volume is listed only on request. + let mut all: Vec = isb::volume::orphans(&c, "default", true) + .unwrap() + .into_iter() + .map(|v| v.name) + .collect(); + all.sort(); + let mut want = vec![kept.clone(), data.clone()]; + want.sort(); + assert_eq!(all, want); +}