diff --git a/Cargo.lock b/Cargo.lock index 310358cc..5f8a3de1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1329,7 +1329,7 @@ checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] name = "isb" -version = "4.0.1" +version = "4.1.0" dependencies = [ "age", "clap", @@ -1352,7 +1352,7 @@ dependencies = [ [[package]] name = "isb-apps" -version = "4.0.1" +version = "4.1.0" dependencies = [ "age", "base64 0.23.1", @@ -1376,7 +1376,7 @@ dependencies = [ [[package]] name = "isb-core" -version = "4.0.1" +version = "4.1.0" dependencies = [ "age", "base64 0.23.1", @@ -1403,7 +1403,7 @@ dependencies = [ [[package]] name = "isb-daemon" -version = "4.0.1" +version = "4.1.0" dependencies = [ "age", "httparse", @@ -1423,7 +1423,7 @@ dependencies = [ [[package]] name = "isb-egress" -version = "4.0.1" +version = "4.1.0" dependencies = [ "base64 0.23.1", "httparse", @@ -1436,7 +1436,7 @@ dependencies = [ [[package]] name = "isb-server" -version = "4.0.1" +version = "4.1.0" dependencies = [ "argon2", "base64 0.23.1", @@ -1459,7 +1459,7 @@ dependencies = [ [[package]] name = "isb-tui" -version = "4.0.1" +version = "4.1.0" dependencies = [ "isb-core", "libc", diff --git a/Cargo.toml b/Cargo.toml index 471f807a..60b0c108 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,7 +55,7 @@ members = ["crates/*"] default-members = [".", "crates/*"] [workspace.package] -version = "4.0.1" +version = "4.1.0" edition = "2024" rust-version = "1.89" license = "LicenseRef-EXA-SL-1.0" @@ -63,12 +63,12 @@ repository = "https://github.com/execution-associates/isb" homepage = "https://github.com/execution-associates/isb" [workspace.dependencies] -isb-core = { path = "crates/isb-core", version = "=4.0.1" } -isb-server = { path = "crates/isb-server", version = "=4.0.1" } -isb-apps = { path = "crates/isb-apps", version = "=4.0.1" } -isb-daemon = { path = "crates/isb-daemon", version = "=4.0.1" } -isb-tui = { path = "crates/isb-tui", version = "=4.0.1" } -isb-egress = { path = "crates/isb-egress", version = "=4.0.1" } +isb-core = { path = "crates/isb-core", version = "=4.1.0" } +isb-server = { path = "crates/isb-server", version = "=4.1.0" } +isb-apps = { path = "crates/isb-apps", version = "=4.1.0" } +isb-daemon = { path = "crates/isb-daemon", version = "=4.1.0" } +isb-tui = { path = "crates/isb-tui", version = "=4.1.0" } +isb-egress = { path = "crates/isb-egress", version = "=4.1.0" } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/crates/isb-apps/src/build/mod.rs b/crates/isb-apps/src/build/mod.rs index 0cbe55bd..45a534c2 100644 --- a/crates/isb-apps/src/build/mod.rs +++ b/crates/isb-apps/src/build/mod.rs @@ -517,13 +517,15 @@ fn create_sandbox( "profiles": ["default"], }); let t = remaining(deadline, "creating the build sandbox")?; + let (body, pin) = crate::ipam::pin_for_create(oc, crate::org::disk::sized_root(oc, body))?; oc.mutate( "POST", "/1.0/instances", - Some(&crate::org::disk::sized_root(oc, body)), + Some(&body), &format!("create build sandbox {name}"), t.min(oc.get_timeouts().create.max(Duration::from_secs(600))), )?; + pin.map_or(Ok(()), |p| p.verify(oc, name))?; let t = remaining(deadline, "starting the build sandbox")?; oc.mutate( "PUT", diff --git a/crates/isb-apps/src/notify/mod.rs b/crates/isb-apps/src/notify/mod.rs index cda314e9..25d9cf1d 100644 --- a/crates/isb-apps/src/notify/mod.rs +++ b/crates/isb-apps/src/notify/mod.rs @@ -1,4 +1,4 @@ -//! Notifications: per-org channels (webhook, Slack, Discord, Telegram, +//! Notifications: per-org channels (webhook, Slack, Discord, Mattermost, Telegram, //! email) told about events of chosen kinds (`deploy.*`, `health.*`, //! `backup.*`, `job.*`, `cert.*`, `monitor.*`; see //! [`crate::stack::controller::Event`]). @@ -428,7 +428,8 @@ impl Notifier { } if let Provider::Webhook { url_secret, .. } | Provider::Slack { url_secret } - | Provider::Discord { url_secret } = &c.provider + | Provider::Discord { url_secret } + | Provider::Mattermost { url_secret } = &c.provider { let v = self .secret_value(org, url_secret) diff --git a/crates/isb-apps/src/notify/provider.rs b/crates/isb-apps/src/notify/provider.rs index fa8f59d5..116a1de4 100644 --- a/crates/isb-apps/src/notify/provider.rs +++ b/crates/isb-apps/src/notify/provider.rs @@ -22,6 +22,9 @@ pub enum Provider { Slack { url_secret: String }, /// A Discord webhook (`https://discord.com/api/webhooks/...`). Discord { url_secret: String }, + /// A Mattermost incoming webhook (`https:///hooks/...`), or any + /// other https endpoint that takes Slack's `{"text": ...}`. + Mattermost { url_secret: String }, /// A Telegram bot message to one chat. Telegram { token_secret: String, @@ -49,6 +52,7 @@ impl Provider { Provider::Webhook { .. } => "webhook", Provider::Slack { .. } => "slack", Provider::Discord { .. } => "discord", + Provider::Mattermost { .. } => "mattermost", Provider::Telegram { .. } => "telegram", Provider::Email { .. } => "email", } @@ -63,7 +67,9 @@ impl Provider { } => std::iter::once(url_secret.as_str()) .chain(signing_secret.as_deref()) .collect(), - Provider::Slack { url_secret } | Provider::Discord { url_secret } => vec![url_secret], + Provider::Slack { url_secret } + | Provider::Discord { url_secret } + | Provider::Mattermost { url_secret } => vec![url_secret], Provider::Telegram { token_secret, .. } => vec![token_secret], Provider::Email { password_secret, .. @@ -187,7 +193,7 @@ impl Message { } } -/// Validate a Slack, Discord or webhook URL read from a secret. +/// Validate a Slack, Discord, Mattermost or webhook URL read from a secret. fn checked_url(p: &Provider, url: &str) -> Result { let t = parse_url(url).map_err(|e| SendError::permanent(format!("{} URL: {e}", p.kind())))?; let want = |hosts: &[&str], prefix: &str| -> Result<(), SendError> { @@ -216,6 +222,14 @@ fn checked_url(p: &Provider, url: &str) -> Result { ], "/api/webhooks/", )?, + // A self-hosted server: any host, but never in the clear (the URL + // is the credential). + Provider::Mattermost { .. } if !t.https => { + return Err(SendError::permanent(format!( + "a mattermost URL must be https; this one for {} is not", + t.host + ))); + } _ => {} } Ok(t) @@ -249,6 +263,13 @@ fn check_bot_token(t: &str) -> Result<(), SendError> { /// Where Telegram's Bot API lives. pub const TELEGRAM_API: &str = "https://api.telegram.org"; +/// Mattermost notifies whoever an `@name`, `@channel` or `@all` in the text +/// names; event text must never ping anyone, so each `@` is followed by a +/// zero-width space. +fn no_mentions(s: &str) -> String { + s.replace('@', "@\u{200b}") +} + /// Slack's mrkdwn treats &, < and > specially. fn slack_escape(s: &str) -> String { s.replace('&', "&") @@ -353,6 +374,19 @@ pub fn request( body: serde_json::to_vec(&body).expect("json"), }) } + Provider::Mattermost { url_secret } => { + let url = secret(url_secret)?; + checked_url(p, &url)?; + // Markdown: the title in bold, then the rest of the text. + let full = no_mentions(&text(m)); + let (title, rest) = full.split_once('\n').unwrap_or((&full, "")); + let body = json!({"text": cap(&format!("**{title}**\n{rest}"), 16000)}); + Ok(Request { + url, + headers: json_headers(vec![]), + body: serde_json::to_vec(&body).expect("json"), + }) + } Provider::Discord { url_secret } => { let url = secret(url_secret)?; checked_url(p, &url)?; @@ -418,6 +452,8 @@ mod tests { "SLACK" => "https://hooks.slack.com/services/T0/B0/xyz".into(), "DISCORD" => "https://discord.com/api/webhooks/1/abc".into(), "TG" => "123456:ABC-def_1".into(), + "MM" => "https://chat.example.org/hooks/abc123".into(), + "MMPLAIN" => "http://chat.example.org/hooks/abc123".into(), "BADSLACK" => "https://evil.example/services/x".into(), _ => return Err(SendError::permanent(format!("no secret {name}"))), }) @@ -503,6 +539,33 @@ mod tests { assert!(e.is_err()); } + #[test] + fn mattermost_takes_slacks_text_on_any_https_server_and_never_pings() { + let p = |s: &str| Provider::Mattermost { + url_secret: s.into(), + }; + let mut m = msg(); + m.message = "deployment 3 failed for @channel and @all".into(); + let r = request(&p("MM"), &m, &secrets).unwrap(); + assert_eq!(r.url, "https://chat.example.org/hooks/abc123"); + let b = body(&r); + let t = b["text"].as_str().unwrap(); + assert!(t.starts_with("**"), "{t}"); + assert!(t.contains("deployment 3 failed"), "{t}"); + assert!(!t.contains("@channel") && !t.contains("@all"), "{t}"); + assert_eq!(b.as_object().unwrap().len(), 1, "{b}"); + let e = request(&p("MMPLAIN"), &m, &secrets).unwrap_err(); + assert!(e.message.contains("https"), "{}", e.message); + assert!(!e.message.contains("abc123"), "the URL is a credential"); + assert!(check_url_value(&p("MM"), "https://chat.example.org/hooks/x").is_ok()); + let long = Message { + message: "x".repeat(20000), + ..msg() + }; + let b = body(&request(&p("MM"), &long, &secrets).unwrap()); + assert!(b["text"].as_str().unwrap().chars().count() <= 16000); + } + #[test] fn discord_never_pings() { let r = request( diff --git a/crates/isb-core/src/ipam.rs b/crates/isb-core/src/ipam.rs new file mode 100644 index 00000000..73444b78 --- /dev/null +++ b/crates/isb-core/src/ipam.rs @@ -0,0 +1,755 @@ +//! Instance addresses on isb's org bridges: pinned, never only leased. +//! +//! incus' bridges hand out addresses by DHCP with a one-hour lease, and an +//! OCI container never renews its lease. An hour after one starts, dnsmasq +//! considers its address free and gives it to the next instance that asks: +//! on 2026-11-09 a new instance in org exa got the address a four-week-old +//! Postgres held, and the apps using it broke until the newcomer moved. A +//! NIC's `ipv4.address` makes dnsmasq keep the address for that NIC alone, +//! whatever happens to the lease, so isb pins every address on an org +//! bridge: +//! +//! - At create ([`pin_for_create`]), on an address no instance on the +//! bridge holds or is pinned to, checked again once the instance exists +//! ([`Pin::verify`]): incus accepts two NICs pinned to one address, then +//! refuses to start either. +//! - The daemon's check ([`watch`]), every minute, pins an instance that +//! has no pin at the address it has (in place: it keeps running, and +//! keeps its address), gives a stopped one a free address, and records +//! each pin in `user.isb.ipv4.`, which adopts pins set by hand. It +//! raises `net.failed` when two instances share an address, when a +//! pinned instance has another one, or when an instance cannot be pinned. +//! +//! Only isb's org projects (`isb-`) and their managed bridges are +//! touched. Egress bridges (`isbbrx*`) hold one sandbox each, so nothing on +//! them can collide; they are left to DHCP. + +use std::collections::{BTreeMap, BTreeSet}; +use std::net::Ipv4Addr; +use std::sync::{Mutex, OnceLock}; +use std::time::Duration; + +use serde_json::{Value, json}; + +use crate::client::{Client, encode_segment}; +use crate::error::{Error, Result}; +use crate::org::OrgId; +use crate::plan::Props; + +/// The config key recording the address isb pinned a NIC to. +pub fn pin_key(device: &str) -> String { + format!("user.isb.ipv4.{device}") +} + +/// How often the daemon checks every org's addresses. +pub const CHECK_EVERY: Duration = Duration::from_secs(60); + +/// A managed bridge's IPv4 subnet. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Subnet { + gateway: Ipv4Addr, + prefix: u8, +} + +impl Subnet { + /// From `GET /1.0/networks/`: a managed bridge with an IPv4 + /// address (`10.1.2.1/24`), not an egress bridge. + pub fn from_network(net: &Value) -> Option { + let name = net["name"].as_str().unwrap_or_default(); + if net["type"].as_str() != Some("bridge") + || net["managed"] == Value::Bool(false) + || name.starts_with(crate::egress::plumb::NET_PREFIX) + { + return None; + } + let (ip, prefix) = net["config"]["ipv4.address"].as_str()?.split_once('/')?; + let prefix: u8 = prefix.trim().parse().ok()?; + (8..=30).contains(&prefix).then_some(())?; + Some(Subnet { + gateway: ip.trim().parse().ok()?, + prefix, + }) + } + + fn mask(&self) -> u32 { + u32::MAX << (32 - self.prefix) + } + + pub fn contains(&self, ip: Ipv4Addr) -> bool { + u32::from(ip) & self.mask() == u32::from(self.gateway) & self.mask() + } + + /// An address an instance may take: in the subnet, and neither its + /// network, broadcast nor gateway address. + fn assignable(&self, ip: Ipv4Addr) -> bool { + let host = u32::from(ip) & !self.mask(); + self.contains(ip) && ip != self.gateway && host != 0 && host != !self.mask() + } + + /// A free address, searched from a point `seed` picks, so two callers + /// with different seeds rarely race for one. + pub fn pick_free(&self, used: &BTreeSet, seed: &str) -> Option { + let base = u32::from(self.gateway) & self.mask(); + let size = !self.mask() as u64 + 1; + let start = fnv(seed) % size; + (0..size) + .map(|i| Ipv4Addr::from(base + ((start + i) % size) as u32)) + .find(|ip| self.assignable(*ip) && !used.contains(ip)) + } +} + +fn fnv(s: &str) -> u64 { + s.bytes().fold(0xcbf29ce484222325, |h, b| { + (h ^ b as u64).wrapping_mul(0x100000001b3) + }) +} + +/// One instance's NIC on a bridge, from an instance listed with +/// `recursion=2`. +#[derive(Debug, Clone)] +pub struct Claim { + pub project: String, + pub instance: String, + pub device: String, + /// The NIC's `ipv4.address`. + pub pin: Option, + /// What [`pin_key`] records. + pub recorded: Option, + /// Its IPv4 addresses on the bridge's subnet (running instances only). + pub addrs: Vec, + pub running: bool, + /// The NIC as the instance has it (expanded: from a profile or its own). + pub props: Props, + pub stack: Option, + pub service: Option, + pub created_at: String, +} + +impl Claim { + fn id(&self) -> (String, String) { + (self.project.clone(), self.instance.clone()) + } +} + +/// Every NIC on `network` among `instances` (in isb's org projects). +pub fn claims(instances: &[Value], network: &str, subnet: &Subnet) -> Vec { + let mut out = Vec::new(); + for i in instances { + let project = i["project"].as_str().unwrap_or("default"); + if OrgId::from_incus_project(project).is_none() { + continue; + } + let Some(devs) = i["expanded_devices"].as_object() else { + continue; + }; + for (dev, d) in devs { + if d["type"] != "nic" || d["network"].as_str() != Some(network) { + continue; + } + out.push(claim(i, project, dev, d, subnet)); + } + } + out +} + +fn claim(i: &Value, project: &str, dev: &str, d: &Value, subnet: &Subnet) -> Claim { + let cfg = &i["config"]; + let label = |k: &str| cfg[format!("user.{k}")].as_str().map(String::from); + let running = i["status"].as_str() == Some("Running"); + Claim { + project: project.to_string(), + instance: i["name"].as_str().unwrap_or_default().to_string(), + device: dev.to_string(), + pin: d["ipv4.address"].as_str().and_then(|a| a.parse().ok()), + recorded: cfg[pin_key(dev)].as_str().map(String::from), + addrs: if running { + nic_addrs(i, dev, d, subnet) + } else { + Vec::new() + }, + running, + props: d + .as_object() + .map(|m| { + m.iter() + .filter_map(|(k, v)| Some((k.clone(), v.as_str()?.to_string()))) + .collect() + }) + .unwrap_or_default(), + stack: label("isb.stack"), + service: label("isb.service"), + created_at: i["created_at"].as_str().unwrap_or_default().to_string(), + } +} + +/// The NIC's addresses on `subnet`, from the instance's state: the guest +/// interface with the NIC's MAC (a VM names it enp5s0, say), else the one +/// named as the NIC says. +fn nic_addrs(i: &Value, dev: &str, d: &Value, subnet: &Subnet) -> Vec { + let Some(ifaces) = i["state"]["network"].as_object() else { + return Vec::new(); + }; + let mac = i["config"][format!("volatile.{dev}.hwaddr")] + .as_str() + .map(str::to_ascii_lowercase); + let by_mac = mac.and_then(|m| { + ifaces + .values() + .find(|v| v["hwaddr"].as_str().map(str::to_ascii_lowercase) == Some(m.clone())) + }); + let iface = by_mac.or_else(|| ifaces.get(d["name"].as_str().unwrap_or(dev))); + iface + .and_then(|v| v["addresses"].as_array()) + .into_iter() + .flatten() + .filter(|a| a["family"] == "inet") + .filter_map(|a| a["address"].as_str()?.parse().ok()) + .filter(|ip| subnet.contains(*ip)) + .collect() +} + +/// Who holds each address: its pin, and what a running instance has. +fn holders(claims: &[Claim]) -> BTreeMap> { + let mut out: BTreeMap> = BTreeMap::new(); + for c in claims { + for ip in c.pin.iter().chain(&c.addrs) { + out.entry(*ip).or_default().insert(c.id()); + } + } + out +} + +/// What the check changes. +#[derive(Debug, Clone, PartialEq)] +pub enum Fix { + /// Record a pin isb did not set (one set by hand) as isb's. + Adopt { + project: String, + instance: String, + device: String, + ip: Ipv4Addr, + }, + /// Pin a NIC: at the address it has when running, else a free one. + Pin { + project: String, + instance: String, + device: String, + props: Props, + ip: Ipv4Addr, + running: bool, + }, +} + +/// What the check reports (as `net.failed`). +#[derive(Debug, Clone, PartialEq)] +pub struct Finding { + /// The same problem has the same key on every pass. + pub key: String, + pub project: String, + pub instance: String, + pub stack: Option, + pub service: Option, + pub message: String, +} + +impl Finding { + fn about(c: &Claim, key: String, message: String) -> Finding { + Finding { + key, + project: c.project.clone(), + instance: c.instance.clone(), + stack: c.stack.clone(), + service: c.service.clone(), + message, + } + } +} + +/// Decide the fixes and findings for one bridge's claims. `reserved`: +/// addresses being given to instances that do not exist yet. +pub fn plan( + network: &str, + subnet: &Subnet, + claims: &[Claim], + reserved: &BTreeSet, +) -> (Vec, Vec) { + let held = holders(claims); + let mut findings = conflicts(network, claims, &held); + let mut used: BTreeSet = held + .keys() + .copied() + .chain(reserved.iter().copied()) + .collect(); + let mut fixes = Vec::new(); + for c in claims { + if let Some(pin) = c.pin { + // A shared pin is reported above, never made isb's. + let alone = held.get(&pin).is_some_and(|h| h.len() == 1); + if alone && c.recorded.as_deref() != Some(pin.to_string().as_str()) { + fixes.push(Fix::Adopt { + project: c.project.clone(), + instance: c.instance.clone(), + device: c.device.clone(), + ip: pin, + }); + } + if c.running && !c.addrs.is_empty() && !c.addrs.contains(&pin) { + let have = join(&c.addrs); + findings.push(Finding::about( + c, + format!( + "drift {} {} {} {pin} {have}", + c.project, c.instance, c.device + ), + format!( + "{}'s {} is pinned to {pin} but has {have}: its next start takes {pin}", + c.instance, c.device + ), + )); + } + continue; + } + let ip = match (c.running, c.addrs.as_slice()) { + // Its own address, unless another instance holds it too (a + // conflict, reported above): pinning it would keep it. + (true, [a]) if held.get(a).is_some_and(|h| h.len() == 1) => Some(*a), + // Starting (no address yet), or sharing one: the next pass. + (true, _) => None, + (false, _) => { + let free = subnet.pick_free(&used, &format!("{}/{}", c.project, c.instance)); + if free.is_none() { + findings.push(Finding::about( + c, + format!("full {} {}", c.project, c.instance), + format!( + "{}: no free address on {network} to pin {} to", + c.instance, c.device + ), + )); + } + free + } + }; + if let Some(ip) = ip { + used.insert(ip); + let mut props = c.props.clone(); + props.insert("ipv4.address".into(), ip.to_string()); + fixes.push(Fix::Pin { + project: c.project.clone(), + instance: c.instance.clone(), + device: c.device.clone(), + props, + ip, + running: c.running, + }); + } + } + (fixes, findings) +} + +fn join(ips: &[Ipv4Addr]) -> String { + ips.iter() + .map(Ipv4Addr::to_string) + .collect::>() + .join(", ") +} + +/// Every address two or more instances hold, as one finding about the +/// newest of them (most likely the one that took it). +fn conflicts( + network: &str, + claims: &[Claim], + held: &BTreeMap>, +) -> Vec { + let mut out = Vec::new(); + for (ip, ids) in held.iter().filter(|(_, h)| h.len() > 1) { + let mut who: Vec<&Claim> = claims.iter().filter(|c| ids.contains(&c.id())).collect(); + who.sort_by(|a, b| b.created_at.cmp(&a.created_at)); + who.dedup_by(|a, b| a.id() == b.id()); + let how = |c: &Claim| match (c.pin == Some(*ip), c.addrs.contains(ip)) { + (true, true) => "pinned, has it", + (true, false) => "pinned", + _ => "has it", + }; + let list: Vec = who + .iter() + .map(|c| format!("{} ({})", c.instance, how(c))) + .collect(); + let names: Vec<&str> = who.iter().map(|c| c.instance.as_str()).collect(); + out.push(Finding::about( + who[0], + format!("conflict {network} {ip} {}", names.join(" ")), + format!( + "{ip} on {network} is held by {}: traffic for one reaches another. isb moves \ + neither; pin one elsewhere (incus config device set {} ipv4.address=) \ + and restart it", + list.join(", "), + who[0].device + ), + )); + } + out +} + +/// Addresses being given to instances in the middle of their create, by +/// network: what this process's other creates and checks must not pick. +fn reserved() -> &'static Mutex>> { + static R: OnceLock>>> = OnceLock::new(); + R.get_or_init(Mutex::default) +} + +fn reserved_on(network: &str) -> BTreeSet { + let r = reserved().lock().unwrap_or_else(|e| e.into_inner()); + r.get(network).cloned().unwrap_or_default() +} + +/// Every instance with its state: on every project when the client may +/// list them (the daemon), else on its own. +fn list(client: &Client) -> Result> { + let v = match client.get("/1.0/instances?recursion=2&all-projects=true") { + Ok(v) => v, + Err(_) => client.get("/1.0/instances?recursion=2")?, + }; + Ok(v.as_array().cloned().unwrap_or_default()) +} + +/// An address pinned for an instance being created. Held until the create +/// is done; [`Pin::verify`] checks nobody else took it in between. +pub struct Pin { + network: String, + subnet: Subnet, + device: String, + ip: Ipv4Addr, +} + +impl Drop for Pin { + fn drop(&mut self) { + let mut r = reserved().lock().unwrap_or_else(|e| e.into_inner()); + if let Some(s) = r.get_mut(&self.network) { + s.remove(&self.ip); + } + } +} + +impl Pin { + pub fn ip(&self) -> Ipv4Addr { + self.ip + } + + /// After the create: when another instance holds the address too (a + /// create by another process that picked it at the same moment), move + /// this one, which has not started yet. + pub fn verify(mut self, client: &Client, name: &str) -> Result<()> { + let me = (client.project_name().to_string(), name.to_string()); + for _ in 0..3 { + let all = claims(&list(client)?, &self.network, &self.subnet); + let held = holders(&all); + if held + .get(&self.ip) + .is_none_or(|h| h.iter().all(|id| *id == me)) + { + return Ok(()); + } + let mut used: BTreeSet = held.keys().copied().collect(); + used.extend(reserved_on(&self.network)); + let ip = self + .subnet + .pick_free(&used, &format!("{name}/{}", self.ip)) + .ok_or_else(|| Error::invalid(format!("no free address on {}", self.network)))?; + set_pin(client, name, &self.device, None, ip)?; + self.reserve(ip); + } + Err(Error::invalid(format!( + "{name}: could not find an address on {} no other instance takes", + self.network + ))) + } + + fn reserve(&mut self, ip: Ipv4Addr) { + let mut r = reserved().lock().unwrap_or_else(|e| e.into_inner()); + let s = r.entry(self.network.clone()).or_default(); + s.remove(&self.ip); + s.insert(ip); + self.ip = ip; + } +} + +/// Pin `eth0` in a create body (`POST /1.0/instances`) for an instance in +/// one of isb's org projects whose `eth0` joins a managed bridge: the +/// body's own NIC, else its profiles'. Anything else is returned as is. +pub fn pin_for_create(client: &Client, mut body: Value) -> Result<(Value, Option)> { + if OrgId::from_incus_project(client.project_name()).is_none() { + return Ok((body, None)); + } + let Some(mut nic) = eth0(client, &body)? else { + return Ok((body, None)); + }; + let Some(network) = nic["network"].as_str().map(String::from) else { + return Ok((body, None)); + }; + if !nic["ipv4.address"].is_null() { + return Ok((body, None)); + } + let path = format!("/1.0/networks/{}", encode_segment(&network)); + let Some(subnet) = client + .get_opt(&path)? + .as_ref() + .and_then(Subnet::from_network) + else { + return Ok((body, None)); + }; + let name = body["name"].as_str().unwrap_or_default().to_string(); + let held = holders(&claims(&list(client)?, &network, &subnet)); + let ip = { + let mut r = reserved().lock().unwrap_or_else(|e| e.into_inner()); + let res = r.entry(network.clone()).or_default(); + let used: BTreeSet = held.keys().chain(res.iter()).copied().collect(); + let ip = subnet + .pick_free(&used, &format!("{}/{name}", client.project_name())) + .ok_or_else(|| Error::invalid(format!("{name}: no free address on {network}")))?; + res.insert(ip); + ip + }; + let pin = Pin { + network, + subnet, + device: "eth0".into(), + ip, + }; + nic["ipv4.address"] = json!(ip.to_string()); + body["devices"]["eth0"] = nic; + body["config"][pin_key("eth0")] = json!(ip.to_string()); + Ok((body, Some(pin))) +} + +/// The `eth0` NIC an instance created from `body` gets: its own, else the +/// last of its profiles' (a later profile overrides an earlier one). +fn eth0(client: &Client, body: &Value) -> Result> { + if let Some(d) = body["devices"].get("eth0") { + return Ok((d["type"] == "nic").then(|| d.clone())); + } + let mut nic = None; + for p in body["profiles"].as_array().into_iter().flatten() { + let Some(p) = p.as_str() else { continue }; + let path = format!("/1.0/profiles/{}", encode_segment(p)); + if let Some(d) = client + .get_opt(&path)? + .and_then(|v| v["devices"].get("eth0").cloned()) + { + nic = (d["type"] == "nic").then_some(d); + } + } + Ok(nic) +} + +/// Pin an instance's NIC to `ip`, in place: `props` is the NIC to write +/// (from a profile), or `None` to change the instance's own. +fn set_pin( + client: &Client, + name: &str, + device: &str, + props: Option<&Props>, + ip: Ipv4Addr, +) -> Result<()> { + crate::sandbox::update_instance( + client, + name, + &format!("pin {name} {device}"), + &mut |config, devices| { + let mut nic = match props { + Some(p) => json!(p), + None => devices + .get(device) + .cloned() + .ok_or_else(|| Error::invalid(format!("{name} has no device {device}")))?, + }; + nic["ipv4.address"] = json!(ip.to_string()); + devices.insert(device.to_string(), nic); + config.insert(pin_key(device), json!(ip.to_string())); + Ok(()) + }, + ) +} + +/// A desired NIC that names no address is satisfied by one pinned to any: +/// the pin is isb's, not the spec's (for [`crate::plan::device_matches`]). +pub fn ignore_pin(desired: &Props, actual: &mut Props) { + if desired.get("type").is_some_and(|t| t == "nic") && !desired.contains_key("ipv4.address") { + actual.remove("ipv4.address"); + } +} + +/// Whether an instance's device is a NIC isb pinned (never pruned). +pub fn is_pin(device: &str, props: &Props, config: &Props) -> bool { + props.get("type").is_some_and(|t| t == "nic") + && props.get("ipv4.address").is_some() + && props.get("ipv4.address") == config.get(&pin_key(device)) +} + +/// An instance's devices but the NICs isb pinned: what pruning may remove. +pub fn unpinned<'a>( + devices: &'a BTreeMap, + config: &'a Props, +) -> impl Iterator { + devices.iter().filter(|(n, p)| !is_pin(n, p, config)) +} + +/// One pass over every org bridge: apply the fixes, return the findings. +/// `only`: the bridges of this incus project alone (a test's org). `report` +/// hears each fix. +pub fn check( + client: &Client, + only: Option<&str>, + report: &mut dyn FnMut(&str), +) -> Result> { + let mut nets: BTreeMap = client + .get("/1.0/networks?recursion=1")? + .as_array() + .into_iter() + .flatten() + .filter_map(|n| Some((n["name"].as_str()?.to_string(), Subnet::from_network(n)?))) + .collect(); + let instances = list(client)?; + if let Some(p) = only { + let mine = |i: &Value| i["project"].as_str() == Some(p); + let used: BTreeSet<&str> = instances + .iter() + .filter(|i| mine(i)) + .filter_map(|i| i["expanded_devices"].as_object()) + .flat_map(|d| d.values().filter_map(|n| n["network"].as_str())) + .collect(); + let used: BTreeSet = used.into_iter().map(String::from).collect(); + nets.retain(|n, _| used.contains(n)); + } + let mut findings = Vec::new(); + for (network, subnet) in &nets { + let cl = claims(&instances, network, subnet); + if cl.is_empty() { + continue; + } + let (fixes, found) = plan(network, subnet, &cl, &reserved_on(network)); + findings.extend(found); + for fix in fixes { + if let Some(f) = apply(client, &fix, &cl, report) { + findings.push(f); + } + } + } + Ok(findings) +} + +/// Apply one fix; a failure is a finding. +fn apply( + client: &Client, + fix: &Fix, + cl: &[Claim], + report: &mut dyn FnMut(&str), +) -> Option { + let (project, instance, device, r) = match fix { + Fix::Adopt { + project, + instance, + device, + ip, + } => { + let c = client.clone().project(project.as_str()); + let r = crate::sandbox::update_instance( + &c, + instance, + &format!("adopt {instance}'s pin"), + &mut |config, _| { + config.insert(pin_key(device), json!(ip.to_string())); + Ok(()) + }, + ); + if r.is_ok() { + report(&format!( + "{project}/{instance}: {device}'s pin to {ip} is isb's now" + )); + } + (project, instance, device, r) + } + Fix::Pin { + project, + instance, + device, + props, + ip, + running, + } => { + let c = client.clone().project(project.as_str()); + let r = set_pin(&c, instance, device, Some(props), *ip); + if r.is_ok() { + let how = if *running { + "its address" + } else { + "a free address" + }; + report(&format!( + "{project}/{instance}: pinned {device} to {ip} ({how})" + )); + } + (project, instance, device, r) + } + }; + r.err().map(|e| { + let c = cl + .iter() + .find(|c| c.project == *project && c.instance == *instance); + let mut f = Finding { + key: format!("unpinned {project} {instance} {device}"), + project: project.clone(), + instance: instance.clone(), + stack: None, + service: None, + message: format!( + "{instance}'s {device} has no pin isb could set ({e}): once its lease lapses, \ + another instance can be given its address" + ), + }; + if let Some(c) = c { + (f.stack, f.service) = (c.stack.clone(), c.service.clone()); + } + f + }) +} + +/// The daemon's check, on a thread of its own: [`check`] every +/// [`CHECK_EVERY`], raising `net.failed` once per problem until it is gone. +pub fn watch(client: Client, ctl: crate::stack::Controller) { + let _ = std::thread::Builder::new() + .name("isb-ipam".into()) + .spawn(move || { + let mut raised: BTreeSet = BTreeSet::new(); + loop { + let mut log = |m: &str| eprintln!("isb serve: addresses: {m}"); + match check(&client, None, &mut log) { + Ok(found) => { + let now: BTreeSet = found.iter().map(|f| f.key.clone()).collect(); + for f in found.iter().filter(|f| !raised.contains(&f.key)) { + raise(&ctl, f); + } + raised = now; + } + Err(e) => log(&format!("check failed: {e}")), + } + std::thread::sleep(CHECK_EVERY); + } + }); +} + +fn raise(ctl: &crate::stack::Controller, f: &Finding) { + let org = OrgId::from_incus_project(&f.project).unwrap_or_else(OrgId::default_org); + let stack = crate::stack::qualified(&org, f.stack.as_deref().unwrap_or_default()); + ctl.instance_event( + "net.failed", + "error", + &stack, + f.service.as_deref().unwrap_or_default(), + &f.instance, + f.message.clone(), + ); +} + +#[cfg(test)] +mod tests; diff --git a/crates/isb-core/src/ipam/tests.rs b/crates/isb-core/src/ipam/tests.rs new file mode 100644 index 00000000..fafb6826 --- /dev/null +++ b/crates/isb-core/src/ipam/tests.rs @@ -0,0 +1,287 @@ +use super::*; +use serde_json::json; + +fn bridge(name: &str, addr: &str) -> Value { + json!({"name": name, "type": "bridge", "managed": true, "config": {"ipv4.address": addr}}) +} + +fn net() -> Subnet { + Subnet::from_network(&bridge("isbbr1", "10.83.115.1/24")).unwrap() +} + +fn ip(s: &str) -> Ipv4Addr { + s.parse().unwrap() +} + +/// An instance as `GET /1.0/instances?recursion=2` lists it, on isbbr1: +/// `pin` its NIC's ipv4.address, `addr` the address it has (running). +fn inst(name: &str, pin: Option<&str>, addr: Option<&str>, created: &str) -> Value { + let mut nic = json!({"type": "nic", "network": "isbbr1", "name": "eth0"}); + if let Some(p) = pin { + nic["ipv4.address"] = json!(p); + } + let state = match addr { + Some(a) => json!({"network": { + "lo": {"hwaddr": "", "addresses": [{"family": "inet", "address": "127.0.0.1"}]}, + "eth0": {"hwaddr": "10:66:6a:00:00:01", "addresses": [ + {"family": "inet", "address": a}, {"family": "inet6", "address": "fe80::1"}]}}}), + None => json!(null), + }; + json!({ + "name": name, "project": "isb-exa", "created_at": created, + "status": if addr.is_some() { "Running" } else { "Stopped" }, + "config": {"volatile.eth0.hwaddr": "10:66:6a:00:00:01", "user.isb.stack": "analytics", + "user.isb.service": "postgres"}, + "expanded_devices": {"root": {"type": "disk"}, "eth0": nic}, + "state": state, + }) +} + +fn plan_of(insts: &[Value]) -> (Vec, Vec) { + plan( + "isbbr1", + &net(), + &claims(insts, "isbbr1", &net()), + &BTreeSet::new(), + ) +} + +#[test] +fn a_subnet_is_a_managed_bridge_with_an_ipv4_address() { + let s = net(); + assert!(s.contains(ip("10.83.115.43"))); + assert!(!s.contains(ip("10.83.116.43"))); + assert!( + Subnet::from_network(&bridge("isbbrxabcd", "10.1.2.1/24")).is_none(), + "egress" + ); + assert!(Subnet::from_network(&bridge("isbbr1", "none")).is_none()); + let mut unmanaged = bridge("br0", "10.1.2.1/24"); + unmanaged["managed"] = json!(false); + assert!(Subnet::from_network(&unmanaged).is_none()); +} + +#[test] +fn a_free_address_is_never_the_gateway_network_broadcast_or_used() { + let s = Subnet::from_network(&bridge("isbbr1", "10.9.9.1/29")).unwrap(); + // .0 network, .1 gateway, .7 broadcast: .2-.6 are the hosts. + let mut used = BTreeSet::new(); + let mut got = Vec::new(); + while let Some(a) = s.pick_free(&used, "x") { + used.insert(a); + got.push(a); + } + got.sort(); + let want: Vec = (2..=6).map(|h| Ipv4Addr::new(10, 9, 9, h)).collect(); + assert_eq!(got, want); +} + +#[test] +fn seeds_spread_picks_over_the_subnet() { + let s = net(); + let used = BTreeSet::new(); + let picks: BTreeSet = (0..20) + .filter_map(|i| s.pick_free(&used, &format!("isb-exa/web-1-{i}"))) + .collect(); + assert!(picks.len() > 15, "{picks:?}"); +} + +#[test] +fn a_claim_reads_pin_address_and_labels() { + let c = claims( + &[inst("pg", Some("10.83.115.43"), Some("10.83.115.43"), "a")], + "isbbr1", + &net(), + ); + assert_eq!(c.len(), 1); + assert_eq!(c[0].pin, Some(ip("10.83.115.43"))); + assert_eq!(c[0].addrs, [ip("10.83.115.43")]); + assert_eq!(c[0].stack.as_deref(), Some("analytics")); + // Another network, and a project outside isb's orgs, are not claims. + assert!(claims(&[inst("pg", None, None, "a")], "isbbr2", &net()).is_empty()); + let mut other = inst("pg", None, None, "a"); + other["project"] = json!("default"); + assert!(claims(&[other], "isbbr1", &net()).is_empty()); +} + +#[test] +fn a_vms_address_is_found_by_mac() { + let mut v = inst("vm", None, Some("10.83.115.9"), "a"); + let eth0 = v["state"]["network"]["eth0"].take(); + v["state"]["network"]["enp5s0"] = eth0; + v["state"]["network"] + .as_object_mut() + .unwrap() + .remove("eth0"); + let c = claims(&[v], "isbbr1", &net()); + assert_eq!(c[0].addrs, [ip("10.83.115.9")]); +} + +#[test] +fn the_incident_is_a_conflict_about_the_newcomer() { + // 2026-11-09: the gateway got the address analytics-postgres held. + let (fixes, found) = plan_of(&[ + inst( + "analytics-production-analytics-postgres-1-0a5e", + None, + Some("10.83.115.43"), + "2026-10-12", + ), + inst( + "mcp-gateway-1-2ee0", + None, + Some("10.83.115.43"), + "2026-11-09", + ), + ]); + assert!( + fixes.is_empty(), + "neither is pinned at a shared address: {fixes:?}" + ); + assert_eq!(found.len(), 1); + assert_eq!(found[0].instance, "mcp-gateway-1-2ee0"); + assert!( + found[0].message.contains("10.83.115.43"), + "{}", + found[0].message + ); + assert!( + found[0] + .message + .contains("analytics-production-analytics-postgres-1-0a5e") + ); +} + +#[test] +fn a_pin_and_another_instances_address_conflict() { + // A stopped instance pinned to an address a running one has: it could + // not start. + let (fixes, found) = plan_of(&[ + inst("a", Some("10.83.115.43"), None, "1"), + inst("b", None, Some("10.83.115.43"), "2"), + ]); + assert!(fixes.is_empty(), "a shared pin is not adopted: {fixes:?}"); + assert_eq!(found.len(), 1); + assert!( + found[0].message.contains("a (pinned)"), + "{}", + found[0].message + ); + assert!( + found[0].message.contains("b (has it)"), + "{}", + found[0].message + ); +} + +#[test] +fn a_running_instance_is_pinned_at_its_own_address() { + let (fixes, found) = plan_of(&[inst("web", None, Some("10.83.115.7"), "1")]); + assert!(found.is_empty()); + let [ + Fix::Pin { + ip: a, + props, + running: true, + .. + }, + ] = fixes.as_slice() + else { + panic!("{fixes:?}"); + }; + assert_eq!(*a, ip("10.83.115.7")); + assert_eq!(props["network"], "isbbr1"); + assert_eq!(props["ipv4.address"], "10.83.115.7"); +} + +#[test] +fn a_stopped_instance_gets_a_free_address() { + let reserved = BTreeSet::from([ip("10.83.115.9")]); + let insts = [ + inst("pg", Some("10.83.115.8"), Some("10.83.115.8"), "1"), + inst("old", None, None, "2"), + ]; + let cl = claims(&insts, "isbbr1", &net()); + let (fixes, _) = plan("isbbr1", &net(), &cl, &reserved); + let pinned: Vec = fixes + .iter() + .filter_map(|f| match f { + Fix::Pin { instance, ip, .. } if instance == "old" => Some(*ip), + _ => None, + }) + .collect(); + assert_eq!(pinned.len(), 1); + assert!(![ip("10.83.115.8"), ip("10.83.115.9"), ip("10.83.115.1")].contains(&pinned[0])); +} + +#[test] +fn a_pin_set_by_hand_is_adopted_once() { + let mut v = inst("pg", Some("10.83.115.43"), Some("10.83.115.43"), "1"); + let (fixes, found) = plan_of(std::slice::from_ref(&v)); + assert!(found.is_empty()); + assert_eq!( + fixes, + [Fix::Adopt { + project: "isb-exa".into(), + instance: "pg".into(), + device: "eth0".into(), + ip: ip("10.83.115.43"), + }] + ); + v["config"]["user.isb.ipv4.eth0"] = json!("10.83.115.43"); + assert_eq!(plan_of(&[v]), (vec![], vec![])); +} + +#[test] +fn a_pin_the_instance_does_not_have_is_drift() { + let mut v = inst("e", Some("10.83.115.28"), Some("10.83.115.7"), "1"); + v["config"]["user.isb.ipv4.eth0"] = json!("10.83.115.28"); + let (fixes, found) = plan_of(&[v]); + assert!(fixes.is_empty()); + assert_eq!(found.len(), 1); + assert!( + found[0] + .message + .contains("pinned to 10.83.115.28 but has 10.83.115.7") + ); +} + +#[test] +fn a_starting_instance_waits_for_its_address() { + let mut v = inst("web", None, Some("10.83.115.7"), "1"); + v["state"]["network"]["eth0"]["addresses"] = json!([]); + assert_eq!(plan_of(&[v]), (vec![], vec![])); +} + +#[test] +fn findings_keep_their_key_from_pass_to_pass() { + let insts = [ + inst("a", None, Some("10.83.115.43"), "1"), + inst("b", None, Some("10.83.115.43"), "2"), + ]; + assert_eq!(plan_of(&insts).1[0].key, plan_of(&insts).1[0].key); +} + +#[test] +fn the_diff_ignores_and_never_prunes_isbs_pin() { + let desired = Props::from([ + ("type".into(), "nic".into()), + ("network".into(), "isbbr1".into()), + ]); + let mut actual = desired.clone(); + actual.insert("ipv4.address".into(), "10.83.115.43".into()); + let mut a = actual.clone(); + ignore_pin(&desired, &mut a); + assert_eq!(a, desired); + // A spec that names an address keeps it compared. + let mut want = desired.clone(); + want.insert("ipv4.address".into(), "10.83.115.44".into()); + let mut a = actual.clone(); + ignore_pin(&want, &mut a); + assert_ne!(a, want); + + let config = Props::from([("user.isb.ipv4.eth0".into(), "10.83.115.43".into())]); + assert!(is_pin("eth0", &actual, &config)); + assert!(!is_pin("eth0", &actual, &Props::new())); + assert!(!is_pin("eth0", &desired, &config)); +} diff --git a/crates/isb-core/src/lib.rs b/crates/isb-core/src/lib.rs index bc43aa45..cd3bfd19 100644 --- a/crates/isb-core/src/lib.rs +++ b/crates/isb-core/src/lib.rs @@ -19,6 +19,7 @@ pub mod idmap; pub mod image_check; pub mod ingress; pub mod interp; +pub mod ipam; pub mod lock; pub mod machine; pub mod metrics; diff --git a/crates/isb-core/src/plan.rs b/crates/isb-core/src/plan.rs index f3ee164a..37cf4e5c 100644 --- a/crates/isb-core/src/plan.rs +++ b/crates/isb-core/src/plan.rs @@ -1189,6 +1189,7 @@ fn normalize(p: &Props) -> Props { pub fn device_matches(desired: &DesiredDevice, actual: &Props) -> bool { let mut d = normalize(&desired.props); let mut a = normalize(actual); + crate::ipam::ignore_pin(&d, &mut a); // initial.copy only acts the first time a volume is used, so a disk that // differs in nothing else is correct, and replacing it would remount it. if d.get("type").map(String::as_str) == Some("disk") { @@ -1420,7 +1421,7 @@ pub fn diff( } if opts.prune_devices { - for (name, props) in &actual.devices { + for (name, props) in crate::ipam::unpinned(&actual.devices, &actual.config) { if name != "root" && !desired.devices.contains_key(name) && !claimed.contains(name) { actions.push(Action::RemoveDevice { device: name.clone(), diff --git a/crates/isb-core/src/sandbox.rs b/crates/isb-core/src/sandbox.rs index 5376f5dd..fa1a2a32 100644 --- a/crates/isb-core/src/sandbox.rs +++ b/crates/isb-core/src/sandbox.rs @@ -505,7 +505,8 @@ fn create_instance(client: &Client, desired: &Desired, report: &mut dyn FnMut(&s "devices": devices, "profiles": desired.profiles, }); - let body = crate::org::disk::sized_root(client, body); + let (body, pin) = + crate::ipam::pin_for_create(client, crate::org::disk::sized_root(client, body))?; match client.mutate( "POST", "/1.0/instances", @@ -513,7 +514,7 @@ fn create_instance(client: &Client, desired: &Desired, report: &mut dyn FnMut(&s &step, client.timeouts.create, ) { - Ok(_) => return Ok(()), + Ok(_) => return pin.map_or(Ok(()), |p| p.verify(client, &desired.name)), Err(e) if e.is_timeout() => { report(&format!("{e}")); if let Error::OperationTimeout { operation, .. } = &e { @@ -524,7 +525,7 @@ fn create_instance(client: &Client, desired: &Desired, report: &mut dyn FnMut(&s .is_ok() { report(&format!("{step}: finished late; keeping it")); - return Ok(()); + return pin.map_or(Ok(()), |p| p.verify(client, &desired.name)); } } if let Err(ce) = cleanup_half_created(client, &desired.name, &token, report) { diff --git a/crates/isb-core/src/stack/controller.rs b/crates/isb-core/src/stack/controller.rs index 5e558ccc..1aa2c7c5 100644 --- a/crates/isb-core/src/stack/controller.rs +++ b/crates/isb-core/src/stack/controller.rs @@ -183,6 +183,7 @@ pub struct Event { /// dotted, `.`. In use: `deploy.succeeded`, /// `deploy.failed`, `health.unhealthy`, `health.recovered`, /// `restart.failed` (a crash loop), `dns.failed` (no nameserver), + /// `net.failed` (two instances share an address, or one has no pin), /// `backup.succeeded`, `backup.failed`, `restore.succeeded`, /// `restore.failed`, `job.succeeded`, `job.failed`, `cert.issued`, /// `cert.failed`, `secret.rotated` (a new secret version reached a @@ -547,6 +548,22 @@ impl Controller { .emit_kind(Some(kind), level, stack, service, None, message); } + /// Record an event of a known kind about one instance (`stack` + /// qualified; `service` empty for an instance no stack owns). + pub fn instance_event( + &self, + kind: &str, + level: &str, + stack: &str, + service: &str, + instance: &str, + message: String, + ) { + eprintln!("isb serve: {stack}: {instance}: {message}"); + self.inner + .emit_kind(Some(kind), level, stack, service, Some(instance), message); + } + pub fn note(&self, level: &str, stack: &str, message: String) { eprintln!("isb serve: {stack}: {message}"); self.inner.emit(level, stack, "", None, message); diff --git a/crates/isb-daemon/src/daemon/mod.rs b/crates/isb-daemon/src/daemon/mod.rs index 5e2095f3..c60dd94e 100644 --- a/crates/isb-daemon/src/daemon/mod.rs +++ b/crates/isb-daemon/src/daemon/mod.rs @@ -419,6 +419,7 @@ pub fn serve(client: Client, cfg: ServeConfig) -> Result<()> { let sandboxes = sandboxes::Sandboxes::new(&cfg.state_dir, client.clone(), recorder.clone()); let org_api = org_api::OrgApi::new(client.clone(), cfg.org_api_port); let (egress, stop_egress) = egress::start(&cfg, &client, &secrets)?; + crate::ipam::watch(client.clone(), ctl.clone()); let meta = crate::stack::deployments::StackMeta::new(&cfg.state_dir); meta.recover(); let d = Arc::new(Daemon { diff --git a/crates/isb-daemon/src/daemon/notify.rs b/crates/isb-daemon/src/daemon/notify.rs index 554ccbe6..b2eba59f 100644 --- a/crates/isb-daemon/src/daemon/notify.rs +++ b/crates/isb-daemon/src/daemon/notify.rs @@ -11,7 +11,7 @@ use crate::metrics_history::{History, Query}; use crate::notify::{Channel, Notifier, Provider, Rule, Settings}; use crate::server::{Caller, Registry, Tool}; -const PROVIDER_DESC: &str = "Where messages go, by `type`: {\"type\": \"webhook\", \"url_secret\": NAME, \"signing_secret\": NAME?} (JSON POST; with a signing secret, X-Isb-Signature: sha256=HMAC of the body), {\"type\": \"slack\", \"url_secret\": NAME} (an incoming-webhook URL), {\"type\": \"discord\", \"url_secret\": NAME}, {\"type\": \"telegram\", \"token_secret\": NAME, \"chat_id\": \"-100...\"}, {\"type\": \"email\", \"host\", \"port\"?, \"tls\": \"starttls\"|\"tls\"|\"none\", \"username\"?, \"password_secret\"?, \"from\", \"to\": [..]}. Every URL, token and password is an org secret, named here (secret_create first), never a value."; +const PROVIDER_DESC: &str = "Where messages go, by `type`: {\"type\": \"webhook\", \"url_secret\": NAME, \"signing_secret\": NAME?} (JSON POST; with a signing secret, X-Isb-Signature: sha256=HMAC of the body), {\"type\": \"slack\", \"url_secret\": NAME} (an incoming-webhook URL), {\"type\": \"discord\", \"url_secret\": NAME}, {\"type\": \"mattermost\", \"url_secret\": NAME} (an incoming-webhook URL, https on any server), {\"type\": \"telegram\", \"token_secret\": NAME, \"chat_id\": \"-100...\"}, {\"type\": \"email\", \"host\", \"port\"?, \"tls\": \"starttls\"|\"tls\"|\"none\", \"username\"?, \"password_secret\"?, \"from\", \"to\": [..]}. Every URL, token and password is an org secret, named here (secret_create first), never a value."; const RULES_DESC: &str = "Which events: a list of {\"events\": [glob, ...] (deploy.*, health.*, backup.*, job.*, cert.*, *.failed, *), \"projects\"?: [..], \"apps\"?: [..], \"stacks\"?: [..]}; any rule matching sends. Default: [{\"events\": [\"*\"]}]."; @@ -33,7 +33,7 @@ pub fn register(r: &mut Registry, n: Notifier, h: History, apps: Apps) -> Result r.register( Tool::new( "notification_channel_create", - "Add a notification channel to the org: a destination (webhook, Slack, Discord, Telegram, email) and rules choosing which events it hears about. Only the org's own events reach it. Its secrets must exist.", + "Add a notification channel to the org: a destination (webhook, Slack, Discord, Mattermost, Telegram, email) and rules choosing which events it hears about. Only the org's own events reach it. Its secrets must exist.", obj( json!({ "name": {"type": "string", "description": "[a-z0-9-], starting with a letter, at most 40."}, diff --git a/docs/concepts/orgs.md b/docs/concepts/orgs.md index 9283dac0..f1c292d2 100644 --- a/docs/concepts/orgs.md +++ b/docs/concepts/orgs.md @@ -216,6 +216,22 @@ The org's ACL decides where its instances may go: Traffic into an org's instances is not filtered by the ACL: the host, and the daemon's load balancer, reach them. +### Addresses + +Every instance's address on the org bridge is pinned (its NIC's +`ipv4.address`), so it is the instance's alone for as long as it exists. A +DHCP lease alone is not enough: incus' bridges lease for an hour, an OCI +container never renews, and dnsmasq then gives its address to the next +instance that asks. isb pins an address no other instance on the bridge +holds or is pinned to when it creates an instance, and records it in +`user.isb.ipv4.eth0`. Once a minute `isb serve` pins any instance without a +pin at the address it has, in place (it keeps running and keeps its +address), gives a stopped one a free address, and adopts pins set by hand. +It raises `net.failed` when two instances share an address, when a pinned +instance has another one, or when it cannot pin one; it moves no instance +itself. An instance with an [egress policy](../guides/egress.md) has a +bridge of its own and is left to DHCP. + A host with a default-deny firewall (ufw) needs `sudo isb host setup` once, so org bridges get DHCP, DNS and egress while staying apart from each other: see [Host firewall](../operations/host-setup.md#host-firewall). diff --git a/docs/guides/notifications.md b/docs/guides/notifications.md index 41c4001e..8e7822a7 100644 --- a/docs/guides/notifications.md +++ b/docs/guides/notifications.md @@ -1,6 +1,6 @@ --- title: Notifications -description: Tell an org's people about failed deploys, unhealthy services, certificates, backups and jobs, through webhooks, Slack, Discord, Telegram or email. +description: Tell an org's people about failed deploys, unhealthy services, certificates, backups and jobs, through webhooks, Slack, Discord, Mattermost, Telegram or email. order: 9 --- @@ -22,6 +22,7 @@ isb notify deliveries ops # what was sent, retried, failed ```text isb notify create NAME (--webhook URL_SECRET [--signing-secret S] | --slack URL_SECRET | --discord URL_SECRET + | --mattermost URL_SECRET | --telegram TOKEN_SECRET --chat-id ID | --smtp-host H [--smtp-port P] [--smtp-tls starttls|tls|none] [--smtp-user U --smtp-password-secret S] --from ADDR --to ADDR...) [--events deploy.*,health.*] [--app-project P]... [--app A]... [--stack S]... [--disabled] @@ -48,11 +49,12 @@ touching the channel. | `webhook` | `url_secret`, optional `signing_secret` | A JSON `POST` ([below](#the-webhook-body)) to any http(s) URL. | | `slack` | `url_secret`: an incoming-webhook URL, `https://hooks.slack.com/services/...` | `{"text": ...}`, the title in bold. | | `discord` | `url_secret`: `https://discord.com/api/webhooks/...` | `{"content": ..., "allowed_mentions": {"parse": []}}` (never pings). | +| `mattermost` | `url_secret`: an incoming-webhook URL on any server, `https:///hooks/...` | `{"text": ...}`, the title in bold; every `@` is followed by a zero-width space, so event text never pings anyone. Any https endpoint that takes Slack's `{"text": ...}` works too. | | `telegram` | `token_secret` (a bot token), `chat_id` (`-100...` or `@channel`) | `sendMessage` to the Bot API, plain text. | | `email` | `host`, `port` (default by `tls`), `tls`: `starttls` (587, default), `tls` (465) or `none` (25); `username` + `password_secret`; `from`; `to` (1 to 20) | A plain-text mail through that SMTP server. A password is never sent without TLS. | Slack and Discord URLs are checked against their hosts when the channel is -saved and again at every send; a Telegram token must look like one +saved and again at every send, and a Mattermost URL must be https; a Telegram token must look like one (`123456:ABC...`), so it cannot change the request path. ### The webhook body @@ -89,7 +91,7 @@ ok = hmac.compare_digest(sig, "sha256=" + hmac.new(key, body, hashlib.sha256).he old replays. Events whose producer adds structure carry it in `details`: `monitor.*` events have the URL, HTTP status, latency, error, downtime and a link to the monitor ([Uptime monitoring](uptime.md#the-notification)); Slack, -Discord, Telegram and email show the latency and the link too. +Discord, Mattermost, Telegram and email show the latency and the link too. ## Rules @@ -119,6 +121,7 @@ shortest globs (`*`, `deploy.*`, `*.failed`). | `health.recovered` | A replica of that service is healthy again. | | `restart.failed` | A replica keeps exiting: its fourth exit in a row within 10 minutes of a start. The daemon starts it again with a backoff (10 s, doubling to 5 minutes); raised once per crash loop ([What keeps it running](../concepts/stacks.md#what-keeps-it-running)). | | `dns.failed` | An OCI replica started with a `/etc/resolv.conf` that names no nameserver, so its DNS lookups fail until it is restarted. | +| `net.failed` | Two instances on an org bridge hold one address (pinned or in use), a pinned instance has another address than its pin, or an instance could not be pinned ([Addresses](../concepts/orgs.md#addresses)). Raised once per problem, while it lasts. | | `cert.issued`, `cert.failed` | The ingress got a certificate for a domain, or failed to ([Domains and ingress](domains.md)). | | `backup.succeeded`, `backup.failed`, `job.succeeded`, `job.failed` | [Database and volume backups](databases.md#backups) and [scheduled jobs](jobs.md). | | `monitor.down`, `monitor.up` | An [uptime monitor](uptime.md) went down (its failure threshold reached), or came back (with the downtime). Once per incident; a flapping monitor is held until it settles. The body's `details` carry the URL, status, latency, error and a link. | diff --git a/docs/operations/troubleshooting.md b/docs/operations/troubleshooting.md index 73522c05..9f5c600d 100644 --- a/docs/operations/troubleshooting.md +++ b/docs/operations/troubleshooting.md @@ -82,6 +82,7 @@ More in [isb on macOS](../getting-started/macos.md). | A service is `waiting` | Its `depends_on` is not met. | Look at the dependency's state. | | Published ports pause during a daemon restart | The load balancer and ingress live in the daemon; apps do not. | Expected; they return within seconds. | | Apps cannot reach each other by name | Service names need an org with its own network: the `default` org on a host whose incus `default` project held workloads first has none. On a fresh org, the host may lack the service-name directory. | Use an org of its own; run `sudo isb host setup`, after which a running `isb serve` turns names on within a minute (`isb org create ORG` again does it without the daemon) ([Setting up a host](host-setup.md#service-names)). | +| `net.failed`: an address is held by two instances | Two instances on the org bridge have one address, so traffic for one reaches the other. isb moves neither. | Pin the one that should move elsewhere (`incus config device set INSTANCE eth0 ipv4.address=FREE`) and restart it ([Addresses](../concepts/orgs.md#addresses)). | | A new org's instances get no address or no internet | A default-deny firewall drops DHCP, DNS and forwarding on new bridges. | `sudo isb host setup` ([Host firewall](host-setup.md#host-firewall)). | | A builder image build stops with `got no IPv4 address on incusbr0` | The same firewall drops DHCP and forwarding on incus' own bridge, where those builds run. | `sudo isb host setup`, then build again. | | `isb serve` or `isb host setup` warns that incus is too old for OCI images | The incus package is older than 6.3 (Ubuntu 24.04's is 6.0), so `docker:` images cannot run. | Install incus from the Zabbly stable repository ([Install isb](../getting-started/install.md#1-incus)). | diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 779dbd31..1ae824d3 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -516,7 +516,7 @@ prints the build id instead. See [Builds](../guides/builds.md). ```text isb notify create NAME (--webhook URL_SECRET [--signing-secret S] | --slack URL_SECRET - | --discord URL_SECRET | --telegram TOKEN_SECRET --chat-id ID + | --discord URL_SECRET | --mattermost URL_SECRET | --telegram TOKEN_SECRET --chat-id ID | --smtp-host H [--smtp-port P] [--smtp-tls starttls|tls|none] [--smtp-user U --smtp-password-secret S] --from ADDR --to ADDR...) [--events deploy.*,health.*] [--app-project P]... [--app A]... [--stack S]... diff --git a/docs/reference/mcp-tools.md b/docs/reference/mcp-tools.md index 1812a7f0..4898b7e1 100644 --- a/docs/reference/mcp-tools.md +++ b/docs/reference/mcp-tools.md @@ -293,7 +293,7 @@ and owners. | Tool | Who | Does | |---|---|---| -| `notification_channel_create` | member | `name`, `provider` (webhook, Slack, Discord, Telegram, email; secrets named, never values), `rules`, `enabled`. | +| `notification_channel_create` | member | `name`, `provider` (webhook, Slack, Discord, Mattermost, Telegram, email; secrets named, never values), `rules`, `enabled`. | | `notification_channel_list`, `notification_channel_get` | viewer | Channels with their last delivery. | | `notification_channel_update` | member | Replace `provider`, `rules` or `enabled`; fields left out are kept. | | `notification_channel_delete` | member | The channel and its delivery log (its secrets stay). | diff --git a/docs/reference/rpc.md b/docs/reference/rpc.md index 6bda1a7e..be32b746 100644 --- a/docs/reference/rpc.md +++ b/docs/reference/rpc.md @@ -19,7 +19,7 @@ it; relative paths resolve against that directory. When it starts, the server writes one hello line: ```json -{"isb": "4.0.1", "protocol": 1} +{"isb": "4.1.0", "protocol": 1} ``` A client checks `protocol` and refuses versions it does not know. @@ -205,7 +205,7 @@ All take `files` (default `./isb.yaml` plus `./isb.override.yaml` if present), ## Example session ```text -<- {"isb":"4.0.1","protocol":1} +<- {"isb":"4.1.0","protocol":1} -> {"id":1,"method":"sandbox.ensure","params":{"spec":{"name":"web","image":"dev-base","cpus":2}}} <- {"id":1,"event":"progress","data":"web: creating from dev-base"} <- {"id":1,"event":"progress","data":"web: starting"} diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 21867996..42656984 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "isb-sdk" -version = "4.0.1" +version = "4.1.0" description = "Python SDK for isb: declarative incus sandboxes (containers and VMs)" readme = "README.md" license = "MIT AND LicenseRef-EXA-SL-1.0" diff --git a/sdk/python/src/isb/__init__.py b/sdk/python/src/isb/__init__.py index 7e7781a8..64f2ffc5 100644 --- a/sdk/python/src/isb/__init__.py +++ b/sdk/python/src/isb/__init__.py @@ -62,7 +62,7 @@ async def main(): from ._util import Duration from .volumes import Volumes -__version__ = "4.0.1" +__version__ = "4.1.0" __all__ = [ "PROTOCOL", diff --git a/sdk/python/tests/test_unit.py b/sdk/python/tests/test_unit.py index 33309462..7084020c 100644 --- a/sdk/python/tests/test_unit.py +++ b/sdk/python/tests/test_unit.py @@ -222,7 +222,7 @@ async def test_death_fails_pending_requests(self) -> None: path = fake_server( self.tmp.name, """\ - echo '{"isb":"4.0.1","protocol":1}' + echo '{"isb":"4.1.0","protocol":1}' read line echo "boom" >&2 exit 3 @@ -243,7 +243,7 @@ async def test_close_fails_pending(self) -> None: path = fake_server( self.tmp.name, """\ - echo '{"isb":"4.0.1","protocol":1}' + echo '{"isb":"4.1.0","protocol":1}' cat >/dev/null """, ) @@ -261,7 +261,7 @@ async def test_events_and_out_of_order_replies(self) -> None: path = fake_server( self.tmp.name, """\ - echo '{"isb":"4.0.1","protocol":1}' + echo '{"isb":"4.1.0","protocol":1}' read a read b echo 'not json' diff --git a/sdk/python/uv.lock b/sdk/python/uv.lock index b2abc01d..2ef5e40a 100644 --- a/sdk/python/uv.lock +++ b/sdk/python/uv.lock @@ -72,7 +72,7 @@ wheels = [ [[package]] name = "isb-sdk" -version = "4.0.1" +version = "4.1.0" source = { editable = "." } [package.dev-dependencies] diff --git a/sdk/typescript/bun.lock b/sdk/typescript/bun.lock index 7d67ab47..a9b03bd7 100644 --- a/sdk/typescript/bun.lock +++ b/sdk/typescript/bun.lock @@ -12,8 +12,8 @@ "typescript": "^5.9.0", }, "optionalDependencies": { - "@execution-associates/isb-linux-arm64": "4.0.1", - "@execution-associates/isb-linux-x64": "4.0.1", + "@execution-associates/isb-linux-arm64": "4.1.0", + "@execution-associates/isb-linux-x64": "4.1.0", }, }, }, diff --git a/sdk/typescript/npm/linux-arm64/package.json b/sdk/typescript/npm/linux-arm64/package.json index f7e2e867..48bc72b8 100644 --- a/sdk/typescript/npm/linux-arm64/package.json +++ b/sdk/typescript/npm/linux-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@execution-associates/isb-linux-arm64", - "version": "4.0.1", + "version": "4.1.0", "description": "The static isb binary for Linux arm64 (aarch64-unknown-linux-musl), used by @execution-associates/isb", "license": "SEE LICENSE IN LICENSE", "repository": { diff --git a/sdk/typescript/npm/linux-x64/package.json b/sdk/typescript/npm/linux-x64/package.json index 41859430..ee087c3b 100644 --- a/sdk/typescript/npm/linux-x64/package.json +++ b/sdk/typescript/npm/linux-x64/package.json @@ -1,6 +1,6 @@ { "name": "@execution-associates/isb-linux-x64", - "version": "4.0.1", + "version": "4.1.0", "description": "The static isb binary for Linux x64 (x86_64-unknown-linux-musl), used by @execution-associates/isb", "license": "SEE LICENSE IN LICENSE", "repository": { diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json index a93956f7..b6ad358b 100644 --- a/sdk/typescript/package.json +++ b/sdk/typescript/package.json @@ -1,6 +1,6 @@ { "name": "@execution-associates/isb", - "version": "4.0.1", + "version": "4.1.0", "description": "TypeScript SDK for isb: declarative incus sandboxes (containers and VMs)", "license": "MIT", "type": "module", @@ -51,8 +51,8 @@ "prepack": "bun run build" }, "optionalDependencies": { - "@execution-associates/isb-linux-arm64": "4.0.1", - "@execution-associates/isb-linux-x64": "4.0.1" + "@execution-associates/isb-linux-arm64": "4.1.0", + "@execution-associates/isb-linux-x64": "4.1.0" }, "devDependencies": { "@biomejs/biome": "^2.2.0", diff --git a/src/bin/isb.rs b/src/bin/isb.rs index 9da9ff06..e544863e 100644 --- a/src/bin/isb.rs +++ b/src/bin/isb.rs @@ -358,7 +358,7 @@ enum Cmd { #[command(subcommand)] Registry(RegistryCmd), /// Notification channels of an org on the `isb serve` daemon: webhook, - /// Slack, Discord, Telegram, email (docs/guides/notifications.md). + /// Slack, Discord, Mattermost, Telegram, email (docs/guides/notifications.md). #[command(subcommand)] Notify(notify::NotifyCmd), /// An org's sandboxes on the `isb serve` daemon: list them, push one's diff --git a/src/bin/isb/notify.rs b/src/bin/isb/notify.rs index b1a4e8fe..77951b0d 100644 --- a/src/bin/isb/notify.rs +++ b/src/bin/isb/notify.rs @@ -79,6 +79,9 @@ pub struct DestOne { /// A Discord webhook URL held in this secret. #[arg(long, value_name = "URL_SECRET")] discord: Option, + /// A Mattermost incoming webhook URL (https, any server) held in this secret. + #[arg(long, value_name = "URL_SECRET")] + mattermost: Option, /// A Telegram bot token held in this secret (with --chat-id). #[arg(long, value_name = "TOKEN_SECRET")] telegram: Option, @@ -163,6 +166,9 @@ fn provider(d: Dest) -> Result { if let Some(s) = o.discord { return Ok(json!({"type": "discord", "url_secret": s})); } + if let Some(s) = o.mattermost { + return Ok(json!({"type": "mattermost", "url_secret": s})); + } if let Some(s) = o.telegram { let chat = d .chat_id @@ -200,7 +206,7 @@ fn with_org(org: &Option, mut args: Value) -> Value { fn describe(p: &Value) -> String { match p["type"].as_str().unwrap_or_default() { "webhook" => format!("webhook ${}", p["url_secret"].as_str().unwrap_or_default()), - "slack" | "discord" => format!( + "slack" | "discord" | "mattermost" => format!( "{} ${}", p["type"].as_str().unwrap_or_default(), p["url_secret"].as_str().unwrap_or_default() diff --git a/src/lib.rs b/src/lib.rs index 5ef0c4b4..fa4e67cf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,8 +34,9 @@ pub use isb_apps::{app, backup, build, jobs, monitor, notify, s3, template, volu #[doc(inline)] pub use isb_core::{ balance, client, compose, cron, discovery, egress, error, exec, foreground, idmap, image_check, - ingress, interp, lock, machine, metrics, metrics_history, net, org, owner, plan, registry, rpc, - sandbox, secrets, self_update, sftp, shorthand, spec, stack, supervise, volume, x, + ingress, interp, ipam, lock, machine, metrics, metrics_history, net, org, owner, plan, + registry, rpc, sandbox, secrets, self_update, sftp, shorthand, spec, stack, supervise, volume, + x, }; #[doc(inline)] pub use isb_daemon::{daemon, sandbox_life}; diff --git a/tests/ipam.rs b/tests/ipam.rs new file mode 100644 index 00000000..27ea7611 --- /dev/null +++ b/tests/ipam.rs @@ -0,0 +1,201 @@ +//! Addresses on an org bridge against a real incusd +//! (docs/concepts/orgs.md#addresses): a sandbox is pinned to an address no +//! other instance holds when it is created; the daemon's check pins an +//! instance made behind isb's back at the address it has without +//! restarting it, gives a stopped one a free address, adopts a pin set by +//! hand, and reports two instances holding one address. +//! +//! Gated like the other integration tests (`ISB_INTEGRATION=1`; build in a +//! sandbox with `cargo test --no-run`, run the binary on the host). Works in +//! the test org (`common::test_org`), removed afterwards, pass or fail; the +//! check is held to that org's bridge. + +#[allow(dead_code, reason = "each test binary uses some of the shared helpers")] +mod common; + +use std::net::Ipv4Addr; +use std::time::{Duration, Instant}; + +use isb::Client; +use serde_json::{Value, json}; + +const NAMES: [&str; 5] = [ + "isb-test-ip-a", + "isb-test-ip-b", + "isb-test-ip-raw", + "isb-test-ip-dup", + "isb-test-ip-cold", +]; + +struct Rm(Client); +impl Drop for Rm { + fn drop(&mut self) { + for n in NAMES { + let _ = isb::Sandbox::remove(&self.0, n, true); + } + } +} + +fn inst(oc: &Client, name: &str) -> Value { + oc.get(&format!("/1.0/instances/{name}")).unwrap() +} + +fn pin(oc: &Client, name: &str) -> Option { + inst(oc, name)["devices"]["eth0"]["ipv4.address"] + .as_str() + .map(String::from) +} + +fn recorded(oc: &Client, name: &str) -> Option { + inst(oc, name)["config"]["user.isb.ipv4.eth0"] + .as_str() + .map(String::from) +} + +/// The instance's IPv4 address on eth0, once it has one, and its PID. +fn address(oc: &Client, name: &str) -> (String, i64) { + let deadline = Instant::now() + Duration::from_secs(90); + loop { + let st = oc.get(&format!("/1.0/instances/{name}/state")).unwrap(); + let a = st["network"]["eth0"]["addresses"] + .as_array() + .into_iter() + .flatten() + .find(|a| a["family"] == "inet") + .and_then(|a| a["address"].as_str().map(String::from)); + if let Some(a) = a { + return (a, st["pid"].as_i64().unwrap_or_default()); + } + assert!(Instant::now() < deadline, "{name} got no address"); + std::thread::sleep(Duration::from_secs(1)); + } +} + +/// An instance created behind isb's back (no pin), with `devices`. +fn create_raw(oc: &Client, name: &str, devices: Value, start: bool) { + let body = json!({ + "name": name, "type": "container", + "source": {"type": "image", "alias": common::image()}, + "config": {"user.isb-test": "1"}, "devices": devices, "profiles": ["default"], + }); + let body = isb::org::disk::sized_root(oc, body); + oc.mutate( + "POST", + "/1.0/instances", + Some(&body), + "create", + Duration::from_secs(300), + ) + .unwrap(); + if start { + oc.mutate( + "PUT", + &format!("/1.0/instances/{name}/state"), + Some(&json!({"action": "start", "timeout": 30})), + "start", + Duration::from_secs(120), + ) + .unwrap(); + } +} + +fn check(base: &Client, oc: &Client) -> Vec { + isb::ipam::check(base, Some(oc.project_name()), &mut |m| { + eprintln!("check: {m}") + }) + .unwrap() +} + +#[test] +fn addresses_on_an_org_bridge_are_pinned_and_checked() { + if !common::enabled() { + return; + } + let base = Client::new(); + let oc = common::test_org_client(&base); + let _rm = Rm(oc.clone()); + + // Created by isb: pinned at create, the pin recorded and taken. + let spec = |n: &str| { + let mut s = isb::SandboxSpec::new(n, common::image()); + s.labels.insert("isb-test".into(), "1".into()); + s + }; + isb::Sandbox::create(&oc, &spec("isb-test-ip-a")).unwrap(); + isb::Sandbox::create(&oc, &spec("isb-test-ip-b")).unwrap(); + let (a, b) = ( + address(&oc, "isb-test-ip-a").0, + address(&oc, "isb-test-ip-b").0, + ); + assert_ne!(a, b); + for (n, ip) in [("isb-test-ip-a", &a), ("isb-test-ip-b", &b)] { + assert_eq!(pin(&oc, n).as_ref(), Some(ip), "{n}"); + assert_eq!(recorded(&oc, n).as_ref(), Some(ip), "{n}"); + } + // Planning it again changes nothing, even pruning devices: the pin is + // not drift from the spec. + let d = isb::sandbox::resolve( + &oc, + &spec("isb-test-ip-a"), + &isb::plan::VolumeDefs::new(), + std::path::Path::new("/"), + ) + .unwrap(); + let prune = isb::plan::DiffOptions { + prune_devices: true, + }; + let plan = isb::sandbox::plan_desired(&oc, &d, prune).unwrap(); + assert!(plan.is_noop(), "{plan:?}"); + + // Behind isb's back: pinned in place by the check, same address, same + // process. + create_raw(&oc, "isb-test-ip-raw", json!({}), true); + assert_eq!(pin(&oc, "isb-test-ip-raw"), None); + let (raw, pid) = address(&oc, "isb-test-ip-raw"); + let found = check(&base, &oc); + assert!(found.is_empty(), "{found:?}"); + assert_eq!(pin(&oc, "isb-test-ip-raw").as_ref(), Some(&raw)); + assert_eq!(recorded(&oc, "isb-test-ip-raw").as_ref(), Some(&raw)); + assert_eq!( + address(&oc, "isb-test-ip-raw"), + (raw.clone(), pid), + "restarted or moved" + ); + + // A pin whose record is gone (one set by hand) is adopted. + isb::sandbox::update_instance(&oc, "isb-test-ip-a", "unrecord", &mut |c, _| { + c.remove("user.isb.ipv4.eth0"); + Ok(()) + }) + .unwrap(); + assert!(check(&base, &oc).is_empty()); + assert_eq!(recorded(&oc, "isb-test-ip-a").as_ref(), Some(&a)); + + // A stopped instance without a pin gets a free address. + create_raw(&oc, "isb-test-ip-cold", json!({}), false); + assert!(check(&base, &oc).is_empty()); + let cold = pin(&oc, "isb-test-ip-cold").expect("pinned"); + assert!(![&a, &b, &raw].contains(&&cold), "{cold}"); + cold.parse::().unwrap(); + + // Pinned by hand to b's address (stopped: incus would refuse to start + // it): reported, nothing moved. + let mut nic = inst(&oc, "isb-test-ip-b")["expanded_devices"]["eth0"].clone(); + nic["ipv4.address"] = json!(b); + create_raw(&oc, "isb-test-ip-dup", json!({"eth0": nic}), false); + let found = check(&base, &oc); + assert_eq!(found.len(), 1, "{found:?}"); + assert!(found[0].message.contains(&b), "{}", found[0].message); + assert!( + found[0].message.contains("isb-test-ip-dup"), + "{}", + found[0].message + ); + assert_eq!( + recorded(&oc, "isb-test-ip-dup"), + None, + "a shared pin is not isb's" + ); + assert_eq!(pin(&oc, "isb-test-ip-b").as_ref(), Some(&b)); + assert_eq!(address(&oc, "isb-test-ip-b").0, b); +} diff --git a/web/openapi.json b/web/openapi.json index c398d19e..560eb589 100644 --- a/web/openapi.json +++ b/web/openapi.json @@ -9938,7 +9938,7 @@ }, "/api/v1/tools/notification_channel_create": { "post": { - "description": "Add a notification channel to the org: a destination (webhook, Slack, Discord, Telegram, email) and rules choosing which events it hears about. Only the org's own events reach it. Its secrets must exist.", + "description": "Add a notification channel to the org: a destination (webhook, Slack, Discord, Mattermost, Telegram, email) and rules choosing which events it hears about. Only the org's own events reach it. Its secrets must exist.", "operationId": "notification_channel_create", "requestBody": { "content": { @@ -9959,7 +9959,7 @@ "type": "string" }, "provider": { - "description": "Where messages go, by `type`: {\"type\": \"webhook\", \"url_secret\": NAME, \"signing_secret\": NAME?} (JSON POST; with a signing secret, X-Isb-Signature: sha256=HMAC of the body), {\"type\": \"slack\", \"url_secret\": NAME} (an incoming-webhook URL), {\"type\": \"discord\", \"url_secret\": NAME}, {\"type\": \"telegram\", \"token_secret\": NAME, \"chat_id\": \"-100...\"}, {\"type\": \"email\", \"host\", \"port\"?, \"tls\": \"starttls\"|\"tls\"|\"none\", \"username\"?, \"password_secret\"?, \"from\", \"to\": [..]}. Every URL, token and password is an org secret, named here (secret_create first), never a value.", + "description": "Where messages go, by `type`: {\"type\": \"webhook\", \"url_secret\": NAME, \"signing_secret\": NAME?} (JSON POST; with a signing secret, X-Isb-Signature: sha256=HMAC of the body), {\"type\": \"slack\", \"url_secret\": NAME} (an incoming-webhook URL), {\"type\": \"discord\", \"url_secret\": NAME}, {\"type\": \"mattermost\", \"url_secret\": NAME} (an incoming-webhook URL, https on any server), {\"type\": \"telegram\", \"token_secret\": NAME, \"chat_id\": \"-100...\"}, {\"type\": \"email\", \"host\", \"port\"?, \"tls\": \"starttls\"|\"tls\"|\"none\", \"username\"?, \"password_secret\"?, \"from\", \"to\": [..]}. Every URL, token and password is an org secret, named here (secret_create first), never a value.", "type": "object" }, "rules": { @@ -10217,7 +10217,7 @@ "type": "string" }, "provider": { - "description": "Where messages go, by `type`: {\"type\": \"webhook\", \"url_secret\": NAME, \"signing_secret\": NAME?} (JSON POST; with a signing secret, X-Isb-Signature: sha256=HMAC of the body), {\"type\": \"slack\", \"url_secret\": NAME} (an incoming-webhook URL), {\"type\": \"discord\", \"url_secret\": NAME}, {\"type\": \"telegram\", \"token_secret\": NAME, \"chat_id\": \"-100...\"}, {\"type\": \"email\", \"host\", \"port\"?, \"tls\": \"starttls\"|\"tls\"|\"none\", \"username\"?, \"password_secret\"?, \"from\", \"to\": [..]}. Every URL, token and password is an org secret, named here (secret_create first), never a value.", + "description": "Where messages go, by `type`: {\"type\": \"webhook\", \"url_secret\": NAME, \"signing_secret\": NAME?} (JSON POST; with a signing secret, X-Isb-Signature: sha256=HMAC of the body), {\"type\": \"slack\", \"url_secret\": NAME} (an incoming-webhook URL), {\"type\": \"discord\", \"url_secret\": NAME}, {\"type\": \"mattermost\", \"url_secret\": NAME} (an incoming-webhook URL, https on any server), {\"type\": \"telegram\", \"token_secret\": NAME, \"chat_id\": \"-100...\"}, {\"type\": \"email\", \"host\", \"port\"?, \"tls\": \"starttls\"|\"tls\"|\"none\", \"username\"?, \"password_secret\"?, \"from\", \"to\": [..]}. Every URL, token and password is an org secret, named here (secret_create first), never a value.", "type": "object" }, "rules": { diff --git a/web/src/api/openapi.gen.ts b/web/src/api/openapi.gen.ts index 680f2d2f..db83915a 100644 --- a/web/src/api/openapi.gen.ts +++ b/web/src/api/openapi.gen.ts @@ -2583,7 +2583,7 @@ export interface paths { put?: never; /** * Create a notification channel - * @description Add a notification channel to the org: a destination (webhook, Slack, Discord, Telegram, email) and rules choosing which events it hears about. Only the org's own events reach it. Its secrets must exist. + * @description Add a notification channel to the org: a destination (webhook, Slack, Discord, Mattermost, Telegram, email) and rules choosing which events it hears about. Only the org's own events reach it. Its secrets must exist. */ post: operations["notification_channel_create"]; delete?: never; @@ -10457,7 +10457,7 @@ export interface operations { name: string; /** @description The org to act in (default: default). */ org?: string; - /** @description Where messages go, by `type`: {"type": "webhook", "url_secret": NAME, "signing_secret": NAME?} (JSON POST; with a signing secret, X-Isb-Signature: sha256=HMAC of the body), {"type": "slack", "url_secret": NAME} (an incoming-webhook URL), {"type": "discord", "url_secret": NAME}, {"type": "telegram", "token_secret": NAME, "chat_id": "-100..."}, {"type": "email", "host", "port"?, "tls": "starttls"|"tls"|"none", "username"?, "password_secret"?, "from", "to": [..]}. Every URL, token and password is an org secret, named here (secret_create first), never a value. */ + /** @description Where messages go, by `type`: {"type": "webhook", "url_secret": NAME, "signing_secret": NAME?} (JSON POST; with a signing secret, X-Isb-Signature: sha256=HMAC of the body), {"type": "slack", "url_secret": NAME} (an incoming-webhook URL), {"type": "discord", "url_secret": NAME}, {"type": "mattermost", "url_secret": NAME} (an incoming-webhook URL, https on any server), {"type": "telegram", "token_secret": NAME, "chat_id": "-100..."}, {"type": "email", "host", "port"?, "tls": "starttls"|"tls"|"none", "username"?, "password_secret"?, "from", "to": [..]}. Every URL, token and password is an org secret, named here (secret_create first), never a value. */ provider: Record; /** @description Which events: a list of {"events": [glob, ...] (deploy.*, health.*, backup.*, job.*, cert.*, *.failed, *), "projects"?: [..], "apps"?: [..], "stacks"?: [..]}; any rule matching sends. Default: [{"events": ["*"]}]. */ rules?: Record[]; @@ -10617,7 +10617,7 @@ export interface operations { name: string; /** @description The org to act in (default: default). */ org?: string; - /** @description Where messages go, by `type`: {"type": "webhook", "url_secret": NAME, "signing_secret": NAME?} (JSON POST; with a signing secret, X-Isb-Signature: sha256=HMAC of the body), {"type": "slack", "url_secret": NAME} (an incoming-webhook URL), {"type": "discord", "url_secret": NAME}, {"type": "telegram", "token_secret": NAME, "chat_id": "-100..."}, {"type": "email", "host", "port"?, "tls": "starttls"|"tls"|"none", "username"?, "password_secret"?, "from", "to": [..]}. Every URL, token and password is an org secret, named here (secret_create first), never a value. */ + /** @description Where messages go, by `type`: {"type": "webhook", "url_secret": NAME, "signing_secret": NAME?} (JSON POST; with a signing secret, X-Isb-Signature: sha256=HMAC of the body), {"type": "slack", "url_secret": NAME} (an incoming-webhook URL), {"type": "discord", "url_secret": NAME}, {"type": "mattermost", "url_secret": NAME} (an incoming-webhook URL, https on any server), {"type": "telegram", "token_secret": NAME, "chat_id": "-100..."}, {"type": "email", "host", "port"?, "tls": "starttls"|"tls"|"none", "username"?, "password_secret"?, "from", "to": [..]}. Every URL, token and password is an org secret, named here (secret_create first), never a value. */ provider?: Record; /** @description Which events: a list of {"events": [glob, ...] (deploy.*, health.*, backup.*, job.*, cert.*, *.failed, *), "projects"?: [..], "apps"?: [..], "stacks"?: [..]}; any rule matching sends. Default: [{"events": ["*"]}]. */ rules?: Record[]; diff --git a/web/src/notifications/api.ts b/web/src/notifications/api.ts index 310623c1..539adb5d 100644 --- a/web/src/notifications/api.ts +++ b/web/src/notifications/api.ts @@ -7,6 +7,7 @@ export type Provider = | { type: "webhook"; url_secret: string; signing_secret?: string } | { type: "slack"; url_secret: string } | { type: "discord"; url_secret: string } + | { type: "mattermost"; url_secret: string } | { type: "telegram"; token_secret: string; chat_id: string } | { type: "email"; @@ -57,6 +58,7 @@ export const PROVIDERS: { id: ProviderType; label: string; hint: string }[] = [ { id: "webhook", label: "Webhook", hint: "A signed JSON POST to any URL" }, { id: "slack", label: "Slack", hint: "An incoming webhook" }, { id: "discord", label: "Discord", hint: "A channel webhook" }, + { id: "mattermost", label: "Mattermost", hint: "An incoming webhook on any server" }, { id: "telegram", label: "Telegram", hint: "A bot message to a chat" }, { id: "email", label: "Email", hint: "Mail through an SMTP server" }, ]; @@ -89,6 +91,11 @@ export const EVENT_GROUPS: { subject: string; label: string; kinds: { kind: stri label: "DNS", kinds: [{ kind: "dns.failed", label: "No nameserver" }], }, + { + subject: "net", + label: "Addresses", + kinds: [{ kind: "net.failed", label: "Shared or unpinned address" }], + }, { subject: "backup", label: "Backups", @@ -220,6 +227,7 @@ export function providerSecrets(p: Provider): string[] { return [p.url_secret, p.signing_secret].filter(Boolean) as string[]; case "slack": case "discord": + case "mattermost": return [p.url_secret]; case "telegram": return [p.token_secret]; @@ -234,6 +242,7 @@ export function providerSummary(p: Provider): string { return `URL in ${p.url_secret}${p.signing_secret ? `, signed with ${p.signing_secret}` : ""}`; case "slack": case "discord": + case "mattermost": return `URL in ${p.url_secret}`; case "telegram": return `chat ${p.chat_id}, token in ${p.token_secret}`; diff --git a/web/src/notifications/channel-dialog.tsx b/web/src/notifications/channel-dialog.tsx index c538ba49..758497ac 100644 --- a/web/src/notifications/channel-dialog.tsx +++ b/web/src/notifications/channel-dialog.tsx @@ -60,6 +60,7 @@ function pfOf(p?: Provider): PF { return { ...emptyPF, url_secret: p.url_secret, signing_secret: p.signing_secret ?? "" }; case "slack": case "discord": + case "mattermost": return { ...emptyPF, url_secret: p.url_secret }; case "telegram": return { ...emptyPF, token_secret: p.token_secret, chat_id: p.chat_id }; @@ -78,7 +79,8 @@ export function providerOf(type: ProviderType, f: PF): { provider: Provider } | return { provider: { type, url_secret: f.url_secret.trim(), ...(f.signing_secret.trim() ? { signing_secret: f.signing_secret.trim() } : {}) } }; } case "slack": - case "discord": { + case "discord": + case "mattermost": { const e = need(f.url_secret, "Name the secret holding the webhook URL."); return e ? { error: e } : { provider: { type, url_secret: f.url_secret.trim() } }; } @@ -236,13 +238,19 @@ export function ChannelDialog({ org, existing, open, onOpenChange }: { org: stri setP({ signing_secret: v })} names={names} hint="Signs the body: X-Isb-Signature: sha256=HMAC." /> )} - {(type === "slack" || type === "discord") && ( + {(type === "slack" || type === "discord" || type === "mattermost") && ( setP({ url_secret: v })} names={names} - hint={type === "slack" ? "Holds https://hooks.slack.com/services/…" : "Holds https://discord.com/api/webhooks/…"} + hint={ + type === "slack" + ? "Holds https://hooks.slack.com/services/…" + : type === "discord" + ? "Holds https://discord.com/api/webhooks/…" + : "Holds https:///hooks/…" + } /> )} {type === "telegram" && ( diff --git a/web/src/notifications/icons.ts b/web/src/notifications/icons.ts index cb04c101..db0540b7 100644 --- a/web/src/notifications/icons.ts +++ b/web/src/notifications/icons.ts @@ -1,11 +1,12 @@ // One icon per channel destination, for the cards and the channel dialog. -import { type Bell, Hash, Mail, MessageCircle, Send, Webhook } from "lucide-react"; +import { type Bell, Hash, Mail, MessageCircle, MessagesSquare, Send, Webhook } from "lucide-react"; import type { ProviderType } from "./api"; export const PROVIDER_ICON: Record = { webhook: Webhook, slack: Hash, discord: MessageCircle, + mattermost: MessagesSquare, telegram: Send, email: Mail, }; diff --git a/web/src/notifications/notifications-page.tsx b/web/src/notifications/notifications-page.tsx index 2732d05f..930b0618 100644 --- a/web/src/notifications/notifications-page.tsx +++ b/web/src/notifications/notifications-page.tsx @@ -1,4 +1,4 @@ -// /orgs/:org/notifications: the org's channels (webhook, Slack, Discord, +// /orgs/:org/notifications: the org's channels (webhook, Slack, Discord, Mattermost, // Telegram, email), the events each hears about, Test, and each one's // delivery log. Platform admins also see the server-wide private-target // switch. @@ -96,7 +96,7 @@ export function NotificationsPage() { ) } > - A channel is a destination (a webhook, Slack, Discord, Telegram or email) and the events it hears about. URLs and tokens stay in the org's secrets. + A channel is a destination (a webhook, Slack, Discord, Mattermost, Telegram or email) and the events it hears about. URLs and tokens stay in the org's secrets. ) : (