From 05a41ba97af13f1cbe9a6ab2d4a305a48838acd3 Mon Sep 17 00:00:00 2001 From: Stephan Fitzpatrick Date: Mon, 9 Nov 2026 10:49:26 +0000 Subject: [PATCH 1/3] Pin every instance's address on an org bridge; raise net.failed From the 2026-11-09 incident (a new instance in org exa got the address analytics-postgres had held since 10/12): incus' bridges lease for an hour and an OCI container never renews, so dnsmasq re-hands its address. - isb pins eth0's ipv4.address at create (sandboxes, stack replicas, build sandboxes) on an address no instance on the bridge holds or is pinned to, and records it in user.isb.ipv4.eth0. A per-process reservation covers creates in flight; after the create the address is checked again and moved before first start if another process took it (incus accepts two NICs pinned to one address, then refuses to start either). - isb serve checks every org bridge each minute: pins an unpinned instance at the address it has, in place (no restart), gives a stopped one a free address, and adopts pins set by hand (the live stopgap's). It raises net.failed when two instances hold one address, when a pinned instance has another, or when one cannot be pinned. It moves no instance itself. - The diff ignores isb's pin on a NIC whose spec names no address and never prunes it. Egress bridges (one sandbox each) stay on DHCP. Tests: unit tests for the subnet, claims and plan; tests/ipam.rs against a real incusd in a throwaway org. Claude-Session: https://claude.ai/code/session_01LJ21XsZE86ywhpFMvYVWmE --- crates/isb-apps/src/build/mod.rs | 4 +- crates/isb-core/src/ipam.rs | 755 ++++++++++++++++++++++++ crates/isb-core/src/ipam/tests.rs | 287 +++++++++ crates/isb-core/src/lib.rs | 1 + crates/isb-core/src/plan.rs | 3 +- crates/isb-core/src/sandbox.rs | 7 +- crates/isb-core/src/stack/controller.rs | 17 + crates/isb-daemon/src/daemon/mod.rs | 1 + docs/concepts/orgs.md | 16 + docs/guides/notifications.md | 1 + docs/operations/troubleshooting.md | 1 + src/lib.rs | 5 +- tests/ipam.rs | 201 +++++++ web/src/notifications/api.ts | 5 + 14 files changed, 1297 insertions(+), 7 deletions(-) create mode 100644 crates/isb-core/src/ipam.rs create mode 100644 crates/isb-core/src/ipam/tests.rs create mode 100644 tests/ipam.rs diff --git a/crates/isb-apps/src/build/mod.rs b/crates/isb-apps/src/build/mod.rs index 0cbe55bd..45a534c2 100644 --- a/crates/isb-apps/src/build/mod.rs +++ b/crates/isb-apps/src/build/mod.rs @@ -517,13 +517,15 @@ fn create_sandbox( "profiles": ["default"], }); let t = remaining(deadline, "creating the build sandbox")?; + let (body, pin) = crate::ipam::pin_for_create(oc, crate::org::disk::sized_root(oc, body))?; oc.mutate( "POST", "/1.0/instances", - Some(&crate::org::disk::sized_root(oc, body)), + Some(&body), &format!("create build sandbox {name}"), t.min(oc.get_timeouts().create.max(Duration::from_secs(600))), )?; + pin.map_or(Ok(()), |p| p.verify(oc, name))?; let t = remaining(deadline, "starting the build sandbox")?; oc.mutate( "PUT", diff --git a/crates/isb-core/src/ipam.rs b/crates/isb-core/src/ipam.rs new file mode 100644 index 00000000..73444b78 --- /dev/null +++ b/crates/isb-core/src/ipam.rs @@ -0,0 +1,755 @@ +//! Instance addresses on isb's org bridges: pinned, never only leased. +//! +//! incus' bridges hand out addresses by DHCP with a one-hour lease, and an +//! OCI container never renews its lease. An hour after one starts, dnsmasq +//! considers its address free and gives it to the next instance that asks: +//! on 2026-11-09 a new instance in org exa got the address a four-week-old +//! Postgres held, and the apps using it broke until the newcomer moved. A +//! NIC's `ipv4.address` makes dnsmasq keep the address for that NIC alone, +//! whatever happens to the lease, so isb pins every address on an org +//! bridge: +//! +//! - At create ([`pin_for_create`]), on an address no instance on the +//! bridge holds or is pinned to, checked again once the instance exists +//! ([`Pin::verify`]): incus accepts two NICs pinned to one address, then +//! refuses to start either. +//! - The daemon's check ([`watch`]), every minute, pins an instance that +//! has no pin at the address it has (in place: it keeps running, and +//! keeps its address), gives a stopped one a free address, and records +//! each pin in `user.isb.ipv4.`, which adopts pins set by hand. It +//! raises `net.failed` when two instances share an address, when a +//! pinned instance has another one, or when an instance cannot be pinned. +//! +//! Only isb's org projects (`isb-`) and their managed bridges are +//! touched. Egress bridges (`isbbrx*`) hold one sandbox each, so nothing on +//! them can collide; they are left to DHCP. + +use std::collections::{BTreeMap, BTreeSet}; +use std::net::Ipv4Addr; +use std::sync::{Mutex, OnceLock}; +use std::time::Duration; + +use serde_json::{Value, json}; + +use crate::client::{Client, encode_segment}; +use crate::error::{Error, Result}; +use crate::org::OrgId; +use crate::plan::Props; + +/// The config key recording the address isb pinned a NIC to. +pub fn pin_key(device: &str) -> String { + format!("user.isb.ipv4.{device}") +} + +/// How often the daemon checks every org's addresses. +pub const CHECK_EVERY: Duration = Duration::from_secs(60); + +/// A managed bridge's IPv4 subnet. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Subnet { + gateway: Ipv4Addr, + prefix: u8, +} + +impl Subnet { + /// From `GET /1.0/networks/`: a managed bridge with an IPv4 + /// address (`10.1.2.1/24`), not an egress bridge. + pub fn from_network(net: &Value) -> Option { + let name = net["name"].as_str().unwrap_or_default(); + if net["type"].as_str() != Some("bridge") + || net["managed"] == Value::Bool(false) + || name.starts_with(crate::egress::plumb::NET_PREFIX) + { + return None; + } + let (ip, prefix) = net["config"]["ipv4.address"].as_str()?.split_once('/')?; + let prefix: u8 = prefix.trim().parse().ok()?; + (8..=30).contains(&prefix).then_some(())?; + Some(Subnet { + gateway: ip.trim().parse().ok()?, + prefix, + }) + } + + fn mask(&self) -> u32 { + u32::MAX << (32 - self.prefix) + } + + pub fn contains(&self, ip: Ipv4Addr) -> bool { + u32::from(ip) & self.mask() == u32::from(self.gateway) & self.mask() + } + + /// An address an instance may take: in the subnet, and neither its + /// network, broadcast nor gateway address. + fn assignable(&self, ip: Ipv4Addr) -> bool { + let host = u32::from(ip) & !self.mask(); + self.contains(ip) && ip != self.gateway && host != 0 && host != !self.mask() + } + + /// A free address, searched from a point `seed` picks, so two callers + /// with different seeds rarely race for one. + pub fn pick_free(&self, used: &BTreeSet, seed: &str) -> Option { + let base = u32::from(self.gateway) & self.mask(); + let size = !self.mask() as u64 + 1; + let start = fnv(seed) % size; + (0..size) + .map(|i| Ipv4Addr::from(base + ((start + i) % size) as u32)) + .find(|ip| self.assignable(*ip) && !used.contains(ip)) + } +} + +fn fnv(s: &str) -> u64 { + s.bytes().fold(0xcbf29ce484222325, |h, b| { + (h ^ b as u64).wrapping_mul(0x100000001b3) + }) +} + +/// One instance's NIC on a bridge, from an instance listed with +/// `recursion=2`. +#[derive(Debug, Clone)] +pub struct Claim { + pub project: String, + pub instance: String, + pub device: String, + /// The NIC's `ipv4.address`. + pub pin: Option, + /// What [`pin_key`] records. + pub recorded: Option, + /// Its IPv4 addresses on the bridge's subnet (running instances only). + pub addrs: Vec, + pub running: bool, + /// The NIC as the instance has it (expanded: from a profile or its own). + pub props: Props, + pub stack: Option, + pub service: Option, + pub created_at: String, +} + +impl Claim { + fn id(&self) -> (String, String) { + (self.project.clone(), self.instance.clone()) + } +} + +/// Every NIC on `network` among `instances` (in isb's org projects). +pub fn claims(instances: &[Value], network: &str, subnet: &Subnet) -> Vec { + let mut out = Vec::new(); + for i in instances { + let project = i["project"].as_str().unwrap_or("default"); + if OrgId::from_incus_project(project).is_none() { + continue; + } + let Some(devs) = i["expanded_devices"].as_object() else { + continue; + }; + for (dev, d) in devs { + if d["type"] != "nic" || d["network"].as_str() != Some(network) { + continue; + } + out.push(claim(i, project, dev, d, subnet)); + } + } + out +} + +fn claim(i: &Value, project: &str, dev: &str, d: &Value, subnet: &Subnet) -> Claim { + let cfg = &i["config"]; + let label = |k: &str| cfg[format!("user.{k}")].as_str().map(String::from); + let running = i["status"].as_str() == Some("Running"); + Claim { + project: project.to_string(), + instance: i["name"].as_str().unwrap_or_default().to_string(), + device: dev.to_string(), + pin: d["ipv4.address"].as_str().and_then(|a| a.parse().ok()), + recorded: cfg[pin_key(dev)].as_str().map(String::from), + addrs: if running { + nic_addrs(i, dev, d, subnet) + } else { + Vec::new() + }, + running, + props: d + .as_object() + .map(|m| { + m.iter() + .filter_map(|(k, v)| Some((k.clone(), v.as_str()?.to_string()))) + .collect() + }) + .unwrap_or_default(), + stack: label("isb.stack"), + service: label("isb.service"), + created_at: i["created_at"].as_str().unwrap_or_default().to_string(), + } +} + +/// The NIC's addresses on `subnet`, from the instance's state: the guest +/// interface with the NIC's MAC (a VM names it enp5s0, say), else the one +/// named as the NIC says. +fn nic_addrs(i: &Value, dev: &str, d: &Value, subnet: &Subnet) -> Vec { + let Some(ifaces) = i["state"]["network"].as_object() else { + return Vec::new(); + }; + let mac = i["config"][format!("volatile.{dev}.hwaddr")] + .as_str() + .map(str::to_ascii_lowercase); + let by_mac = mac.and_then(|m| { + ifaces + .values() + .find(|v| v["hwaddr"].as_str().map(str::to_ascii_lowercase) == Some(m.clone())) + }); + let iface = by_mac.or_else(|| ifaces.get(d["name"].as_str().unwrap_or(dev))); + iface + .and_then(|v| v["addresses"].as_array()) + .into_iter() + .flatten() + .filter(|a| a["family"] == "inet") + .filter_map(|a| a["address"].as_str()?.parse().ok()) + .filter(|ip| subnet.contains(*ip)) + .collect() +} + +/// Who holds each address: its pin, and what a running instance has. +fn holders(claims: &[Claim]) -> BTreeMap> { + let mut out: BTreeMap> = BTreeMap::new(); + for c in claims { + for ip in c.pin.iter().chain(&c.addrs) { + out.entry(*ip).or_default().insert(c.id()); + } + } + out +} + +/// What the check changes. +#[derive(Debug, Clone, PartialEq)] +pub enum Fix { + /// Record a pin isb did not set (one set by hand) as isb's. + Adopt { + project: String, + instance: String, + device: String, + ip: Ipv4Addr, + }, + /// Pin a NIC: at the address it has when running, else a free one. + Pin { + project: String, + instance: String, + device: String, + props: Props, + ip: Ipv4Addr, + running: bool, + }, +} + +/// What the check reports (as `net.failed`). +#[derive(Debug, Clone, PartialEq)] +pub struct Finding { + /// The same problem has the same key on every pass. + pub key: String, + pub project: String, + pub instance: String, + pub stack: Option, + pub service: Option, + pub message: String, +} + +impl Finding { + fn about(c: &Claim, key: String, message: String) -> Finding { + Finding { + key, + project: c.project.clone(), + instance: c.instance.clone(), + stack: c.stack.clone(), + service: c.service.clone(), + message, + } + } +} + +/// Decide the fixes and findings for one bridge's claims. `reserved`: +/// addresses being given to instances that do not exist yet. +pub fn plan( + network: &str, + subnet: &Subnet, + claims: &[Claim], + reserved: &BTreeSet, +) -> (Vec, Vec) { + let held = holders(claims); + let mut findings = conflicts(network, claims, &held); + let mut used: BTreeSet = held + .keys() + .copied() + .chain(reserved.iter().copied()) + .collect(); + let mut fixes = Vec::new(); + for c in claims { + if let Some(pin) = c.pin { + // A shared pin is reported above, never made isb's. + let alone = held.get(&pin).is_some_and(|h| h.len() == 1); + if alone && c.recorded.as_deref() != Some(pin.to_string().as_str()) { + fixes.push(Fix::Adopt { + project: c.project.clone(), + instance: c.instance.clone(), + device: c.device.clone(), + ip: pin, + }); + } + if c.running && !c.addrs.is_empty() && !c.addrs.contains(&pin) { + let have = join(&c.addrs); + findings.push(Finding::about( + c, + format!( + "drift {} {} {} {pin} {have}", + c.project, c.instance, c.device + ), + format!( + "{}'s {} is pinned to {pin} but has {have}: its next start takes {pin}", + c.instance, c.device + ), + )); + } + continue; + } + let ip = match (c.running, c.addrs.as_slice()) { + // Its own address, unless another instance holds it too (a + // conflict, reported above): pinning it would keep it. + (true, [a]) if held.get(a).is_some_and(|h| h.len() == 1) => Some(*a), + // Starting (no address yet), or sharing one: the next pass. + (true, _) => None, + (false, _) => { + let free = subnet.pick_free(&used, &format!("{}/{}", c.project, c.instance)); + if free.is_none() { + findings.push(Finding::about( + c, + format!("full {} {}", c.project, c.instance), + format!( + "{}: no free address on {network} to pin {} to", + c.instance, c.device + ), + )); + } + free + } + }; + if let Some(ip) = ip { + used.insert(ip); + let mut props = c.props.clone(); + props.insert("ipv4.address".into(), ip.to_string()); + fixes.push(Fix::Pin { + project: c.project.clone(), + instance: c.instance.clone(), + device: c.device.clone(), + props, + ip, + running: c.running, + }); + } + } + (fixes, findings) +} + +fn join(ips: &[Ipv4Addr]) -> String { + ips.iter() + .map(Ipv4Addr::to_string) + .collect::>() + .join(", ") +} + +/// Every address two or more instances hold, as one finding about the +/// newest of them (most likely the one that took it). +fn conflicts( + network: &str, + claims: &[Claim], + held: &BTreeMap>, +) -> Vec { + let mut out = Vec::new(); + for (ip, ids) in held.iter().filter(|(_, h)| h.len() > 1) { + let mut who: Vec<&Claim> = claims.iter().filter(|c| ids.contains(&c.id())).collect(); + who.sort_by(|a, b| b.created_at.cmp(&a.created_at)); + who.dedup_by(|a, b| a.id() == b.id()); + let how = |c: &Claim| match (c.pin == Some(*ip), c.addrs.contains(ip)) { + (true, true) => "pinned, has it", + (true, false) => "pinned", + _ => "has it", + }; + let list: Vec = who + .iter() + .map(|c| format!("{} ({})", c.instance, how(c))) + .collect(); + let names: Vec<&str> = who.iter().map(|c| c.instance.as_str()).collect(); + out.push(Finding::about( + who[0], + format!("conflict {network} {ip} {}", names.join(" ")), + format!( + "{ip} on {network} is held by {}: traffic for one reaches another. isb moves \ + neither; pin one elsewhere (incus config device set {} ipv4.address=) \ + and restart it", + list.join(", "), + who[0].device + ), + )); + } + out +} + +/// Addresses being given to instances in the middle of their create, by +/// network: what this process's other creates and checks must not pick. +fn reserved() -> &'static Mutex>> { + static R: OnceLock>>> = OnceLock::new(); + R.get_or_init(Mutex::default) +} + +fn reserved_on(network: &str) -> BTreeSet { + let r = reserved().lock().unwrap_or_else(|e| e.into_inner()); + r.get(network).cloned().unwrap_or_default() +} + +/// Every instance with its state: on every project when the client may +/// list them (the daemon), else on its own. +fn list(client: &Client) -> Result> { + let v = match client.get("/1.0/instances?recursion=2&all-projects=true") { + Ok(v) => v, + Err(_) => client.get("/1.0/instances?recursion=2")?, + }; + Ok(v.as_array().cloned().unwrap_or_default()) +} + +/// An address pinned for an instance being created. Held until the create +/// is done; [`Pin::verify`] checks nobody else took it in between. +pub struct Pin { + network: String, + subnet: Subnet, + device: String, + ip: Ipv4Addr, +} + +impl Drop for Pin { + fn drop(&mut self) { + let mut r = reserved().lock().unwrap_or_else(|e| e.into_inner()); + if let Some(s) = r.get_mut(&self.network) { + s.remove(&self.ip); + } + } +} + +impl Pin { + pub fn ip(&self) -> Ipv4Addr { + self.ip + } + + /// After the create: when another instance holds the address too (a + /// create by another process that picked it at the same moment), move + /// this one, which has not started yet. + pub fn verify(mut self, client: &Client, name: &str) -> Result<()> { + let me = (client.project_name().to_string(), name.to_string()); + for _ in 0..3 { + let all = claims(&list(client)?, &self.network, &self.subnet); + let held = holders(&all); + if held + .get(&self.ip) + .is_none_or(|h| h.iter().all(|id| *id == me)) + { + return Ok(()); + } + let mut used: BTreeSet = held.keys().copied().collect(); + used.extend(reserved_on(&self.network)); + let ip = self + .subnet + .pick_free(&used, &format!("{name}/{}", self.ip)) + .ok_or_else(|| Error::invalid(format!("no free address on {}", self.network)))?; + set_pin(client, name, &self.device, None, ip)?; + self.reserve(ip); + } + Err(Error::invalid(format!( + "{name}: could not find an address on {} no other instance takes", + self.network + ))) + } + + fn reserve(&mut self, ip: Ipv4Addr) { + let mut r = reserved().lock().unwrap_or_else(|e| e.into_inner()); + let s = r.entry(self.network.clone()).or_default(); + s.remove(&self.ip); + s.insert(ip); + self.ip = ip; + } +} + +/// Pin `eth0` in a create body (`POST /1.0/instances`) for an instance in +/// one of isb's org projects whose `eth0` joins a managed bridge: the +/// body's own NIC, else its profiles'. Anything else is returned as is. +pub fn pin_for_create(client: &Client, mut body: Value) -> Result<(Value, Option)> { + if OrgId::from_incus_project(client.project_name()).is_none() { + return Ok((body, None)); + } + let Some(mut nic) = eth0(client, &body)? else { + return Ok((body, None)); + }; + let Some(network) = nic["network"].as_str().map(String::from) else { + return Ok((body, None)); + }; + if !nic["ipv4.address"].is_null() { + return Ok((body, None)); + } + let path = format!("/1.0/networks/{}", encode_segment(&network)); + let Some(subnet) = client + .get_opt(&path)? + .as_ref() + .and_then(Subnet::from_network) + else { + return Ok((body, None)); + }; + let name = body["name"].as_str().unwrap_or_default().to_string(); + let held = holders(&claims(&list(client)?, &network, &subnet)); + let ip = { + let mut r = reserved().lock().unwrap_or_else(|e| e.into_inner()); + let res = r.entry(network.clone()).or_default(); + let used: BTreeSet = held.keys().chain(res.iter()).copied().collect(); + let ip = subnet + .pick_free(&used, &format!("{}/{name}", client.project_name())) + .ok_or_else(|| Error::invalid(format!("{name}: no free address on {network}")))?; + res.insert(ip); + ip + }; + let pin = Pin { + network, + subnet, + device: "eth0".into(), + ip, + }; + nic["ipv4.address"] = json!(ip.to_string()); + body["devices"]["eth0"] = nic; + body["config"][pin_key("eth0")] = json!(ip.to_string()); + Ok((body, Some(pin))) +} + +/// The `eth0` NIC an instance created from `body` gets: its own, else the +/// last of its profiles' (a later profile overrides an earlier one). +fn eth0(client: &Client, body: &Value) -> Result> { + if let Some(d) = body["devices"].get("eth0") { + return Ok((d["type"] == "nic").then(|| d.clone())); + } + let mut nic = None; + for p in body["profiles"].as_array().into_iter().flatten() { + let Some(p) = p.as_str() else { continue }; + let path = format!("/1.0/profiles/{}", encode_segment(p)); + if let Some(d) = client + .get_opt(&path)? + .and_then(|v| v["devices"].get("eth0").cloned()) + { + nic = (d["type"] == "nic").then_some(d); + } + } + Ok(nic) +} + +/// Pin an instance's NIC to `ip`, in place: `props` is the NIC to write +/// (from a profile), or `None` to change the instance's own. +fn set_pin( + client: &Client, + name: &str, + device: &str, + props: Option<&Props>, + ip: Ipv4Addr, +) -> Result<()> { + crate::sandbox::update_instance( + client, + name, + &format!("pin {name} {device}"), + &mut |config, devices| { + let mut nic = match props { + Some(p) => json!(p), + None => devices + .get(device) + .cloned() + .ok_or_else(|| Error::invalid(format!("{name} has no device {device}")))?, + }; + nic["ipv4.address"] = json!(ip.to_string()); + devices.insert(device.to_string(), nic); + config.insert(pin_key(device), json!(ip.to_string())); + Ok(()) + }, + ) +} + +/// A desired NIC that names no address is satisfied by one pinned to any: +/// the pin is isb's, not the spec's (for [`crate::plan::device_matches`]). +pub fn ignore_pin(desired: &Props, actual: &mut Props) { + if desired.get("type").is_some_and(|t| t == "nic") && !desired.contains_key("ipv4.address") { + actual.remove("ipv4.address"); + } +} + +/// Whether an instance's device is a NIC isb pinned (never pruned). +pub fn is_pin(device: &str, props: &Props, config: &Props) -> bool { + props.get("type").is_some_and(|t| t == "nic") + && props.get("ipv4.address").is_some() + && props.get("ipv4.address") == config.get(&pin_key(device)) +} + +/// An instance's devices but the NICs isb pinned: what pruning may remove. +pub fn unpinned<'a>( + devices: &'a BTreeMap, + config: &'a Props, +) -> impl Iterator { + devices.iter().filter(|(n, p)| !is_pin(n, p, config)) +} + +/// One pass over every org bridge: apply the fixes, return the findings. +/// `only`: the bridges of this incus project alone (a test's org). `report` +/// hears each fix. +pub fn check( + client: &Client, + only: Option<&str>, + report: &mut dyn FnMut(&str), +) -> Result> { + let mut nets: BTreeMap = client + .get("/1.0/networks?recursion=1")? + .as_array() + .into_iter() + .flatten() + .filter_map(|n| Some((n["name"].as_str()?.to_string(), Subnet::from_network(n)?))) + .collect(); + let instances = list(client)?; + if let Some(p) = only { + let mine = |i: &Value| i["project"].as_str() == Some(p); + let used: BTreeSet<&str> = instances + .iter() + .filter(|i| mine(i)) + .filter_map(|i| i["expanded_devices"].as_object()) + .flat_map(|d| d.values().filter_map(|n| n["network"].as_str())) + .collect(); + let used: BTreeSet = used.into_iter().map(String::from).collect(); + nets.retain(|n, _| used.contains(n)); + } + let mut findings = Vec::new(); + for (network, subnet) in &nets { + let cl = claims(&instances, network, subnet); + if cl.is_empty() { + continue; + } + let (fixes, found) = plan(network, subnet, &cl, &reserved_on(network)); + findings.extend(found); + for fix in fixes { + if let Some(f) = apply(client, &fix, &cl, report) { + findings.push(f); + } + } + } + Ok(findings) +} + +/// Apply one fix; a failure is a finding. +fn apply( + client: &Client, + fix: &Fix, + cl: &[Claim], + report: &mut dyn FnMut(&str), +) -> Option { + let (project, instance, device, r) = match fix { + Fix::Adopt { + project, + instance, + device, + ip, + } => { + let c = client.clone().project(project.as_str()); + let r = crate::sandbox::update_instance( + &c, + instance, + &format!("adopt {instance}'s pin"), + &mut |config, _| { + config.insert(pin_key(device), json!(ip.to_string())); + Ok(()) + }, + ); + if r.is_ok() { + report(&format!( + "{project}/{instance}: {device}'s pin to {ip} is isb's now" + )); + } + (project, instance, device, r) + } + Fix::Pin { + project, + instance, + device, + props, + ip, + running, + } => { + let c = client.clone().project(project.as_str()); + let r = set_pin(&c, instance, device, Some(props), *ip); + if r.is_ok() { + let how = if *running { + "its address" + } else { + "a free address" + }; + report(&format!( + "{project}/{instance}: pinned {device} to {ip} ({how})" + )); + } + (project, instance, device, r) + } + }; + r.err().map(|e| { + let c = cl + .iter() + .find(|c| c.project == *project && c.instance == *instance); + let mut f = Finding { + key: format!("unpinned {project} {instance} {device}"), + project: project.clone(), + instance: instance.clone(), + stack: None, + service: None, + message: format!( + "{instance}'s {device} has no pin isb could set ({e}): once its lease lapses, \ + another instance can be given its address" + ), + }; + if let Some(c) = c { + (f.stack, f.service) = (c.stack.clone(), c.service.clone()); + } + f + }) +} + +/// The daemon's check, on a thread of its own: [`check`] every +/// [`CHECK_EVERY`], raising `net.failed` once per problem until it is gone. +pub fn watch(client: Client, ctl: crate::stack::Controller) { + let _ = std::thread::Builder::new() + .name("isb-ipam".into()) + .spawn(move || { + let mut raised: BTreeSet = BTreeSet::new(); + loop { + let mut log = |m: &str| eprintln!("isb serve: addresses: {m}"); + match check(&client, None, &mut log) { + Ok(found) => { + let now: BTreeSet = found.iter().map(|f| f.key.clone()).collect(); + for f in found.iter().filter(|f| !raised.contains(&f.key)) { + raise(&ctl, f); + } + raised = now; + } + Err(e) => log(&format!("check failed: {e}")), + } + std::thread::sleep(CHECK_EVERY); + } + }); +} + +fn raise(ctl: &crate::stack::Controller, f: &Finding) { + let org = OrgId::from_incus_project(&f.project).unwrap_or_else(OrgId::default_org); + let stack = crate::stack::qualified(&org, f.stack.as_deref().unwrap_or_default()); + ctl.instance_event( + "net.failed", + "error", + &stack, + f.service.as_deref().unwrap_or_default(), + &f.instance, + f.message.clone(), + ); +} + +#[cfg(test)] +mod tests; diff --git a/crates/isb-core/src/ipam/tests.rs b/crates/isb-core/src/ipam/tests.rs new file mode 100644 index 00000000..fafb6826 --- /dev/null +++ b/crates/isb-core/src/ipam/tests.rs @@ -0,0 +1,287 @@ +use super::*; +use serde_json::json; + +fn bridge(name: &str, addr: &str) -> Value { + json!({"name": name, "type": "bridge", "managed": true, "config": {"ipv4.address": addr}}) +} + +fn net() -> Subnet { + Subnet::from_network(&bridge("isbbr1", "10.83.115.1/24")).unwrap() +} + +fn ip(s: &str) -> Ipv4Addr { + s.parse().unwrap() +} + +/// An instance as `GET /1.0/instances?recursion=2` lists it, on isbbr1: +/// `pin` its NIC's ipv4.address, `addr` the address it has (running). +fn inst(name: &str, pin: Option<&str>, addr: Option<&str>, created: &str) -> Value { + let mut nic = json!({"type": "nic", "network": "isbbr1", "name": "eth0"}); + if let Some(p) = pin { + nic["ipv4.address"] = json!(p); + } + let state = match addr { + Some(a) => json!({"network": { + "lo": {"hwaddr": "", "addresses": [{"family": "inet", "address": "127.0.0.1"}]}, + "eth0": {"hwaddr": "10:66:6a:00:00:01", "addresses": [ + {"family": "inet", "address": a}, {"family": "inet6", "address": "fe80::1"}]}}}), + None => json!(null), + }; + json!({ + "name": name, "project": "isb-exa", "created_at": created, + "status": if addr.is_some() { "Running" } else { "Stopped" }, + "config": {"volatile.eth0.hwaddr": "10:66:6a:00:00:01", "user.isb.stack": "analytics", + "user.isb.service": "postgres"}, + "expanded_devices": {"root": {"type": "disk"}, "eth0": nic}, + "state": state, + }) +} + +fn plan_of(insts: &[Value]) -> (Vec, Vec) { + plan( + "isbbr1", + &net(), + &claims(insts, "isbbr1", &net()), + &BTreeSet::new(), + ) +} + +#[test] +fn a_subnet_is_a_managed_bridge_with_an_ipv4_address() { + let s = net(); + assert!(s.contains(ip("10.83.115.43"))); + assert!(!s.contains(ip("10.83.116.43"))); + assert!( + Subnet::from_network(&bridge("isbbrxabcd", "10.1.2.1/24")).is_none(), + "egress" + ); + assert!(Subnet::from_network(&bridge("isbbr1", "none")).is_none()); + let mut unmanaged = bridge("br0", "10.1.2.1/24"); + unmanaged["managed"] = json!(false); + assert!(Subnet::from_network(&unmanaged).is_none()); +} + +#[test] +fn a_free_address_is_never_the_gateway_network_broadcast_or_used() { + let s = Subnet::from_network(&bridge("isbbr1", "10.9.9.1/29")).unwrap(); + // .0 network, .1 gateway, .7 broadcast: .2-.6 are the hosts. + let mut used = BTreeSet::new(); + let mut got = Vec::new(); + while let Some(a) = s.pick_free(&used, "x") { + used.insert(a); + got.push(a); + } + got.sort(); + let want: Vec = (2..=6).map(|h| Ipv4Addr::new(10, 9, 9, h)).collect(); + assert_eq!(got, want); +} + +#[test] +fn seeds_spread_picks_over_the_subnet() { + let s = net(); + let used = BTreeSet::new(); + let picks: BTreeSet = (0..20) + .filter_map(|i| s.pick_free(&used, &format!("isb-exa/web-1-{i}"))) + .collect(); + assert!(picks.len() > 15, "{picks:?}"); +} + +#[test] +fn a_claim_reads_pin_address_and_labels() { + let c = claims( + &[inst("pg", Some("10.83.115.43"), Some("10.83.115.43"), "a")], + "isbbr1", + &net(), + ); + assert_eq!(c.len(), 1); + assert_eq!(c[0].pin, Some(ip("10.83.115.43"))); + assert_eq!(c[0].addrs, [ip("10.83.115.43")]); + assert_eq!(c[0].stack.as_deref(), Some("analytics")); + // Another network, and a project outside isb's orgs, are not claims. + assert!(claims(&[inst("pg", None, None, "a")], "isbbr2", &net()).is_empty()); + let mut other = inst("pg", None, None, "a"); + other["project"] = json!("default"); + assert!(claims(&[other], "isbbr1", &net()).is_empty()); +} + +#[test] +fn a_vms_address_is_found_by_mac() { + let mut v = inst("vm", None, Some("10.83.115.9"), "a"); + let eth0 = v["state"]["network"]["eth0"].take(); + v["state"]["network"]["enp5s0"] = eth0; + v["state"]["network"] + .as_object_mut() + .unwrap() + .remove("eth0"); + let c = claims(&[v], "isbbr1", &net()); + assert_eq!(c[0].addrs, [ip("10.83.115.9")]); +} + +#[test] +fn the_incident_is_a_conflict_about_the_newcomer() { + // 2026-11-09: the gateway got the address analytics-postgres held. + let (fixes, found) = plan_of(&[ + inst( + "analytics-production-analytics-postgres-1-0a5e", + None, + Some("10.83.115.43"), + "2026-10-12", + ), + inst( + "mcp-gateway-1-2ee0", + None, + Some("10.83.115.43"), + "2026-11-09", + ), + ]); + assert!( + fixes.is_empty(), + "neither is pinned at a shared address: {fixes:?}" + ); + assert_eq!(found.len(), 1); + assert_eq!(found[0].instance, "mcp-gateway-1-2ee0"); + assert!( + found[0].message.contains("10.83.115.43"), + "{}", + found[0].message + ); + assert!( + found[0] + .message + .contains("analytics-production-analytics-postgres-1-0a5e") + ); +} + +#[test] +fn a_pin_and_another_instances_address_conflict() { + // A stopped instance pinned to an address a running one has: it could + // not start. + let (fixes, found) = plan_of(&[ + inst("a", Some("10.83.115.43"), None, "1"), + inst("b", None, Some("10.83.115.43"), "2"), + ]); + assert!(fixes.is_empty(), "a shared pin is not adopted: {fixes:?}"); + assert_eq!(found.len(), 1); + assert!( + found[0].message.contains("a (pinned)"), + "{}", + found[0].message + ); + assert!( + found[0].message.contains("b (has it)"), + "{}", + found[0].message + ); +} + +#[test] +fn a_running_instance_is_pinned_at_its_own_address() { + let (fixes, found) = plan_of(&[inst("web", None, Some("10.83.115.7"), "1")]); + assert!(found.is_empty()); + let [ + Fix::Pin { + ip: a, + props, + running: true, + .. + }, + ] = fixes.as_slice() + else { + panic!("{fixes:?}"); + }; + assert_eq!(*a, ip("10.83.115.7")); + assert_eq!(props["network"], "isbbr1"); + assert_eq!(props["ipv4.address"], "10.83.115.7"); +} + +#[test] +fn a_stopped_instance_gets_a_free_address() { + let reserved = BTreeSet::from([ip("10.83.115.9")]); + let insts = [ + inst("pg", Some("10.83.115.8"), Some("10.83.115.8"), "1"), + inst("old", None, None, "2"), + ]; + let cl = claims(&insts, "isbbr1", &net()); + let (fixes, _) = plan("isbbr1", &net(), &cl, &reserved); + let pinned: Vec = fixes + .iter() + .filter_map(|f| match f { + Fix::Pin { instance, ip, .. } if instance == "old" => Some(*ip), + _ => None, + }) + .collect(); + assert_eq!(pinned.len(), 1); + assert!(![ip("10.83.115.8"), ip("10.83.115.9"), ip("10.83.115.1")].contains(&pinned[0])); +} + +#[test] +fn a_pin_set_by_hand_is_adopted_once() { + let mut v = inst("pg", Some("10.83.115.43"), Some("10.83.115.43"), "1"); + let (fixes, found) = plan_of(std::slice::from_ref(&v)); + assert!(found.is_empty()); + assert_eq!( + fixes, + [Fix::Adopt { + project: "isb-exa".into(), + instance: "pg".into(), + device: "eth0".into(), + ip: ip("10.83.115.43"), + }] + ); + v["config"]["user.isb.ipv4.eth0"] = json!("10.83.115.43"); + assert_eq!(plan_of(&[v]), (vec![], vec![])); +} + +#[test] +fn a_pin_the_instance_does_not_have_is_drift() { + let mut v = inst("e", Some("10.83.115.28"), Some("10.83.115.7"), "1"); + v["config"]["user.isb.ipv4.eth0"] = json!("10.83.115.28"); + let (fixes, found) = plan_of(&[v]); + assert!(fixes.is_empty()); + assert_eq!(found.len(), 1); + assert!( + found[0] + .message + .contains("pinned to 10.83.115.28 but has 10.83.115.7") + ); +} + +#[test] +fn a_starting_instance_waits_for_its_address() { + let mut v = inst("web", None, Some("10.83.115.7"), "1"); + v["state"]["network"]["eth0"]["addresses"] = json!([]); + assert_eq!(plan_of(&[v]), (vec![], vec![])); +} + +#[test] +fn findings_keep_their_key_from_pass_to_pass() { + let insts = [ + inst("a", None, Some("10.83.115.43"), "1"), + inst("b", None, Some("10.83.115.43"), "2"), + ]; + assert_eq!(plan_of(&insts).1[0].key, plan_of(&insts).1[0].key); +} + +#[test] +fn the_diff_ignores_and_never_prunes_isbs_pin() { + let desired = Props::from([ + ("type".into(), "nic".into()), + ("network".into(), "isbbr1".into()), + ]); + let mut actual = desired.clone(); + actual.insert("ipv4.address".into(), "10.83.115.43".into()); + let mut a = actual.clone(); + ignore_pin(&desired, &mut a); + assert_eq!(a, desired); + // A spec that names an address keeps it compared. + let mut want = desired.clone(); + want.insert("ipv4.address".into(), "10.83.115.44".into()); + let mut a = actual.clone(); + ignore_pin(&want, &mut a); + assert_ne!(a, want); + + let config = Props::from([("user.isb.ipv4.eth0".into(), "10.83.115.43".into())]); + assert!(is_pin("eth0", &actual, &config)); + assert!(!is_pin("eth0", &actual, &Props::new())); + assert!(!is_pin("eth0", &desired, &config)); +} diff --git a/crates/isb-core/src/lib.rs b/crates/isb-core/src/lib.rs index bc43aa45..cd3bfd19 100644 --- a/crates/isb-core/src/lib.rs +++ b/crates/isb-core/src/lib.rs @@ -19,6 +19,7 @@ pub mod idmap; pub mod image_check; pub mod ingress; pub mod interp; +pub mod ipam; pub mod lock; pub mod machine; pub mod metrics; diff --git a/crates/isb-core/src/plan.rs b/crates/isb-core/src/plan.rs index f3ee164a..37cf4e5c 100644 --- a/crates/isb-core/src/plan.rs +++ b/crates/isb-core/src/plan.rs @@ -1189,6 +1189,7 @@ fn normalize(p: &Props) -> Props { pub fn device_matches(desired: &DesiredDevice, actual: &Props) -> bool { let mut d = normalize(&desired.props); let mut a = normalize(actual); + crate::ipam::ignore_pin(&d, &mut a); // initial.copy only acts the first time a volume is used, so a disk that // differs in nothing else is correct, and replacing it would remount it. if d.get("type").map(String::as_str) == Some("disk") { @@ -1420,7 +1421,7 @@ pub fn diff( } if opts.prune_devices { - for (name, props) in &actual.devices { + for (name, props) in crate::ipam::unpinned(&actual.devices, &actual.config) { if name != "root" && !desired.devices.contains_key(name) && !claimed.contains(name) { actions.push(Action::RemoveDevice { device: name.clone(), diff --git a/crates/isb-core/src/sandbox.rs b/crates/isb-core/src/sandbox.rs index 5376f5dd..fa1a2a32 100644 --- a/crates/isb-core/src/sandbox.rs +++ b/crates/isb-core/src/sandbox.rs @@ -505,7 +505,8 @@ fn create_instance(client: &Client, desired: &Desired, report: &mut dyn FnMut(&s "devices": devices, "profiles": desired.profiles, }); - let body = crate::org::disk::sized_root(client, body); + let (body, pin) = + crate::ipam::pin_for_create(client, crate::org::disk::sized_root(client, body))?; match client.mutate( "POST", "/1.0/instances", @@ -513,7 +514,7 @@ fn create_instance(client: &Client, desired: &Desired, report: &mut dyn FnMut(&s &step, client.timeouts.create, ) { - Ok(_) => return Ok(()), + Ok(_) => return pin.map_or(Ok(()), |p| p.verify(client, &desired.name)), Err(e) if e.is_timeout() => { report(&format!("{e}")); if let Error::OperationTimeout { operation, .. } = &e { @@ -524,7 +525,7 @@ fn create_instance(client: &Client, desired: &Desired, report: &mut dyn FnMut(&s .is_ok() { report(&format!("{step}: finished late; keeping it")); - return Ok(()); + return pin.map_or(Ok(()), |p| p.verify(client, &desired.name)); } } if let Err(ce) = cleanup_half_created(client, &desired.name, &token, report) { diff --git a/crates/isb-core/src/stack/controller.rs b/crates/isb-core/src/stack/controller.rs index 5e558ccc..1aa2c7c5 100644 --- a/crates/isb-core/src/stack/controller.rs +++ b/crates/isb-core/src/stack/controller.rs @@ -183,6 +183,7 @@ pub struct Event { /// dotted, `.`. In use: `deploy.succeeded`, /// `deploy.failed`, `health.unhealthy`, `health.recovered`, /// `restart.failed` (a crash loop), `dns.failed` (no nameserver), + /// `net.failed` (two instances share an address, or one has no pin), /// `backup.succeeded`, `backup.failed`, `restore.succeeded`, /// `restore.failed`, `job.succeeded`, `job.failed`, `cert.issued`, /// `cert.failed`, `secret.rotated` (a new secret version reached a @@ -547,6 +548,22 @@ impl Controller { .emit_kind(Some(kind), level, stack, service, None, message); } + /// Record an event of a known kind about one instance (`stack` + /// qualified; `service` empty for an instance no stack owns). + pub fn instance_event( + &self, + kind: &str, + level: &str, + stack: &str, + service: &str, + instance: &str, + message: String, + ) { + eprintln!("isb serve: {stack}: {instance}: {message}"); + self.inner + .emit_kind(Some(kind), level, stack, service, Some(instance), message); + } + pub fn note(&self, level: &str, stack: &str, message: String) { eprintln!("isb serve: {stack}: {message}"); self.inner.emit(level, stack, "", None, message); diff --git a/crates/isb-daemon/src/daemon/mod.rs b/crates/isb-daemon/src/daemon/mod.rs index 5e2095f3..c60dd94e 100644 --- a/crates/isb-daemon/src/daemon/mod.rs +++ b/crates/isb-daemon/src/daemon/mod.rs @@ -419,6 +419,7 @@ pub fn serve(client: Client, cfg: ServeConfig) -> Result<()> { let sandboxes = sandboxes::Sandboxes::new(&cfg.state_dir, client.clone(), recorder.clone()); let org_api = org_api::OrgApi::new(client.clone(), cfg.org_api_port); let (egress, stop_egress) = egress::start(&cfg, &client, &secrets)?; + crate::ipam::watch(client.clone(), ctl.clone()); let meta = crate::stack::deployments::StackMeta::new(&cfg.state_dir); meta.recover(); let d = Arc::new(Daemon { diff --git a/docs/concepts/orgs.md b/docs/concepts/orgs.md index 9283dac0..f1c292d2 100644 --- a/docs/concepts/orgs.md +++ b/docs/concepts/orgs.md @@ -216,6 +216,22 @@ The org's ACL decides where its instances may go: Traffic into an org's instances is not filtered by the ACL: the host, and the daemon's load balancer, reach them. +### Addresses + +Every instance's address on the org bridge is pinned (its NIC's +`ipv4.address`), so it is the instance's alone for as long as it exists. A +DHCP lease alone is not enough: incus' bridges lease for an hour, an OCI +container never renews, and dnsmasq then gives its address to the next +instance that asks. isb pins an address no other instance on the bridge +holds or is pinned to when it creates an instance, and records it in +`user.isb.ipv4.eth0`. Once a minute `isb serve` pins any instance without a +pin at the address it has, in place (it keeps running and keeps its +address), gives a stopped one a free address, and adopts pins set by hand. +It raises `net.failed` when two instances share an address, when a pinned +instance has another one, or when it cannot pin one; it moves no instance +itself. An instance with an [egress policy](../guides/egress.md) has a +bridge of its own and is left to DHCP. + A host with a default-deny firewall (ufw) needs `sudo isb host setup` once, so org bridges get DHCP, DNS and egress while staying apart from each other: see [Host firewall](../operations/host-setup.md#host-firewall). diff --git a/docs/guides/notifications.md b/docs/guides/notifications.md index 41c4001e..a6207249 100644 --- a/docs/guides/notifications.md +++ b/docs/guides/notifications.md @@ -119,6 +119,7 @@ shortest globs (`*`, `deploy.*`, `*.failed`). | `health.recovered` | A replica of that service is healthy again. | | `restart.failed` | A replica keeps exiting: its fourth exit in a row within 10 minutes of a start. The daemon starts it again with a backoff (10 s, doubling to 5 minutes); raised once per crash loop ([What keeps it running](../concepts/stacks.md#what-keeps-it-running)). | | `dns.failed` | An OCI replica started with a `/etc/resolv.conf` that names no nameserver, so its DNS lookups fail until it is restarted. | +| `net.failed` | Two instances on an org bridge hold one address (pinned or in use), a pinned instance has another address than its pin, or an instance could not be pinned ([Addresses](../concepts/orgs.md#addresses)). Raised once per problem, while it lasts. | | `cert.issued`, `cert.failed` | The ingress got a certificate for a domain, or failed to ([Domains and ingress](domains.md)). | | `backup.succeeded`, `backup.failed`, `job.succeeded`, `job.failed` | [Database and volume backups](databases.md#backups) and [scheduled jobs](jobs.md). | | `monitor.down`, `monitor.up` | An [uptime monitor](uptime.md) went down (its failure threshold reached), or came back (with the downtime). Once per incident; a flapping monitor is held until it settles. The body's `details` carry the URL, status, latency, error and a link. | diff --git a/docs/operations/troubleshooting.md b/docs/operations/troubleshooting.md index 73522c05..9f5c600d 100644 --- a/docs/operations/troubleshooting.md +++ b/docs/operations/troubleshooting.md @@ -82,6 +82,7 @@ More in [isb on macOS](../getting-started/macos.md). | A service is `waiting` | Its `depends_on` is not met. | Look at the dependency's state. | | Published ports pause during a daemon restart | The load balancer and ingress live in the daemon; apps do not. | Expected; they return within seconds. | | Apps cannot reach each other by name | Service names need an org with its own network: the `default` org on a host whose incus `default` project held workloads first has none. On a fresh org, the host may lack the service-name directory. | Use an org of its own; run `sudo isb host setup`, after which a running `isb serve` turns names on within a minute (`isb org create ORG` again does it without the daemon) ([Setting up a host](host-setup.md#service-names)). | +| `net.failed`: an address is held by two instances | Two instances on the org bridge have one address, so traffic for one reaches the other. isb moves neither. | Pin the one that should move elsewhere (`incus config device set INSTANCE eth0 ipv4.address=FREE`) and restart it ([Addresses](../concepts/orgs.md#addresses)). | | A new org's instances get no address or no internet | A default-deny firewall drops DHCP, DNS and forwarding on new bridges. | `sudo isb host setup` ([Host firewall](host-setup.md#host-firewall)). | | A builder image build stops with `got no IPv4 address on incusbr0` | The same firewall drops DHCP and forwarding on incus' own bridge, where those builds run. | `sudo isb host setup`, then build again. | | `isb serve` or `isb host setup` warns that incus is too old for OCI images | The incus package is older than 6.3 (Ubuntu 24.04's is 6.0), so `docker:` images cannot run. | Install incus from the Zabbly stable repository ([Install isb](../getting-started/install.md#1-incus)). | diff --git a/src/lib.rs b/src/lib.rs index 5ef0c4b4..fa4e67cf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,8 +34,9 @@ pub use isb_apps::{app, backup, build, jobs, monitor, notify, s3, template, volu #[doc(inline)] pub use isb_core::{ balance, client, compose, cron, discovery, egress, error, exec, foreground, idmap, image_check, - ingress, interp, lock, machine, metrics, metrics_history, net, org, owner, plan, registry, rpc, - sandbox, secrets, self_update, sftp, shorthand, spec, stack, supervise, volume, x, + ingress, interp, ipam, lock, machine, metrics, metrics_history, net, org, owner, plan, + registry, rpc, sandbox, secrets, self_update, sftp, shorthand, spec, stack, supervise, volume, + x, }; #[doc(inline)] pub use isb_daemon::{daemon, sandbox_life}; diff --git a/tests/ipam.rs b/tests/ipam.rs new file mode 100644 index 00000000..27ea7611 --- /dev/null +++ b/tests/ipam.rs @@ -0,0 +1,201 @@ +//! Addresses on an org bridge against a real incusd +//! (docs/concepts/orgs.md#addresses): a sandbox is pinned to an address no +//! other instance holds when it is created; the daemon's check pins an +//! instance made behind isb's back at the address it has without +//! restarting it, gives a stopped one a free address, adopts a pin set by +//! hand, and reports two instances holding one address. +//! +//! Gated like the other integration tests (`ISB_INTEGRATION=1`; build in a +//! sandbox with `cargo test --no-run`, run the binary on the host). Works in +//! the test org (`common::test_org`), removed afterwards, pass or fail; the +//! check is held to that org's bridge. + +#[allow(dead_code, reason = "each test binary uses some of the shared helpers")] +mod common; + +use std::net::Ipv4Addr; +use std::time::{Duration, Instant}; + +use isb::Client; +use serde_json::{Value, json}; + +const NAMES: [&str; 5] = [ + "isb-test-ip-a", + "isb-test-ip-b", + "isb-test-ip-raw", + "isb-test-ip-dup", + "isb-test-ip-cold", +]; + +struct Rm(Client); +impl Drop for Rm { + fn drop(&mut self) { + for n in NAMES { + let _ = isb::Sandbox::remove(&self.0, n, true); + } + } +} + +fn inst(oc: &Client, name: &str) -> Value { + oc.get(&format!("/1.0/instances/{name}")).unwrap() +} + +fn pin(oc: &Client, name: &str) -> Option { + inst(oc, name)["devices"]["eth0"]["ipv4.address"] + .as_str() + .map(String::from) +} + +fn recorded(oc: &Client, name: &str) -> Option { + inst(oc, name)["config"]["user.isb.ipv4.eth0"] + .as_str() + .map(String::from) +} + +/// The instance's IPv4 address on eth0, once it has one, and its PID. +fn address(oc: &Client, name: &str) -> (String, i64) { + let deadline = Instant::now() + Duration::from_secs(90); + loop { + let st = oc.get(&format!("/1.0/instances/{name}/state")).unwrap(); + let a = st["network"]["eth0"]["addresses"] + .as_array() + .into_iter() + .flatten() + .find(|a| a["family"] == "inet") + .and_then(|a| a["address"].as_str().map(String::from)); + if let Some(a) = a { + return (a, st["pid"].as_i64().unwrap_or_default()); + } + assert!(Instant::now() < deadline, "{name} got no address"); + std::thread::sleep(Duration::from_secs(1)); + } +} + +/// An instance created behind isb's back (no pin), with `devices`. +fn create_raw(oc: &Client, name: &str, devices: Value, start: bool) { + let body = json!({ + "name": name, "type": "container", + "source": {"type": "image", "alias": common::image()}, + "config": {"user.isb-test": "1"}, "devices": devices, "profiles": ["default"], + }); + let body = isb::org::disk::sized_root(oc, body); + oc.mutate( + "POST", + "/1.0/instances", + Some(&body), + "create", + Duration::from_secs(300), + ) + .unwrap(); + if start { + oc.mutate( + "PUT", + &format!("/1.0/instances/{name}/state"), + Some(&json!({"action": "start", "timeout": 30})), + "start", + Duration::from_secs(120), + ) + .unwrap(); + } +} + +fn check(base: &Client, oc: &Client) -> Vec { + isb::ipam::check(base, Some(oc.project_name()), &mut |m| { + eprintln!("check: {m}") + }) + .unwrap() +} + +#[test] +fn addresses_on_an_org_bridge_are_pinned_and_checked() { + if !common::enabled() { + return; + } + let base = Client::new(); + let oc = common::test_org_client(&base); + let _rm = Rm(oc.clone()); + + // Created by isb: pinned at create, the pin recorded and taken. + let spec = |n: &str| { + let mut s = isb::SandboxSpec::new(n, common::image()); + s.labels.insert("isb-test".into(), "1".into()); + s + }; + isb::Sandbox::create(&oc, &spec("isb-test-ip-a")).unwrap(); + isb::Sandbox::create(&oc, &spec("isb-test-ip-b")).unwrap(); + let (a, b) = ( + address(&oc, "isb-test-ip-a").0, + address(&oc, "isb-test-ip-b").0, + ); + assert_ne!(a, b); + for (n, ip) in [("isb-test-ip-a", &a), ("isb-test-ip-b", &b)] { + assert_eq!(pin(&oc, n).as_ref(), Some(ip), "{n}"); + assert_eq!(recorded(&oc, n).as_ref(), Some(ip), "{n}"); + } + // Planning it again changes nothing, even pruning devices: the pin is + // not drift from the spec. + let d = isb::sandbox::resolve( + &oc, + &spec("isb-test-ip-a"), + &isb::plan::VolumeDefs::new(), + std::path::Path::new("/"), + ) + .unwrap(); + let prune = isb::plan::DiffOptions { + prune_devices: true, + }; + let plan = isb::sandbox::plan_desired(&oc, &d, prune).unwrap(); + assert!(plan.is_noop(), "{plan:?}"); + + // Behind isb's back: pinned in place by the check, same address, same + // process. + create_raw(&oc, "isb-test-ip-raw", json!({}), true); + assert_eq!(pin(&oc, "isb-test-ip-raw"), None); + let (raw, pid) = address(&oc, "isb-test-ip-raw"); + let found = check(&base, &oc); + assert!(found.is_empty(), "{found:?}"); + assert_eq!(pin(&oc, "isb-test-ip-raw").as_ref(), Some(&raw)); + assert_eq!(recorded(&oc, "isb-test-ip-raw").as_ref(), Some(&raw)); + assert_eq!( + address(&oc, "isb-test-ip-raw"), + (raw.clone(), pid), + "restarted or moved" + ); + + // A pin whose record is gone (one set by hand) is adopted. + isb::sandbox::update_instance(&oc, "isb-test-ip-a", "unrecord", &mut |c, _| { + c.remove("user.isb.ipv4.eth0"); + Ok(()) + }) + .unwrap(); + assert!(check(&base, &oc).is_empty()); + assert_eq!(recorded(&oc, "isb-test-ip-a").as_ref(), Some(&a)); + + // A stopped instance without a pin gets a free address. + create_raw(&oc, "isb-test-ip-cold", json!({}), false); + assert!(check(&base, &oc).is_empty()); + let cold = pin(&oc, "isb-test-ip-cold").expect("pinned"); + assert!(![&a, &b, &raw].contains(&&cold), "{cold}"); + cold.parse::().unwrap(); + + // Pinned by hand to b's address (stopped: incus would refuse to start + // it): reported, nothing moved. + let mut nic = inst(&oc, "isb-test-ip-b")["expanded_devices"]["eth0"].clone(); + nic["ipv4.address"] = json!(b); + create_raw(&oc, "isb-test-ip-dup", json!({"eth0": nic}), false); + let found = check(&base, &oc); + assert_eq!(found.len(), 1, "{found:?}"); + assert!(found[0].message.contains(&b), "{}", found[0].message); + assert!( + found[0].message.contains("isb-test-ip-dup"), + "{}", + found[0].message + ); + assert_eq!( + recorded(&oc, "isb-test-ip-dup"), + None, + "a shared pin is not isb's" + ); + assert_eq!(pin(&oc, "isb-test-ip-b").as_ref(), Some(&b)); + assert_eq!(address(&oc, "isb-test-ip-b").0, b); +} diff --git a/web/src/notifications/api.ts b/web/src/notifications/api.ts index 310623c1..41194ffa 100644 --- a/web/src/notifications/api.ts +++ b/web/src/notifications/api.ts @@ -89,6 +89,11 @@ export const EVENT_GROUPS: { subject: string; label: string; kinds: { kind: stri label: "DNS", kinds: [{ kind: "dns.failed", label: "No nameserver" }], }, + { + subject: "net", + label: "Addresses", + kinds: [{ kind: "net.failed", label: "Shared or unpinned address" }], + }, { subject: "backup", label: "Backups", From 73f94d38ed4a34e39cf1ac0e2eee168d1c906496 Mon Sep 17 00:00:00 2001 From: Stephan Fitzpatrick Date: Mon, 9 Nov 2026 11:06:43 +0000 Subject: [PATCH 2/3] notify: a mattermost provider (Slack's {"text": ...} on any https server) The slack provider takes only hooks.slack.com, and Mattermost answers the generic webhook body with 400. `mattermost` posts {"text": ...} (the title in bold) to an incoming-webhook URL on any server, https only, and puts a zero-width space after every @ so event text never pings anyone. CLI --mattermost, the channel_create tool, the web UI's channel dialog and the docs know it. Claude-Session: https://claude.ai/code/session_01LJ21XsZE86ywhpFMvYVWmE --- crates/isb-apps/src/notify/mod.rs | 5 +- crates/isb-apps/src/notify/provider.rs | 67 +++++++++++++++++++- crates/isb-daemon/src/daemon/notify.rs | 4 +- docs/guides/notifications.md | 8 ++- docs/reference/cli.md | 2 +- docs/reference/mcp-tools.md | 2 +- src/bin/isb.rs | 2 +- src/bin/isb/notify.rs | 8 ++- web/openapi.json | 6 +- web/src/api/openapi.gen.ts | 6 +- web/src/notifications/api.ts | 4 ++ web/src/notifications/channel-dialog.tsx | 14 +++- web/src/notifications/icons.ts | 3 +- web/src/notifications/notifications-page.tsx | 4 +- 14 files changed, 110 insertions(+), 25 deletions(-) diff --git a/crates/isb-apps/src/notify/mod.rs b/crates/isb-apps/src/notify/mod.rs index cda314e9..25d9cf1d 100644 --- a/crates/isb-apps/src/notify/mod.rs +++ b/crates/isb-apps/src/notify/mod.rs @@ -1,4 +1,4 @@ -//! Notifications: per-org channels (webhook, Slack, Discord, Telegram, +//! Notifications: per-org channels (webhook, Slack, Discord, Mattermost, Telegram, //! email) told about events of chosen kinds (`deploy.*`, `health.*`, //! `backup.*`, `job.*`, `cert.*`, `monitor.*`; see //! [`crate::stack::controller::Event`]). @@ -428,7 +428,8 @@ impl Notifier { } if let Provider::Webhook { url_secret, .. } | Provider::Slack { url_secret } - | Provider::Discord { url_secret } = &c.provider + | Provider::Discord { url_secret } + | Provider::Mattermost { url_secret } = &c.provider { let v = self .secret_value(org, url_secret) diff --git a/crates/isb-apps/src/notify/provider.rs b/crates/isb-apps/src/notify/provider.rs index fa8f59d5..116a1de4 100644 --- a/crates/isb-apps/src/notify/provider.rs +++ b/crates/isb-apps/src/notify/provider.rs @@ -22,6 +22,9 @@ pub enum Provider { Slack { url_secret: String }, /// A Discord webhook (`https://discord.com/api/webhooks/...`). Discord { url_secret: String }, + /// A Mattermost incoming webhook (`https:///hooks/...`), or any + /// other https endpoint that takes Slack's `{"text": ...}`. + Mattermost { url_secret: String }, /// A Telegram bot message to one chat. Telegram { token_secret: String, @@ -49,6 +52,7 @@ impl Provider { Provider::Webhook { .. } => "webhook", Provider::Slack { .. } => "slack", Provider::Discord { .. } => "discord", + Provider::Mattermost { .. } => "mattermost", Provider::Telegram { .. } => "telegram", Provider::Email { .. } => "email", } @@ -63,7 +67,9 @@ impl Provider { } => std::iter::once(url_secret.as_str()) .chain(signing_secret.as_deref()) .collect(), - Provider::Slack { url_secret } | Provider::Discord { url_secret } => vec![url_secret], + Provider::Slack { url_secret } + | Provider::Discord { url_secret } + | Provider::Mattermost { url_secret } => vec![url_secret], Provider::Telegram { token_secret, .. } => vec![token_secret], Provider::Email { password_secret, .. @@ -187,7 +193,7 @@ impl Message { } } -/// Validate a Slack, Discord or webhook URL read from a secret. +/// Validate a Slack, Discord, Mattermost or webhook URL read from a secret. fn checked_url(p: &Provider, url: &str) -> Result { let t = parse_url(url).map_err(|e| SendError::permanent(format!("{} URL: {e}", p.kind())))?; let want = |hosts: &[&str], prefix: &str| -> Result<(), SendError> { @@ -216,6 +222,14 @@ fn checked_url(p: &Provider, url: &str) -> Result { ], "/api/webhooks/", )?, + // A self-hosted server: any host, but never in the clear (the URL + // is the credential). + Provider::Mattermost { .. } if !t.https => { + return Err(SendError::permanent(format!( + "a mattermost URL must be https; this one for {} is not", + t.host + ))); + } _ => {} } Ok(t) @@ -249,6 +263,13 @@ fn check_bot_token(t: &str) -> Result<(), SendError> { /// Where Telegram's Bot API lives. pub const TELEGRAM_API: &str = "https://api.telegram.org"; +/// Mattermost notifies whoever an `@name`, `@channel` or `@all` in the text +/// names; event text must never ping anyone, so each `@` is followed by a +/// zero-width space. +fn no_mentions(s: &str) -> String { + s.replace('@', "@\u{200b}") +} + /// Slack's mrkdwn treats &, < and > specially. fn slack_escape(s: &str) -> String { s.replace('&', "&") @@ -353,6 +374,19 @@ pub fn request( body: serde_json::to_vec(&body).expect("json"), }) } + Provider::Mattermost { url_secret } => { + let url = secret(url_secret)?; + checked_url(p, &url)?; + // Markdown: the title in bold, then the rest of the text. + let full = no_mentions(&text(m)); + let (title, rest) = full.split_once('\n').unwrap_or((&full, "")); + let body = json!({"text": cap(&format!("**{title}**\n{rest}"), 16000)}); + Ok(Request { + url, + headers: json_headers(vec![]), + body: serde_json::to_vec(&body).expect("json"), + }) + } Provider::Discord { url_secret } => { let url = secret(url_secret)?; checked_url(p, &url)?; @@ -418,6 +452,8 @@ mod tests { "SLACK" => "https://hooks.slack.com/services/T0/B0/xyz".into(), "DISCORD" => "https://discord.com/api/webhooks/1/abc".into(), "TG" => "123456:ABC-def_1".into(), + "MM" => "https://chat.example.org/hooks/abc123".into(), + "MMPLAIN" => "http://chat.example.org/hooks/abc123".into(), "BADSLACK" => "https://evil.example/services/x".into(), _ => return Err(SendError::permanent(format!("no secret {name}"))), }) @@ -503,6 +539,33 @@ mod tests { assert!(e.is_err()); } + #[test] + fn mattermost_takes_slacks_text_on_any_https_server_and_never_pings() { + let p = |s: &str| Provider::Mattermost { + url_secret: s.into(), + }; + let mut m = msg(); + m.message = "deployment 3 failed for @channel and @all".into(); + let r = request(&p("MM"), &m, &secrets).unwrap(); + assert_eq!(r.url, "https://chat.example.org/hooks/abc123"); + let b = body(&r); + let t = b["text"].as_str().unwrap(); + assert!(t.starts_with("**"), "{t}"); + assert!(t.contains("deployment 3 failed"), "{t}"); + assert!(!t.contains("@channel") && !t.contains("@all"), "{t}"); + assert_eq!(b.as_object().unwrap().len(), 1, "{b}"); + let e = request(&p("MMPLAIN"), &m, &secrets).unwrap_err(); + assert!(e.message.contains("https"), "{}", e.message); + assert!(!e.message.contains("abc123"), "the URL is a credential"); + assert!(check_url_value(&p("MM"), "https://chat.example.org/hooks/x").is_ok()); + let long = Message { + message: "x".repeat(20000), + ..msg() + }; + let b = body(&request(&p("MM"), &long, &secrets).unwrap()); + assert!(b["text"].as_str().unwrap().chars().count() <= 16000); + } + #[test] fn discord_never_pings() { let r = request( diff --git a/crates/isb-daemon/src/daemon/notify.rs b/crates/isb-daemon/src/daemon/notify.rs index 554ccbe6..b2eba59f 100644 --- a/crates/isb-daemon/src/daemon/notify.rs +++ b/crates/isb-daemon/src/daemon/notify.rs @@ -11,7 +11,7 @@ use crate::metrics_history::{History, Query}; use crate::notify::{Channel, Notifier, Provider, Rule, Settings}; use crate::server::{Caller, Registry, Tool}; -const PROVIDER_DESC: &str = "Where messages go, by `type`: {\"type\": \"webhook\", \"url_secret\": NAME, \"signing_secret\": NAME?} (JSON POST; with a signing secret, X-Isb-Signature: sha256=HMAC of the body), {\"type\": \"slack\", \"url_secret\": NAME} (an incoming-webhook URL), {\"type\": \"discord\", \"url_secret\": NAME}, {\"type\": \"telegram\", \"token_secret\": NAME, \"chat_id\": \"-100...\"}, {\"type\": \"email\", \"host\", \"port\"?, \"tls\": \"starttls\"|\"tls\"|\"none\", \"username\"?, \"password_secret\"?, \"from\", \"to\": [..]}. Every URL, token and password is an org secret, named here (secret_create first), never a value."; +const PROVIDER_DESC: &str = "Where messages go, by `type`: {\"type\": \"webhook\", \"url_secret\": NAME, \"signing_secret\": NAME?} (JSON POST; with a signing secret, X-Isb-Signature: sha256=HMAC of the body), {\"type\": \"slack\", \"url_secret\": NAME} (an incoming-webhook URL), {\"type\": \"discord\", \"url_secret\": NAME}, {\"type\": \"mattermost\", \"url_secret\": NAME} (an incoming-webhook URL, https on any server), {\"type\": \"telegram\", \"token_secret\": NAME, \"chat_id\": \"-100...\"}, {\"type\": \"email\", \"host\", \"port\"?, \"tls\": \"starttls\"|\"tls\"|\"none\", \"username\"?, \"password_secret\"?, \"from\", \"to\": [..]}. Every URL, token and password is an org secret, named here (secret_create first), never a value."; const RULES_DESC: &str = "Which events: a list of {\"events\": [glob, ...] (deploy.*, health.*, backup.*, job.*, cert.*, *.failed, *), \"projects\"?: [..], \"apps\"?: [..], \"stacks\"?: [..]}; any rule matching sends. Default: [{\"events\": [\"*\"]}]."; @@ -33,7 +33,7 @@ pub fn register(r: &mut Registry, n: Notifier, h: History, apps: Apps) -> Result r.register( Tool::new( "notification_channel_create", - "Add a notification channel to the org: a destination (webhook, Slack, Discord, Telegram, email) and rules choosing which events it hears about. Only the org's own events reach it. Its secrets must exist.", + "Add a notification channel to the org: a destination (webhook, Slack, Discord, Mattermost, Telegram, email) and rules choosing which events it hears about. Only the org's own events reach it. Its secrets must exist.", obj( json!({ "name": {"type": "string", "description": "[a-z0-9-], starting with a letter, at most 40."}, diff --git a/docs/guides/notifications.md b/docs/guides/notifications.md index a6207249..8e7822a7 100644 --- a/docs/guides/notifications.md +++ b/docs/guides/notifications.md @@ -1,6 +1,6 @@ --- title: Notifications -description: Tell an org's people about failed deploys, unhealthy services, certificates, backups and jobs, through webhooks, Slack, Discord, Telegram or email. +description: Tell an org's people about failed deploys, unhealthy services, certificates, backups and jobs, through webhooks, Slack, Discord, Mattermost, Telegram or email. order: 9 --- @@ -22,6 +22,7 @@ isb notify deliveries ops # what was sent, retried, failed ```text isb notify create NAME (--webhook URL_SECRET [--signing-secret S] | --slack URL_SECRET | --discord URL_SECRET + | --mattermost URL_SECRET | --telegram TOKEN_SECRET --chat-id ID | --smtp-host H [--smtp-port P] [--smtp-tls starttls|tls|none] [--smtp-user U --smtp-password-secret S] --from ADDR --to ADDR...) [--events deploy.*,health.*] [--app-project P]... [--app A]... [--stack S]... [--disabled] @@ -48,11 +49,12 @@ touching the channel. | `webhook` | `url_secret`, optional `signing_secret` | A JSON `POST` ([below](#the-webhook-body)) to any http(s) URL. | | `slack` | `url_secret`: an incoming-webhook URL, `https://hooks.slack.com/services/...` | `{"text": ...}`, the title in bold. | | `discord` | `url_secret`: `https://discord.com/api/webhooks/...` | `{"content": ..., "allowed_mentions": {"parse": []}}` (never pings). | +| `mattermost` | `url_secret`: an incoming-webhook URL on any server, `https:///hooks/...` | `{"text": ...}`, the title in bold; every `@` is followed by a zero-width space, so event text never pings anyone. Any https endpoint that takes Slack's `{"text": ...}` works too. | | `telegram` | `token_secret` (a bot token), `chat_id` (`-100...` or `@channel`) | `sendMessage` to the Bot API, plain text. | | `email` | `host`, `port` (default by `tls`), `tls`: `starttls` (587, default), `tls` (465) or `none` (25); `username` + `password_secret`; `from`; `to` (1 to 20) | A plain-text mail through that SMTP server. A password is never sent without TLS. | Slack and Discord URLs are checked against their hosts when the channel is -saved and again at every send; a Telegram token must look like one +saved and again at every send, and a Mattermost URL must be https; a Telegram token must look like one (`123456:ABC...`), so it cannot change the request path. ### The webhook body @@ -89,7 +91,7 @@ ok = hmac.compare_digest(sig, "sha256=" + hmac.new(key, body, hashlib.sha256).he old replays. Events whose producer adds structure carry it in `details`: `monitor.*` events have the URL, HTTP status, latency, error, downtime and a link to the monitor ([Uptime monitoring](uptime.md#the-notification)); Slack, -Discord, Telegram and email show the latency and the link too. +Discord, Mattermost, Telegram and email show the latency and the link too. ## Rules diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 779dbd31..1ae824d3 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -516,7 +516,7 @@ prints the build id instead. See [Builds](../guides/builds.md). ```text isb notify create NAME (--webhook URL_SECRET [--signing-secret S] | --slack URL_SECRET - | --discord URL_SECRET | --telegram TOKEN_SECRET --chat-id ID + | --discord URL_SECRET | --mattermost URL_SECRET | --telegram TOKEN_SECRET --chat-id ID | --smtp-host H [--smtp-port P] [--smtp-tls starttls|tls|none] [--smtp-user U --smtp-password-secret S] --from ADDR --to ADDR...) [--events deploy.*,health.*] [--app-project P]... [--app A]... [--stack S]... diff --git a/docs/reference/mcp-tools.md b/docs/reference/mcp-tools.md index 1812a7f0..4898b7e1 100644 --- a/docs/reference/mcp-tools.md +++ b/docs/reference/mcp-tools.md @@ -293,7 +293,7 @@ and owners. | Tool | Who | Does | |---|---|---| -| `notification_channel_create` | member | `name`, `provider` (webhook, Slack, Discord, Telegram, email; secrets named, never values), `rules`, `enabled`. | +| `notification_channel_create` | member | `name`, `provider` (webhook, Slack, Discord, Mattermost, Telegram, email; secrets named, never values), `rules`, `enabled`. | | `notification_channel_list`, `notification_channel_get` | viewer | Channels with their last delivery. | | `notification_channel_update` | member | Replace `provider`, `rules` or `enabled`; fields left out are kept. | | `notification_channel_delete` | member | The channel and its delivery log (its secrets stay). | diff --git a/src/bin/isb.rs b/src/bin/isb.rs index 9da9ff06..e544863e 100644 --- a/src/bin/isb.rs +++ b/src/bin/isb.rs @@ -358,7 +358,7 @@ enum Cmd { #[command(subcommand)] Registry(RegistryCmd), /// Notification channels of an org on the `isb serve` daemon: webhook, - /// Slack, Discord, Telegram, email (docs/guides/notifications.md). + /// Slack, Discord, Mattermost, Telegram, email (docs/guides/notifications.md). #[command(subcommand)] Notify(notify::NotifyCmd), /// An org's sandboxes on the `isb serve` daemon: list them, push one's diff --git a/src/bin/isb/notify.rs b/src/bin/isb/notify.rs index b1a4e8fe..77951b0d 100644 --- a/src/bin/isb/notify.rs +++ b/src/bin/isb/notify.rs @@ -79,6 +79,9 @@ pub struct DestOne { /// A Discord webhook URL held in this secret. #[arg(long, value_name = "URL_SECRET")] discord: Option, + /// A Mattermost incoming webhook URL (https, any server) held in this secret. + #[arg(long, value_name = "URL_SECRET")] + mattermost: Option, /// A Telegram bot token held in this secret (with --chat-id). #[arg(long, value_name = "TOKEN_SECRET")] telegram: Option, @@ -163,6 +166,9 @@ fn provider(d: Dest) -> Result { if let Some(s) = o.discord { return Ok(json!({"type": "discord", "url_secret": s})); } + if let Some(s) = o.mattermost { + return Ok(json!({"type": "mattermost", "url_secret": s})); + } if let Some(s) = o.telegram { let chat = d .chat_id @@ -200,7 +206,7 @@ fn with_org(org: &Option, mut args: Value) -> Value { fn describe(p: &Value) -> String { match p["type"].as_str().unwrap_or_default() { "webhook" => format!("webhook ${}", p["url_secret"].as_str().unwrap_or_default()), - "slack" | "discord" => format!( + "slack" | "discord" | "mattermost" => format!( "{} ${}", p["type"].as_str().unwrap_or_default(), p["url_secret"].as_str().unwrap_or_default() diff --git a/web/openapi.json b/web/openapi.json index c398d19e..560eb589 100644 --- a/web/openapi.json +++ b/web/openapi.json @@ -9938,7 +9938,7 @@ }, "/api/v1/tools/notification_channel_create": { "post": { - "description": "Add a notification channel to the org: a destination (webhook, Slack, Discord, Telegram, email) and rules choosing which events it hears about. Only the org's own events reach it. Its secrets must exist.", + "description": "Add a notification channel to the org: a destination (webhook, Slack, Discord, Mattermost, Telegram, email) and rules choosing which events it hears about. Only the org's own events reach it. Its secrets must exist.", "operationId": "notification_channel_create", "requestBody": { "content": { @@ -9959,7 +9959,7 @@ "type": "string" }, "provider": { - "description": "Where messages go, by `type`: {\"type\": \"webhook\", \"url_secret\": NAME, \"signing_secret\": NAME?} (JSON POST; with a signing secret, X-Isb-Signature: sha256=HMAC of the body), {\"type\": \"slack\", \"url_secret\": NAME} (an incoming-webhook URL), {\"type\": \"discord\", \"url_secret\": NAME}, {\"type\": \"telegram\", \"token_secret\": NAME, \"chat_id\": \"-100...\"}, {\"type\": \"email\", \"host\", \"port\"?, \"tls\": \"starttls\"|\"tls\"|\"none\", \"username\"?, \"password_secret\"?, \"from\", \"to\": [..]}. Every URL, token and password is an org secret, named here (secret_create first), never a value.", + "description": "Where messages go, by `type`: {\"type\": \"webhook\", \"url_secret\": NAME, \"signing_secret\": NAME?} (JSON POST; with a signing secret, X-Isb-Signature: sha256=HMAC of the body), {\"type\": \"slack\", \"url_secret\": NAME} (an incoming-webhook URL), {\"type\": \"discord\", \"url_secret\": NAME}, {\"type\": \"mattermost\", \"url_secret\": NAME} (an incoming-webhook URL, https on any server), {\"type\": \"telegram\", \"token_secret\": NAME, \"chat_id\": \"-100...\"}, {\"type\": \"email\", \"host\", \"port\"?, \"tls\": \"starttls\"|\"tls\"|\"none\", \"username\"?, \"password_secret\"?, \"from\", \"to\": [..]}. Every URL, token and password is an org secret, named here (secret_create first), never a value.", "type": "object" }, "rules": { @@ -10217,7 +10217,7 @@ "type": "string" }, "provider": { - "description": "Where messages go, by `type`: {\"type\": \"webhook\", \"url_secret\": NAME, \"signing_secret\": NAME?} (JSON POST; with a signing secret, X-Isb-Signature: sha256=HMAC of the body), {\"type\": \"slack\", \"url_secret\": NAME} (an incoming-webhook URL), {\"type\": \"discord\", \"url_secret\": NAME}, {\"type\": \"telegram\", \"token_secret\": NAME, \"chat_id\": \"-100...\"}, {\"type\": \"email\", \"host\", \"port\"?, \"tls\": \"starttls\"|\"tls\"|\"none\", \"username\"?, \"password_secret\"?, \"from\", \"to\": [..]}. Every URL, token and password is an org secret, named here (secret_create first), never a value.", + "description": "Where messages go, by `type`: {\"type\": \"webhook\", \"url_secret\": NAME, \"signing_secret\": NAME?} (JSON POST; with a signing secret, X-Isb-Signature: sha256=HMAC of the body), {\"type\": \"slack\", \"url_secret\": NAME} (an incoming-webhook URL), {\"type\": \"discord\", \"url_secret\": NAME}, {\"type\": \"mattermost\", \"url_secret\": NAME} (an incoming-webhook URL, https on any server), {\"type\": \"telegram\", \"token_secret\": NAME, \"chat_id\": \"-100...\"}, {\"type\": \"email\", \"host\", \"port\"?, \"tls\": \"starttls\"|\"tls\"|\"none\", \"username\"?, \"password_secret\"?, \"from\", \"to\": [..]}. Every URL, token and password is an org secret, named here (secret_create first), never a value.", "type": "object" }, "rules": { diff --git a/web/src/api/openapi.gen.ts b/web/src/api/openapi.gen.ts index 680f2d2f..db83915a 100644 --- a/web/src/api/openapi.gen.ts +++ b/web/src/api/openapi.gen.ts @@ -2583,7 +2583,7 @@ export interface paths { put?: never; /** * Create a notification channel - * @description Add a notification channel to the org: a destination (webhook, Slack, Discord, Telegram, email) and rules choosing which events it hears about. Only the org's own events reach it. Its secrets must exist. + * @description Add a notification channel to the org: a destination (webhook, Slack, Discord, Mattermost, Telegram, email) and rules choosing which events it hears about. Only the org's own events reach it. Its secrets must exist. */ post: operations["notification_channel_create"]; delete?: never; @@ -10457,7 +10457,7 @@ export interface operations { name: string; /** @description The org to act in (default: default). */ org?: string; - /** @description Where messages go, by `type`: {"type": "webhook", "url_secret": NAME, "signing_secret": NAME?} (JSON POST; with a signing secret, X-Isb-Signature: sha256=HMAC of the body), {"type": "slack", "url_secret": NAME} (an incoming-webhook URL), {"type": "discord", "url_secret": NAME}, {"type": "telegram", "token_secret": NAME, "chat_id": "-100..."}, {"type": "email", "host", "port"?, "tls": "starttls"|"tls"|"none", "username"?, "password_secret"?, "from", "to": [..]}. Every URL, token and password is an org secret, named here (secret_create first), never a value. */ + /** @description Where messages go, by `type`: {"type": "webhook", "url_secret": NAME, "signing_secret": NAME?} (JSON POST; with a signing secret, X-Isb-Signature: sha256=HMAC of the body), {"type": "slack", "url_secret": NAME} (an incoming-webhook URL), {"type": "discord", "url_secret": NAME}, {"type": "mattermost", "url_secret": NAME} (an incoming-webhook URL, https on any server), {"type": "telegram", "token_secret": NAME, "chat_id": "-100..."}, {"type": "email", "host", "port"?, "tls": "starttls"|"tls"|"none", "username"?, "password_secret"?, "from", "to": [..]}. Every URL, token and password is an org secret, named here (secret_create first), never a value. */ provider: Record; /** @description Which events: a list of {"events": [glob, ...] (deploy.*, health.*, backup.*, job.*, cert.*, *.failed, *), "projects"?: [..], "apps"?: [..], "stacks"?: [..]}; any rule matching sends. Default: [{"events": ["*"]}]. */ rules?: Record[]; @@ -10617,7 +10617,7 @@ export interface operations { name: string; /** @description The org to act in (default: default). */ org?: string; - /** @description Where messages go, by `type`: {"type": "webhook", "url_secret": NAME, "signing_secret": NAME?} (JSON POST; with a signing secret, X-Isb-Signature: sha256=HMAC of the body), {"type": "slack", "url_secret": NAME} (an incoming-webhook URL), {"type": "discord", "url_secret": NAME}, {"type": "telegram", "token_secret": NAME, "chat_id": "-100..."}, {"type": "email", "host", "port"?, "tls": "starttls"|"tls"|"none", "username"?, "password_secret"?, "from", "to": [..]}. Every URL, token and password is an org secret, named here (secret_create first), never a value. */ + /** @description Where messages go, by `type`: {"type": "webhook", "url_secret": NAME, "signing_secret": NAME?} (JSON POST; with a signing secret, X-Isb-Signature: sha256=HMAC of the body), {"type": "slack", "url_secret": NAME} (an incoming-webhook URL), {"type": "discord", "url_secret": NAME}, {"type": "mattermost", "url_secret": NAME} (an incoming-webhook URL, https on any server), {"type": "telegram", "token_secret": NAME, "chat_id": "-100..."}, {"type": "email", "host", "port"?, "tls": "starttls"|"tls"|"none", "username"?, "password_secret"?, "from", "to": [..]}. Every URL, token and password is an org secret, named here (secret_create first), never a value. */ provider?: Record; /** @description Which events: a list of {"events": [glob, ...] (deploy.*, health.*, backup.*, job.*, cert.*, *.failed, *), "projects"?: [..], "apps"?: [..], "stacks"?: [..]}; any rule matching sends. Default: [{"events": ["*"]}]. */ rules?: Record[]; diff --git a/web/src/notifications/api.ts b/web/src/notifications/api.ts index 41194ffa..539adb5d 100644 --- a/web/src/notifications/api.ts +++ b/web/src/notifications/api.ts @@ -7,6 +7,7 @@ export type Provider = | { type: "webhook"; url_secret: string; signing_secret?: string } | { type: "slack"; url_secret: string } | { type: "discord"; url_secret: string } + | { type: "mattermost"; url_secret: string } | { type: "telegram"; token_secret: string; chat_id: string } | { type: "email"; @@ -57,6 +58,7 @@ export const PROVIDERS: { id: ProviderType; label: string; hint: string }[] = [ { id: "webhook", label: "Webhook", hint: "A signed JSON POST to any URL" }, { id: "slack", label: "Slack", hint: "An incoming webhook" }, { id: "discord", label: "Discord", hint: "A channel webhook" }, + { id: "mattermost", label: "Mattermost", hint: "An incoming webhook on any server" }, { id: "telegram", label: "Telegram", hint: "A bot message to a chat" }, { id: "email", label: "Email", hint: "Mail through an SMTP server" }, ]; @@ -225,6 +227,7 @@ export function providerSecrets(p: Provider): string[] { return [p.url_secret, p.signing_secret].filter(Boolean) as string[]; case "slack": case "discord": + case "mattermost": return [p.url_secret]; case "telegram": return [p.token_secret]; @@ -239,6 +242,7 @@ export function providerSummary(p: Provider): string { return `URL in ${p.url_secret}${p.signing_secret ? `, signed with ${p.signing_secret}` : ""}`; case "slack": case "discord": + case "mattermost": return `URL in ${p.url_secret}`; case "telegram": return `chat ${p.chat_id}, token in ${p.token_secret}`; diff --git a/web/src/notifications/channel-dialog.tsx b/web/src/notifications/channel-dialog.tsx index c538ba49..758497ac 100644 --- a/web/src/notifications/channel-dialog.tsx +++ b/web/src/notifications/channel-dialog.tsx @@ -60,6 +60,7 @@ function pfOf(p?: Provider): PF { return { ...emptyPF, url_secret: p.url_secret, signing_secret: p.signing_secret ?? "" }; case "slack": case "discord": + case "mattermost": return { ...emptyPF, url_secret: p.url_secret }; case "telegram": return { ...emptyPF, token_secret: p.token_secret, chat_id: p.chat_id }; @@ -78,7 +79,8 @@ export function providerOf(type: ProviderType, f: PF): { provider: Provider } | return { provider: { type, url_secret: f.url_secret.trim(), ...(f.signing_secret.trim() ? { signing_secret: f.signing_secret.trim() } : {}) } }; } case "slack": - case "discord": { + case "discord": + case "mattermost": { const e = need(f.url_secret, "Name the secret holding the webhook URL."); return e ? { error: e } : { provider: { type, url_secret: f.url_secret.trim() } }; } @@ -236,13 +238,19 @@ export function ChannelDialog({ org, existing, open, onOpenChange }: { org: stri setP({ signing_secret: v })} names={names} hint="Signs the body: X-Isb-Signature: sha256=HMAC." /> )} - {(type === "slack" || type === "discord") && ( + {(type === "slack" || type === "discord" || type === "mattermost") && ( setP({ url_secret: v })} names={names} - hint={type === "slack" ? "Holds https://hooks.slack.com/services/…" : "Holds https://discord.com/api/webhooks/…"} + hint={ + type === "slack" + ? "Holds https://hooks.slack.com/services/…" + : type === "discord" + ? "Holds https://discord.com/api/webhooks/…" + : "Holds https:///hooks/…" + } /> )} {type === "telegram" && ( diff --git a/web/src/notifications/icons.ts b/web/src/notifications/icons.ts index cb04c101..db0540b7 100644 --- a/web/src/notifications/icons.ts +++ b/web/src/notifications/icons.ts @@ -1,11 +1,12 @@ // One icon per channel destination, for the cards and the channel dialog. -import { type Bell, Hash, Mail, MessageCircle, Send, Webhook } from "lucide-react"; +import { type Bell, Hash, Mail, MessageCircle, MessagesSquare, Send, Webhook } from "lucide-react"; import type { ProviderType } from "./api"; export const PROVIDER_ICON: Record = { webhook: Webhook, slack: Hash, discord: MessageCircle, + mattermost: MessagesSquare, telegram: Send, email: Mail, }; diff --git a/web/src/notifications/notifications-page.tsx b/web/src/notifications/notifications-page.tsx index 2732d05f..930b0618 100644 --- a/web/src/notifications/notifications-page.tsx +++ b/web/src/notifications/notifications-page.tsx @@ -1,4 +1,4 @@ -// /orgs/:org/notifications: the org's channels (webhook, Slack, Discord, +// /orgs/:org/notifications: the org's channels (webhook, Slack, Discord, Mattermost, // Telegram, email), the events each hears about, Test, and each one's // delivery log. Platform admins also see the server-wide private-target // switch. @@ -96,7 +96,7 @@ export function NotificationsPage() { ) } > - A channel is a destination (a webhook, Slack, Discord, Telegram or email) and the events it hears about. URLs and tokens stay in the org's secrets. + A channel is a destination (a webhook, Slack, Discord, Mattermost, Telegram or email) and the events it hears about. URLs and tokens stay in the org's secrets. ) : ( From 935691860b2bf0f4feba852ef1c616fd10447d74 Mon Sep 17 00:00:00 2001 From: Stephan Fitzpatrick Date: Mon, 9 Nov 2026 11:17:50 +0000 Subject: [PATCH 3/3] isb 4.1.0 Claude-Session: https://claude.ai/code/session_01LJ21XsZE86ywhpFMvYVWmE --- Cargo.lock | 14 +++++++------- Cargo.toml | 14 +++++++------- docs/reference/rpc.md | 4 ++-- sdk/python/pyproject.toml | 2 +- sdk/python/src/isb/__init__.py | 2 +- sdk/python/tests/test_unit.py | 6 +++--- sdk/python/uv.lock | 2 +- sdk/typescript/bun.lock | 4 ++-- sdk/typescript/npm/linux-arm64/package.json | 2 +- sdk/typescript/npm/linux-x64/package.json | 2 +- sdk/typescript/package.json | 6 +++--- 11 files changed, 29 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 310358cc..5f8a3de1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1329,7 +1329,7 @@ checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] name = "isb" -version = "4.0.1" +version = "4.1.0" dependencies = [ "age", "clap", @@ -1352,7 +1352,7 @@ dependencies = [ [[package]] name = "isb-apps" -version = "4.0.1" +version = "4.1.0" dependencies = [ "age", "base64 0.23.1", @@ -1376,7 +1376,7 @@ dependencies = [ [[package]] name = "isb-core" -version = "4.0.1" +version = "4.1.0" dependencies = [ "age", "base64 0.23.1", @@ -1403,7 +1403,7 @@ dependencies = [ [[package]] name = "isb-daemon" -version = "4.0.1" +version = "4.1.0" dependencies = [ "age", "httparse", @@ -1423,7 +1423,7 @@ dependencies = [ [[package]] name = "isb-egress" -version = "4.0.1" +version = "4.1.0" dependencies = [ "base64 0.23.1", "httparse", @@ -1436,7 +1436,7 @@ dependencies = [ [[package]] name = "isb-server" -version = "4.0.1" +version = "4.1.0" dependencies = [ "argon2", "base64 0.23.1", @@ -1459,7 +1459,7 @@ dependencies = [ [[package]] name = "isb-tui" -version = "4.0.1" +version = "4.1.0" dependencies = [ "isb-core", "libc", diff --git a/Cargo.toml b/Cargo.toml index 471f807a..60b0c108 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,7 +55,7 @@ members = ["crates/*"] default-members = [".", "crates/*"] [workspace.package] -version = "4.0.1" +version = "4.1.0" edition = "2024" rust-version = "1.89" license = "LicenseRef-EXA-SL-1.0" @@ -63,12 +63,12 @@ repository = "https://github.com/execution-associates/isb" homepage = "https://github.com/execution-associates/isb" [workspace.dependencies] -isb-core = { path = "crates/isb-core", version = "=4.0.1" } -isb-server = { path = "crates/isb-server", version = "=4.0.1" } -isb-apps = { path = "crates/isb-apps", version = "=4.0.1" } -isb-daemon = { path = "crates/isb-daemon", version = "=4.0.1" } -isb-tui = { path = "crates/isb-tui", version = "=4.0.1" } -isb-egress = { path = "crates/isb-egress", version = "=4.0.1" } +isb-core = { path = "crates/isb-core", version = "=4.1.0" } +isb-server = { path = "crates/isb-server", version = "=4.1.0" } +isb-apps = { path = "crates/isb-apps", version = "=4.1.0" } +isb-daemon = { path = "crates/isb-daemon", version = "=4.1.0" } +isb-tui = { path = "crates/isb-tui", version = "=4.1.0" } +isb-egress = { path = "crates/isb-egress", version = "=4.1.0" } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/docs/reference/rpc.md b/docs/reference/rpc.md index 6bda1a7e..be32b746 100644 --- a/docs/reference/rpc.md +++ b/docs/reference/rpc.md @@ -19,7 +19,7 @@ it; relative paths resolve against that directory. When it starts, the server writes one hello line: ```json -{"isb": "4.0.1", "protocol": 1} +{"isb": "4.1.0", "protocol": 1} ``` A client checks `protocol` and refuses versions it does not know. @@ -205,7 +205,7 @@ All take `files` (default `./isb.yaml` plus `./isb.override.yaml` if present), ## Example session ```text -<- {"isb":"4.0.1","protocol":1} +<- {"isb":"4.1.0","protocol":1} -> {"id":1,"method":"sandbox.ensure","params":{"spec":{"name":"web","image":"dev-base","cpus":2}}} <- {"id":1,"event":"progress","data":"web: creating from dev-base"} <- {"id":1,"event":"progress","data":"web: starting"} diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 21867996..42656984 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "isb-sdk" -version = "4.0.1" +version = "4.1.0" description = "Python SDK for isb: declarative incus sandboxes (containers and VMs)" readme = "README.md" license = "MIT AND LicenseRef-EXA-SL-1.0" diff --git a/sdk/python/src/isb/__init__.py b/sdk/python/src/isb/__init__.py index 7e7781a8..64f2ffc5 100644 --- a/sdk/python/src/isb/__init__.py +++ b/sdk/python/src/isb/__init__.py @@ -62,7 +62,7 @@ async def main(): from ._util import Duration from .volumes import Volumes -__version__ = "4.0.1" +__version__ = "4.1.0" __all__ = [ "PROTOCOL", diff --git a/sdk/python/tests/test_unit.py b/sdk/python/tests/test_unit.py index 33309462..7084020c 100644 --- a/sdk/python/tests/test_unit.py +++ b/sdk/python/tests/test_unit.py @@ -222,7 +222,7 @@ async def test_death_fails_pending_requests(self) -> None: path = fake_server( self.tmp.name, """\ - echo '{"isb":"4.0.1","protocol":1}' + echo '{"isb":"4.1.0","protocol":1}' read line echo "boom" >&2 exit 3 @@ -243,7 +243,7 @@ async def test_close_fails_pending(self) -> None: path = fake_server( self.tmp.name, """\ - echo '{"isb":"4.0.1","protocol":1}' + echo '{"isb":"4.1.0","protocol":1}' cat >/dev/null """, ) @@ -261,7 +261,7 @@ async def test_events_and_out_of_order_replies(self) -> None: path = fake_server( self.tmp.name, """\ - echo '{"isb":"4.0.1","protocol":1}' + echo '{"isb":"4.1.0","protocol":1}' read a read b echo 'not json' diff --git a/sdk/python/uv.lock b/sdk/python/uv.lock index b2abc01d..2ef5e40a 100644 --- a/sdk/python/uv.lock +++ b/sdk/python/uv.lock @@ -72,7 +72,7 @@ wheels = [ [[package]] name = "isb-sdk" -version = "4.0.1" +version = "4.1.0" source = { editable = "." } [package.dev-dependencies] diff --git a/sdk/typescript/bun.lock b/sdk/typescript/bun.lock index 7d67ab47..a9b03bd7 100644 --- a/sdk/typescript/bun.lock +++ b/sdk/typescript/bun.lock @@ -12,8 +12,8 @@ "typescript": "^5.9.0", }, "optionalDependencies": { - "@execution-associates/isb-linux-arm64": "4.0.1", - "@execution-associates/isb-linux-x64": "4.0.1", + "@execution-associates/isb-linux-arm64": "4.1.0", + "@execution-associates/isb-linux-x64": "4.1.0", }, }, }, diff --git a/sdk/typescript/npm/linux-arm64/package.json b/sdk/typescript/npm/linux-arm64/package.json index f7e2e867..48bc72b8 100644 --- a/sdk/typescript/npm/linux-arm64/package.json +++ b/sdk/typescript/npm/linux-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@execution-associates/isb-linux-arm64", - "version": "4.0.1", + "version": "4.1.0", "description": "The static isb binary for Linux arm64 (aarch64-unknown-linux-musl), used by @execution-associates/isb", "license": "SEE LICENSE IN LICENSE", "repository": { diff --git a/sdk/typescript/npm/linux-x64/package.json b/sdk/typescript/npm/linux-x64/package.json index 41859430..ee087c3b 100644 --- a/sdk/typescript/npm/linux-x64/package.json +++ b/sdk/typescript/npm/linux-x64/package.json @@ -1,6 +1,6 @@ { "name": "@execution-associates/isb-linux-x64", - "version": "4.0.1", + "version": "4.1.0", "description": "The static isb binary for Linux x64 (x86_64-unknown-linux-musl), used by @execution-associates/isb", "license": "SEE LICENSE IN LICENSE", "repository": { diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json index a93956f7..b6ad358b 100644 --- a/sdk/typescript/package.json +++ b/sdk/typescript/package.json @@ -1,6 +1,6 @@ { "name": "@execution-associates/isb", - "version": "4.0.1", + "version": "4.1.0", "description": "TypeScript SDK for isb: declarative incus sandboxes (containers and VMs)", "license": "MIT", "type": "module", @@ -51,8 +51,8 @@ "prepack": "bun run build" }, "optionalDependencies": { - "@execution-associates/isb-linux-arm64": "4.0.1", - "@execution-associates/isb-linux-x64": "4.0.1" + "@execution-associates/isb-linux-arm64": "4.1.0", + "@execution-associates/isb-linux-x64": "4.1.0" }, "devDependencies": { "@biomejs/biome": "^2.2.0",