From 8f3edcd254d99af287735e4e723994655e01922f Mon Sep 17 00:00:00 2001 From: Stephan Fitzpatrick Date: Mon, 9 Nov 2026 12:03:41 +0000 Subject: [PATCH] isb 4.1.1: report a NIC incus lost the host side of, pin it after a restart 4.1.0's first pass on titan could not pin analytics-capture in org exa: incus had lost its volatile.eth0.host_name, so changing its running NIC fails ("Unknown or missing host side veth device"), and an org project (restricted.containers.lowlevel: block) may not set the key back. The check retried the PUT every minute. Now such an instance is not retried: net.failed says what happened, that isb pins it after its next restart (a restart gives the key back), and how to pin it without one. capture itself was pinned in place by hand that way (same PID and address, port 3000 answering). Tests: a unit test, and tests/ipam.rs loses an instance's host_name, sees it reported, restarts it and sees it pinned; the test now holds whether or not an isb serve on the host checks the test org first. Claude-Session: https://claude.ai/code/session_01LJ21XsZE86ywhpFMvYVWmE --- Cargo.lock | 14 +++---- Cargo.toml | 14 +++---- crates/isb-core/src/ipam.rs | 33 ++++++++++++++++ crates/isb-core/src/ipam/tests.rs | 37 +++++++++++++++++- docs/concepts/orgs.md | 5 +++ docs/operations/troubleshooting.md | 1 + docs/reference/rpc.md | 4 +- sdk/python/pyproject.toml | 2 +- sdk/python/src/isb/__init__.py | 2 +- sdk/python/tests/test_unit.py | 6 +-- sdk/python/uv.lock | 2 +- sdk/typescript/bun.lock | 4 +- sdk/typescript/npm/linux-arm64/package.json | 2 +- sdk/typescript/npm/linux-x64/package.json | 2 +- sdk/typescript/package.json | 6 +-- tests/ipam.rs | 43 +++++++++++++++++++-- 16 files changed, 143 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5f8a3de..b89bfea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1329,7 +1329,7 @@ checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] name = "isb" -version = "4.1.0" +version = "4.1.1" dependencies = [ "age", "clap", @@ -1352,7 +1352,7 @@ dependencies = [ [[package]] name = "isb-apps" -version = "4.1.0" +version = "4.1.1" dependencies = [ "age", "base64 0.23.1", @@ -1376,7 +1376,7 @@ dependencies = [ [[package]] name = "isb-core" -version = "4.1.0" +version = "4.1.1" dependencies = [ "age", "base64 0.23.1", @@ -1403,7 +1403,7 @@ dependencies = [ [[package]] name = "isb-daemon" -version = "4.1.0" +version = "4.1.1" dependencies = [ "age", "httparse", @@ -1423,7 +1423,7 @@ dependencies = [ [[package]] name = "isb-egress" -version = "4.1.0" +version = "4.1.1" dependencies = [ "base64 0.23.1", "httparse", @@ -1436,7 +1436,7 @@ dependencies = [ [[package]] name = "isb-server" -version = "4.1.0" +version = "4.1.1" dependencies = [ "argon2", "base64 0.23.1", @@ -1459,7 +1459,7 @@ dependencies = [ [[package]] name = "isb-tui" -version = "4.1.0" +version = "4.1.1" dependencies = [ "isb-core", "libc", diff --git a/Cargo.toml b/Cargo.toml index 60b0c10..3037d44 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,7 +55,7 @@ members = ["crates/*"] default-members = [".", "crates/*"] [workspace.package] -version = "4.1.0" +version = "4.1.1" edition = "2024" rust-version = "1.89" license = "LicenseRef-EXA-SL-1.0" @@ -63,12 +63,12 @@ repository = "https://github.com/execution-associates/isb" homepage = "https://github.com/execution-associates/isb" [workspace.dependencies] -isb-core = { path = "crates/isb-core", version = "=4.1.0" } -isb-server = { path = "crates/isb-server", version = "=4.1.0" } -isb-apps = { path = "crates/isb-apps", version = "=4.1.0" } -isb-daemon = { path = "crates/isb-daemon", version = "=4.1.0" } -isb-tui = { path = "crates/isb-tui", version = "=4.1.0" } -isb-egress = { path = "crates/isb-egress", version = "=4.1.0" } +isb-core = { path = "crates/isb-core", version = "=4.1.1" } +isb-server = { path = "crates/isb-server", version = "=4.1.1" } +isb-apps = { path = "crates/isb-apps", version = "=4.1.1" } +isb-daemon = { path = "crates/isb-daemon", version = "=4.1.1" } +isb-tui = { path = "crates/isb-tui", version = "=4.1.1" } +isb-egress = { path = "crates/isb-egress", version = "=4.1.1" } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/crates/isb-core/src/ipam.rs b/crates/isb-core/src/ipam.rs index 73444b7..74ef5e2 100644 --- a/crates/isb-core/src/ipam.rs +++ b/crates/isb-core/src/ipam.rs @@ -117,6 +117,10 @@ pub struct Claim { pub recorded: Option, /// Its IPv4 addresses on the bridge's subnet (running instances only). pub addrs: Vec, + /// Running, but incus has lost the NIC's host side name + /// (`volatile..host_name`): incus cannot change the NIC until the + /// instance restarts, and an org project may not set the name back. + pub veth_lost: bool, pub running: bool, /// The NIC as the instance has it (expanded: from a profile or its own). pub props: Props, @@ -168,6 +172,7 @@ fn claim(i: &Value, project: &str, dev: &str, d: &Value, subnet: &Subnet) -> Cla Vec::new() }, running, + veth_lost: running && cfg[format!("volatile.{dev}.host_name")].is_null(), props: d .as_object() .map(|m| { @@ -309,6 +314,10 @@ pub fn plan( } continue; } + if c.veth_lost { + findings.push(veth_lost(c)); + continue; + } let ip = match (c.running, c.addrs.as_slice()) { // Its own address, unless another instance holds it too (a // conflict, reported above): pinning it would keep it. @@ -347,6 +356,30 @@ pub fn plan( (fixes, findings) } +/// An unpinned instance whose NIC incus cannot change while it runs +/// (2026-11-09, analytics-capture in org exa): what to do about it. +fn veth_lost(c: &Claim) -> Finding { + let (i, d, p) = (&c.instance, &c.device, &c.project); + Finding::about( + c, + format!("veth {p} {i} {d}"), + format!( + "{i}'s {d} cannot be pinned while it runs: incus lost its host side name \ + (volatile.{d}.host_name), and an org project may not set it. isb pins it after its \ + next restart; until then {} can be given to another instance once its lease lapses. \ + To pin it without a restart: incus project set {p} restricted.containers.lowlevel=allow, \ + incus config set {i} volatile.{d}.host_name=, incus project unset {p} \ + restricted.containers.lowlevel; the next check pins it", + if c.addrs.is_empty() { + "its address".to_string() + } else { + join(&c.addrs) + } + ), + ) +} + fn join(ips: &[Ipv4Addr]) -> String { ips.iter() .map(Ipv4Addr::to_string) diff --git a/crates/isb-core/src/ipam/tests.rs b/crates/isb-core/src/ipam/tests.rs index fafb682..1d7ee8c 100644 --- a/crates/isb-core/src/ipam/tests.rs +++ b/crates/isb-core/src/ipam/tests.rs @@ -30,7 +30,8 @@ fn inst(name: &str, pin: Option<&str>, addr: Option<&str>, created: &str) -> Val json!({ "name": name, "project": "isb-exa", "created_at": created, "status": if addr.is_some() { "Running" } else { "Stopped" }, - "config": {"volatile.eth0.hwaddr": "10:66:6a:00:00:01", "user.isb.stack": "analytics", + "config": {"volatile.eth0.hwaddr": "10:66:6a:00:00:01", "volatile.eth0.host_name": "veth1234", + "user.isb.stack": "analytics", "user.isb.service": "postgres"}, "expanded_devices": {"root": {"type": "disk"}, "eth0": nic}, "state": state, @@ -285,3 +286,37 @@ fn the_diff_ignores_and_never_prunes_isbs_pin() { assert!(!is_pin("eth0", &actual, &Props::new())); assert!(!is_pin("eth0", &desired, &config)); } + +#[test] +fn a_nic_incus_lost_the_host_side_of_is_reported_not_retried() { + // 2026-11-09: analytics-capture had no volatile.eth0.host_name, and + // incus refused to change its NIC ("Unknown or missing host side veth + // device") on every pass. + let mut v = inst("capture", None, Some("10.83.115.175"), "1"); + v["config"] + .as_object_mut() + .unwrap() + .remove("volatile.eth0.host_name"); + let (fixes, found) = plan_of(std::slice::from_ref(&v)); + assert!(fixes.is_empty(), "{fixes:?}"); + assert_eq!(found.len(), 1); + assert!(found[0].key.starts_with("veth "), "{}", found[0].key); + assert!( + found[0].message.contains("10.83.115.175"), + "{}", + found[0].message + ); + assert!(found[0].message.contains("restricted.containers.lowlevel")); + // Stopped, it has no host side at all, and is pinned as any other. + v["status"] = json!("Stopped"); + v["state"] = json!(null); + let (fixes, found) = plan_of(&[v]); + assert!(found.is_empty(), "{found:?}"); + assert!(matches!(fixes.as_slice(), [Fix::Pin { .. }]), "{fixes:?}"); + // With the name back, it is pinned in place. + let (fixes, _) = plan_of(&[inst("capture", None, Some("10.83.115.175"), "1")]); + assert!( + matches!(fixes.as_slice(), [Fix::Pin { running: true, .. }]), + "{fixes:?}" + ); +} diff --git a/docs/concepts/orgs.md b/docs/concepts/orgs.md index f1c292d..d66280c 100644 --- a/docs/concepts/orgs.md +++ b/docs/concepts/orgs.md @@ -232,6 +232,11 @@ instance has another one, or when it cannot pin one; it moves no instance itself. An instance with an [egress policy](../guides/egress.md) has a bridge of its own and is left to DHCP. +incus cannot change a running instance's NIC once it has lost the NIC's +host side name (`volatile.eth0.host_name`), and an org project may not set +that key. isb reports such an instance (`net.failed`) and pins it after its +next restart; the message says how to pin it without one. + A host with a default-deny firewall (ufw) needs `sudo isb host setup` once, so org bridges get DHCP, DNS and egress while staying apart from each other: see [Host firewall](../operations/host-setup.md#host-firewall). diff --git a/docs/operations/troubleshooting.md b/docs/operations/troubleshooting.md index 9f5c600..a78cd82 100644 --- a/docs/operations/troubleshooting.md +++ b/docs/operations/troubleshooting.md @@ -83,6 +83,7 @@ More in [isb on macOS](../getting-started/macos.md). | Published ports pause during a daemon restart | The load balancer and ingress live in the daemon; apps do not. | Expected; they return within seconds. | | Apps cannot reach each other by name | Service names need an org with its own network: the `default` org on a host whose incus `default` project held workloads first has none. On a fresh org, the host may lack the service-name directory. | Use an org of its own; run `sudo isb host setup`, after which a running `isb serve` turns names on within a minute (`isb org create ORG` again does it without the daemon) ([Setting up a host](host-setup.md#service-names)). | | `net.failed`: an address is held by two instances | Two instances on the org bridge have one address, so traffic for one reaches the other. isb moves neither. | Pin the one that should move elsewhere (`incus config device set INSTANCE eth0 ipv4.address=FREE`) and restart it ([Addresses](../concepts/orgs.md#addresses)). | +| `net.failed`: an instance `cannot be pinned while it runs` | incus lost the host side name of its NIC (`volatile.eth0.host_name`), so it cannot change the NIC until a restart, and an org project may not set the key. | Restart it, or without a restart: `incus project set isb-ORG restricted.containers.lowlevel=allow`, `incus config set INSTANCE volatile.eth0.host_name=VETH` (the `host_name` that `incus query /1.0/instances/INSTANCE/state?project=isb-ORG` gives eth0), `incus project unset isb-ORG restricted.containers.lowlevel`; the next check pins it ([Addresses](../concepts/orgs.md#addresses)). | | A new org's instances get no address or no internet | A default-deny firewall drops DHCP, DNS and forwarding on new bridges. | `sudo isb host setup` ([Host firewall](host-setup.md#host-firewall)). | | A builder image build stops with `got no IPv4 address on incusbr0` | The same firewall drops DHCP and forwarding on incus' own bridge, where those builds run. | `sudo isb host setup`, then build again. | | `isb serve` or `isb host setup` warns that incus is too old for OCI images | The incus package is older than 6.3 (Ubuntu 24.04's is 6.0), so `docker:` images cannot run. | Install incus from the Zabbly stable repository ([Install isb](../getting-started/install.md#1-incus)). | diff --git a/docs/reference/rpc.md b/docs/reference/rpc.md index be32b74..cf448bb 100644 --- a/docs/reference/rpc.md +++ b/docs/reference/rpc.md @@ -19,7 +19,7 @@ it; relative paths resolve against that directory. When it starts, the server writes one hello line: ```json -{"isb": "4.1.0", "protocol": 1} +{"isb": "4.1.1", "protocol": 1} ``` A client checks `protocol` and refuses versions it does not know. @@ -205,7 +205,7 @@ All take `files` (default `./isb.yaml` plus `./isb.override.yaml` if present), ## Example session ```text -<- {"isb":"4.1.0","protocol":1} +<- {"isb":"4.1.1","protocol":1} -> {"id":1,"method":"sandbox.ensure","params":{"spec":{"name":"web","image":"dev-base","cpus":2}}} <- {"id":1,"event":"progress","data":"web: creating from dev-base"} <- {"id":1,"event":"progress","data":"web: starting"} diff --git a/sdk/python/pyproject.toml b/sdk/python/pyproject.toml index 4265698..f86d103 100644 --- a/sdk/python/pyproject.toml +++ b/sdk/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "isb-sdk" -version = "4.1.0" +version = "4.1.1" description = "Python SDK for isb: declarative incus sandboxes (containers and VMs)" readme = "README.md" license = "MIT AND LicenseRef-EXA-SL-1.0" diff --git a/sdk/python/src/isb/__init__.py b/sdk/python/src/isb/__init__.py index 64f2ffc..dadd65e 100644 --- a/sdk/python/src/isb/__init__.py +++ b/sdk/python/src/isb/__init__.py @@ -62,7 +62,7 @@ async def main(): from ._util import Duration from .volumes import Volumes -__version__ = "4.1.0" +__version__ = "4.1.1" __all__ = [ "PROTOCOL", diff --git a/sdk/python/tests/test_unit.py b/sdk/python/tests/test_unit.py index 7084020..4c06942 100644 --- a/sdk/python/tests/test_unit.py +++ b/sdk/python/tests/test_unit.py @@ -222,7 +222,7 @@ async def test_death_fails_pending_requests(self) -> None: path = fake_server( self.tmp.name, """\ - echo '{"isb":"4.1.0","protocol":1}' + echo '{"isb":"4.1.1","protocol":1}' read line echo "boom" >&2 exit 3 @@ -243,7 +243,7 @@ async def test_close_fails_pending(self) -> None: path = fake_server( self.tmp.name, """\ - echo '{"isb":"4.1.0","protocol":1}' + echo '{"isb":"4.1.1","protocol":1}' cat >/dev/null """, ) @@ -261,7 +261,7 @@ async def test_events_and_out_of_order_replies(self) -> None: path = fake_server( self.tmp.name, """\ - echo '{"isb":"4.1.0","protocol":1}' + echo '{"isb":"4.1.1","protocol":1}' read a read b echo 'not json' diff --git a/sdk/python/uv.lock b/sdk/python/uv.lock index 2ef5e40..9b3b40e 100644 --- a/sdk/python/uv.lock +++ b/sdk/python/uv.lock @@ -72,7 +72,7 @@ wheels = [ [[package]] name = "isb-sdk" -version = "4.1.0" +version = "4.1.1" source = { editable = "." } [package.dev-dependencies] diff --git a/sdk/typescript/bun.lock b/sdk/typescript/bun.lock index a9b03bd..8abe00d 100644 --- a/sdk/typescript/bun.lock +++ b/sdk/typescript/bun.lock @@ -12,8 +12,8 @@ "typescript": "^5.9.0", }, "optionalDependencies": { - "@execution-associates/isb-linux-arm64": "4.1.0", - "@execution-associates/isb-linux-x64": "4.1.0", + "@execution-associates/isb-linux-arm64": "4.1.1", + "@execution-associates/isb-linux-x64": "4.1.1", }, }, }, diff --git a/sdk/typescript/npm/linux-arm64/package.json b/sdk/typescript/npm/linux-arm64/package.json index 48bc72b..34f0b12 100644 --- a/sdk/typescript/npm/linux-arm64/package.json +++ b/sdk/typescript/npm/linux-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@execution-associates/isb-linux-arm64", - "version": "4.1.0", + "version": "4.1.1", "description": "The static isb binary for Linux arm64 (aarch64-unknown-linux-musl), used by @execution-associates/isb", "license": "SEE LICENSE IN LICENSE", "repository": { diff --git a/sdk/typescript/npm/linux-x64/package.json b/sdk/typescript/npm/linux-x64/package.json index ee087c3..c7a45e2 100644 --- a/sdk/typescript/npm/linux-x64/package.json +++ b/sdk/typescript/npm/linux-x64/package.json @@ -1,6 +1,6 @@ { "name": "@execution-associates/isb-linux-x64", - "version": "4.1.0", + "version": "4.1.1", "description": "The static isb binary for Linux x64 (x86_64-unknown-linux-musl), used by @execution-associates/isb", "license": "SEE LICENSE IN LICENSE", "repository": { diff --git a/sdk/typescript/package.json b/sdk/typescript/package.json index b6ad358..619fab1 100644 --- a/sdk/typescript/package.json +++ b/sdk/typescript/package.json @@ -1,6 +1,6 @@ { "name": "@execution-associates/isb", - "version": "4.1.0", + "version": "4.1.1", "description": "TypeScript SDK for isb: declarative incus sandboxes (containers and VMs)", "license": "MIT", "type": "module", @@ -51,8 +51,8 @@ "prepack": "bun run build" }, "optionalDependencies": { - "@execution-associates/isb-linux-arm64": "4.1.0", - "@execution-associates/isb-linux-x64": "4.1.0" + "@execution-associates/isb-linux-arm64": "4.1.1", + "@execution-associates/isb-linux-x64": "4.1.1" }, "devDependencies": { "@biomejs/biome": "^2.2.0", diff --git a/tests/ipam.rs b/tests/ipam.rs index 27ea761..db3fc15 100644 --- a/tests/ipam.rs +++ b/tests/ipam.rs @@ -8,7 +8,8 @@ //! Gated like the other integration tests (`ISB_INTEGRATION=1`; build in a //! sandbox with `cargo test --no-run`, run the binary on the host). Works in //! the test org (`common::test_org`), removed afterwards, pass or fail; the -//! check is held to that org's bridge. +//! check is held to that org's bridge. An `isb serve` on the host checks +//! the test org too, so each step asserts what holds whichever ran first. #[allow(dead_code, reason = "each test binary uses some of the shared helpers")] mod common; @@ -19,12 +20,13 @@ use std::time::{Duration, Instant}; use isb::Client; use serde_json::{Value, json}; -const NAMES: [&str; 5] = [ +const NAMES: [&str; 6] = [ "isb-test-ip-a", "isb-test-ip-b", "isb-test-ip-raw", "isb-test-ip-dup", "isb-test-ip-cold", + "isb-test-ip-lost", ]; struct Rm(Client); @@ -148,9 +150,8 @@ fn addresses_on_an_org_bridge_are_pinned_and_checked() { assert!(plan.is_noop(), "{plan:?}"); // Behind isb's back: pinned in place by the check, same address, same - // process. + // process. (A daemon running on the host may pin it first.) create_raw(&oc, "isb-test-ip-raw", json!({}), true); - assert_eq!(pin(&oc, "isb-test-ip-raw"), None); let (raw, pid) = address(&oc, "isb-test-ip-raw"); let found = check(&base, &oc); assert!(found.is_empty(), "{found:?}"); @@ -198,4 +199,38 @@ fn addresses_on_an_org_bridge_are_pinned_and_checked() { ); assert_eq!(pin(&oc, "isb-test-ip-b").as_ref(), Some(&b)); assert_eq!(address(&oc, "isb-test-ip-b").0, b); + + lost_host_name(&base, &oc); +} + +/// incus lost the host side name of a running instance's NIC (as +/// analytics-capture's on 2026-11-09): reported, not pinned; pinned in +/// place once a restart gives it back. +fn lost_host_name(base: &Client, oc: &Client) { + create_raw(oc, "isb-test-ip-lost", json!({}), true); + isb::sandbox::update_instance(oc, "isb-test-ip-lost", "lose host_name", &mut |c, _| { + c.remove("volatile.eth0.host_name"); + Ok(()) + }) + .unwrap(); + address(oc, "isb-test-ip-lost"); + let found = check(base, oc); + let lost: Vec<_> = found + .iter() + .filter(|f| f.instance == "isb-test-ip-lost") + .collect(); + assert_eq!(lost.len(), 1, "{found:?}"); + assert!(lost[0].key.starts_with("veth "), "{lost:?}"); + assert_eq!(pin(oc, "isb-test-ip-lost"), None); + isb::Sandbox::get(oc, "isb-test-ip-lost") + .unwrap() + .restart() + .unwrap(); + let (ip, _) = address(oc, "isb-test-ip-lost"); + let found = check(base, oc); + assert!( + found.iter().all(|f| f.instance != "isb-test-ip-lost"), + "{found:?}" + ); + assert_eq!(pin(oc, "isb-test-ip-lost"), Some(ip)); }