[WIP] Per-router V2Bs

Cargo.toml

@@ -81,7 +81,7 @@ version_check = "0.9"
 zerocopy = { version = "0.8", features = ["derive"] }
 zone = { git = "https://github.com/oxidecomputer/zone" }
 ztest = { git = "https://github.com/oxidecomputer/falcon", branch = "main" }
-poptrie = { git = "https://github.com/oxidecomputer/poptrie", branch = "main" }
+poptrie = { git = "https://github.com/nicolaskagami/poptrie", branch = "main" }
 
 [profile.dev]
 opt-level = 1

bin/opteadm/Cargo.toml

@@ -23,6 +23,7 @@ postcard.workspace = true
 serde.workspace = true
 tabwriter.workspace = true
 thiserror.workspace = true
+uuid.workspace = true
 
 [build-dependencies]
 anyhow.workspace = true

bin/opteadm/src/bin/opteadm.rs

@@ -36,6 +36,7 @@ use oxide_vpc::api::DEFAULT_MULTICAST_VNI;
 use oxide_vpc::api::DelRouterEntryReq;
 use oxide_vpc::api::DelRouterEntryResp;
 use oxide_vpc::api::DhcpCfg;
+use oxide_vpc::api::DumpRouterListReq;
 use oxide_vpc::api::ExternalIpCfg;
 use oxide_vpc::api::Filters as FirewallFilters;
 use oxide_vpc::api::FirewallAction;
@@ -56,13 +57,16 @@ use oxide_vpc::api::RemFwRuleReq;
 use oxide_vpc::api::RemoveCidrResp;
 use oxide_vpc::api::Replication;
 use oxide_vpc::api::RouterClass;
+use oxide_vpc::api::RouterId;
+use oxide_vpc::api::RouterList;
 use oxide_vpc::api::RouterTarget;
 use oxide_vpc::api::SNat4Cfg;
 use oxide_vpc::api::SNat6Cfg;
 use oxide_vpc::api::SetExternalIpsReq;
 use oxide_vpc::api::SetFwRulesReq;
 use oxide_vpc::api::SetMcast2PhysReq;
 use oxide_vpc::api::SetMcastForwardingReq;
+use oxide_vpc::api::SetRouterListReq;
 use oxide_vpc::api::SetVirt2BoundaryReq;
 use oxide_vpc::api::SetVirt2PhysReq;
 use oxide_vpc::api::SourceFilter;
@@ -78,6 +82,7 @@ use std::io;
 use std::io::Write;
 use std::str::FromStr;
 use tabwriter::TabWriter;
+use uuid::Uuid;
 
 /// Administer the Oxide Packet Transformation Engine (OPTE)
 #[derive(Debug, Parser)]
@@ -240,10 +245,39 @@ enum Command {
     },
 
     /// Set a virtual-to-boundary mapping
-    SetV2B { prefix: IpCidr, tunnel_endpoint: Vec<Ipv6Addr> },
+    SetV2B {
+        prefix: IpCidr,
+        tunnel_endpoint: Vec<Ipv6Addr>,
+        /// The router whose table to modify (default router if omitted)
+        #[arg(long)]
+        router_id: Option<Uuid>,
+    },
 
     /// Clear a virtual-to-boundary mapping
-    ClearV2B { prefix: IpCidr, tunnel_endpoint: Vec<Ipv6Addr> },
+    ClearV2B {
+        prefix: IpCidr,
+        tunnel_endpoint: Vec<Ipv6Addr>,
+        /// The router whose table to modify (default router if omitted)
+        #[arg(long)]
+        router_id: Option<Uuid>,
+    },
+
+    /// Replace a port's prioritized router list for boundary TEP
+    /// selection.
+    ///
+    /// Entries are `<priority>=<router uuid>` or `<priority>=default`,
+    /// e.g. `set-router-list -p opte0 10=6fe93b02-… 1000=default`.
+    SetRouterList {
+        #[arg(short)]
+        port: String,
+        entries: Vec<RouterListEntryArg>,
+    },
+
+    /// Show a port's prioritized router list.
+    DumpRouterList {
+        #[arg(short)]
+        port: String,
+    },
 
     /// Set a multicast-to-physical (M2P) mapping
     ///
@@ -470,6 +504,35 @@ impl From<Filters> for FirewallFilters {
     }
 }
 
+/// One router-list entry: `<priority>=<router uuid>` or
+/// `<priority>=default`.
+#[derive(Clone, Debug)]
+struct RouterListEntryArg {
+    priority: u16,
+    router: RouterId,
+}
+
+impl FromStr for RouterListEntryArg {
+    type Err = String;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        let (prio, rtr) = s.split_once('=').ok_or_else(|| {
+            format!("expected <priority>=<uuid|default>, got {s}")
+        })?;
+        let priority = prio
+            .parse::<u16>()
+            .map_err(|e| format!("bad priority {prio}: {e}"))?;
+        let router = match rtr {
+            "default" => None,
+            uuid => Some(
+                uuid.parse::<Uuid>()
+                    .map_err(|e| format!("bad router id {uuid}: {e}"))?,
+            ),
+        };
+        Ok(Self { priority, router })
+    }
+}
+
 #[derive(Debug, Parser)]
 struct RouterRule {
     /// The OPTE port to which the route change is applied.
@@ -912,30 +975,52 @@ fn main() -> anyhow::Result<()> {
             hdl.clear_v2p(&req)?;
         }
 
-        Command::SetV2B { prefix, tunnel_endpoint } => {
+        Command::SetV2B { prefix, tunnel_endpoint, router_id } => {
             let tep = tunnel_endpoint
                 .into_iter()
                 .map(|ip| TunnelEndpoint {
                     ip,
                     vni: Vni::new(BOUNDARY_SERVICES_VNI).unwrap(),
                 })
                 .collect();
-            let req = SetVirt2BoundaryReq { vip: prefix, tep };
+            let req = SetVirt2BoundaryReq { router_id, vip: prefix, tep };
             hdl.set_v2b(&req)?;
         }
 
-        Command::ClearV2B { prefix, tunnel_endpoint } => {
+        Command::ClearV2B { prefix, tunnel_endpoint, router_id } => {
             let tep = tunnel_endpoint
                 .into_iter()
                 .map(|ip| TunnelEndpoint {
                     ip,
                     vni: Vni::new(BOUNDARY_SERVICES_VNI).unwrap(),
                 })
                 .collect();
-            let req = ClearVirt2BoundaryReq { vip: prefix, tep };
+            let req = ClearVirt2BoundaryReq { router_id, vip: prefix, tep };
             hdl.clear_v2b(&req)?;
         }
 
+        Command::SetRouterList { port, entries } => {
+            let list = RouterList::new(
+                entries.into_iter().map(|e| (e.priority, e.router)),
+            )
+            .map_err(|e| anyhow::anyhow!("invalid router list: {e}"))?;
+            let req = SetRouterListReq { port_name: port, list };
+            hdl.set_router_list(&req)?;
+        }
+
+        Command::DumpRouterList { port } => {
+            let resp =
+                hdl.dump_router_list(&DumpRouterListReq { port_name: port })?;
+            println!("PRIORITY\tROUTER");
+            for (prio, rtr) in resp.list.entries() {
+                let rtr = match rtr {
+                    None => "default".to_string(),
+                    Some(id) => id.to_string(),
+                };
+                println!("{prio}\t{rtr}");
+            }
+        }
+
         Command::SetM2P { group, underlay } => {
             let req = SetMcast2PhysReq { group, underlay };
             hdl.set_m2p(&req)?;
@@ -1123,3 +1208,29 @@ fn main() -> anyhow::Result<()> {
 
     Ok(())
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn router_list_entry_parses() {
+        let e: RouterListEntryArg = "1000=default".parse().unwrap();
+        assert_eq!(e.priority, 1000);
+        assert_eq!(e.router, None);
+
+        let id = "6fe93b02-9b02-4c40-9dbd-33b50b3ba9f4";
+        let e: RouterListEntryArg = format!("10={id}").parse().unwrap();
+        assert_eq!(e.priority, 10);
+        assert_eq!(e.router, Some(id.parse().unwrap()));
+    }
+
+    #[test]
+    fn router_list_entry_rejects_bad_input() {
+        assert!("10".parse::<RouterListEntryArg>().is_err());
+        assert!("=default".parse::<RouterListEntryArg>().is_err());
+        assert!("banana=default".parse::<RouterListEntryArg>().is_err());
+        assert!("70000=default".parse::<RouterListEntryArg>().is_err());
+        assert!("10=not-a-uuid".parse::<RouterListEntryArg>().is_err());
+    }
+}

crates/opte-api/Cargo.toml

@@ -15,6 +15,7 @@ illumos-sys-hdrs.workspace = true
 
 ingot.workspace = true
 ipnetwork = { workspace = true, optional = true }
+poptrie.workspace = true
 postcard.workspace = true
 serde.workspace = true
 

crates/opte-api/src/cmd.rs

@@ -116,6 +116,12 @@ pub enum OpteCmd {
     McastUnsubscribeAll = 108,
     /// Read out all M2P (multicast group -> underlay multicast) mappings.
     DumpMcast2Phys = 109,
+
+    /// Replace a port's prioritized router list for boundary TEP
+    /// selection.
+    SetRouterList = 110,
+    /// Read a port's prioritized router list.
+    DumpRouterList = 111,
 }
 
 impl TryFrom<c_int> for OpteCmd {
@@ -158,6 +164,8 @@ impl TryFrom<c_int> for OpteCmd {
             107 => Ok(Self::DumpMcastSubscriptions),
             108 => Ok(Self::McastUnsubscribeAll),
             109 => Ok(Self::DumpMcast2Phys),
+            110 => Ok(Self::SetRouterList),
+            111 => Ok(Self::DumpRouterList),
             _ => Err(()),
         }
     }

crates/opte-api/src/ip.rs

@@ -1346,6 +1346,18 @@ impl From<Ipv4Cidr> for ipnetwork::Ipv4Network {
     }
 }
 
+impl poptrie::Prefix for Ipv4Cidr {
+    type ADDRESS = u32;
+
+    fn address(&self) -> Self::ADDRESS {
+        u32::from_be_bytes(self.ip.inner)
+    }
+
+    fn prefix_length(&self) -> u8 {
+        self.prefix_len.val()
+    }
+}
+
 /// An IPv6 CIDR.
 #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
 pub struct Ipv6Cidr {
@@ -1533,6 +1545,18 @@ impl NetworkRepr<[u8; 16]> for Ipv6Addr {
     }
 }
 
+impl poptrie::Prefix for Ipv6Cidr {
+    type ADDRESS = u128;
+
+    fn address(&self) -> Self::ADDRESS {
+        u128::from_be_bytes(self.ip.bytes())
+    }
+
+    fn prefix_length(&self) -> u8 {
+        self.prefix_len.val()
+    }
+}
+
 #[cfg(test)]
 mod test {
     use super::*;

crates/opte-api/src/lib.rs

@@ -51,7 +51,7 @@ pub use ulp::*;
 ///
 /// We rely on CI and the check-api-version.sh script to verify that
 /// this number is incremented anytime the oxide-api code changes.
-pub const API_VERSION: u64 = 41;
+pub const API_VERSION: u64 = 42;
 
 /// Major version of the OPTE package.
 pub const MAJOR_VERSION: u64 = 0;

lib/opte-ioctl/src/lib.rs

@@ -42,6 +42,8 @@ use oxide_vpc::api::DetachSubnetResp;
 use oxide_vpc::api::DumpMcast2PhysResp;
 use oxide_vpc::api::DumpMcastForwardingResp;
 use oxide_vpc::api::DumpMcastSubscriptionsResp;
+use oxide_vpc::api::DumpRouterListReq;
+use oxide_vpc::api::DumpRouterListResp;
 use oxide_vpc::api::DumpVirt2BoundaryResp;
 use oxide_vpc::api::DumpVirt2PhysResp;
 use oxide_vpc::api::IpCidr;
@@ -56,6 +58,7 @@ use oxide_vpc::api::SetExternalIpsReq;
 use oxide_vpc::api::SetFwRulesReq;
 use oxide_vpc::api::SetMcast2PhysReq;
 use oxide_vpc::api::SetMcastForwardingReq;
+use oxide_vpc::api::SetRouterListReq;
 use oxide_vpc::api::SetVirt2BoundaryReq;
 use oxide_vpc::api::SetVirt2PhysReq;
 use oxide_vpc::api::VpcCfg;
@@ -252,6 +255,24 @@ impl OpteHdl {
         run_cmd_ioctl(self.device.as_raw_fd(), cmd, None::<&()>)
     }
 
+    /// Replace a port's prioritized router list.
+    pub fn set_router_list(
+        &self,
+        req: &SetRouterListReq,
+    ) -> Result<NoResp, Error> {
+        let cmd = OpteCmd::SetRouterList;
+        run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req))
+    }
+
+    /// Read a port's prioritized router list.
+    pub fn dump_router_list(
+        &self,
+        req: &DumpRouterListReq,
+    ) -> Result<DumpRouterListResp, Error> {
+        let cmd = OpteCmd::DumpRouterList;
+        run_cmd_ioctl(self.device.as_raw_fd(), cmd, Some(&req))
+    }
+
     /// Set a multicast forwarding entry.
     pub fn set_mcast_fwd(
         &self,