diff --git a/nvx-rs/README.md b/nvx-rs/README.md index 0311c2294..04ead4666 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -694,6 +694,30 @@ abrupt exit leaves, and the release makes the socket's absence durable before the lease ends, and removes the record and `restore.sock`, as it removes `clone-scratch.raw`. +`Restoring::activate` then runs the clone's restore sequence through the +adapter's framing of its control session, a `CloneControl`, by a +restore-to-Ready deadline that runs from the launch, `RESTORE_TO_READY_TIMEOUT`, +60 seconds by default: it reads the restore-ready event as soon as it comes, +refusing another process's connection and bytes that differ from the event, and +reading nothing past it, since OpenVMM on Windows waits until the host has read +it, and then closes its end; only then connects to the clone's control endpoint, +whose console bounds the host's authentication from the connection on; attaches +with the launch's capability; sends the record, read again from +`clone-activation.bin`, which must still be byte for byte the launch's, in the +adapter's Activate, as the session's first request; and checks that Ready +reports the record's transaction ID. It replaces a session that the adapter +lost, as `CloneControl::lost` tells, with a new one that sends the same record, +in at most `MAX_ACTIVATION_SESSIONS` sessions by the same deadline, and retries +nothing else. + +`Activated::start` then starts the sandbox by a Ready-to-Start deadline from the +check of Ready, `READY_TO_START_TIMEOUT`, 30 seconds by default, never sending +the start again, and `Started::commit` commits the launch, which writes its +runtime record. A failure ends the launch as `Starting::abandon` does; a crash +before the commit leaves the launch's journal, whose recovery ends the VMM, +while the lease lasts until the release, and a crash after it leaves a VM that +reattaches, and keeps its lease while it runs. + `Store::tombstone` quarantines a generation, which takes no new lease while its leases keep working, and `Store::retire` deletes it once its leases have drained, crash-safely, removing the records of leases that no clone binding diff --git a/nvx-rs/src/host/platform/linux.rs b/nvx-rs/src/host/platform/linux.rs index 1157a689f..eae6c4477 100644 --- a/nvx-rs/src/host/platform/linux.rs +++ b/nvx-rs/src/host/platform/linux.rs @@ -857,11 +857,13 @@ pub(crate) fn control_endpoint(socket_path: &Path) -> io::Result { /// The listener at which the launch of a clone awaits the restore-ready event of its VMM: a Unix /// socket, which the VMM connects to once, and which queues its connection until the host -/// accepts it. +/// accepts it, then the connection that the host accepted, from which it reads the event. #[derive(Debug)] pub(crate) struct ReadyListener { /// The listening socket, which the VMM finds as long as it is open. - _socket: UnixListener, + socket: UnixListener, + /// The VMM's connection, once the host accepted it. + connection: Option, } /// Listens at the Unix socket `socket_path`, in place of a socket that an earlier launch left @@ -877,7 +879,90 @@ pub(crate) fn listen_ready(socket_path: &Path) -> io::Result<(String, ReadyListe _ => {} } let socket = UnixListener::bind(socket_path)?; - Ok((endpoint, ReadyListener { _socket: socket })) + // The host polls the socket before it accepts, so an accept never blocks. + socket.set_nonblocking(true)?; + let listener = ReadyListener { + socket, + connection: None, + }; + Ok((endpoint, listener)) +} + +impl ReadyListener { + /// Waits at most `timeout` for what the VMM, the process `pid`, sends to the endpoint, and + /// appends it to `received`, which then holds at most `limit` bytes. Returns whether more may + /// arrive: `false` once the VMM closed its end of the connection. + /// + /// The endpoint's first connection must be the VMM's: one of another process fails with + /// [`io::ErrorKind::PermissionDenied`]. + pub(crate) fn receive( + &mut self, + pid: u32, + received: &mut Vec, + limit: usize, + timeout: Duration, + ) -> io::Result { + let deadline = Instant::now() + timeout; + if self.connection.is_none() { + if !sys::wait_readable(&self.socket, timeout)? { + return Ok(true); + } + let connection = match self.socket.accept() { + Ok((connection, _)) => connection, + Err(error) + if matches!( + error.kind(), + io::ErrorKind::WouldBlock | io::ErrorKind::Interrupted + ) => + { + return Ok(true); + } + Err(error) => return Err(error), + }; + let peer = sys::peer_pid(&connection)?; + if peer != pid { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + format!( + "process {peer} connected to the restore-ready endpoint, not the VMM, \ + process {pid}" + ), + )); + } + connection.set_nonblocking(true)?; + self.connection = Some(connection); + } + let connection = self + .connection + .as_mut() + .expect("the host accepted the VMM's connection"); + let mut buffer = [0; 64]; + let room = limit.saturating_sub(received.len()).min(buffer.len()); + if room == 0 + || !sys::wait_readable( + connection, + deadline.saturating_duration_since(Instant::now()), + )? + { + return Ok(true); + } + match connection.read(&mut buffer[..room]) { + Ok(0) => Ok(false), + Ok(count) => { + received.extend_from_slice(&buffer[..count]); + Ok(true) + } + Err(error) + if matches!( + error.kind(), + io::ErrorKind::WouldBlock | io::ErrorKind::Interrupted + ) => + { + Ok(true) + } + Err(error) => Err(error), + } + } } /// Connects to a control endpoint served by the process `expected_pid`. diff --git a/nvx-rs/src/host/platform/other.rs b/nvx-rs/src/host/platform/other.rs index 443b748db..06fc9ce1a 100644 --- a/nvx-rs/src/host/platform/other.rs +++ b/nvx-rs/src/host/platform/other.rs @@ -252,6 +252,18 @@ pub(crate) fn listen_ready(_socket_path: &Path) -> io::Result<(String, ReadyList Err(unsupported()) } +impl ReadyListener { + pub(crate) fn receive( + &mut self, + _pid: u32, + _received: &mut Vec, + _limit: usize, + _timeout: Duration, + ) -> io::Result { + Err(unsupported()) + } +} + pub(crate) fn connect_endpoint( _endpoint: &str, _expected_pid: u32, diff --git a/nvx-rs/src/host/platform/windows.rs b/nvx-rs/src/host/platform/windows.rs index b5dd0fa6d..f249d9343 100644 --- a/nvx-rs/src/host/platform/windows.rs +++ b/nvx-rs/src/host/platform/windows.rs @@ -15,10 +15,10 @@ use windows_sys::Win32::Foundation::{DUPLICATE_SAME_ACCESS, DuplicateHandle}; use windows_sys::Win32::Foundation::{ ERROR_ACCESS_DENIED, ERROR_ALREADY_EXISTS, ERROR_BROKEN_PIPE, ERROR_FILE_NOT_FOUND, ERROR_INVALID_PARAMETER, ERROR_IO_PENDING, ERROR_MORE_DATA, ERROR_NO_DATA, - ERROR_OPERATION_ABORTED, ERROR_PIPE_BUSY, ERROR_PIPE_NOT_CONNECTED, ERROR_SHARING_VIOLATION, - ERROR_SUCCESS, FILETIME, FreeLibrary, GENERIC_ALL, GENERIC_READ, GENERIC_WRITE, - GetHandleInformation, GetLastError, HANDLE, HANDLE_FLAG_INHERIT, INVALID_HANDLE_VALUE, - SetHandleInformation, WAIT_OBJECT_0, WAIT_TIMEOUT, + ERROR_OPERATION_ABORTED, ERROR_PIPE_BUSY, ERROR_PIPE_CONNECTED, ERROR_PIPE_NOT_CONNECTED, + ERROR_SHARING_VIOLATION, ERROR_SUCCESS, FILETIME, FreeLibrary, GENERIC_ALL, GENERIC_READ, + GENERIC_WRITE, GetHandleInformation, GetLastError, HANDLE, HANDLE_FLAG_INHERIT, + INVALID_HANDLE_VALUE, SetHandleInformation, WAIT_OBJECT_0, WAIT_TIMEOUT, }; use windows_sys::Win32::Security::{ ACCESS_ALLOWED_ACE, ACE_HEADER, ACL, ACL_REVISION, ACL_SIZE_INFORMATION, AclSizeInformation, @@ -50,8 +50,8 @@ use windows_sys::Win32::System::LibraryLoader::{ GetProcAddress, LOAD_LIBRARY_SEARCH_SYSTEM32, LoadLibraryExW, }; use windows_sys::Win32::System::Pipes::{ - CreateNamedPipeW, GetNamedPipeServerProcessId, PIPE_READMODE_BYTE, PIPE_REJECT_REMOTE_CLIENTS, - PIPE_TYPE_BYTE, PIPE_WAIT, WaitNamedPipeW, + ConnectNamedPipe, CreateNamedPipeW, GetNamedPipeClientProcessId, GetNamedPipeServerProcessId, + PIPE_READMODE_BYTE, PIPE_REJECT_REMOTE_CLIENTS, PIPE_TYPE_BYTE, PIPE_WAIT, WaitNamedPipeW, }; use windows_sys::Win32::System::Registry::{ HKEY_LOCAL_MACHINE, RRF_RT_REG_DWORD, RRF_RT_REG_SZ, RegGetValueW, @@ -64,7 +64,7 @@ use windows_sys::Win32::System::Threading::{ EXTENDED_STARTUPINFO_PRESENT, GetCurrentProcess, GetProcessTimes, INFINITE, InitializeProcThreadAttributeList, LPPROC_THREAD_ATTRIBUTE_LIST, OpenProcess, OpenProcessToken, PROC_THREAD_ATTRIBUTE_HANDLE_LIST, PROCESS_CREATION_FLAGS, PROCESS_INFORMATION, - PROCESS_QUERY_LIMITED_INFORMATION, PROCESS_SYNCHRONIZE, PROCESS_TERMINATE, + PROCESS_QUERY_LIMITED_INFORMATION, PROCESS_SYNCHRONIZE, PROCESS_TERMINATE, ResetEvent, STARTF_USESTDHANDLES, STARTUPINFOEXW, TerminateProcess, UpdateProcThreadAttribute, WaitForSingleObject, }; @@ -1781,12 +1781,16 @@ pub(crate) fn control_endpoint(_socket_path: &Path) -> io::Result { const READY_BUFFER_LEN: u32 = 4096; /// The listener at which the launch of a clone awaits the restore-ready event of its VMM: the -/// one instance of a named pipe, inbound, which only the current user may open, from this host -/// alone, and which takes the connection of the VMM before the host accepts it. +/// one instance of a named pipe, inbound and overlapped, which only the current user may open, +/// from this host alone, and which takes the connection of the VMM before the host accepts it. #[derive(Debug)] pub(crate) struct ReadyListener { /// The pipe's instance, which the VMM finds as long as it is open. - _pipe: OwnedHandle, + pipe: OwnedHandle, + /// The manual-reset event of the overlapped operations on the pipe. + event: OwnedHandle, + /// Whether the host accepted the VMM's connection. + connected: bool, } /// Listens at a fresh named pipe, whose name it draws at random, as the restore-ready endpoint of @@ -1805,7 +1809,7 @@ pub(crate) fn listen_ready(_socket_path: &Path) -> io::Result<(String, ReadyList let handle = unsafe { CreateNamedPipeW( name.as_ptr(), - PIPE_ACCESS_INBOUND | FILE_FLAG_FIRST_PIPE_INSTANCE, + PIPE_ACCESS_INBOUND | FILE_FLAG_FIRST_PIPE_INSTANCE | FILE_FLAG_OVERLAPPED, PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_WAIT | PIPE_REJECT_REMOTE_CLIENTS, 1, 0, @@ -1816,7 +1820,90 @@ pub(crate) fn listen_ready(_socket_path: &Path) -> io::Result<(String, ReadyList }; owned(handle).ok_or_else(io::Error::last_os_error) })??; - Ok((endpoint, ReadyListener { _pipe: pipe })) + // SAFETY: creates an unnamed manual-reset event with default security. + let event = owned(unsafe { CreateEventW(ptr::null(), 1, 0, ptr::null()) }) + .ok_or_else(io::Error::last_os_error)?; + let listener = ReadyListener { + pipe, + event, + connected: false, + }; + Ok((endpoint, listener)) +} + +impl ReadyListener { + /// Waits at most `timeout` for what the VMM, the process `pid`, sends to the endpoint, and + /// appends it to `received`, which then holds at most `limit` bytes. Returns whether more may + /// arrive: `false` once the VMM closed its end of the pipe. + /// + /// The pipe's client must be the VMM: another process fails with + /// [`io::ErrorKind::PermissionDenied`]. + pub(crate) fn receive( + &mut self, + pid: u32, + received: &mut Vec, + limit: usize, + timeout: Duration, + ) -> io::Result { + let deadline = Instant::now() + timeout; + let (pipe, event) = (raw(&self.pipe), raw(&self.event)); + if !self.connected { + // ConnectNamedPipe does not promise to reset the event before it starts, as ReadFile + // and WriteFile do, so it is reset here. + // SAFETY: the event belongs to the listener, and no operation that uses it is pending. + if unsafe { ResetEvent(event) } == 0 { + return Err(io::Error::last_os_error()); + } + // A wait that times out cancels the connection's operation, and the next wait issues + // it again, which a client that connected meanwhile completes at once. + // SAFETY: the operation's `OVERLAPPED` outlives it, as `overlapped` guarantees. + let connection = overlapped(pipe, event, Some(timeout), |pipe, overlapped| unsafe { + ConnectNamedPipe(pipe, overlapped) + }); + match connection { + Ok(_) => {} + Err(error) if error.raw_os_error() == Some(ERROR_PIPE_CONNECTED as i32) => {} + Err(error) if error.kind() == io::ErrorKind::TimedOut => return Ok(true), + Err(error) => return Err(error), + } + let mut client = 0u32; + // SAFETY: the handle is a pipe's server end, and the output pointer is valid. + if unsafe { GetNamedPipeClientProcessId(pipe, &mut client) } == 0 { + return Err(io::Error::last_os_error()); + } + if client != pid { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + format!( + "process {client} connected to the restore-ready endpoint, not the VMM, \ + process {pid}" + ), + )); + } + self.connected = true; + } + let mut buffer = [0u8; 64]; + let room = limit.saturating_sub(received.len()).min(buffer.len()); + if room == 0 { + return Ok(true); + } + let (data, length) = (buffer.as_mut_ptr(), room as u32); + let left = deadline.saturating_duration_since(Instant::now()); + // SAFETY: `data` is valid for `length` bytes until the operation completes, which + // `overlapped` guarantees before it returns. + let read = overlapped(pipe, event, Some(left), |pipe, overlapped| unsafe { + ReadFile(pipe, data, length, ptr::null_mut(), overlapped) + }); + match read { + Ok(0) => Ok(false), + Ok(count) => { + received.extend_from_slice(&buffer[..count]); + Ok(true) + } + Err(error) if error.kind() == io::ErrorKind::TimedOut => Ok(true), + Err(error) => Err(error), + } + } } /// Connects to a named pipe served by the process `expected_pid`. @@ -1922,45 +2009,55 @@ struct PipeTransport { } impl PipeTransport { - /// Runs one overlapped operation to completion or timeout. - /// - /// `start` issues the operation. The function always waits for the operation to finish, - /// cancelling it on timeout, so the `OVERLAPPED` and the caller's buffer outlive it. + /// Runs one overlapped operation on the pipe to completion or timeout, as [`overlapped`] + /// runs it. fn overlapped( &mut self, timeout: Option, start: impl FnOnce(HANDLE, *mut OVERLAPPED) -> i32, ) -> io::Result { - let pipe = raw(&self.pipe); - // SAFETY: OVERLAPPED is plain data for which all-zero bytes are a valid initial state. - let mut overlapped: OVERLAPPED = unsafe { std::mem::zeroed() }; - // The event can be reused: ReadFile and WriteFile reset it to nonsignaled before they - // start an operation. - overlapped.hEvent = raw(&self.event); - if start(pipe, &raw mut overlapped) == 0 { - let error = last_error(); - if error != ERROR_IO_PENDING { - return completion_error(error, 0, false); - } - } - let wait = timeout.map_or(INFINITE, |timeout| { - u32::try_from(timeout.as_millis()) - .unwrap_or(INFINITE - 1) - .clamp(1, INFINITE - 1) - }); - // SAFETY: the event belongs to this transport. - let signaled = unsafe { WaitForSingleObject(overlapped.hEvent, wait) } == WAIT_OBJECT_0; - if !signaled { - // SAFETY: cancels only the operation that uses this OVERLAPPED. - unsafe { CancelIoEx(pipe, &overlapped) }; - } - let mut transferred = 0u32; - // SAFETY: waits for the operation to finish, so nothing references `overlapped` after. - if unsafe { GetOverlappedResult(pipe, &overlapped, &mut transferred, 1) } != 0 { - return Ok(transferred as usize); + overlapped(raw(&self.pipe), raw(&self.event), timeout, start) + } +} + +/// Runs one overlapped operation on `pipe` to completion or timeout, waiting on `event`, a +/// manual-reset event that no other operation uses meanwhile, and which the operation resets to +/// nonsignaled before it starts, as ReadFile and WriteFile do. +/// +/// `start` issues the operation. The function always waits for the operation to finish, +/// cancelling it on timeout, so the `OVERLAPPED` and the caller's buffer outlive it. +fn overlapped( + pipe: HANDLE, + event: HANDLE, + timeout: Option, + start: impl FnOnce(HANDLE, *mut OVERLAPPED) -> i32, +) -> io::Result { + // SAFETY: OVERLAPPED is plain data for which all-zero bytes are a valid initial state. + let mut overlapped: OVERLAPPED = unsafe { std::mem::zeroed() }; + overlapped.hEvent = event; + if start(pipe, &raw mut overlapped) == 0 { + let error = last_error(); + if error != ERROR_IO_PENDING { + return completion_error(error, 0, false); } - completion_error(last_error(), transferred as usize, !signaled) } + let wait = timeout.map_or(INFINITE, |timeout| { + u32::try_from(timeout.as_millis()) + .unwrap_or(INFINITE - 1) + .clamp(1, INFINITE - 1) + }); + // SAFETY: the event belongs to the caller. + let signaled = unsafe { WaitForSingleObject(overlapped.hEvent, wait) } == WAIT_OBJECT_0; + if !signaled { + // SAFETY: cancels only the operation that uses this OVERLAPPED. + unsafe { CancelIoEx(pipe, &overlapped) }; + } + let mut transferred = 0u32; + // SAFETY: waits for the operation to finish, so nothing references `overlapped` after. + if unsafe { GetOverlappedResult(pipe, &overlapped, &mut transferred, 1) } != 0 { + return Ok(transferred as usize); + } + completion_error(last_error(), transferred as usize, !signaled) } /// Maps a completed pipe operation's Win32 status to Rust I/O semantics. diff --git a/nvx-rs/src/host/supervisor/mod.rs b/nvx-rs/src/host/supervisor/mod.rs index 49299a3e2..5d1b81005 100644 --- a/nvx-rs/src/host/supervisor/mod.rs +++ b/nvx-rs/src/host/supervisor/mod.rs @@ -124,7 +124,14 @@ //! socket that the lease reserves in the generation's payload, named after the launch, which //! [`Starting::endpoint`] and the runtime record name, as every launch's endpoint, rather than //! at the sandbox's own socket. -//! 3. Once the clone's VM has exited, as a stop, a termination, or a reattach verifies, +//! 3. Still under that lock, the caller runs the clone's restore sequence: +//! [`Restoring::activate`](crate::host::template::Restoring::activate) waits for the VMM's +//! restore-ready event, activates the clone through the adapter's framing of its control +//! session, and checks its Ready, by a deadline that runs from the launch; +//! [`Activated::start`](crate::host::template::Activated::start) starts its sandbox; and +//! [`Started::commit`](crate::host::template::Started::commit) commits the launch, as +//! [`Starting::commit`] does. A failure ends the launch, as [`Starting::abandon`] does. +//! 4. Once the clone's VM has exited, as a stop, a termination, or a reattach verifies, //! [`Store::release`](crate::host::template::Store::release) ends the lease, under the //! sandbox's lock, taken through the lease's path, //! [`Lease::sandbox`](crate::host::template::Lease::sandbox), as the acquisition took it. The @@ -158,9 +165,10 @@ //! - After the acquisition, the binding reserves a launch that never began, and that no later lock //! begins: a release ends the lease at once, and another acquisition, which checks the //! generation's eligibility again, reserves another launch. -//! - After the launch's journal, before or after the spawn, [`reattach`] ends the launch, as -//! [`LockedStore::recover`] does, and the lease lasts until it has: a launch that cannot be -//! ended yet is [`Reattached::Indeterminate`], and keeps it. +//! - After the launch's journal, before or after the spawn, or in the clone's restore sequence, +//! before its commit, [`reattach`] ends the launch, as [`LockedStore::recover`] does, and the +//! lease lasts until it has: a launch that cannot be ended yet is +//! [`Reattached::Indeterminate`], and keeps it. //! - After the commit, the runtime record names the VM, which [`reattach`] holds again: the lease //! lasts as long as the VM runs, until a stop or a termination verifies its exit. //! - During a release, the binding goes first, durably, and leaves the record of the lease, which diff --git a/nvx-rs/src/host/sys/mod.rs b/nvx-rs/src/host/sys/mod.rs index 8212248b7..bd580b635 100644 --- a/nvx-rs/src/host/sys/mod.rs +++ b/nvx-rs/src/host/sys/mod.rs @@ -34,7 +34,9 @@ pub(crate) use process::{ #[cfg(all(test, target_os = "linux"))] pub(crate) use process::{clear_close_on_exec, mark_listed_descriptors_on_spawn}; #[cfg(target_os = "linux")] -pub(crate) use socket::{connect_nonblocking, peer_pid, send_nonblocking, wait_writable}; +pub(crate) use socket::{ + connect_nonblocking, peer_pid, send_nonblocking, wait_readable, wait_writable, +}; /// Returns the effective user and group IDs of this process. pub(crate) fn effective_ids() -> (u32, u32) { diff --git a/nvx-rs/src/host/sys/socket.rs b/nvx-rs/src/host/sys/socket.rs index 4ab42b7a5..de9ba558d 100644 --- a/nvx-rs/src/host/sys/socket.rs +++ b/nvx-rs/src/host/sys/socket.rs @@ -54,6 +54,20 @@ pub(crate) fn send_nonblocking(stream: &UnixStream, data: &[u8]) -> io::Result io::Result { + // Round up, so that a wait shorter than a millisecond does not spin. + let timeout = + PollTimeout::try_from(timeout.as_nanos().div_ceil(1_000_000)).unwrap_or(PollTimeout::MAX); + let mut descriptors = [PollFd::new(descriptor.as_fd(), PollFlags::POLLIN)]; + match poll(&mut descriptors, timeout) { + Ok(0) | Err(Errno::EINTR) => Ok(false), + Ok(_) => Ok(true), + Err(error) => Err(error.into()), + } +} + /// Waits until `stream` has room to send, a signal arrives, or `timeout` passes (`None` waits /// indefinitely). pub(crate) fn wait_writable(stream: &UnixStream, timeout: Option) -> io::Result<()> { diff --git a/nvx-rs/src/host/template/clone.rs b/nvx-rs/src/host/template/clone.rs index ac78de75c..a99d49872 100644 --- a/nvx-rs/src/host/template/clone.rs +++ b/nvx-rs/src/host/template/clone.rs @@ -9,8 +9,13 @@ mod tests; mod launch; mod record; +mod restore; pub use self::launch::Restoring; +pub use self::restore::{ + Activated, CloneControl, MAX_ACTIVATION_SESSIONS, READY_TO_START_TIMEOUT, RESTORE_READY_EVENT, + RESTORE_TO_READY_TIMEOUT, RestoreDeadlines, Started, +}; use std::fmt; use std::fs::{self, File}; diff --git a/nvx-rs/src/host/template/clone/launch.rs b/nvx-rs/src/host/template/clone/launch.rs index bb6b9d399..2354f622b 100644 --- a/nvx-rs/src/host/template/clone/launch.rs +++ b/nvx-rs/src/host/template/clone/launch.rs @@ -5,6 +5,7 @@ use std::ffi::{OsStr, OsString}; use std::fmt; use std::path::{Path, PathBuf}; +use std::time::Instant; use super::{ClonePlan, CloneUpper, MIB, SPARSE_WHAT, check_geometry}; use crate::activation::ActivationRecord; @@ -67,7 +68,9 @@ impl ClonePlan { /// which a relative path names from the current directory, restores the leased generation's /// snapshot with the clone's devices, under the runtime identity `runtime`, which must be the /// template's, [`TemplateKey::runtime`], and receives the plan's capability. Returns the - /// launch, whose VMM runs, as a [`Restoring`]. + /// launch, whose VMM runs, as a [`Restoring`], whose restore sequence, + /// [`Restoring::activate`], activates the clone, under a deadline that starts as this + /// launch does. /// /// The launch first checks, without writing anything, that `acquired` holds the plan's lease, /// under the lock that acquired it, before the launch that it reserves began, and that the @@ -185,6 +188,8 @@ impl ClonePlan { runtime: &RuntimeIdentity, hooks: Hooks<'_>, ) -> Result> { + // The restore-to-Ready deadline of the clone's restore sequence starts here. + let launched = Instant::now(); let Acquired { lease, locked } = acquired; self.check_launch(lease, locked, runtime)?; let openvmm = artifacts::absolute(openvmm)?; @@ -201,7 +206,8 @@ impl ClonePlan { devices, } = self; let devices = device_arguments(&devices)?; - write_private(&locked.clone_activation_path(), &activation.encode())?; + let record = locked.clone_activation_path(); + write_private(&record, &activation.encode())?; let (ready, listener) = platform::listen_ready(&locked.restore_ready_path()).map_err(|error| { Error::io(format!( @@ -250,10 +256,12 @@ impl ClonePlan { drop(claim); Ok(Restoring { starting: starting?, + launched, ready, listener, image, activation, + record, capability, }) } @@ -488,7 +496,9 @@ impl Restore<'_> { /// The launch of a clone whose VMM restores the leased generation's snapshot, as /// [`ClonePlan::launch`] returns it: the [`Starting`] launch of the VMM, whose journal records /// the VMM's identity, the endpoint at which the launch awaits the VMM's restore-ready event, the -/// clone's activation record and capability, and its image, open for reading. +/// clone's activation record and capability, and its image, open for reading. Its restore +/// sequence, [`activate`](Self::activate), then [`Activated::start`](super::Activated::start), +/// takes the clone to its sandbox's start. /// /// It borrows the sandbox's locked store exclusively, as [`Starting`] does. Dropping it abandons /// the launch, as [`abandon`](Self::abandon) does, ignoring failures, and stops listening for the @@ -496,17 +506,21 @@ impl Restore<'_> { /// [`LockedStore::recover`]. pub struct Restoring<'s> { /// The launch, whose journal records the VMM. - starting: Starting<'s>, + pub(super) starting: Starting<'s>, + /// When the launch began, at which the restore-to-Ready deadline starts. + pub(super) launched: Instant, /// The endpoint of the restore-ready event, as OpenVMM's `--restore-ready-path` takes it. - ready: String, + pub(super) ready: String, /// The listener of the restore-ready event. - listener: ReadyListener, + pub(super) listener: ReadyListener, /// The clone's image, open for reading, which the plan checked. - image: CheckedDistro, + pub(super) image: CheckedDistro, /// The clone's activation record. - activation: ActivationRecord, + pub(super) activation: ActivationRecord, + /// The path of the activation record in the sandbox, which the launch wrote. + pub(super) record: PathBuf, /// The capability of the launch. - capability: [u8; CAPABILITY_LEN], + pub(super) capability: [u8; CAPABILITY_LEN], } impl<'s> Restoring<'s> { diff --git a/nvx-rs/src/host/template/clone/restore.rs b/nvx-rs/src/host/template/clone/restore.rs new file mode 100644 index 000000000..f2ef00613 --- /dev/null +++ b/nvx-rs/src/host/template/clone/restore.rs @@ -0,0 +1,671 @@ +//! The restore sequence of a clone, from its launch, a [`Restoring`], to its sandbox's start, +//! through the adapter's framing of the clone's control session, a [`CloneControl`]: +//! [`Restoring::activate`] activates the clone, [`Activated::start`] starts its sandbox, and +//! [`Started::commit`] commits its launch, as the documentation of [`Restoring::activate`] +//! describes. + +use std::fmt; +use std::io; +use std::path::Path; +use std::thread; +use std::time::{Duration, Instant}; + +use super::launch::Restoring; +use crate::activation::{ActivationRecord, MAX_RECORD_BYTES, ReadyActivation}; +use crate::error::{Error, Result}; +use crate::host::platform::{ReadyListener, Transport}; +use crate::host::sandbox::distro::CheckedDistro; +use crate::host::session::{self, SessionError}; +use crate::host::store::Recovery; +use crate::host::store::regular::{open_regular, read_at_most}; +use crate::host::supervisor::{Starting, Vmm}; +use crate::protocol::CAPABILITY_LEN; + +/// The restore-ready event that OpenVMM writes to a restore's `--restore-ready-path` once the +/// restored VM has started, which [`Restoring::activate`] waits for. +pub const RESTORE_READY_EVENT: &[u8] = b"OPENVMM_RESTORE_READY_V1\n"; + +/// The default restore-to-Ready deadline of a clone's restore sequence: how long after the clone's +/// launch its Ready may come. +/// +/// It covers OpenVMM's restore, the guest's repair, the host's attachment, and the activation of +/// the clone's root, which assembles it as a cold boot does, with a wide margin on the CI runs of +/// the slowest backend, WHP, whose restores ran within a third of a second, and whose 25 VMs of the +/// test adapter, at once, assembled their layered roots, applied their mapping tables, and started +/// within 19 seconds in all. It is the test adapter's deadline of a cold launch. +pub const RESTORE_TO_READY_TIMEOUT: Duration = Duration::from_secs(60); + +/// The default Ready-to-Start deadline of a clone's restore sequence: how long after the check of +/// the clone's Ready its sandbox may start, which takes a few of the guest's answers, as the CI +/// runs of [`RESTORE_TO_READY_TIMEOUT`] show. It is the test adapter's deadline of a session's +/// answers. +pub const READY_TO_START_TIMEOUT: Duration = Duration::from_secs(30); + +/// The most sessions in which [`Restoring::activate`] sends a clone's Activate: its first session, +/// and those that replace a session that the adapter lost. +pub const MAX_ACTIVATION_SESSIONS: usize = 3; + +/// How often the wait for the restore-ready event checks whether the VMM still runs. +const EXIT_CHECK_INTERVAL: Duration = Duration::from_millis(50); + +/// How long the sequence waits before a session that replaces a lost one, so that the VMM's +/// control console notices the end of the lost session, which frees the console's one attachment. +const SESSION_RETRY_INTERVAL: Duration = Duration::from_millis(50); + +/// The deadlines of a clone's restore sequence, each a duration from the step at which it starts. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RestoreDeadlines { + /// The restore-to-Ready deadline: how long after the clone's launch its Ready may come, which + /// bounds the wait for the restore-ready event, the connections to the clone's control + /// endpoint, the adapter's attachments, and the Activates. + pub ready: Duration, + /// The Ready-to-Start deadline: how long after the check of the clone's Ready its sandbox may + /// start. + pub start: Duration, +} + +impl Default for RestoreDeadlines { + /// Returns the default deadlines: [`RESTORE_TO_READY_TIMEOUT`] and [`READY_TO_START_TIMEOUT`]. + fn default() -> Self { + Self { + ready: RESTORE_TO_READY_TIMEOUT, + start: READY_TO_START_TIMEOUT, + } + } +} + +/// The adapter's framing of a clone's control session, through which the clone's restore +/// sequence, [`Restoring::activate`], activates the clone and starts its sandbox, over the +/// session of [`nvxs`](crate::host::nvxs), say. +pub trait CloneControl { + /// A session of the adapter's framing over one connection to the clone's control endpoint. + type Session; + /// A failure of the adapter's framing or of its session. + type Error: std::error::Error + Send + Sync + 'static; + + /// Attaches a session of the adapter's framing over `transport`, a fresh connection to the + /// clone's control endpoint, with the launch's `capability`, and waits until `deadline` while + /// the session's broker waits for the guest, as + /// [`HostConnection::attach`](crate::host::nvxs::HostConnection::attach) does. Sends no + /// request. + fn attach( + &mut self, + transport: Box, + capability: &[u8; CAPABILITY_LEN], + deadline: Instant, + ) -> Result; + + /// Sends `record`, the clone's activation record in its canonical encoding, in the framing's + /// Activate, as the first request of `session`, and waits until `deadline` for the clone's + /// Ready. Returns the activation that Ready reports, or `None` for a Ready that reports + /// none, once the adapter checked its own identities in Ready, such as its build, and + /// refused a Ready that does not match them. + fn activate( + &mut self, + session: &mut Self::Session, + record: &[u8], + deadline: Instant, + ) -> Result, Self::Error>; + + /// Starts the clone's sandbox over `session`, once the clone is ready: sends what the + /// framing sends before the start, such as the mappings that the record announces, then the + /// start, and waits until `deadline` for its confirmation. + fn start(&mut self, session: &mut Self::Session, deadline: Instant) -> Result<(), Self::Error>; + + /// Returns whether `error`, with which [`attach`](Self::attach) or + /// [`activate`](Self::activate) failed, lost the session, as a reset of the session, the end + /// of its stream, or a failure of its transport do, rather than another failure, such as the + /// guest's refusal or the passing of the deadline: a clone that answers no more in a lost + /// session may still answer in another. + fn lost(&self, error: &Self::Error) -> bool; +} + +impl<'s> Restoring<'s> { + /// Runs the clone's restore sequence until the clone is ready, through the adapter's framing + /// of the clone's control session, `control`, under the restore-to-Ready deadline, + /// `deadlines.ready` after the launch, [`ClonePlan::launch`](super::ClonePlan::launch), in + /// order: + /// + /// 1. Waits for OpenVMM's restore-ready event, [`RESTORE_READY_EVENT`], at the endpoint at + /// which the launch listens, [`ready_endpoint`](Self::ready_endpoint), and reads it as soon + /// as it comes: OpenVMM connects to the endpoint before the restored VM runs, writes the + /// event once the restored VM has started, after which, on Windows, it waits until the + /// host has read the event, and then closes its end, which carries nothing else. The wait + /// refuses a connection of another process than the VMM, and bytes that differ from the + /// event, reads nothing past the event, and ends once the VMM has exited. + /// 2. Reads the clone's activation record again where the launch wrote it, + /// [`CLONE_ACTIVATION_NAME`](crate::host::store::CLONE_ACTIVATION_NAME) in the sandbox, + /// which must still be, byte for byte, the record of the launch, + /// [`activation`](Self::activation), in its canonical encoding: the only record that the + /// clone's restore sends. + /// 3. Connects to the clone's control endpoint, [`Starting::endpoint`], which the VMM serves, + /// as [`session::connect`] does, only after the event: the VMM's control console bounds the + /// host's authentication from its connection on, which the restore may hold up until its + /// event. + /// 4. Attaches the adapter's session over the connection with the launch's capability, + /// [`capability`](Self::capability), which the VMM authenticates, and sends the record in + /// the adapter's Activate, as the session's first request. + /// 5. Checks the activation that the clone's Ready reports, after the adapter checked its own + /// identities in it: the transaction ID of the record's restore section, as + /// [`ReadyActivation::of`] returns it. + /// + /// Returns the [`Activated`] clone, whose Ready-to-Start deadline, `deadlines.start`, starts + /// at the check of its Ready, and whose sandbox [`Activated::start`] starts, after which + /// [`Started::commit`] commits the launch, as the cold boot of a sandbox commits its launch + /// once it checked its VM, with [`Starting::commit`]. + /// + /// # Retries + /// + /// A clone answers an Activate of its record, byte for byte, with the same Ready, without + /// activating again, also in a later session once it is ready, and refuses any other Activate + /// for good, as does the reset of its session before it is ready. So once the adapter lost the + /// session, as [`CloneControl::lost`] tells, while it attached it or awaited the clone's + /// Ready, the sequence waits briefly, reads the record again, connects again, and attaches a + /// new session, in which it sends the record again, byte for byte, and checks the Ready again: + /// a clone that was ready answers it, and a clone that took the reset of the lost session + /// before then has ended, so that the connection fails once its VMM has exited. It does so + /// until the restore-to-Ready deadline, which no retry extends, in at most + /// [`MAX_ACTIVATION_SESSIONS`] sessions in all, and never after another failure: a refusal, a + /// Ready that does not match the record, a record that changed, and the passing of the + /// deadline are final. + /// + /// # Failures + /// + /// A step that fails ends the launch, as [`abandon`](Self::abandon) does, which terminates the + /// VMM, and ends the launch once nothing of it runs, and fails with the step's error: + /// + /// - [`ErrorKind::TimedOut`](crate::ErrorKind::TimedOut) once the restore-to-Ready deadline + /// passed; + /// - [`ErrorKind::Exited`](crate::ErrorKind::Exited) once the VMM exited before its event, or + /// before a connection; + /// - [`ErrorKind::Protocol`](crate::ErrorKind::Protocol) for a connection to the restore-ready + /// endpoint of another process than the VMM, bytes there that differ from the event, a + /// control endpoint that another process serves, a failure of the adapter's framing, a + /// session that it lost in the last of the sessions, and a Ready that reports another + /// activation than the record's, or none; + /// - [`ErrorKind::Io`](crate::ErrorKind::Io) if the endpoints fail otherwise, and if the + /// sandbox's activation record cannot be read, or is no longer the launch's; + /// - [`ErrorKind::Unsupported`](crate::ErrorKind::Unsupported) for a deadline beyond the + /// host's clock. + /// + /// The error says so if the launch could not be ended, whose journal then remains for + /// [`LockedStore::recover`](crate::host::store::LockedStore::recover), and the lease with it, + /// as [Crashes](crate::host::supervisor#crashes) describes. + pub fn activate( + self, + control: &mut C, + deadlines: RestoreDeadlines, + ) -> Result> { + let Self { + starting, + launched, + ready: _, + mut listener, + image, + activation, + record, + capability, + } = self; + let Some(deadline) = launched.checked_add(deadlines.ready) else { + let error = beyond_clock("restore-to-Ready", deadlines.ready); + return Err(abandoned(starting, error)); + }; + if let Err(error) = wait_event(&mut listener, &starting, deadline, deadlines.ready) { + return Err(abandoned(starting, error)); + } + // OpenVMM sends one event, and the endpoint takes nothing more. + drop(listener); + let activation_session = ActivationSession { + capability: &capability, + record: &record, + encoded: activation.encode(), + expected: ReadyActivation::of(&activation), + deadline, + timeout: deadlines.ready, + }; + let mut sessions = 0; + let session = loop { + sessions += 1; + match activation_session.run(control, &starting) { + Ok(session) => break session, + Err(Attempt::Lost(_)) if sessions < MAX_ACTIVATION_SESSIONS => { + let left = deadline.saturating_duration_since(Instant::now()); + thread::sleep(SESSION_RETRY_INTERVAL.min(left)); + } + Err(Attempt::Lost(error) | Attempt::Failed(error)) => { + return Err(abandoned(starting, error)); + } + } + }; + let Some(start) = Instant::now().checked_add(deadlines.start) else { + drop(session); + let error = beyond_clock("Ready-to-Start", deadlines.start); + return Err(abandoned(starting, error)); + }; + Ok(Activated { + session, + starting, + deadline: start, + image, + activation, + }) + } +} + +/// A clone that its restore sequence activated, as [`Restoring::activate`] returns it: its guest +/// answered the Activate of its record with a Ready that reports the record's activation, and +/// awaits the start of its sandbox, which [`start`](Self::start) asks for, by the Ready-to-Start +/// deadline, [`deadline`](Self::deadline). +/// +/// It holds the adapter's session in which the clone answered, and the launch, uncommitted, +/// through which it borrows the sandbox's locked store exclusively, as [`Starting`] does. +/// Dropping it abandons the launch, as [`abandon`](Self::abandon) does, ignoring failures; a crash +/// of the caller leaves the launch's journal instead, for +/// [`LockedStore::recover`](crate::host::store::LockedStore::recover). +pub struct Activated<'s, S> { + /// The adapter's session, in which the clone answered its Activate. + session: S, + /// The launch, whose journal records the VMM. + starting: Starting<'s>, + /// The Ready-to-Start deadline. + deadline: Instant, + /// The clone's image, open for reading, which the plan checked. + image: CheckedDistro, + /// The clone's activation record. + activation: ActivationRecord, +} + +impl<'s, S> Activated<'s, S> { + /// Returns the launch, whose VMM runs, and whose journal records its identity. + pub fn starting(&self) -> &Starting<'s> { + &self.starting + } + + /// Returns the adapter's session, in which the clone answered its Activate. + pub fn session(&mut self) -> &mut S { + &mut self.session + } + + /// Returns the Ready-to-Start deadline, by which the clone's sandbox must start. + pub fn deadline(&self) -> Instant { + self.deadline + } + + /// Returns the clone's activation record, with which the clone is ready. + pub fn activation(&self) -> &ActivationRecord { + &self.activation + } + + /// Returns the clone's image, open for reading, which the plan checked. + pub fn image(&self) -> &CheckedDistro { + &self.image + } + + /// Starts the clone's sandbox through the adapter's framing, `control`, in the session in + /// which the clone answered, by the Ready-to-Start deadline, and returns the [`Started`] + /// clone, whose launch the caller commits. + /// + /// The start is not sent again: a guest that started its sandbox refuses another start, so a + /// session that fails once the start was sent leaves the start unknown. A failure ends the + /// launch, as [`abandon`](Self::abandon) does, and fails with + /// [`ErrorKind::TimedOut`](crate::ErrorKind::TimedOut) once the deadline passed, and with + /// [`ErrorKind::Protocol`](crate::ErrorKind::Protocol) before, and says so if the launch + /// could not be ended, as [`Restoring::activate`] does. + pub fn start>(self, control: &mut C) -> Result> { + let Self { + mut session, + starting, + deadline, + image, + activation, + } = self; + match control.start(&mut session, deadline) { + Ok(()) => Ok(Started { + session, + starting, + image, + activation, + }), + Err(source) => { + drop(session); + let error = if Instant::now() >= deadline { + Error::timed_out( + "the clone's sandbox did not start by its Ready-to-Start deadline", + ) + } else { + Error::protocol("cannot start the clone's sandbox") + }; + Err(abandoned(starting, error.with_source(source))) + } + } + } + + /// Ends the launch, as [`Starting::abandon`] does, which terminates its VMM and ends the + /// launch once nothing of it runs, once the adapter's session has ended. + pub fn abandon(self) -> Result { + let Self { + session, starting, .. + } = self; + drop(session); + starting.abandon() + } +} + +impl fmt::Debug for Activated<'_, S> { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("Activated") + .field("starting", &self.starting) + .field("deadline", &self.deadline) + .field("image", &self.image) + .field("activation", &self.activation) + .finish_non_exhaustive() + } +} + +/// A clone whose sandbox started, as [`Activated::start`] returns it, whose launch the caller +/// [commits](Self::commit), once it checked anything else that it requires of a launch, or +/// [abandons](Self::abandon). +/// +/// It holds the adapter's session, and the launch, as [`Activated`] does. Dropping it abandons the +/// launch, as [`abandon`](Self::abandon) does, ignoring failures; a crash of the caller leaves the +/// launch's journal instead, for +/// [`LockedStore::recover`](crate::host::store::LockedStore::recover). +pub struct Started<'s, S> { + /// The adapter's session, in which the clone's sandbox started. + session: S, + /// The launch, whose journal records the VMM. + starting: Starting<'s>, + /// The clone's image, open for reading, which the plan checked. + image: CheckedDistro, + /// The clone's activation record. + activation: ActivationRecord, +} + +impl<'s, S> Started<'s, S> { + /// Returns the launch, whose VMM runs, and whose journal records its identity. + pub fn starting(&self) -> &Starting<'s> { + &self.starting + } + + /// Returns the adapter's session, in which the clone's sandbox started. + pub fn session(&mut self) -> &mut S { + &mut self.session + } + + /// Returns the clone's activation record, with which the clone is ready. + pub fn activation(&self) -> &ActivationRecord { + &self.activation + } + + /// Returns the clone's image, open for reading, which the plan checked. + pub fn image(&self) -> &CheckedDistro { + &self.image + } + + /// Commits the launch, as [`Starting::commit`] does, which writes its runtime record, and + /// returns the VM, with the adapter's session. The clone's image closes. + /// + /// Fails as [`Starting::commit`] fails, which ends the VMM. + pub fn commit(self) -> Result<(Vmm, S)> { + let Self { + session, starting, .. + } = self; + let vmm = starting.commit()?; + Ok((vmm, session)) + } + + /// Ends the launch, as [`Activated::abandon`] does. + pub fn abandon(self) -> Result { + let Self { + session, starting, .. + } = self; + drop(session); + starting.abandon() + } +} + +impl fmt::Debug for Started<'_, S> { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("Started") + .field("starting", &self.starting) + .field("image", &self.image) + .field("activation", &self.activation) + .finish_non_exhaustive() + } +} + +/// What every session of a clone's activation takes: the launch's capability, the sandbox's +/// activation record, and the activation that its Ready must report, by the restore-to-Ready +/// deadline. +struct ActivationSession<'a> { + /// The capability of the launch. + capability: &'a [u8; CAPABILITY_LEN], + /// The path of the activation record in the sandbox. + record: &'a Path, + /// The canonical encoding of the launch's activation record. + encoded: Vec, + /// The activation that the clone's Ready must report. + expected: ReadyActivation, + /// The restore-to-Ready deadline. + deadline: Instant, + /// The restore-to-Ready deadline's duration from the launch. + timeout: Duration, +} + +/// How a session of a clone's activation failed. +enum Attempt { + /// The adapter lost the session, after which another session may find the clone ready. + Lost(Error), + /// Any other failure, which is final. + Failed(Error), +} + +impl ActivationSession<'_> { + /// Activates the clone of `starting` in a new session of `control`, as + /// [`Restoring::activate`] describes, and returns the session once the clone is ready. + fn run( + &self, + control: &mut C, + starting: &Starting<'_>, + ) -> std::result::Result { + let record = self.read_record().map_err(Attempt::Failed)?; + let transport = self.connect(starting).map_err(Attempt::Failed)?; + let attached = control.attach(transport, self.capability, self.deadline); + let mut session = + attached.map_err(|error| self.failed(control, error, "attach its control session"))?; + let reported = control + .activate(&mut session, &record, self.deadline) + .map_err(|error| self.failed(control, error, "activate it"))?; + self.check(reported).map_err(Attempt::Failed)?; + Ok(session) + } + + /// Connects to the control endpoint of the VMM of `starting`, as [`session::connect`] does, + /// by the deadline. + fn connect(&self, starting: &Starting<'_>) -> Result> { + let reference = starting.process().reference(); + let endpoint = starting.endpoint(); + session::connect(endpoint, reference.pid, reference.start_time, self.deadline).map_err( + |error| { + let message = format!("cannot connect to the clone's control endpoint {endpoint}"); + match error { + SessionError::TimedOut => Error::timed_out(format!( + "{message} within {:?} of its launch", + self.timeout + )), + SessionError::ProcessExited => { + Error::exited(format!("{message}: its VMM exited")) + } + SessionError::Io(source) => Error::io(message).with_source(source), + error => Error::protocol(message).with_source(error), + } + }, + ) + } + + /// Reads the activation record in the sandbox, which must be, byte for byte, the encoding of + /// the launch's record, and returns it. + fn read_record(&self) -> Result> { + let path = self.record; + let bytes = open_regular(path) + .and_then(|file| read_at_most(file, MAX_RECORD_BYTES as u64, "an activation record")) + .map_err(|error| { + Error::io(format!( + "cannot read the clone's activation record {}", + path.display() + )) + .with_source(error) + })?; + if bytes != self.encoded { + return Err(Error::io(format!( + "{} no longer holds the activation record that the clone's launch wrote, the only \ + record that its restore sends", + path.display() + ))); + } + Ok(bytes) + } + + /// Checks `reported`, the activation that the clone's Ready reports, against the record's. + fn check(&self, reported: Option) -> Result<()> { + let expected = &self.expected; + let Some(reported) = reported else { + return Err(Error::protocol( + "the clone's Ready reports no activation, as a VM that boots cold reports none", + )); + }; + if reported == *expected { + return Ok(()); + } + let transaction = |activation: &ReadyActivation| { + activation + .transaction + .map_or_else(|| "none".to_owned(), |transaction| transaction.to_string()) + }; + Err(Error::protocol(format!( + "the clone's Ready reports the transaction {}, not its record's transaction {}", + transaction(&reported), + transaction(expected) + ))) + } + + /// Returns how a session failed with `error`, which `control` returned as it tried to `what`. + fn failed(&self, control: &C, error: C::Error, what: &str) -> Attempt { + if Instant::now() >= self.deadline { + return Attempt::Failed( + Error::timed_out(format!( + "the clone was not ready within {:?} of its launch: cannot {what}", + self.timeout + )) + .with_source(error), + ); + } + let lost = control.lost(&error); + let message = if lost { + format!("the clone's control session was lost: cannot {what}") + } else { + format!("cannot {what}") + }; + let error = Error::protocol(message).with_source(error); + if lost { + Attempt::Lost(error) + } else { + Attempt::Failed(error) + } + } +} + +/// Waits until `deadline`, `timeout` after the launch, for the restore-ready event of the VMM of +/// `starting` at `listener`, as [`Restoring::activate`] describes. +fn wait_event( + listener: &mut ReadyListener, + starting: &Starting<'_>, + deadline: Instant, + timeout: Duration, +) -> Result<()> { + let process = starting.process(); + let pid = process.reference().pid; + // OpenVMM writes the event once, and then closes its end, so the wait reads the event's bytes + // and nothing past them. + let length = RESTORE_READY_EVENT.len(); + let mut received = Vec::with_capacity(length); + while received.len() < length { + let left = deadline.saturating_duration_since(Instant::now()); + if left.is_zero() { + return Err(Error::timed_out(format!( + "the clone's VMM sent no restore-ready event within {timeout:?} of its launch" + ))); + } + let before = received.len(); + let open = listener + .receive(pid, &mut received, length, left.min(EXIT_CHECK_INTERVAL)) + .map_err(|error| { + let message = "cannot receive the restore-ready event of the clone's VMM"; + if error.kind() == io::ErrorKind::PermissionDenied { + Error::protocol(message).with_source(error) + } else { + Error::io(message).with_source(error) + } + })?; + if !RESTORE_READY_EVENT.starts_with(&received) { + return Err(Error::protocol(format!( + "the clone's VMM sent {:?} at its restore-ready endpoint, which is not OpenVMM's \ + restore-ready event", + String::from_utf8_lossy(&received) + ))); + } + if received.len() == length { + break; + } + if !open { + return Err(Error::protocol(format!( + "the clone's VMM closed its restore-ready endpoint after {} bytes of the event", + received.len() + ))); + } + if received.len() == before { + let exited = process.has_exited().map_err(|error| { + Error::io("cannot tell whether the clone's VMM still runs").with_source(error) + })?; + if exited { + return Err(Error::exited( + "the clone's VMM exited before its restore-ready event, as OpenVMM does once \ + it cannot restore the snapshot", + )); + } + } + } + Ok(()) +} + +/// Returns the error of the deadline `name`, `timeout` after its start, which this host's clock +/// cannot represent. +fn beyond_clock(name: &str, timeout: Duration) -> Error { + Error::unsupported(format!( + "the {name} deadline of a clone's restore, {timeout:?} after its start, is beyond this \ + host's clock" + )) +} + +/// Ends `starting`, a launch whose restore sequence failed with `error`, as +/// [`Starting::abandon`] does, and returns `error`, which says why the launch could not be ended +/// if it could not: its journal then remains for recovery. +fn abandoned(starting: Starting<'_>, error: Error) -> Error { + let reason = match starting.abandon() { + Ok(Recovery::Indeterminate(reason)) => reason, + Ok(_) => return error, + Err(failure) => failure.to_string(), + }; + let code = error.code(); + let (_, message, source) = error.into_parts(); + let error = Error::new( + code, + format!("{message}; the launch could not be ended, and remains for recovery: {reason}"), + ); + match source { + Some(source) => error.with_source(source), + None => error, + } +} diff --git a/nvx-rs/src/host/template/clone/tests.rs b/nvx-rs/src/host/template/clone/tests.rs index 063f7cc2b..0aeb338ad 100644 --- a/nvx-rs/src/host/template/clone/tests.rs +++ b/nvx-rs/src/host/template/clone/tests.rs @@ -1,8 +1,10 @@ //! Tests of the plans of clones: the checks of their images, uppers, and memory targets against //! the generation that they lease, which read no content, the claims of their scratch images, and -//! the sparse scratch device of an upper in RAM; and, in [`launch`], of their launches. +//! the sparse scratch device of an upper in RAM; in [`launch`], of their launches; and in +//! [`restore`], of their restore sequences. mod launch; +mod restore; use std::io::Read; use std::net::{IpAddr, Ipv4Addr}; diff --git a/nvx-rs/src/host/template/clone/tests/launch.rs b/nvx-rs/src/host/template/clone/tests/launch.rs index dc2495364..af9233a2c 100644 --- a/nvx-rs/src/host/template/clone/tests/launch.rs +++ b/nvx-rs/src/host/template/clone/tests/launch.rs @@ -3,7 +3,8 @@ //! [`restore_vmm_role`] stands in, listening beside the snapshot on Linux as OpenVMM does; the //! checks that keep a VMM from starting before the clone's binding is durable, or with a control //! socket that OpenVMM or the host could not use; the removal of that socket; and the crashes of -//! a launch's caller, which a [`launch_role`] runs. +//! a launch's caller, which a [`launch_role`] runs, in the launch and in the restore sequence that +//! follows it, around the clone's Ready and the start of its sandbox. use std::cell::RefCell; use std::env; @@ -14,8 +15,9 @@ use std::thread; use std::time::{Duration, Instant}; use super::*; -use crate::activation::StoreGeneration; +use crate::activation::{ReadyActivation, StoreGeneration}; use crate::host::artifacts; +use crate::host::platform::Transport; use crate::host::process::{Process, ProcessRef}; use crate::host::store::{ CLONE_ACTIVATION_NAME, CLONE_BINDING_NAME, CLONE_SCRATCH_NAME, JOURNAL_NAME, LaunchState, @@ -55,19 +57,24 @@ const ARGUMENTS: &str = "arguments.json"; /// File in which a [`restore_vmm_role`] writes, once it has tried to send its restore-ready /// event, the capability that it read, in hexadecimal, and whether it sent the event. const STARTED: &str = "started"; -/// The restore-ready event that OpenVMM sends. -const READY_EVENT: &[u8] = b"OPENVMM_RESTORE_READY_V1\n"; +/// File in the working directory of a [`restore_vmm_role`] whose words change how it sends its +/// restore-ready event, as [`VmmDir::plan`] describes them. Without it, the VMM sends the event at +/// once, and closes its end of the endpoint. +const EVENT_PLAN: &str = "event-plan"; +/// File in which a [`restore_vmm_role`] whose plan asks it to flush its event as OpenVMM does +/// notes that the flush returned: on Windows, once the host read the event. +const FLUSHED: &str = "flushed"; /// The OpenVMM executable that the tests' launches name, which [`fake_vmm`] replaces. -const OPENVMM: &str = "openvmm"; +pub(super) const OPENVMM: &str = "openvmm"; /// Longest wait for a VMM that is expected to start or exit, for a sandbox's lock, or for a /// reattach to become determinate. -const TIMEOUT: Duration = Duration::from_secs(60); +pub(super) const TIMEOUT: Duration = Duration::from_secs(60); /// Longest life of a [`restore_vmm_role`] that nothing ends, so that a failed test leaves no /// process. const VMM_LIFETIME: Duration = Duration::from_secs(180); /// Returns `bytes` in lowercase hexadecimal. -fn hex(bytes: &[u8]) -> String { +pub(super) fn hex(bytes: &[u8]) -> String { bytes.iter().map(|byte| format!("{byte:02x}")).collect() } @@ -88,7 +95,7 @@ fn role(name: &str) -> Vec { /// Gives the snapshot of the generation of `fixture` its RAM and saved state beside its manifest, /// as OpenVMM publishes them, and returns the snapshot's directory, and the content of each of /// its files, by name. -fn complete_snapshot(fixture: &Fixture) -> (PathBuf, Vec<(&'static str, Vec)>) { +pub(super) fn complete_snapshot(fixture: &Fixture) -> (PathBuf, Vec<(&'static str, Vec)>) { let snapshot = fixture.store.snapshot_dir(&fixture.generation).unwrap(); let mut contents = Vec::new(); for name in SNAPSHOT_ARTIFACTS { @@ -115,7 +122,7 @@ fn seals(snapshot: &Path) -> Vec<(&'static str, FileSeal)> { /// Returns the runtime identity that captured the template of `fixture`, under which its clones /// launch. -fn runtime_of(fixture: &Fixture) -> RuntimeIdentity { +pub(super) fn runtime_of(fixture: &Fixture) -> RuntimeIdentity { fixture .store .read_key(&fixture.generation) @@ -125,7 +132,7 @@ fn runtime_of(fixture: &Fixture) -> RuntimeIdentity { } /// Returns whether a socket, or another file, is at `path`. -fn exists(path: &Path) -> bool { +pub(super) fn exists(path: &Path) -> bool { fs::symlink_metadata(path).is_ok() } @@ -167,7 +174,7 @@ fn assert_unlaunched(fixture: &Fixture, acquired: &Acquired) { /// Asserts that the sandbox in `sandbox` holds none of the files that the launch of a clone /// leaves for its release. -fn assert_released(sandbox: &Path) { +pub(super) fn assert_released(sandbox: &Path) { for name in [ CLONE_SCRATCH_NAME, CLONE_ACTIVATION_NAME, @@ -185,7 +192,7 @@ fn never_spawned(_: VmmCommand<'_>) -> VmmCommand<'_> { /// The command that a launch made for its VMM, before [`fake_vmm`] replaced it, and what the /// sandbox held as the VMM was about to start. #[derive(Debug)] -struct Made { +pub(super) struct Made { /// The VMM's executable. program: PathBuf, /// The VMM's arguments. @@ -204,7 +211,7 @@ struct Made { /// keeps the command that the launch made in `made`, writes its arguments in [`ARGUMENTS`] in /// `vmm`, and runs the role there instead, with the descriptors or handles that the command /// passes. -fn fake_vmm<'a>( +pub(super) fn fake_vmm<'a>( vmm: &'a Path, made: &'a RefCell>, ) -> impl for<'h> Fn(VmmCommand<'h>) -> VmmCommand<'h> + 'a { @@ -235,16 +242,16 @@ fn fake_vmm<'a>( /// The working directory of a [`restore_vmm_role`], which it marks. Dropping it ends the VMM, and /// waits for it to exit, so that a failed test leaves no process behind. -struct VmmDir { +pub(super) struct VmmDir { /// The reference of the VMM, once the test knows it. - reference: Option, + pub(super) reference: Option, /// The directory, which is removed after the VMM exits. dir: tempfile::TempDir, } impl VmmDir { /// Creates the directory, and marks it. - fn new() -> Self { + pub(super) fn new() -> Self { let dir = tempfile::tempdir().unwrap(); fs::write(dir.path().join(VMM_MARKER), b"").unwrap(); Self { @@ -254,13 +261,13 @@ impl VmmDir { } /// Returns the path of the directory. - fn path(&self) -> &Path { + pub(super) fn path(&self) -> &Path { self.dir.path() } /// Waits until the VMM has tried to send its restore-ready event, and returns what it wrote /// in [`STARTED`]. - fn started(&self) -> String { + pub(super) fn started(&self) -> String { let path = self.path().join(STARTED); let deadline = Instant::now() + TIMEOUT; loop { @@ -276,13 +283,13 @@ impl VmmDir { } /// Asks the VMM to exit. - fn end(&self) { + pub(super) fn end(&self) { fs::write(self.path().join(EXIT_MARKER), b"").unwrap(); } /// Asks the VMM, which runs, to exit, and waits until it has. It opens the VMM's process /// before it asks, since a process that has exited no longer opens. - fn end_and_wait(&self) { + pub(super) fn end_and_wait(&self) { let reference = self.reference.as_ref().expect("the test knows the VMM"); let process = Process::open(reference) .unwrap() @@ -290,6 +297,25 @@ impl VmmDir { self.end(); assert!(process.wait(Instant::now() + TIMEOUT).unwrap()); } + + /// Changes how the VMM sends its restore-ready event, before it starts, by the words of + /// `plan`: + /// + /// - `delay:MS` sends it after MS milliseconds; + /// - `none` sends nothing, and runs on; + /// - `exit` sends nothing, and exits at once; + /// - `other` sends a version of the event that OpenVMM does not send; + /// - `partial` sends the first bytes of the event, then closes its end of the endpoint; + /// - `flush` flushes the event as OpenVMM does, which waits on Windows until the host read + /// it, before the VMM writes [`STARTED`], then notes it in [`FLUSHED`]. + pub(super) fn plan(&self, plan: &str) { + fs::write(self.path().join(EVENT_PLAN), plan).unwrap(); + } + + /// Returns whether the VMM's flush of its event returned. + pub(super) fn flushed(&self) -> bool { + self.path().join(FLUSHED).exists() + } } impl Drop for VmmDir { @@ -310,9 +336,11 @@ impl Drop for VmmDir { /// restore-ready event, and the snapshot that it restores, among the arguments that its launch /// made, which [`ARGUMENTS`] holds; holds the snapshot as OpenVMM does, which on Linux listens at /// the control endpoint beside the snapshot, and on Windows opens the snapshot's directory and -/// files sharing nothing but reading; sends the restore-ready event as OpenVMM does; writes -/// [`STARTED`]; then runs until a file named [`EXIT_MARKER`] appears, or for [`VMM_LIFETIME`] at -/// most, and removes its control socket as it exits, as OpenVMM does at a normal exit. +/// files sharing nothing but reading, and serves its control endpoint's named pipe; sends the +/// restore-ready event as OpenVMM does, or as its [plan](VmmDir::plan) says; writes [`STARTED`]; +/// then runs until a file named [`EXIT_MARKER`] appears, or for [`VMM_LIFETIME`] at most, and +/// removes its control socket as it exits, as OpenVMM does at a normal exit. It never answers on +/// its control endpoint. /// /// Where OpenVMM would refuse its arguments, this VMM writes why in [`STARTED`] after its /// capability and whether it sent its event. Where OpenVMM would abort a restore whose event it @@ -327,14 +355,46 @@ fn restore_vmm_role() { } let mut capability = [0; CAPABILITY_LEN]; std::io::stdin().read_exact(&mut capability).unwrap(); + let plan = fs::read_to_string(directory.join(EVENT_PLAN)).unwrap_or_default(); + let words: Vec<&str> = plan.split_whitespace().collect(); + if words.contains(&"exit") { + return; + } let arguments: Vec = serde_json::from_slice(&fs::read(directory.join(ARGUMENTS)).unwrap()).unwrap(); let (held, problem) = match hold_snapshot(&arguments) { Ok(held) => (Some(held), String::new()), Err(problem) => (None, format!(" {problem}")), }; + let delay = words + .iter() + .find_map(|word| word.strip_prefix("delay:")) + .map_or(0, |milliseconds| milliseconds.parse().unwrap()); + thread::sleep(Duration::from_millis(delay)); + let event = if words.contains(&"other") { + b"OPENVMM_RESTORE_READY_V2\n" + } else if words.contains(&"partial") { + &RESTORE_READY_EVENT[..10] + } else { + RESTORE_READY_EVENT + }; let ready = argument(&arguments, "--restore-ready-path").unwrap(); - let sent = send_ready(Path::new(&ready)).is_ok(); + let sent = if words.contains(&"none") { + false + } else { + match send_ready(Path::new(&ready), event) { + Ok(sink) if words.contains(&"flush") => { + // OpenVMM flushes its event, and then closes its end of the endpoint. + #[cfg(windows)] + sink.sync_all().unwrap(); + drop(sink); + fs::write(directory.join(FLUSHED), b"").unwrap(); + true + } + Ok(_) => true, + Err(_) => false, + } + }; let staged = directory.join(format!(".{STARTED}")); fs::write(&staged, format!("{} {sent}{problem}", hex(&capability))).unwrap(); fs::rename(&staged, directory.join(STARTED)).unwrap(); @@ -363,6 +423,9 @@ struct Held { /// The snapshot's directory and files, open, on Windows. #[cfg(windows)] _files: Vec, + /// The instances of the control endpoint's named pipe, on Windows. + #[cfg(windows)] + _control: Vec, } impl Held { @@ -423,7 +486,8 @@ fn hold_snapshot(arguments: &[String]) -> std::result::Result { /// Holds the snapshot that `arguments` restore as OpenVMM does, or returns why OpenVMM would /// refuse them: on Windows, OpenVMM opens the snapshot's directory and files for reading, sharing /// nothing but reading, so that a restore of another clone, which shares reading, reads them too, -/// while no one changes them. +/// while no one changes them. It also serves the named pipe of its control endpoint, as +/// [`control_pipe::serve`] does. #[cfg(windows)] fn hold_snapshot(arguments: &[String]) -> std::result::Result { use std::os::windows::fs::OpenOptionsExt; @@ -444,17 +508,79 @@ fn hold_snapshot(arguments: &[String]) -> std::result::Result { .map_err(|error| format!("cannot open {}: {error}", path.display()))?; files.push(file); } - Ok(Held { _files: files }) + let endpoint = argument(arguments, "--microvm-control-console") + .and_then(|console| console.strip_prefix("listen=").map(str::to_owned)) + .ok_or("no control endpoint")?; + let control = control_pipe::serve(&endpoint) + .map_err(|error| format!("cannot serve the control endpoint {endpoint}: {error}"))?; + Ok(Held { + _files: files, + _control: control, + }) } -/// Sends the restore-ready event to the endpoint at `path`, as OpenVMM does: through a Unix -/// socket, or a named pipe on Windows. -fn send_ready(path: &Path) -> io::Result<()> { +/// The named pipe of the control endpoint that a [`restore_vmm_role`] serves on Windows, as +/// OpenVMM does, through the Win32 calls that the standard library lacks. +#[cfg(windows)] +#[allow(unsafe_code)] +mod control_pipe { + use std::io; + use std::os::windows::io::{FromRawHandle, OwnedHandle}; + + use windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE; + use windows_sys::Win32::Storage::FileSystem::PIPE_ACCESS_DUPLEX; + use windows_sys::Win32::System::Pipes::{ + CreateNamedPipeW, PIPE_READMODE_BYTE, PIPE_TYPE_BYTE, PIPE_WAIT, + }; + + /// How many hosts may connect to the endpoint at once, one to each instance of its pipe: as + /// many as a test's restore sequence connects, at most. + const INSTANCES: u32 = 4; + + /// Creates the instances of the named pipe `endpoint`, `//./pipe/NAME`, each of which a host + /// connects to before the role, which never answers, would accept it. + pub(super) fn serve(endpoint: &str) -> io::Result> { + let name: Vec = endpoint + .replace('/', "\\") + .encode_utf16() + .chain(Some(0)) + .collect(); + (0..INSTANCES) + .map(|_| { + // SAFETY: the name is NUL-terminated, and outlives the call. + let handle = unsafe { + CreateNamedPipeW( + name.as_ptr(), + PIPE_ACCESS_DUPLEX, + PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_WAIT, + INSTANCES, + 4096, + 4096, + 0, + std::ptr::null(), + ) + }; + if handle == INVALID_HANDLE_VALUE { + return Err(io::Error::last_os_error()); + } + // SAFETY: CreateNamedPipeW returned a fresh handle that nothing else owns. + Ok(unsafe { OwnedHandle::from_raw_handle(handle) }) + }) + .collect() + } +} + +/// Sends `event` to the restore-ready endpoint at `path`, as OpenVMM sends its event: through a +/// Unix socket, or a named pipe on Windows. Returns the VMM's end of the endpoint. +fn send_ready(path: &Path, event: &[u8]) -> io::Result { #[cfg(unix)] - let mut sink = std::os::unix::net::UnixStream::connect(path)?; + let mut sink = File::from(std::os::fd::OwnedFd::from( + std::os::unix::net::UnixStream::connect(path)?, + )); #[cfg(windows)] let mut sink = fs::OpenOptions::new().write(true).open(path)?; - sink.write_all(READY_EVENT) + sink.write_all(event)?; + Ok(sink) } #[test] @@ -1200,9 +1326,72 @@ fn launch_role() { if step == "launched" { crash(&step); } + // The restore sequence, through a framing that crashes within the Activate or the start. + let mut control = RoleControl { step: &step }; + let activated = restoring + .activate(&mut control, RestoreDeadlines::default()) + .unwrap(); + if step == "activated" { + crash(&step); + } + let started = activated.start(&mut control).unwrap(); + if step == "started" { + crash(&step); + } + let committed = started.commit().unwrap(); + if step == "committed" { + crash(&step); + } + drop(committed); panic!("the clone's launch did not crash at {step}"); } +/// The adapter's framing of a [`launch_role`], whose clone answers as the record asks, and which +/// crashes as it awaits the clone's Ready, at the step `activating`, or the start of its sandbox, +/// at the step `starting`. +struct RoleControl<'a> { + /// The step at which the role crashes. + step: &'a str, +} + +impl CloneControl for RoleControl<'_> { + type Session = Box; + type Error = io::Error; + + fn attach( + &mut self, + transport: Box, + _capability: &[u8; CAPABILITY_LEN], + _deadline: Instant, + ) -> io::Result> { + Ok(transport) + } + + fn activate( + &mut self, + _session: &mut Box, + record: &[u8], + _deadline: Instant, + ) -> io::Result> { + if self.step == "activating" { + crash(self.step); + } + let record = ActivationRecord::decode(record).unwrap(); + Ok(Some(ReadyActivation::of(&record))) + } + + fn start(&mut self, _session: &mut Box, _deadline: Instant) -> io::Result<()> { + if self.step == "starting" { + crash(self.step); + } + Ok(()) + } + + fn lost(&self, _error: &io::Error) -> bool { + false + } +} + /// Ends this process at once, once it has printed that it crashes at `step`, without unwinding /// or running destructors, as a crash would. fn crash(step: &str) -> ! { @@ -1270,6 +1459,10 @@ impl Crashed { let (image, _) = fixture.image("distro.erofs", IMAGE_LEN); let sandbox = fixture.provision("sandbox"); let mut vmm = VmmDir::new(); + if !matches!(step, "prepared" | "begun" | "launched") { + // The restore sequence reads the event, whose flush waits for that on Windows. + vmm.plan("flush"); + } vmm.reference = run_launch_role(step, &fixture, &sandbox, &image, vmm.path()); let lease = Lease::of(&SandboxStore::open(&sandbox).unwrap()) .unwrap() @@ -1418,3 +1611,63 @@ fn a_launch_whose_caller_crashes_once_its_vmm_started_ends_the_vmm_as_reattach_r assert!(!exists(&crashed.socket())); crashed.releases(&mut locked); } + +#[test] +fn a_restore_whose_caller_crashes_before_its_commit_ends_its_vmm_as_reattach_recovers_it() { + // The caller crashes as it awaits the clone's Ready, once the clone is ready, as it awaits the + // start of the clone's sandbox, and once the sandbox started: each time, the launch has not + // committed, and its journal names the VMM. + for step in ["activating", "activated", "starting", "started"] { + let crashed = Crashed::at(step); + let reference = crashed.vmm.reference.clone().unwrap(); + let process = Process::open(&reference).unwrap().unwrap(); + // The caller read the VMM's restore-ready event, whose flush waited for that on Windows. + assert!(crashed.vmm.started().ends_with(" true"), "{step}"); + assert!(crashed.vmm.flushed(), "{step}"); + let mut locked = crashed.lock(); + let journal = locked.journal().unwrap().unwrap(); + assert_eq!(journal.launch(), crashed.lease.launch(), "{step}"); + assert_eq!(journal.state(), LaunchState::Spawned, "{step}"); + assert_eq!(journal.process(), Some(&reference), "{step}"); + assert!(locked.runtime().unwrap().is_none(), "{step}"); + crashed.kept_its_files(&locked); + // The lease lasts until reattach ended the VMM, through its verified reference. + crashed.refuses_release(&mut locked); + match crashed.reattach(&mut locked) { + Reattached::Exited { + record: None, + recovery: Recovery::Terminated(terminated), + } => assert_eq!(terminated, reference, "{step}"), + other => panic!("unexpected reattach at {step}: {other:?}"), + } + assert!(process.wait(Instant::now() + TIMEOUT).unwrap(), "{step}"); + assert!(!exists(&crashed.socket()), "{step}"); + crashed.releases(&mut locked); + } +} + +#[test] +fn a_restore_whose_caller_crashes_once_it_committed_keeps_its_lease_while_its_vm_runs() { + let crashed = Crashed::at("committed"); + let reference = crashed.vmm.reference.clone().unwrap(); + let mut locked = crashed.lock(); + // The runtime record names the VM, which reattach holds again, and the lease lasts as long as + // the VM runs. + assert!(locked.journal().unwrap().is_none()); + let record = locked.runtime().unwrap().unwrap(); + assert_eq!(record.process(), &reference); + crashed.kept_its_files(&locked); + let vmm = match crashed.reattach(&mut locked) { + Reattached::Live(vmm) => vmm, + other => panic!("unexpected reattach: {other:?}"), + }; + assert_eq!(vmm.process().reference(), &reference); + crashed.refuses_release(&mut locked); + // Once a termination verified the VM's exit, the release ends the lease. + assert_eq!( + vmm.terminate(&mut locked, TIMEOUT).unwrap(), + StopOutcome::Forced + ); + assert!(!exists(&crashed.socket())); + crashed.releases(&mut locked); +} diff --git a/nvx-rs/src/host/template/clone/tests/restore.rs b/nvx-rs/src/host/template/clone/tests/restore.rs new file mode 100644 index 000000000..e905ee0ad --- /dev/null +++ b/nvx-rs/src/host/template/clone/tests/restore.rs @@ -0,0 +1,747 @@ +//! Tests of the restore sequence of clones, [`Restoring::activate`], [`Activated::start`], and +//! [`Started::commit`], with a [`restore_vmm_role`](super::launch) as the clone's VMM, which +//! sends its restore-ready event as its plan says, and a [`FakeControl`] in place of the adapter's +//! framing: the wait for the event, and the connection only once it came; the Activate of the +//! sandbox's record, byte for byte, as the first request of each session; the check of the +//! activation that Ready reports; the start; the sessions that replace those that the adapter lost, +//! after a reset of the session, say, and the clone that ended once it took that reset before it +//! was ready; and the deadlines. The crashes of the sequence's caller are among the launch's +//! tests. + +use std::cell::RefCell; +use std::collections::VecDeque; +use std::thread; +use std::time::{Duration, Instant}; + +use super::launch::{OPENVMM, TIMEOUT, VmmDir, complete_snapshot, fake_vmm}; +use super::*; +use crate::activation::{ReadyActivation, RestoreSection, TransactionId}; +use crate::host::platform::Transport; +use crate::host::process::Process; +use crate::host::store::{CLONE_ACTIVATION_NAME, Recovery}; +use crate::host::supervisor::StopOutcome; +use crate::host::template::clone::launch::Hooks; + +/// How long a VMM whose plan delays its event waits before it sends it. +const DELAY: Duration = Duration::from_millis(300); + +/// A short deadline, which a test expects to pass. +const SHORT: Duration = Duration::from_secs(2); + +/// The restore-to-Ready deadline of a clone that never answers its Activate, which leaves its VMM +/// time to start on a busy host. +const SILENT: Duration = Duration::from_secs(10); + +/// A call that the restore sequence made to a [`FakeControl`]. +#[derive(Clone, Debug, PartialEq, Eq)] +enum Call { + /// An attachment of a session, numbered from 1, with the capability, when the call came, by + /// the deadline. + Attach { + session: usize, + capability: [u8; CAPABILITY_LEN], + at: Instant, + deadline: Instant, + }, + /// An Activate of the record in a session, by the deadline. + Activate { + session: usize, + record: Vec, + deadline: Instant, + }, + /// A start in a session, by the deadline. + Start { session: usize, deadline: Instant }, +} + +/// How a [`FakeControl`] answers an Activate or a start. +enum Answer { + /// With the clone's Ready, which reports the activation of the record that the Activate + /// carries, or with the start's confirmation. + Done, + /// With a Ready that reports this activation, or none. + Report(Option), + /// With this failure. + Fail(Failure), + /// With no answer until the deadline, then a failure that keeps the session. + Silent, +} + +/// A failure of a [`FakeControl`]: one that lost its session, or another. +#[derive(Debug)] +struct Failure { + /// Whether the failure lost the session. + lost: bool, +} + +impl Failure { + /// The loss of the session, as its reset or the end of its stream are. + const LOST: Self = Self { lost: true }; + /// Another failure, which keeps the session, as the guest's refusal does. + const REFUSED: Self = Self { lost: false }; +} + +impl fmt::Display for Failure { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(if self.lost { + "the fake framing lost its session" + } else { + "the fake guest refused" + }) + } +} + +impl std::error::Error for Failure {} + +/// A session of a [`FakeControl`], over a connection to the clone's control endpoint. +struct FakeSession { + /// The session's number, from 1. + number: usize, + /// The connection, which the session holds while it lasts, as a real one does. + _transport: Box, +} + +/// The adapter's framing of the tests: it attaches a session to each connection that the sequence +/// makes, numbered from 1, answers each Activate as the next of its `activates` says, and each +/// start as the next of its `starts` says, as [`Answer::Done`] does once they run out, and records +/// each call. Its `on_activate` runs at each Activate, with the session's number, before the +/// answer. +#[derive(Default)] +struct FakeControl<'a> { + /// The calls, in order. + calls: Vec, + /// The sessions that it attached. + sessions: usize, + /// How it answers the next Activates. + activates: VecDeque, + /// How it answers the next starts. + starts: VecDeque, + /// What runs at each Activate. + on_activate: Option>, +} + +impl<'a> FakeControl<'a> { + /// Returns a framing that answers the Activates as `activates` say. + fn activating(activates: impl IntoIterator) -> Self { + Self { + activates: activates.into_iter().collect(), + ..Self::default() + } + } + + /// Returns the records of the Activates, in order. + fn records(&self) -> Vec<&[u8]> { + self.calls + .iter() + .filter_map(|call| match call { + Call::Activate { record, .. } => Some(record.as_slice()), + _ => None, + }) + .collect() + } + + /// Returns the deadlines of the attachments and Activates, in order. + fn ready_deadlines(&self) -> Vec { + self.calls + .iter() + .filter_map(|call| match call { + Call::Attach { deadline, .. } | Call::Activate { deadline, .. } => Some(*deadline), + Call::Start { .. } => None, + }) + .collect() + } + + /// Checks that each session of the framing attached, then sent an Activate first, in its + /// order, and that the starts came last, in the last session. + fn assert_activates_first(&self) { + let mut session = 0; + let mut pending = false; + for call in &self.calls { + match call { + Call::Attach { + session: number, .. + } => { + assert!(!pending, "{:?}", self.calls); + session += 1; + assert_eq!(*number, session, "{:?}", self.calls); + pending = true; + } + Call::Activate { + session: number, .. + } => { + assert!(pending && *number == session, "{:?}", self.calls); + pending = false; + } + Call::Start { + session: number, .. + } => { + assert!(!pending && *number == session, "{:?}", self.calls); + } + } + } + } +} + +/// Answers `answer` by `deadline`, with the Ready `ready` for [`Answer::Done`]. +fn answered(answer: Answer, ready: T, deadline: Instant) -> Result +where + T: From>, +{ + match answer { + Answer::Done => Ok(ready), + Answer::Report(reported) => Ok(T::from(reported)), + Answer::Fail(failure) => Err(failure), + Answer::Silent => { + thread::sleep(deadline.saturating_duration_since(Instant::now())); + Err(Failure::REFUSED) + } + } +} + +impl CloneControl for FakeControl<'_> { + type Session = FakeSession; + type Error = Failure; + + fn attach( + &mut self, + transport: Box, + capability: &[u8; CAPABILITY_LEN], + deadline: Instant, + ) -> Result { + self.sessions += 1; + self.calls.push(Call::Attach { + session: self.sessions, + capability: *capability, + at: Instant::now(), + deadline, + }); + Ok(FakeSession { + number: self.sessions, + _transport: transport, + }) + } + + fn activate( + &mut self, + session: &mut FakeSession, + record: &[u8], + deadline: Instant, + ) -> Result, Failure> { + self.calls.push(Call::Activate { + session: session.number, + record: record.to_vec(), + deadline, + }); + if let Some(on_activate) = &mut self.on_activate { + on_activate(session.number); + } + let ready = ReadyActivation::of(&ActivationRecord::decode(record).unwrap()); + let answer = self.activates.pop_front().unwrap_or(Answer::Done); + answered(answer, Some(ready), deadline) + } + + fn start(&mut self, session: &mut FakeSession, deadline: Instant) -> Result<(), Failure> { + self.calls.push(Call::Start { + session: session.number, + deadline, + }); + let answer = self.starts.pop_front().unwrap_or(Answer::Done); + answered(answer, Confirmed, deadline).map(|_| ()) + } + + fn lost(&self, error: &Failure) -> bool { + error.lost + } +} + +/// The confirmation of a start, which [`answered`] returns. +struct Confirmed; + +impl From> for Confirmed { + fn from(_: Option) -> Self { + Self + } +} + +/// The instants around the launch of a clone, at which its restore-to-Ready deadline started. +struct Launched { + /// Before the launch. + before: Instant, + /// Once the launch returned. + after: Instant, +} + +impl Launched { + /// Checks that `deadline` is `timeout` after the launch. + fn assert_deadline(&self, deadline: Instant, timeout: Duration) { + assert!( + self.before + timeout <= deadline && deadline <= self.after + timeout, + "{deadline:?} is not {timeout:?} after the launch" + ); + } +} + +/// Launches the clone of `acquired`, which leases the generation of `fixture`, of `image` in RAM, +/// with a [`restore_vmm_role`](super::launch) in `vmm` as its VMM. Returns the launch, and when it +/// happened. +fn launch<'a>( + fixture: &Fixture, + acquired: &'a mut Acquired, + image: &(PathBuf, DistroBinding), + vmm: &mut VmmDir, +) -> (Restoring<'a>, Launched) { + let request = tmpfs_request(&image.0, &image.1); + let plan = fixture.plan(acquired, &request, NO_CONTENT).unwrap(); + let runtime = plan.key().runtime().clone(); + let dir = vmm.path().to_owned(); + let made = RefCell::new(None); + let command = fake_vmm(&dir, &made); + let hooks = Hooks { + command: &command, + ..Hooks::LAUNCH + }; + let before = Instant::now(); + let restoring = plan + .launch_with(acquired, Path::new(OPENVMM), &runtime, hooks) + .unwrap(); + let after = Instant::now(); + vmm.reference = Some(restoring.starting().process().reference().clone()); + (restoring, Launched { before, after }) +} + +/// A fresh fixture whose generation's snapshot is complete, and an image of its clones. +fn fixture() -> (Fixture, (PathBuf, DistroBinding)) { + let fixture = Fixture::new(); + complete_snapshot(&fixture); + let image = fixture.image("distro.erofs", IMAGE_LEN); + (fixture, image) +} + +/// Returns a VMM's working directory, whose VMM sends its event as `plan` says. +fn vmm_with(plan: &str) -> VmmDir { + let vmm = VmmDir::new(); + vmm.plan(plan); + vmm +} + +/// Checks that the VMM of `vmm` has exited. +fn assert_vmm_ended(vmm: &VmmDir) { + let reference = vmm.reference.as_ref().unwrap(); + if let Some(process) = Process::open(reference).unwrap() { + assert!(process.wait(Instant::now() + TIMEOUT).unwrap()); + } +} + +/// Checks that the restore sequence of the clone of `acquired` ended its launch, after its VMM in +/// `vmm`, and left the lease, which a release then ends. +fn assert_launch_ended(fixture: &Fixture, acquired: &mut Acquired, vmm: &VmmDir) { + assert_vmm_ended(vmm); + let locked = &mut acquired.locked; + assert!(locked.journal().unwrap().is_none()); + assert!(locked.runtime().unwrap().is_none()); + assert_eq!(Lease::of(locked).unwrap(), Some(acquired.lease.clone())); + assert_eq!( + fixture.store.release(locked).unwrap(), + Some(acquired.lease.clone()) + ); +} + +#[test] +fn a_clone_is_activated_once_its_restore_is_ready_then_started_and_committed() { + let (fixture, image) = fixture(); + let mut acquired = fixture.acquire("sandbox"); + let record = acquired.locked.clone_activation_path(); + // The VMM sends its event late, and flushes it as OpenVMM does, which waits on Windows until + // the host read it. + let mut vmm = vmm_with(&format!("delay:{} flush", DELAY.as_millis())); + let (restoring, launched) = launch(&fixture, &mut acquired, &image, &mut vmm); + let activation = restoring.activation().clone(); + let capability = *restoring.capability(); + let endpoint = restoring.starting().endpoint().to_owned(); + let mut control = FakeControl::default(); + let deadlines = RestoreDeadlines::default(); + assert_eq!(deadlines.ready, RESTORE_TO_READY_TIMEOUT); + assert_eq!(deadlines.start, READY_TO_START_TIMEOUT); + let activating = Instant::now(); + let mut activated = restoring.activate(&mut control, deadlines).unwrap(); + let ready = Instant::now(); + // The host read the event, and only then connected, and attached one session with the + // launch's capability, whose first request was the Activate of the sandbox's record, byte for + // byte the launch's, by the restore-to-Ready deadline, which started with the launch. + assert!(vmm.started().ends_with(" true")); + assert!(vmm.flushed()); + control.assert_activates_first(); + let [ + Call::Attach { + session: 1, + capability: attached, + at, + deadline, + }, + Call::Activate { + session: 1, + record: sent, + deadline: activated_by, + }, + ] = control.calls.as_slice() + else { + panic!("unexpected calls: {:?}", control.calls); + }; + assert_eq!(attached, &capability); + assert!( + *at >= launched.before + DELAY, + "the host connected before the event" + ); + launched.assert_deadline(*deadline, RESTORE_TO_READY_TIMEOUT); + assert_eq!(activated_by, deadline); + assert_eq!(sent, &activation.encode()); + assert_eq!(&fs::read(&record).unwrap(), sent); + // The Ready-to-Start deadline started at the check of the clone's Ready. + let start = activated.deadline(); + assert!( + activating + READY_TO_START_TIMEOUT <= start && start <= ready + READY_TO_START_TIMEOUT + ); + assert_eq!(activated.activation(), &activation); + assert_eq!(activated.starting().endpoint(), endpoint); + assert_eq!(activated.session().number, 1); + let shown = format!("{activated:?}"); + assert!(shown.starts_with("Activated {"), "{shown}"); + assert!(!shown.contains("capability"), "{shown}"); + // The sandbox starts in the same session, by that deadline, and the launch commits. + let started = activated.start(&mut control).unwrap(); + assert_eq!( + control.calls.last(), + Some(&Call::Start { + session: 1, + deadline: start + }) + ); + let (running, session) = started.commit().unwrap(); + assert_eq!(session.number, 1); + // The runtime record names the VM, whose lease lasts as long as it runs. + let locked = &mut acquired.locked; + let recorded = locked.runtime().unwrap().unwrap(); + assert_eq!(recorded.endpoint(), endpoint); + assert_eq!(running.endpoint(), endpoint); + assert!(locked.journal().unwrap().is_none()); + let error = fixture.store.release(locked).unwrap_err(); + assert_eq!(error.code(), ErrorCode::SandboxRunning, "{error}"); + assert_eq!(Lease::of(locked).unwrap(), Some(acquired.lease.clone())); + assert_eq!( + running.terminate(locked, TIMEOUT).unwrap(), + StopOutcome::Forced + ); + assert_eq!( + fixture.store.release(locked).unwrap(), + Some(acquired.lease.clone()) + ); +} + +/// The Ready that a clone reports, given another transaction than the record's. +type Reported = fn(TransactionId) -> Option; + +#[test] +fn a_ready_that_does_not_report_the_record_s_activation_ends_the_launch() { + let (fixture, image) = fixture(); + let other_transaction = TransactionId::from_bytes([0x6b; TransactionId::LEN]); + let cases: [(&str, Reported); 3] = [ + // Another transaction. + ("transaction", |transaction| { + Some(ReadyActivation { + transaction: Some(transaction), + }) + }), + // A cold boot's activation, without a transaction. + ("cold", |_| Some(ReadyActivation { transaction: None })), + // No activation. + ("none", |_| None), + ]; + for (name, reported) in cases { + let mut acquired = fixture.acquire(name); + let mut vmm = vmm_with("flush"); + let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); + let reported = reported(other_transaction); + let mut control = FakeControl::activating([Answer::Report(reported)]); + let error = restoring + .activate(&mut control, RestoreDeadlines::default()) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Protocol, "{name}: {error}"); + assert!( + error.message().starts_with("the clone's Ready reports"), + "{error}" + ); + // One session, which the host ended, and no start. + assert_eq!(control.calls.len(), 2, "{name}: {:?}", control.calls); + assert_launch_ended(&fixture, &mut acquired, &vmm); + } +} + +#[test] +fn a_session_that_the_adapter_lost_is_replaced_by_one_that_resends_the_same_record_first() { + let (fixture, image) = fixture(); + let mut acquired = fixture.acquire("sandbox"); + let record = acquired.locked.clone_activation_path(); + let mut vmm = vmm_with("flush"); + let (restoring, launched) = launch(&fixture, &mut acquired, &image, &mut vmm); + // The adapter loses its first session as it awaits the clone's Ready, as a reset of the + // session does: the clone, which was ready, answers the Activate of a new session. + let mut control = FakeControl::activating([Answer::Fail(Failure::LOST), Answer::Done]); + let activated = restoring + .activate(&mut control, RestoreDeadlines::default()) + .unwrap(); + control.assert_activates_first(); + assert_eq!(control.sessions, 2); + // Both Activates carry the sandbox's record, byte for byte, by the same deadline, which the + // retry did not extend. + let recorded = fs::read(&record).unwrap(); + assert_eq!(control.records(), [&recorded[..], &recorded[..]]); + let deadlines = control.ready_deadlines(); + assert!(deadlines.iter().all(|deadline| *deadline == deadlines[0])); + launched.assert_deadline(deadlines[0], RESTORE_TO_READY_TIMEOUT); + // The sandbox starts in the new session. + let started = activated.start(&mut control).unwrap(); + assert!(matches!( + control.calls.last(), + Some(Call::Start { session: 2, .. }) + )); + assert_eq!( + started.abandon().unwrap(), + Recovery::Terminated(vmm.reference.clone().unwrap()) + ); + assert_launch_ended(&fixture, &mut acquired, &vmm); +} + +#[test] +fn a_retry_never_sends_another_record_nor_takes_another_activation() { + let (fixture, image) = fixture(); + + // The sandbox's record changed once the first session was lost: the retry sends nothing, and + // connects no more. + let mut acquired = fixture.acquire("changed"); + let record = acquired.locked.clone_activation_path(); + let mut vmm = vmm_with("flush"); + let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); + let original = restoring.activation().clone(); + let restore = original.restore().unwrap(); + let other = ActivationRecord::new( + Some(RestoreSection { + transaction: TransactionId::from_bytes([0x77; TransactionId::LEN]), + template: restore.template, + }), + original.configuration().clone(), + ) + .unwrap(); + let path = record.clone(); + let mut control = FakeControl::activating([Answer::Fail(Failure::LOST)]); + control.on_activate = Some(Box::new(move |_| { + fs::write(&path, other.encode()).unwrap(); + })); + let error = restoring + .activate(&mut control, RestoreDeadlines::default()) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Io, "{error}"); + assert!(error.message().contains("no longer holds"), "{error}"); + assert_eq!(control.sessions, 1); + assert_eq!(control.records(), [&original.encode()[..]]); + assert_launch_ended(&fixture, &mut acquired, &vmm); + + // The Ready of the retry reports another transaction. + let mut acquired = fixture.acquire("diverged"); + let mut vmm = vmm_with("flush"); + let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); + let diverged = ReadyActivation { + transaction: Some(TransactionId::from_bytes([0x78; TransactionId::LEN])), + }; + let mut control = + FakeControl::activating([Answer::Fail(Failure::LOST), Answer::Report(Some(diverged))]); + let error = restoring + .activate(&mut control, RestoreDeadlines::default()) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Protocol, "{error}"); + assert_eq!(control.sessions, 2); + assert_launch_ended(&fixture, &mut acquired, &vmm); +} + +#[test] +fn only_lost_sessions_are_replaced_and_only_so_many_times() { + let (fixture, image) = fixture(); + + // A failure that did not lose the session is final. + let mut acquired = fixture.acquire("refused"); + let mut vmm = vmm_with("flush"); + let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); + let mut control = FakeControl::activating([Answer::Fail(Failure::REFUSED)]); + let error = restoring + .activate(&mut control, RestoreDeadlines::default()) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Protocol, "{error}"); + assert_eq!(error.message(), "cannot activate it"); + assert_eq!(control.sessions, 1); + assert_launch_ended(&fixture, &mut acquired, &vmm); + + // A session lost again and again ends the sequence once the last of its sessions is lost. + let mut acquired = fixture.acquire("lost"); + let mut vmm = vmm_with("flush"); + let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); + let lost = (0..MAX_ACTIVATION_SESSIONS).map(|_| Answer::Fail(Failure::LOST)); + let mut control = FakeControl::activating(lost); + let error = restoring + .activate(&mut control, RestoreDeadlines::default()) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Protocol, "{error}"); + assert!(error.message().contains("was lost"), "{error}"); + control.assert_activates_first(); + assert_eq!(control.sessions, MAX_ACTIVATION_SESSIONS); + assert_launch_ended(&fixture, &mut acquired, &vmm); +} + +#[test] +fn a_clone_that_took_the_reset_of_a_lost_session_before_its_ready_ends_the_retries() { + let (fixture, image) = fixture(); + let mut acquired = fixture.acquire("sandbox"); + let mut vmm = vmm_with("flush"); + let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); + // The reset of the lost session ends a clone that was not ready yet, and with it its VMM: the + // retry cannot connect any more. + let ending = &vmm; + let mut control = FakeControl::activating([Answer::Fail(Failure::LOST)]); + control.on_activate = Some(Box::new(move |_| ending.end_and_wait())); + let error = restoring + .activate(&mut control, RestoreDeadlines::default()) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Exited, "{error}"); + assert_eq!(control.sessions, 1); + drop(control); + assert_launch_ended(&fixture, &mut acquired, &vmm); +} + +#[test] +fn the_restore_to_ready_deadline_runs_from_the_launch_through_every_step() { + let (fixture, image) = fixture(); + let deadlines = RestoreDeadlines { + ready: SHORT, + ..RestoreDeadlines::default() + }; + + // A VMM that sends no event: the wait ends at the deadline, before any connection. + let mut acquired = fixture.acquire("no-event"); + let mut vmm = vmm_with("none"); + let (restoring, launched) = launch(&fixture, &mut acquired, &image, &mut vmm); + let mut control = FakeControl::default(); + let error = restoring.activate(&mut control, deadlines).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::TimedOut, "{error}"); + assert!(launched.before.elapsed() >= SHORT); + assert!(control.calls.is_empty()); + assert_launch_ended(&fixture, &mut acquired, &vmm); + + // A clone that never answers its Activate, after a late event: the attachment and the + // Activate have the deadline that started with the launch. + let mut acquired = fixture.acquire("no-ready"); + let mut vmm = vmm_with(&format!("delay:{} flush", DELAY.as_millis())); + let (restoring, launched) = launch(&fixture, &mut acquired, &image, &mut vmm); + let mut control = FakeControl::activating([Answer::Silent]); + let deadlines = RestoreDeadlines { + ready: SILENT, + ..RestoreDeadlines::default() + }; + let error = restoring.activate(&mut control, deadlines).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::TimedOut, "{error}"); + assert_eq!(control.sessions, 1); + for deadline in control.ready_deadlines() { + launched.assert_deadline(deadline, deadlines.ready); + } + assert!(launched.before.elapsed() >= deadlines.ready); + assert_launch_ended(&fixture, &mut acquired, &vmm); +} + +#[test] +fn the_ready_to_start_deadline_runs_from_the_ready_and_a_start_is_never_resent() { + let (fixture, image) = fixture(); + let deadlines = RestoreDeadlines { + start: SHORT, + ..RestoreDeadlines::default() + }; + + // A start that the guest never confirms ends at the deadline that started at the Ready. + let mut acquired = fixture.acquire("silent"); + let mut vmm = vmm_with("flush"); + let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); + let mut control = FakeControl::default(); + control.starts.push_back(Answer::Silent); + let ready = Instant::now(); + let activated = restoring.activate(&mut control, deadlines).unwrap(); + let start = activated.deadline(); + assert!(start >= ready + SHORT); + let error = activated.start(&mut control).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::TimedOut, "{error}"); + assert!(Instant::now() >= start); + assert_eq!( + control.calls.last(), + Some(&Call::Start { + session: 1, + deadline: start + }) + ); + assert_launch_ended(&fixture, &mut acquired, &vmm); + + // A start that fails, even in a lost session, is not sent again. + let mut acquired = fixture.acquire("lost"); + let mut vmm = vmm_with("flush"); + let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); + let mut control = FakeControl::default(); + control.starts.push_back(Answer::Fail(Failure::LOST)); + let activated = restoring.activate(&mut control, deadlines).unwrap(); + let error = activated.start(&mut control).unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Protocol, "{error}"); + let starts = control + .calls + .iter() + .filter(|call| matches!(call, Call::Start { .. })) + .count(); + assert_eq!((control.sessions, starts), (1, 1)); + assert_launch_ended(&fixture, &mut acquired, &vmm); +} + +#[test] +fn a_vmm_that_exits_or_sends_anything_but_its_event_ends_the_launch() { + let (fixture, image) = fixture(); + // A VMM that exits before its event, as OpenVMM does once it cannot restore the snapshot, ends + // the wait at once; another version of the event, and part of it, are refused. + let cases = [ + ("exit", ErrorKind::Exited), + ("other flush", ErrorKind::Protocol), + ("partial flush", ErrorKind::Protocol), + ]; + for (plan, kind) in cases { + let mut acquired = fixture.acquire(&plan.replace(' ', "-")); + let mut vmm = vmm_with(plan); + let (restoring, launched) = launch(&fixture, &mut acquired, &image, &mut vmm); + let mut control = FakeControl::default(); + let error = restoring + .activate(&mut control, RestoreDeadlines::default()) + .unwrap_err(); + assert_eq!(error.kind(), kind, "{plan}: {error}"); + assert!(launched.before.elapsed() < TIMEOUT, "{plan}"); + assert!(control.calls.is_empty(), "{plan}"); + assert_launch_ended(&fixture, &mut acquired, &vmm); + } + + // Another process than the VMM connects to the restore-ready endpoint. + let mut acquired = fixture.acquire("intruder"); + let mut vmm = vmm_with("none"); + let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); + let ready = PathBuf::from(restoring.ready_endpoint()); + #[cfg(unix)] + let _intruder = std::os::unix::net::UnixStream::connect(&ready).unwrap(); + #[cfg(windows)] + let _intruder = fs::OpenOptions::new().write(true).open(&ready).unwrap(); + let mut control = FakeControl::default(); + let error = restoring + .activate(&mut control, RestoreDeadlines::default()) + .unwrap_err(); + assert_eq!(error.kind(), ErrorKind::Protocol, "{error}"); + let source = std::error::Error::source(&error).unwrap().to_string(); + assert!(source.contains("not the VMM"), "{source}"); + assert!(control.calls.is_empty()); + assert_launch_ended(&fixture, &mut acquired, &vmm); + assert!(!acquired.locked.dir().join(CLONE_ACTIVATION_NAME).exists()); +} diff --git a/nvx-rs/src/host/template/mod.rs b/nvx-rs/src/host/template/mod.rs index 3feb60cd3..00af2578c 100644 --- a/nvx-rs/src/host/template/mod.rs +++ b/nvx-rs/src/host/template/mod.rs @@ -40,6 +40,13 @@ //! - [`ClonePlan::launch`] launches the clone, as the launch that its lease reserves: OpenVMM //! restores the generation's snapshot in place with the clone's devices, and the clone's //! activation record waits in its sandbox, as a [`Restoring`] describes. +//! - [`Restoring::activate`] runs the clone's restore sequence through the adapter's framing of +//! the clone's control session, a [`CloneControl`]: it waits for OpenVMM's restore-ready event, +//! then connects, sends the clone's activation record in the adapter's Activate, and checks the +//! activation that the clone's Ready reports, by a deadline that starts with the launch, and +//! resends the same record in a new session once the adapter lost one. [`Activated::start`] +//! then starts the clone's sandbox by a deadline that starts at its Ready, and +//! [`Started::commit`] commits the launch. //! //! # Canonical encoding //! @@ -71,7 +78,11 @@ mod store; mod test_support; mod wire; -pub use self::clone::{ClonePlan, CloneRequest, CloneUpper, Restoring}; +pub use self::clone::{ + Activated, CloneControl, ClonePlan, CloneRequest, CloneUpper, MAX_ACTIVATION_SESSIONS, + READY_TO_START_TIMEOUT, RESTORE_READY_EVENT, RESTORE_TO_READY_TIMEOUT, RestoreDeadlines, + Restoring, Started, +}; pub use self::eligibility::{Downtime, DowntimeSource, Eligibility, HostTime, MAX_DOWNTIME_NS}; pub use self::key::{CaptureClass, ClassIdentity, Invariants, KeyIdentity, TemplateKey}; pub use self::manifest::{ diff --git a/tests/runtime-adapter/README.md b/tests/runtime-adapter/README.md index d5479aa6b..d89423d5e 100644 --- a/tests/runtime-adapter/README.md +++ b/tests/runtime-adapter/README.md @@ -889,6 +889,10 @@ transport to the session's host endpoint, such as the one that - `configure_root` sends a root configuration, and returns how the guest answered: `Ready`, with the guest agent's build, or the stage of the root that failed; +- `activate` sends a clone's activation record in an `Activate`, and returns + the guest's `Ready`, with the guest agent's build and the clone's + activation. A clone whose activation failed answers nothing, but ends, and + its session with it; - `map` sends a part of the host's mapping table, and returns how the guest answered: `Mapped`, or `MapFailed`, with how many of the table's mappings it applied and the problem; `start` sends `Start`, and checks that the guest @@ -902,11 +906,11 @@ with a new message, in a new attachment. A failure of the session, a send that fails or times out, a malformed message, an answer whose body does not decode, and a message that only the host program -sends, an execution's start, input, end of input, or cancellation included, -end the client, and its later calls fail. A receive that times out doesn't: it -keeps what arrived. A request ends the client on any failure after the framing -accepted it, a timeout included, because an answer that arrived later would -answer the wrong request. +sends, an execution's start, input, end of input, or cancellation, and a +clone's `Activate` included, end the client, and its later calls fail. A +receive that times out doesn't: it keeps what arrived. A request ends the +client on any failure after the framing accepted it, a timeout included, +because an answer that arrived later would answer the wrong request. `host::Executions` is the host program's driver of [executions](#executions), over one attachment of the client: @@ -974,6 +978,27 @@ mapping that failed with how the stop ended, a table that does not encode, which fails before anything is sent, or a failure of the client. The host program [tears the VM down](#commands) after any of them. +A clone, which a restore of a [template's](#template-mode) snapshot started, +takes the host's activation of its record instead of a root configuration: +the guest assembles the clone's root from the record, with the inputs that the +template's capture kept, as it activates the clone. Its first step is +`host::activate` instead of `host::ready`: it sends the clone's activation +record in an `Activate`, the first request of the session, and waits for +`Ready`, whose build it checks as `host::ready` does, and which reports the +clone's activation. Then `host::start_sandbox` runs as after a cold boot. +`host::CloneOrder` runs both +steps as the framing of the restore sequence of `nvx::host::template`, which +checks the activation against the record, and which replaces a session that +the order takes for lost: one that a reset of the broker, the end of its +stream, or a failure of the transport ended while the host attached or +awaited `Ready`. A clone that was ready answers the same record, byte for +byte, with the same `Ready` in the new session, and one that took the reset +before it was ready has ended. OpenVMM's broker ends the stream both to end an +attachment and to refuse one, so the order takes such a refusal for a lost +session too, whose retries the sequence's limit of sessions bounds. Any other +failure is final: a refusal that the broker sends, a deadline that passed, +another build, and a message that breaks the framing. + ## VMs The host program runs the guest agent in VMs of NVX's artifacts, through NVX's @@ -1578,8 +1603,9 @@ Windows. They cover: payloads, a payload beyond the session's limit, which fails at its header, and header faults; - the messages that it refuses: malformed headers, which fail before their - body arrives, and messages that only the host program sends; and a message - beyond the framing's limit, which it refuses to send; + body arrives, and messages that only the host program sends, a clone's + `Activate` among them; and a message beyond the framing's limit, which it + refuses to send; - the answers that its requests check, and a request whose answer arrives too late; - a layered root: the configuration that the client sends, the guest's @@ -1597,6 +1623,16 @@ Windows. They cover: teardown that failed, or as the session ends; and `Maps` or `Start` answered with another kind, or failures whose bodies do not decode, which end the client; +- a clone's order: an attachment that waits through the session's bootstrap, + the record as the first request, after which the host waits for `Ready`, + whose activation it returns, also none, then the table in parts and `Start`; + a session that a later reset, the end of the stream, or a failure of the + transport ended, which the order takes for lost, and whose successor sends + the same record first; and the failures that are final: a refusal in an + ERROR, deadlines that pass while attaching or awaiting `Ready`, another + build, after which the host sends nothing more, an `Activate` answered with + another kind or with an activation that does not decode, and a record beyond + the framing's limit, which fails before it is sent; - the driver of executions, whose guest runs the runner of NVX's execution engine for each start, as the worker does, without a program: two executions at once, one with more output than its credit holds, each diff --git a/tests/runtime-adapter/src/host.rs b/tests/runtime-adapter/src/host.rs index 11cb1211c..339cae9b9 100644 --- a/tests/runtime-adapter/src/host.rs +++ b/tests/runtime-adapter/src/host.rs @@ -22,6 +22,21 @@ //! its root down, as NVX's managed lifecycle stops a guest whose mappings failed. The caller then //! waits for the VM to end, and ends a VM whose guest did not stop, as it ends one after any other //! failure of the order, an [`OrderError`]. +//! +//! # A clone's order +//! +//! A clone, which a restore of a template's snapshot started, takes the host's activation of its +//! record instead of a root configuration: the guest assembles the clone's root from the record, +//! with the inputs that the template's capture kept, as it activates the clone. Its first step is +//! [`activate`], which sends the clone's activation record in an `Activate`, the first request of +//! the session, and waits for the guest agent's `Ready`, whose build it checks as [`ready`] does, +//! and which reports the clone's activation, for the host to check against the record. Then +//! [`start_sandbox`] runs as after a cold boot. [`CloneOrder`] runs both steps as the adapter's +//! framing of `nvx::host::template`'s restore sequence, which checks the activation, and which +//! replaces a session that the order lost, as [`CloneOrder`] tells, while the clone was not ready +//! or its `Ready` had not arrived: the clone answers the same record, byte for byte, with the same +//! `Ready` in a new session, and has ended once the reset of a session that it took before it was +//! ready failed its activation. mod executions; mod map_reply; @@ -30,8 +45,10 @@ mod order_error; use std::fmt; use std::time::Instant; +use nvx::activation::ReadyActivation; use nvx::host::nvxs::{HostConnection, HostConnectionError}; use nvx::host::platform::Transport; +use nvx::host::template::CloneControl; use nvx::protocol::nvxs::Identity; use nvx::protocol::{CAPABILITY_LEN, HostMap}; @@ -119,8 +136,9 @@ impl From for ClientError { /// message in as many of the session's records as it takes, and [`receive`](Self::receive) /// returns the guest agent's next message, whether it spans records or shares one with the next /// message. [`ping`](Self::ping), [`echo`](Self::echo), [`configure_root`](Self::configure_root), -/// [`map`](Self::map), [`start`](Self::start), [`probe`](Self::probe), and [`stop`](Self::stop) -/// each send a request and wait for its answer, and [`Executions`] runs executions over the client. +/// [`activate`](Self::activate), [`map`](Self::map), [`start`](Self::start), +/// [`probe`](Self::probe), and [`stop`](Self::stop) each send a request and wait for its answer, +/// and [`Executions`] runs executions over the client. /// /// Besides the session's own buffers, the client holds at most the rest of one of the guest's /// records and one partial message, whose header [`Decoder`] checks before it takes any byte of @@ -215,7 +233,8 @@ impl Client { | Message::ExecStart(_) | Message::ExecInput(_) | Message::ExecInputEnd(_) - | Message::ExecCancel(_)), + | Message::ExecCancel(_) + | Message::Activate(_)), ) => { return Err(self.fail(ClientError::Unexpected(message.kind()))); } @@ -279,6 +298,22 @@ impl Client { } } + /// Sends `record`, the activation record of a clone, which a restore of a template's snapshot + /// started, in its canonical encoding, in an `Activate`, and waits until `deadline` for the + /// guest agent's answer: `Ready`, with the guest agent's build and the activation that it + /// reports, once it activated the clone. A clone whose activation failed answers nothing, but + /// ends, and its session with it. + /// + /// A record of more than [`MAX_ACTIVATE_BODY`](crate::frame::MAX_ACTIVATE_BODY) bytes fails + /// before anything is sent, and the client stays usable. + pub fn activate(&mut self, record: &[u8], deadline: Instant) -> Result { + let request = Message::Activate(record.to_vec()); + match self.request(&request, deadline)? { + Message::Ready(body) => self.decoded(Ready::decode(&body)), + reply => Err(self.mismatched(&request, &reply)), + } + } + /// Sends `part`, a part of the host's mapping table as [`encode_maps`] encodes it, in a /// `Maps`, and waits until `deadline` for the guest agent's answer: `Mapped` once it applied the /// part's mappings, or `MapFailed` once it refused the part or failed to apply one of them, @@ -447,3 +482,91 @@ pub fn start_sandbox( } client.start(deadline).map_err(OrderError::Client) } + +/// Runs the first step of [a clone's order](self#a-clones-order) over `client`, in place of +/// [`ready`]: sends `record`, the clone's activation record in its canonical encoding, in an +/// `Activate`, which must be the first request of the client's attachment, and waits until +/// `deadline` for the guest agent's `Ready`, which it returns once it names `build`, the guest +/// agent's build that the host ships, with the activation that it reports, which the caller checks +/// against the record. +/// +/// Fails with [`OrderError::Build`] for a guest agent of another build, to which it sends nothing +/// more, and with [`OrderError::Client`] once the client fails: as the session ends once the +/// clone's activation failed, or a reset ended the attachment. +pub fn activate( + client: &mut Client, + record: &[u8], + build: &str, + deadline: Instant, +) -> Result { + let ready = client + .activate(record, deadline) + .map_err(OrderError::Client)?; + if ready.build() != build { + return Err(OrderError::Build { + reported: ready.build().to_owned(), + expected: build.to_owned(), + }); + } + Ok(ready) +} + +/// [A clone's order](self#a-clones-order), as the adapter's framing of the clone's control +/// session, through which `nvx::host::template`'s restore sequence activates the clone and starts +/// its sandbox: it attaches a [`Client`] over each connection to the clone's control endpoint, +/// runs [`activate`] with the guest agent's build that the host ships, then [`start_sandbox`] with +/// the host's mapping table, which the clone's record announces. +/// +/// It takes a session for lost once a reset of the broker, the end of the broker's stream, or a +/// failure of the transport ended it, while the client attached or awaited the clone's `Ready`: +/// the clone may answer the same record in a new session. OpenVMM's broker ends the stream both to +/// end an attachment and to refuse one, as it refuses a host whose authentication outlasts the +/// broker's deadline, so the order takes such a refusal for a lost session too, whose retries the +/// restore sequence's limit of sessions bounds. Any other failure is final: a refusal that the +/// broker sends, a deadline that passed, a guest agent of another build, and a message that breaks +/// the framing. +#[derive(Clone, Copy, Debug)] +pub struct CloneOrder<'a> { + /// The guest agent's build that the host ships. + pub build: &'a str, + /// The host's mapping table, which the clone's activation record announces. + pub table: &'a [HostMap], +} + +impl CloneControl for CloneOrder<'_> { + type Session = Client; + type Error = OrderError; + + fn attach( + &mut self, + transport: Box, + capability: &[u8; CAPABILITY_LEN], + deadline: Instant, + ) -> Result { + Client::attach(transport, capability, deadline).map_err(OrderError::Client) + } + + fn activate( + &mut self, + session: &mut Client, + record: &[u8], + deadline: Instant, + ) -> Result, OrderError> { + activate(session, record, self.build, deadline).map(|ready| ready.activation()) + } + + fn start(&mut self, session: &mut Client, deadline: Instant) -> Result<(), OrderError> { + start_sandbox(session, self.table, deadline) + } + + fn lost(&self, error: &OrderError) -> bool { + matches!( + error, + OrderError::Client(ClientError::Session( + HostConnectionError::Reset(_) + | HostConnectionError::Closed + | HostConnectionError::Io(_) + )) + ) + } +} diff --git a/tests/runtime-adapter/src/host/order_error.rs b/tests/runtime-adapter/src/host/order_error.rs index 6f101e2d1..60a5e6b6c 100644 --- a/tests/runtime-adapter/src/host/order_error.rs +++ b/tests/runtime-adapter/src/host/order_error.rs @@ -1,6 +1,6 @@ //! How the [host's order](super#the-hosts-order) of a VM with a layered root fails, which -//! [`ready`](super::ready) and [`start_sandbox`](super::start_sandbox) return: an -//! [`OrderError`]. +//! [`ready`](super::ready), [`activate`](super::activate), and +//! [`start_sandbox`](super::start_sandbox) return: an [`OrderError`]. use std::fmt; @@ -15,8 +15,8 @@ use crate::root::{MapFailure, RootFailure}; pub enum OrderError { /// The guest agent reported that a stage of the root failed, after which it ends. Root(RootFailure), - /// The guest agent assembled the root, but is of another build than the one that the host - /// ships, so the host sent it nothing more. + /// The guest agent assembled the root, or activated a clone, but is of another build than the + /// one that the host ships, so the host sent it nothing more. Build { /// The build that the guest agent reported in its `Ready`. reported: String, diff --git a/tests/runtime-adapter/src/lib.rs b/tests/runtime-adapter/src/lib.rs index 93a08542b..0f10a0340 100644 --- a/tests/runtime-adapter/src/lib.rs +++ b/tests/runtime-adapter/src/lib.rs @@ -51,7 +51,9 @@ //! captures such a VM through NVX's template store, which publishes the snapshot as a generation //! of a template, in the `capture` module. In each clone, the worker repairs the restore, then //! activates the clone with the host's `Activate`, whose record the guest library applies, answers -//! `Ready` with the activation, and serves the host's order as after a cold boot. +//! `Ready` with the activation, and serves the host's order as after a cold boot. On the host, the +//! `host` module's clone order is the framing through which the restore sequence of +//! `nvx::host::template` activates a clone and starts its sandbox. #[cfg(feature = "host")] pub mod capture; diff --git a/tests/runtime-adapter/tests/host_session.rs b/tests/runtime-adapter/tests/host_session.rs index a74ed462c..d39b7b043 100644 --- a/tests/runtime-adapter/tests/host_session.rs +++ b/tests/runtime-adapter/tests/host_session.rs @@ -10,10 +10,14 @@ //! decoder, the requests of a layered root: its configuration, its probes, and a stop whose //! teardown failed, and the host's order of a layered root: the guest agent's readiness and build, //! the mapping table in parts, and `Start`, and each failure of the order, after which the host -//! sends nothing more, or stops the guest. The broker's guest also runs the runners of NVX's -//! execution engine for the executions that the host program's driver runs, so that the tests cover -//! the driver's output, credit, outcomes, and timers, the refusals that end its run, and a guest -//! that stops reading. +//! sends nothing more, or stops the guest. They also cover a clone's order, the framing of the +//! restore sequence of `nvx::host::template`: the bootstrap of its session, its record as the +//! first request, the `Ready` that reports its activation, the sessions that it takes for lost, +//! after a later reset, the end of the stream, or a failure of the transport, whose successor sends +//! the same record, and the failures that are final. The broker's guest also runs the runners of +//! NVX's execution engine for the executions that the host program's driver runs, so that the tests +//! cover the driver's output, credit, outcomes, and timers, the refusals that end its run, and a +//! guest that stops reading. #![cfg(feature = "host")] use std::collections::{BTreeMap, VecDeque}; @@ -22,12 +26,14 @@ use std::sync::{Arc, Condvar, Mutex, MutexGuard}; use std::thread::{self, JoinHandle}; use std::time::{Duration, Instant}; +use nvx::activation::{ReadyActivation, TransactionId}; use nvx::exec::{ Cause, Key, LaunchFailure, Limits, Outcome, Runner, Settlement, Status, Stream, Timers, Violation, Window, runner, }; use nvx::host::nvxs::HostConnectionError; use nvx::host::platform::Transport; +use nvx::host::template::CloneControl; use nvx::protocol::nvxs::{ ErrorCode, Identity, Record, RecordError, RecordParser, RecordType, SessionError, encode_record, }; @@ -37,9 +43,12 @@ use nvx::protocol::{ }; use runtime_adapter::GUEST_BUILD; use runtime_adapter::exec::{GuestMessage, HostMessage, MAX_EXECUTIONS, Start}; -use runtime_adapter::frame::{Decoder, FrameError, HEADER_LEN, MAX_BODY, MAX_TEXT_BODY, Message}; +use runtime_adapter::frame::{ + Decoder, FrameError, HEADER_LEN, MAX_ACTIVATE_BODY, MAX_BODY, MAX_TEXT_BODY, Message, +}; use runtime_adapter::host::{ - self, Client, ClientError, Executions, ExecutionsError, MAX_OUTPUT_FRAMES, MapReply, OrderError, + self, Client, ClientError, CloneOrder, Executions, ExecutionsError, MAX_OUTPUT_FRAMES, + MapReply, OrderError, }; use runtime_adapter::probe::{Facts, Probe, ProbeReply}; use runtime_adapter::root::{ @@ -1214,6 +1223,7 @@ fn the_client_refuses_messages_that_its_framing_does_not_allow() { HostMessage::Input(key, b"input").encode(), HostMessage::InputEnd(key).encode(), HostMessage::Cancel(key, Cause::Cancelled).encode(), + Message::Activate(activation_record()), ]; for message in host_messages { let (mut broker, mut client) = attached(FIRST, usize::MAX); @@ -1269,9 +1279,10 @@ fn requests_check_their_answers_and_end_the_client_when_they_fail() { let stop: Request = |client| client.stop(deadline()).map(drop); let map: Request = |client| client.map(&maps_part(), deadline()).map(drop); let start: Request = |client| client.start(deadline()); + let activate: Request = |client| client.activate(&activation_record(), deadline()).map(drop); let teardown = RootFailure::new(RootStage::Teardown, "busy").encode(); // Each request, and answers that don't answer it. - let cases: [(&str, Request, Vec, Kinds); 16] = [ + let cases: [(&str, Request, Vec, Kinds); 18] = [ ( "Ping answered with Stopped", |client| client.ping(deadline()), @@ -1371,6 +1382,20 @@ fn requests_check_their_answers_and_end_the_client_when_they_fail() { vec![Message::MapFailed(MapFailure::new(0, "late").encode())], Some((25, 24)), ), + ( + "an Activate answered with Started", + activate, + vec![Message::Started], + Some((27, 26)), + ), + ( + "an Activate answered with a root's failure", + activate, + vec![Message::RootFailed( + RootFailure::new(RootStage::Layout, "late").encode(), + )], + Some((27, 8)), + ), ]; for (name, request, replies, kinds) in cases { let (mut broker, client) = attached(FIRST, usize::MAX); @@ -1496,7 +1521,10 @@ fn answers_whose_bodies_do_not_decode_end_the_client() { }; let stop: Request = |client| client.stop(deadline()).map(drop); let map: Request = |client| client.map(&maps_part(), deadline()).map(drop); - let cases: [(&str, Request, Message, &str); 11] = [ + let activate: Request = |client| client.activate(&activation_record(), deadline()).map(drop); + let mut partial_activation = ready().encode(); + partial_activation.extend([0; 4]); + let cases: [(&str, Request, Message, &str); 12] = [ ( "a readiness without a build", configure, @@ -1563,6 +1591,12 @@ fn answers_whose_bodies_do_not_decode_end_the_client() { Message::MapFailed(vec![0, 0, 0, 0, 0xff]), "a failure of the mapping table is not UTF-8 text", ), + ( + "a clone's readiness with bytes after its build that are not an activation", + activate, + Message::Ready(partial_activation), + "has 4 bytes after its build, which are not an activation", + ), ]; for (name, request, reply, problem) in cases { let (mut broker, client) = attached(FIRST, usize::MAX); @@ -1820,6 +1854,253 @@ fn a_mapping_that_the_guest_fails_ends_the_table_and_the_host_stops_the_guest() ); } +/// Returns the bytes that stand for a clone's activation record in its canonical encoding, which +/// the host's client carries in an `Activate` without decoding them. +fn activation_record() -> Vec { + b"\x01the clone's activation record".to_vec() +} + +/// Returns the activation that the guest of a clone reports in its `Ready`. +fn clone_activation() -> ReadyActivation { + ReadyActivation { + transaction: Some(TransactionId::from_bytes([0x4d; TransactionId::LEN])), + } +} + +/// Returns the `Ready` of a clone, which reports [`clone_activation`] after the guest agent's build +/// that the host program ships. +fn clone_ready() -> Message { + Message::Ready(ready().with_activation(clone_activation()).encode()) +} + +/// Returns the order of a clone of the guest agent's build that the host program ships, with no +/// mapping. +fn clone_order() -> CloneOrder<'static> { + CloneOrder { + build: GUEST_BUILD, + table: &[], + } +} + +/// Runs the first step of a clone's order with `client` in a thread: sends [`activation_record`] in an +/// `Activate`. The thread returns the client with the activation that `Ready` reports, or how the +/// step failed. +fn activating(client: Client) -> JoinHandle<(Client, Result, OrderError>)> { + spawn(client, |client| { + clone_order().activate(client, &activation_record(), deadline()) + }) +} + +#[test] +fn a_clone_s_order_sends_its_record_first_then_starts_its_sandbox_as_after_a_cold_boot() { + // The broker waits for the guest's acknowledgement of the restore's reset, the session's + // bootstrap, before it attaches the host, which waits for it. + let (mut broker, transport) = connect(FIRST, usize::MAX); + let attaching = thread::spawn(move || clone_order().attach(transport, &CAPABILITY, deadline())); + broker.accept(); + for _ in 0..3 { + broker.send(RecordType::Wait, &[]); + thread::sleep(Duration::from_millis(20)); + } + assert!(!attaching.is_finished(), "the order waits for READY"); + broker.send(RecordType::Ready, &[]); + let client = joined(attaching).unwrap(); + assert_eq!(client.identity(), FIRST); + + // The record is the session's first request, after which the host waits for `Ready`, which + // arrives in records of 7 bytes. + let activation = activating(client); + assert_eq!(broker.messages(1), [Message::Activate(activation_record())]); + broker.assert_quiet(); + broker.send_data(&clone_ready().encode().unwrap(), 7); + let (client, activated) = joined(activation); + assert_eq!(activated.unwrap(), Some(clone_activation())); + + // Then the table, in its parts, each once the guest applied the one before, and `Start`. + let table = long_table(); + let parts = encode_maps(&table).unwrap(); + let starting = spawn(client, move |client| { + let mut order = CloneOrder { + build: GUEST_BUILD, + table: &table, + }; + order.start(client, deadline()) + }); + for part in parts { + broker.expect(Message::Maps(part), &Message::Mapped); + } + broker.expect(Message::Start, &Message::Started); + let (client, started) = joined(starting); + started.unwrap(); + drop(client); + assert!(broker.read().is_none(), "the host sent more"); + + // A `Ready` that reports no activation, as a cold boot's does, is returned as none, for the + // restore sequence to refuse; `host::activate` returns the readiness with its build. + let (mut broker, client) = attached(FIRST, usize::MAX); + let activation = activating(client); + broker.expect(Message::Activate(activation_record()), &ready_message()); + let (client, activated) = joined(activation); + assert_eq!(activated.unwrap(), None); + let activation = spawn(client, |client| { + host::activate(client, &activation_record(), GUEST_BUILD, deadline()) + }); + broker.expect(Message::Activate(activation_record()), &clone_ready()); + let readiness = joined(activation).1.unwrap(); + assert_eq!(readiness.build(), GUEST_BUILD); + assert_eq!(readiness.activation(), Some(clone_activation())); +} + +#[test] +fn a_clone_s_session_that_ends_after_its_bootstrap_is_lost_and_the_next_takes_the_same_record() { + let next = Identity { epoch: 2, ..FIRST }; + // The broker resets the session, which a broker announces, or ends its stream, as OpenVMM's + // broker ends an attachment, while the host awaits `Ready`; or the transport fails as the + // host sends the record. + for ending in ["reset", "end of stream", "transport"] { + let (mut broker, client) = attached(FIRST, usize::MAX); + if ending == "transport" { + broker.from_host.close(); + } + let activation = activating(client); + match ending { + "reset" => { + assert_eq!(broker.messages(1), [Message::Activate(activation_record())]); + broker.identity = next; + broker.send(RecordType::Reset, &[]); + } + "end of stream" => { + assert_eq!(broker.messages(1), [Message::Activate(activation_record())]); + broker.close(); + } + _ => {} + } + let (mut client, activated) = joined(activation); + let error = activated.unwrap_err(); + assert!(clone_order().lost(&error), "{ending}: {error:?}"); + assert_ended(&mut client); + + // The order's next session sends the same record, byte for byte, as its first request, + // and the clone, which was ready, answers it with the same `Ready`. + let (mut broker, client) = attached(next, usize::MAX); + let activation = activating(client); + broker.expect(Message::Activate(activation_record()), &clone_ready()); + let (_, activated) = joined(activation); + assert_eq!(activated.unwrap(), Some(clone_activation()), "{ending}"); + } +} + +#[test] +fn only_a_session_that_ended_is_lost_to_a_clone_s_order() { + let not_lost = |error: &OrderError| !clone_order().lost(error); + + // The broker refuses the capability with an ERROR. + let (mut broker, transport) = connect(FIRST, usize::MAX); + let attaching = thread::spawn(move || clone_order().attach(transport, &CAPABILITY, deadline())); + broker.accept(); + broker.send( + RecordType::Error, + &ErrorCode::Authentication.code().to_le_bytes(), + ); + let error = joined(attaching).unwrap_err(); + let rejected = matches!( + error, + OrderError::Client(ClientError::Session(HostConnectionError::Rejected( + ErrorCode::Authentication + ))) + ); + assert!(rejected && not_lost(&error), "{error:?}"); + + // The deadline passes while the broker waits for the guest. + let (mut broker, transport) = connect(FIRST, usize::MAX); + broker.send(RecordType::Wait, &[]); + let attaching = + thread::spawn(move || clone_order().attach(transport, &CAPABILITY, Instant::now() + SHORT)); + let error = joined(attaching).unwrap_err(); + let late = matches!( + error, + OrderError::Client(ClientError::Session(HostConnectionError::TimedOut)) + ); + assert!(late && not_lost(&error), "{error:?}"); + + // The guest agent that activated the clone is of another build: the host sends it nothing + // more. + let (mut broker, client) = attached(FIRST, usize::MAX); + let activation = activating(client); + let another = Ready::new("another-build") + .unwrap() + .with_activation(clone_activation()); + broker.expect( + Message::Activate(activation_record()), + &Message::Ready(another.encode()), + ); + let (client, activated) = joined(activation); + drop(client); + assert!(broker.read().is_none(), "the host sent more"); + let error = activated.unwrap_err(); + let other = matches!( + &error, + OrderError::Build { reported, expected } + if reported == "another-build" && expected == GUEST_BUILD + ); + assert!(other && not_lost(&error), "{error:?}"); + + // `Ready` does not arrive in time. + let (mut broker, client) = attached(FIRST, usize::MAX); + let activation = spawn(client, |client| { + clone_order().activate(client, &activation_record(), Instant::now() + SHORT) + }); + assert_eq!(broker.messages(1), [Message::Activate(activation_record())]); + let (mut client, activated) = joined(activation); + let error = activated.unwrap_err(); + let late = matches!( + error, + OrderError::Client(ClientError::Session(HostConnectionError::TimedOut)) + ); + assert!(late && not_lost(&error), "{error:?}"); + assert_ended(&mut client); + + // The guest answers the record with another kind, or with a `Ready` whose activation does not + // decode. + let mut partial = ready().encode(); + partial.push(1); + for reply in [Message::Pong, Message::Ready(partial)] { + let (mut broker, client) = attached(FIRST, usize::MAX); + let activation = activating(client); + broker.expect(Message::Activate(activation_record()), &reply); + let (mut client, activated) = joined(activation); + let error = activated.unwrap_err(); + let failed = matches!( + &error, + OrderError::Client(ClientError::Reply { + request: 27, + reply: 2 + }) | OrderError::Client(ClientError::Body(_)) + ); + assert!(failed && not_lost(&error), "{reply:?}: {error:?}"); + assert_ended(&mut client); + } + + // A record that the framing refuses fails before it is sent, and the client stays usable. + let (mut broker, client) = attached(FIRST, usize::MAX); + let activation = spawn(client, |client| { + let too_long = vec![0; MAX_ACTIVATE_BODY + 1]; + clone_order().activate(client, &too_long, deadline()) + }); + let (client, activated) = joined(activation); + let error = activated.unwrap_err(); + let expected = FrameError::Length { + kind: 27, + length: MAX_ACTIVATE_BODY + 1, + }; + let refused = + matches!(&error, OrderError::Client(ClientError::Frame(found)) if *found == expected); + assert!(refused && not_lost(&error), "{error:?}"); + let pinging = spawn(client, |client| client.ping(deadline())); + assert_eq!(broker.answer(OUTER_MAX_PAYLOAD), Message::Ping); + joined(pinging).1.unwrap(); +} + /// The chunk of the executions of the tests of the host's driver. const CHUNK: u32 = 16 * 1024;