From 7f1d3468f443ce76c92bc61c736efd2387c25bf3 Mon Sep 17 00:00:00 2001 From: Simon Veith Date: Thu, 2 Apr 2026 15:58:21 +0200 Subject: [PATCH 1/7] DNM: vmm: Rename ConsoleConfig.file to output_file In order to prepare for supporting bidirectional communication over a file-backed virtio-console device, rename the field in `ConsoleConfig` from `file` to `output_file` to clarify its purpose. No functional change intended, but merging this change would likely break API compatibility. Signed-off-by: Simon Veith --- cloud-hypervisor/src/main.rs | 21 +++++++++++++++++++-- fuzz/fuzz_targets/http_api.rs | 4 ++-- vmm/src/config.rs | 31 +++++++++++++++++++++---------- vmm/src/console_devices.rs | 8 ++++---- vmm/src/lib.rs | 4 ++-- vmm/src/vm_config.rs | 10 +++++----- 6 files changed, 53 insertions(+), 25 deletions(-) diff --git a/cloud-hypervisor/src/main.rs b/cloud-hypervisor/src/main.rs index 65eb8cca85..99b8e396d5 100644 --- a/cloud-hypervisor/src/main.rs +++ b/cloud-hypervisor/src/main.rs @@ -1154,14 +1154,14 @@ mod tests { pmem: None, serial: SerialConfig { common: CommonConsoleConfig { - file: None, + output_file: None, mode: ConsoleOutputMode::Null, socket: None, }, }, console: ConsoleConfig { common: CommonConsoleConfig { - file: None, + output_file: None, mode: ConsoleOutputMode::Tty, socket: None, }, @@ -1860,6 +1860,23 @@ mod tests { }"#, true, ), + ( + vec![ + "cloud-hypervisor", + "--kernel", + "/path/to/kernel", + "--serial", + "pty", + "--console", + "file=/path/to/console", + ], + r#"{ + "payload": {"kernel": "/path/to/kernel"}, + "serial": {"mode": "Pty"}, + "console": {"mode": "File", "output_file": "/path/to/console"} + }"#, + true, + ), ( vec![ "cloud-hypervisor", diff --git a/fuzz/fuzz_targets/http_api.rs b/fuzz/fuzz_targets/http_api.rs index 8aa3a00f98..1e8658e00e 100644 --- a/fuzz/fuzz_targets/http_api.rs +++ b/fuzz/fuzz_targets/http_api.rs @@ -180,14 +180,14 @@ impl RequestHandler for StubApiRequestHandler { pmem: None, serial: SerialConfig { common: CommonConsoleConfig { - file: None, + output_file: None, mode: ConsoleOutputMode::Tty, socket: None, }, }, console: ConsoleConfig { common: CommonConsoleConfig { - file: None, + output_file: None, mode: ConsoleOutputMode::Tty, socket: None, }, diff --git a/vmm/src/config.rs b/vmm/src/config.rs index e1654de17a..9c9947b93f 100644 --- a/vmm/src/config.rs +++ b/vmm/src/config.rs @@ -2458,7 +2458,7 @@ impl CommonConsoleConfig { .add_all(Self::VALUE_OPTIONS); parser.parse_subset(console).map_err(map_err)?; - let mut file: Option = None; + let mut output_file: Option = None; let mut socket: Option = None; let mut mode: ConsoleOutputMode = ConsoleOutputMode::Off; @@ -2471,7 +2471,7 @@ impl CommonConsoleConfig { mode = ConsoleOutputMode::Null; } else if parser.is_set("file") { mode = ConsoleOutputMode::File; - file = + output_file = Some(PathBuf::from(parser.get("file").ok_or( Error::Validation(ValidationError::ConsoleFileMissing), )?)); @@ -2484,7 +2484,11 @@ impl CommonConsoleConfig { return Err(Error::ParseConsoleInvalidModeGiven); } - Ok(Self { mode, file, socket }) + Ok(Self { + mode, + output_file, + socket, + }) } pub fn validate(&self) -> ValidationResult<()> { @@ -3531,12 +3535,15 @@ impl VmConfig { warn!("Using TTY output for multiple consoles: {tty_consoles:?}"); } - if self.console.common.mode == ConsoleOutputMode::File && self.console.common.file.is_none() + if self.console.common.mode == ConsoleOutputMode::File + && self.console.common.output_file.is_none() { return Err(ValidationError::ConsoleFileMissing); } - if self.serial.common.mode == ConsoleOutputMode::File && self.serial.common.file.is_none() { + if self.serial.common.mode == ConsoleOutputMode::File + && self.serial.common.output_file.is_none() + { return Err(ValidationError::ConsoleFileMissing); } @@ -5146,8 +5153,12 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}" #[test] fn test_console_parsing() -> Result<()> { - let console_config = |mode, file, socket, iommu| ConsoleConfig { - common: CommonConsoleConfig { file, mode, socket }, + let console_config = |mode, output_file, socket, iommu| ConsoleConfig { + common: CommonConsoleConfig { + output_file, + mode, + socket, + }, pci_common: PciDeviceCommonConfig { iommu, ..Default::default() @@ -6140,14 +6151,14 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}" pmem: None, serial: SerialConfig { common: CommonConsoleConfig { - file: None, + output_file: None, mode: ConsoleOutputMode::Null, socket: None, }, }, console: ConsoleConfig { common: CommonConsoleConfig { - file: None, + output_file: None, mode: ConsoleOutputMode::Tty, socket: None, }, @@ -6266,7 +6277,7 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}" let mut invalid_config = valid_config.clone(); invalid_config.serial.common.mode = ConsoleOutputMode::File; - invalid_config.serial.common.file = None; + invalid_config.serial.common.output_file = None; assert_eq!( invalid_config.validate(), Err(ValidationError::ConsoleFileMissing) diff --git a/vmm/src/console_devices.rs b/vmm/src/console_devices.rs index 764bda03bd..d89a10f679 100644 --- a/vmm/src/console_devices.rs +++ b/vmm/src/console_devices.rs @@ -200,7 +200,7 @@ pub(crate) fn pre_create_console_devices(vmm: &mut Vmm) -> ConsoleDeviceResult { - let file = File::create(vmconfig.console.common.file.as_ref().unwrap()) + let file = File::create(vmconfig.console.common.output_file.as_ref().unwrap()) .map_err(ConsoleDeviceError::CreateConsoleDevice)?; ConsoleTransport::File(Arc::new(file)) } @@ -208,7 +208,7 @@ pub(crate) fn pre_create_console_devices(vmm: &mut Vmm) -> ConsoleDeviceResult ConsoleDeviceResult { - let file = File::create(vmconfig.serial.common.file.as_ref().unwrap()) + let file = File::create(vmconfig.serial.common.output_file.as_ref().unwrap()) .map_err(ConsoleDeviceError::CreateConsoleDevice)?; ConsoleTransport::File(Arc::new(file)) } @@ -272,7 +272,7 @@ pub(crate) fn pre_create_console_devices(vmm: &mut Vmm) -> ConsoleDeviceResult { diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 2df211ca78..0b1fb92d6e 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -3721,14 +3721,14 @@ mod tests { pmem: None, serial: SerialConfig { common: CommonConsoleConfig { - file: None, + output_file: None, mode: ConsoleOutputMode::Null, socket: None, }, }, console: ConsoleConfig { common: CommonConsoleConfig { - file: None, + output_file: None, // Caution: Don't use `Tty` to not mess with users terminal mode: ConsoleOutputMode::Off, socket: None, diff --git a/vmm/src/vm_config.rs b/vmm/src/vm_config.rs index d45495a875..9c380f378a 100644 --- a/vmm/src/vm_config.rs +++ b/vmm/src/vm_config.rs @@ -717,7 +717,7 @@ pub enum ConsoleOutputMode { #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] pub struct CommonConsoleConfig { #[serde(default)] - pub file: Option, + pub output_file: Option, pub mode: ConsoleOutputMode, #[serde(default)] pub socket: Option, @@ -729,8 +729,8 @@ impl ApplyLandlock for CommonConsoleConfig { landlock.add_rule_with_access(Path::new("/dev/pts"), "rw")?; landlock.add_rule_with_access(Path::new("/dev/ptmx"), "rw")?; } - if let Some(file) = &self.file { - landlock.add_rule_with_access(file, "rw")?; + if let Some(output_file) = &self.output_file { + landlock.add_rule_with_access(output_file, "rw")?; } if let Some(socket) = &self.socket { landlock.add_rule_with_access(socket, "rw")?; @@ -754,7 +754,7 @@ impl Default for SerialConfig { fn default() -> Self { Self { common: CommonConsoleConfig { - file: None, + output_file: None, mode: ConsoleOutputMode::Null, socket: None, }, @@ -785,7 +785,7 @@ impl Default for ConsoleConfig { fn default() -> Self { Self { common: CommonConsoleConfig { - file: None, + output_file: None, mode: ConsoleOutputMode::Tty, socket: None, }, From c982291537d702cc1c95d24526c43bc7af6c9a30 Mon Sep 17 00:00:00 2001 From: Simon Veith Date: Thu, 2 Apr 2026 16:29:12 +0200 Subject: [PATCH 2/7] DNM: vmm: Support console input_file option When used as `--console file=`, the virtio-console device is currently unidirectional (output from guest only). Add an optional parameter `input_file=` which allows input into the guest from the given file or pipe as well. Signed-off-by: Simon Veith --- cloud-hypervisor/src/main.rs | 2 ++ fuzz/fuzz_targets/http_api.rs | 2 ++ vmm/src/config.rs | 50 ++++++++++++++++++++++++++++++----- vmm/src/console_devices.rs | 29 +++++++++++++++----- vmm/src/device_manager.rs | 7 +++++ vmm/src/lib.rs | 2 ++ vmm/src/vm_config.rs | 6 +++++ 7 files changed, 85 insertions(+), 13 deletions(-) diff --git a/cloud-hypervisor/src/main.rs b/cloud-hypervisor/src/main.rs index 99b8e396d5..dc26c60e1c 100644 --- a/cloud-hypervisor/src/main.rs +++ b/cloud-hypervisor/src/main.rs @@ -1155,6 +1155,7 @@ mod tests { serial: SerialConfig { common: CommonConsoleConfig { output_file: None, + input_file: None, mode: ConsoleOutputMode::Null, socket: None, }, @@ -1162,6 +1163,7 @@ mod tests { console: ConsoleConfig { common: CommonConsoleConfig { output_file: None, + input_file: None, mode: ConsoleOutputMode::Tty, socket: None, }, diff --git a/fuzz/fuzz_targets/http_api.rs b/fuzz/fuzz_targets/http_api.rs index 1e8658e00e..a516e749ce 100644 --- a/fuzz/fuzz_targets/http_api.rs +++ b/fuzz/fuzz_targets/http_api.rs @@ -181,6 +181,7 @@ impl RequestHandler for StubApiRequestHandler { serial: SerialConfig { common: CommonConsoleConfig { output_file: None, + input_file: None, mode: ConsoleOutputMode::Tty, socket: None, }, @@ -188,6 +189,7 @@ impl RequestHandler for StubApiRequestHandler { console: ConsoleConfig { common: CommonConsoleConfig { output_file: None, + input_file: None, mode: ConsoleOutputMode::Tty, socket: None, }, diff --git a/vmm/src/config.rs b/vmm/src/config.rs index 9c9947b93f..a5c9174ce9 100644 --- a/vmm/src/config.rs +++ b/vmm/src/config.rs @@ -201,6 +201,8 @@ pub enum ValidationError { /// Missing file value for console #[error("Path missing when using file console mode")] ConsoleFileMissing, + #[error("Console input is not supported by this endpoint")] + ConsoleInputUnsupported, /// Missing socket path for console #[error("Path missing when using socket console mode")] ConsoleSocketPathMissing, @@ -2449,7 +2451,7 @@ impl PmemConfig { impl CommonConsoleConfig { const VALUELESS_OPTIONS: &[&str] = &["off", "pty", "tty", "null"]; - const VALUE_OPTIONS: &[&str] = &["file", "socket"]; + const VALUE_OPTIONS: &[&str] = &["file", "socket", "input_file"]; fn parse(console: &str, map_err: impl Fn(OptionParserError) -> Error) -> Result { let mut parser = OptionParser::new(); @@ -2459,6 +2461,7 @@ impl CommonConsoleConfig { parser.parse_subset(console).map_err(map_err)?; let mut output_file: Option = None; + let mut input_file: Option = None; let mut socket: Option = None; let mut mode: ConsoleOutputMode = ConsoleOutputMode::Off; @@ -2475,6 +2478,7 @@ impl CommonConsoleConfig { Some(PathBuf::from(parser.get("file").ok_or( Error::Validation(ValidationError::ConsoleFileMissing), )?)); + input_file = parser.get("input_file").map(PathBuf::from); } else if parser.is_set("socket") { mode = ConsoleOutputMode::Socket; socket = Some(PathBuf::from(parser.get("socket").ok_or( @@ -2487,6 +2491,7 @@ impl CommonConsoleConfig { Ok(Self { mode, output_file, + input_file, socket, }) } @@ -3539,12 +3544,20 @@ impl VmConfig { && self.console.common.output_file.is_none() { return Err(ValidationError::ConsoleFileMissing); + } else if self.console.common.mode != ConsoleOutputMode::File + && self.console.common.input_file.is_some() + { + return Err(ValidationError::ConsoleInputUnsupported); } if self.serial.common.mode == ConsoleOutputMode::File && self.serial.common.output_file.is_none() { return Err(ValidationError::ConsoleFileMissing); + } else if self.serial.common.mode != ConsoleOutputMode::File + && self.serial.common.input_file.is_some() + { + return Err(ValidationError::ConsoleInputUnsupported); } if self.cpus.max_vcpus < self.cpus.boot_vcpus { @@ -5153,9 +5166,10 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}" #[test] fn test_console_parsing() -> Result<()> { - let console_config = |mode, output_file, socket, iommu| ConsoleConfig { + let console_config = |mode, output_file, input_file, socket, iommu| ConsoleConfig { common: CommonConsoleConfig { output_file, + input_file, mode, socket, }, @@ -5169,19 +5183,19 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}" ConsoleConfig::parse("badmode").unwrap_err(); assert_eq!( ConsoleConfig::parse("off")?, - console_config(ConsoleOutputMode::Off, None, None, IommuType::Off) + console_config(ConsoleOutputMode::Off, None, None, None, IommuType::Off) ); assert_eq!( ConsoleConfig::parse("pty")?, - console_config(ConsoleOutputMode::Pty, None, None, IommuType::Off) + console_config(ConsoleOutputMode::Pty, None, None, None, IommuType::Off) ); assert_eq!( ConsoleConfig::parse("tty")?, - console_config(ConsoleOutputMode::Tty, None, None, IommuType::Off) + console_config(ConsoleOutputMode::Tty, None, None, None, IommuType::Off) ); assert_eq!( ConsoleConfig::parse("null")?, - console_config(ConsoleOutputMode::Null, None, None, IommuType::Off) + console_config(ConsoleOutputMode::Null, None, None, None, IommuType::Off) ); assert_eq!( ConsoleConfig::parse("file=/tmp/console")?, @@ -5189,12 +5203,23 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}" ConsoleOutputMode::File, Some(PathBuf::from("/tmp/console")), None, + None, + IommuType::Off + ) + ); + assert_eq!( + ConsoleConfig::parse("file=/tmp/console_out,input_file=/tmp/console_in")?, + console_config( + ConsoleOutputMode::File, + Some(PathBuf::from("/tmp/console_out")), + Some(PathBuf::from("/tmp/console_in")), + None, IommuType::Off ) ); assert_eq!( ConsoleConfig::parse("null,iommu=on")?, - console_config(ConsoleOutputMode::Null, None, None, IommuType::Virtio) + console_config(ConsoleOutputMode::Null, None, None, None, IommuType::Virtio) ); assert_eq!( ConsoleConfig::parse("file=/tmp/console,iommu=on")?, @@ -5202,6 +5227,7 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}" ConsoleOutputMode::File, Some(PathBuf::from("/tmp/console")), None, + None, IommuType::Virtio ) ); @@ -5210,6 +5236,7 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}" console_config( ConsoleOutputMode::Socket, None, + None, Some(PathBuf::from("/tmp/serial.sock")), IommuType::Virtio ) @@ -6152,6 +6179,7 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}" serial: SerialConfig { common: CommonConsoleConfig { output_file: None, + input_file: None, mode: ConsoleOutputMode::Null, socket: None, }, @@ -6159,6 +6187,7 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}" console: ConsoleConfig { common: CommonConsoleConfig { output_file: None, + input_file: None, mode: ConsoleOutputMode::Tty, socket: None, }, @@ -6283,6 +6312,13 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}" Err(ValidationError::ConsoleFileMissing) ); + invalid_config.serial.common.mode = ConsoleOutputMode::Pty; + invalid_config.serial.common.input_file = Some(PathBuf::from("/dev/null")); + assert_eq!( + invalid_config.validate(), + Err(ValidationError::ConsoleInputUnsupported) + ); + let mut invalid_config = valid_config.clone(); invalid_config.serial.common.mode = ConsoleOutputMode::Socket; invalid_config.serial.common.socket = None; diff --git a/vmm/src/console_devices.rs b/vmm/src/console_devices.rs index d89a10f679..86fed8b0e7 100644 --- a/vmm/src/console_devices.rs +++ b/vmm/src/console_devices.rs @@ -71,6 +71,7 @@ type ConsoleDeviceResult = result::Result; #[derive(Clone)] pub enum ConsoleTransport { File(Arc), + FilePair(Arc, Arc), Pty(Arc), Tty(Arc), Null, @@ -200,9 +201,17 @@ pub(crate) fn pre_create_console_devices(vmm: &mut Vmm) -> ConsoleDeviceResult { - let file = File::create(vmconfig.console.common.output_file.as_ref().unwrap()) - .map_err(ConsoleDeviceError::CreateConsoleDevice)?; - ConsoleTransport::File(Arc::new(file)) + let output_file = + File::create(vmconfig.console.common.output_file.as_ref().unwrap()) + .map_err(ConsoleDeviceError::CreateConsoleDevice)?; + match vmconfig.console.common.input_file.as_ref() { + Some(input_file_path) => { + let input_file = File::open(input_file_path) + .map_err(ConsoleDeviceError::CreateConsoleDevice)?; + ConsoleTransport::FilePair(Arc::new(output_file), Arc::new(input_file)) + } + None => ConsoleTransport::File(Arc::new(output_file)), + } } ConsoleOutputMode::Pty => { let (main_fd, sub_fd, path) = @@ -264,9 +273,17 @@ pub(crate) fn pre_create_console_devices(vmm: &mut Vmm) -> ConsoleDeviceResult { - let file = File::create(vmconfig.serial.common.output_file.as_ref().unwrap()) - .map_err(ConsoleDeviceError::CreateConsoleDevice)?; - ConsoleTransport::File(Arc::new(file)) + let output_file = + File::create(vmconfig.serial.common.output_file.as_ref().unwrap()) + .map_err(ConsoleDeviceError::CreateConsoleDevice)?; + match vmconfig.serial.common.input_file.as_ref() { + Some(input_file_path) => { + let input_file = File::open(input_file_path) + .map_err(ConsoleDeviceError::CreateConsoleDevice)?; + ConsoleTransport::FilePair(Arc::new(output_file), Arc::new(input_file)) + } + None => ConsoleTransport::File(Arc::new(output_file)), + } } ConsoleOutputMode::Pty => { let (main_fd, sub_fd, path) = diff --git a/vmm/src/device_manager.rs b/vmm/src/device_manager.rs index b594e17cd0..415fa3172e 100644 --- a/vmm/src/device_manager.rs +++ b/vmm/src/device_manager.rs @@ -2571,6 +2571,9 @@ impl DeviceManager { let mut console_config = self.config.lock().unwrap().console.clone(); let endpoint = match transport { ConsoleTransport::File(file) => Endpoint::File(file), + ConsoleTransport::FilePair(output_file, input_file) => { + Endpoint::FilePair(output_file, input_file) + } ConsoleTransport::Pty(file) => { self.console.resize_pipe = resize_pipe; Endpoint::PtyPair(Arc::new(file.try_clone().unwrap()), file) @@ -2677,6 +2680,9 @@ impl DeviceManager { ConsoleTransport::File(ref file) | ConsoleTransport::Tty(ref file) => { Some(Box::new(Arc::clone(file))) } + ConsoleTransport::FilePair(ref output_file, ref _input_file) => { + Some(Box::new(Arc::clone(output_file))) + } ConsoleTransport::Off | ConsoleTransport::Null | ConsoleTransport::Pty(_) @@ -2713,6 +2719,7 @@ impl DeviceManager { { let debug_console_writer: Option> = match console_info.debug { ConsoleTransport::File(file) | ConsoleTransport::Tty(file) => Some(Box::new(file)), + ConsoleTransport::FilePair(output_file, _input_file) => Some(Box::new(output_file)), ConsoleTransport::Off | ConsoleTransport::Null | ConsoleTransport::Pty(_) diff --git a/vmm/src/lib.rs b/vmm/src/lib.rs index 0b1fb92d6e..5745c27740 100644 --- a/vmm/src/lib.rs +++ b/vmm/src/lib.rs @@ -3722,6 +3722,7 @@ mod tests { serial: SerialConfig { common: CommonConsoleConfig { output_file: None, + input_file: None, mode: ConsoleOutputMode::Null, socket: None, }, @@ -3729,6 +3730,7 @@ mod tests { console: ConsoleConfig { common: CommonConsoleConfig { output_file: None, + input_file: None, // Caution: Don't use `Tty` to not mess with users terminal mode: ConsoleOutputMode::Off, socket: None, diff --git a/vmm/src/vm_config.rs b/vmm/src/vm_config.rs index 9c380f378a..978340bc2d 100644 --- a/vmm/src/vm_config.rs +++ b/vmm/src/vm_config.rs @@ -718,6 +718,7 @@ pub enum ConsoleOutputMode { pub struct CommonConsoleConfig { #[serde(default)] pub output_file: Option, + pub input_file: Option, pub mode: ConsoleOutputMode, #[serde(default)] pub socket: Option, @@ -732,6 +733,9 @@ impl ApplyLandlock for CommonConsoleConfig { if let Some(output_file) = &self.output_file { landlock.add_rule_with_access(output_file, "rw")?; } + if let Some(input_file) = &self.input_file { + landlock.add_rule_with_access(input_file, "rw")?; + } if let Some(socket) = &self.socket { landlock.add_rule_with_access(socket, "rw")?; } @@ -755,6 +759,7 @@ impl Default for SerialConfig { Self { common: CommonConsoleConfig { output_file: None, + input_file: None, mode: ConsoleOutputMode::Null, socket: None, }, @@ -786,6 +791,7 @@ impl Default for ConsoleConfig { Self { common: CommonConsoleConfig { output_file: None, + input_file: None, mode: ConsoleOutputMode::Tty, socket: None, }, From 530738d7f17d7f54e0e4d0bcb82d205ebb8eff97 Mon Sep 17 00:00:00 2001 From: Steven Noonan Date: Mon, 29 Jun 2026 08:03:44 -0700 Subject: [PATCH 3/7] arch: x86_64: derive guest cache topology from the synthesized topology The deterministic cache leaf (0x8000_001d on AMD, 0x4 on Intel) is inherited verbatim from the host via KVM_GET_SUPPORTED_CPUID, so its "logical processors sharing this cache" field still describes the host's last-level cache span (e.g. a whole CCX). When a guest is given a multi-package topology to back a vNUMA layout, the guest's LLC then appears to cross NUMA node boundaries and the kernel warns: sched: CPU #2's llc-sibling CPU #0 is not on the same node! Re-derive the cache-sharing field from the topology we synthesize: per-core caches (L1/L2) share across a core's threads, and the last-level cache shares across a whole package (== one vNUMA node). Cache geometry is left untouched, and single-package guests keep the host's cache topology, which is already self-consistent. Signed-off-by: Steven Noonan --- arch/src/x86_64/mod.rs | 174 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 159 insertions(+), 15 deletions(-) diff --git a/arch/src/x86_64/mod.rs b/arch/src/x86_64/mod.rs index 37ceed7a44..736adbe91a 100644 --- a/arch/src/x86_64/mod.rs +++ b/arch/src/x86_64/mod.rs @@ -1765,7 +1765,88 @@ fn update_cpuid_topology( CpuidPatch::set_cpuid_reg(cpuid, 0x8000_0008, Some(0), CpuidReg::ECX, 0u32); } } + + // Our vNUMA layout maps one guest package per NUMA node. The cache leaves + // are inherited verbatim from the host (KVM_GET_SUPPORTED_CPUID) and still + // describe the host's last-level cache span (e.g. a whole CCX), so on a + // multi-package layout the guest sees an LLC that crosses node boundaries + // and the kernel warns: + // "sched: CPU #N's llc-sibling CPU #M is not on the same node!". + // Re-derive the cache-sharing topology from the synthesized layout so the + // LLC stays contained within a package (== a vNUMA node). Single-package + // guests keep the host's cache topology, which is already consistent. + if packages > 1 { + update_cpuid_cache_topology( + cpuid, + threads_per_core, + cores_per_die, + dies_per_package, + cpu_vendor, + ); + } +} + +/// Re-derive the deterministic cache topology leaf so each cache level's +/// "logical processors sharing this cache" field reflects the synthesized CPU +/// topology rather than the host's. +/// +/// The leaf has an identical layout on Intel (`0x4`) and AMD (`0x8000_001d`): +/// EAX[4:0] cache type (0 = no further caches at this subleaf) +/// EAX[7:5] cache level (1, 2, 3, ...) +/// EAX[25:14] max number of logical processors sharing this cache, minus 1 +/// +/// Per-core caches (levels 1-2) are shared across a core's threads; the +/// last-level cache (level 3+) is shared across a whole package, which is how +/// we expose a single vNUMA node. Cache geometry (size/associativity carried in +/// the other registers) is left untouched. +fn update_cpuid_cache_topology( + cpuid: &mut Vec, + threads_per_core: u16, + cores_per_die: u16, + dies_per_package: u16, + cpu_vendor: CpuVendor, +) { + let leaf: u32 = match cpu_vendor { + CpuVendor::AMD => 0x8000_001d, + // Intel and unknown vendors use the Intel deterministic cache leaf. + _ => 0x4, + }; + + let threads_per_core = threads_per_core as u32; + let threads_per_package = threads_per_core * cores_per_die as u32 * dies_per_package as u32; + + // Snapshot the subleaf indices up front so we can mutate entries in place + // without assuming the host enumerated them contiguously from zero. + let indices: Vec = cpuid + .iter() + .filter(|entry| entry.function == leaf) + .map(|entry| entry.index) + .collect(); + + for index in indices { + let Some(eax) = CpuidPatch::get_cpuid_reg(cpuid, leaf, Some(index), CpuidReg::EAX) else { + continue; + }; + + let cache_type = eax & 0x1f; + if cache_type == 0 { + // Null subleaf: no cache described here. + continue; + } + let cache_level = (eax >> 5) & 0x7; + + let sharing = if cache_level <= 2 { + threads_per_core + } else { + threads_per_package + }; + + // Replace EAX[25:14] with (sharing - 1), preserving every other field. + let eax = (eax & !(0xfff << 14)) | (((sharing - 1) & 0xfff) << 14); + CpuidPatch::set_cpuid_reg(cpuid, leaf, Some(index), CpuidReg::EAX, eax); + } } + #[cfg(test)] mod tests { use linux_loader::loader::bootparam::boot_e820_entry; @@ -1985,6 +2066,16 @@ mod tests { } } + fn cache_entry(leaf: u32, index: u32, cache_type: u32, level: u32, sharing: u32) -> CpuIdEntry { + CpuIdEntry { + function: leaf, + index, + flags: CPUID_FLAG_VALID_INDEX, + eax: cache_type | (level << 5) | ((sharing - 1) << 14), + ..Default::default() + } + } + fn sharing_field(cpuid: &[CpuIdEntry], function: u32, index: u32) -> u32 { let eax = CpuidPatch::get_cpuid_reg(cpuid, function, Some(index), CpuidReg::EAX).unwrap(); (eax >> CACHE_SHARING_SHIFT) & CACHE_SHARING_MASK @@ -1994,7 +2085,8 @@ mod tests { fn test_update_cpuid_cache_topology_intel() { // Guest topology 2:60:1:2, one die spans 128 ID slots for its // 120 vCPUs. The host leaf 0x4 declares a socket wide L3 of 256 - // addressable IDs, wider than the die, so only the L3 is clamped + // addressable IDs, a multi-package guest derives every sharing + // width from its own topology let mut cpuid = vec![ host_cache_subleaf(0x4, 0, 1, 1, (1 << CACHE_SHARING_SHIFT) | CACHE_SELF_INIT), host_cache_subleaf(0x4, 1, 1, 2, (1 << CACHE_SHARING_SHIFT) | CACHE_SELF_INIT), @@ -2011,12 +2103,12 @@ mod tests { let host_l3 = cpuid[3].eax; update_cpuid_topology(&mut cpuid, 2, 60, 1, 2, CpuVendor::Intel, 0); - // L1 and L2 fit inside the die, the host values are kept + // L1 and L2 are shared across the 2 threads of a core assert_eq!(sharing_field(&cpuid, 0x4, 0), 1); assert_eq!(sharing_field(&cpuid, 0x4, 1), 1); - assert_eq!(sharing_field(&cpuid, 0x4, 2), 3); - // L3 is clamped to one die, 1 << 7 addressable IDs - assert_eq!(sharing_field(&cpuid, 0x4, 3), 127); + assert_eq!(sharing_field(&cpuid, 0x4, 2), 1); + // L3 is shared across the 120 vCPUs of the package + assert_eq!(sharing_field(&cpuid, 0x4, 3), 119); let l3 = CpuidPatch::get_cpuid_reg(&cpuid, 0x4, Some(3), CpuidReg::EAX).unwrap(); // The core IDs field holds 64 core ID slots for the 60 cores assert_eq!((l3 >> CACHE_CORE_IDS_SHIFT) & CACHE_CORE_IDS_MASK, 63); @@ -2034,8 +2126,8 @@ mod tests { #[test] fn test_update_cpuid_cache_topology_amd() { // Guest topology 2:96:1:2, one die spans 256 ID slots for its - // 192 vCPUs. The EPYC 9654 host L3 declares a 16 ID CCX, so the - // clamp has nothing to narrow and the host value is kept + // 192 vCPUs. The EPYC 9654 host L3 declares a 16 ID CCX, a + // multi-package guest widens it to the package let mut cpuid = vec![ host_cache_subleaf( 0x8000_001d, @@ -2073,14 +2165,13 @@ mod tests { assert_eq!(sharing_field(&cpuid, 0x8000_001d, 0), 1); assert_eq!(sharing_field(&cpuid, 0x8000_001d, 1), 1); assert_eq!(sharing_field(&cpuid, 0x8000_001d, 2), 1); - // The 16 ID CCX L3 stays intact - assert_eq!(sharing_field(&cpuid, 0x8000_001d, 3), 15); - // The whole descriptor is untouched, the sharing width is kept - // and the core IDs field is reserved on AMD - assert_eq!( - CpuidPatch::get_cpuid_reg(&cpuid, 0x8000_001d, Some(3), CpuidReg::EAX).unwrap(), - host_l3 - ); + // L3 is shared across the 192 vCPUs of the package + assert_eq!(sharing_field(&cpuid, 0x8000_001d, 3), 191); + // Every bit outside the sharing field is untouched and the core + // IDs field is reserved on AMD + let l3 = CpuidPatch::get_cpuid_reg(&cpuid, 0x8000_001d, Some(3), CpuidReg::EAX).unwrap(); + let rewritten = CACHE_SHARING_MASK << CACHE_SHARING_SHIFT; + assert_eq!(l3 & !rewritten, host_l3 & !rewritten); assert_eq!( CpuidPatch::get_cpuid_reg(&cpuid, 0x8000_001d, Some(4), CpuidReg::EAX).unwrap(), 0 @@ -2140,4 +2231,57 @@ mod tests { assert_eq!(sharing_field(&cpuid, 0x4, 3), 0xfff); } + + fn cache_sharing(cpuid: &[CpuIdEntry], leaf: u32, index: u32) -> u32 { + let eax = CpuidPatch::get_cpuid_reg(cpuid, leaf, Some(index), CpuidReg::EAX).unwrap(); + ((eax >> 14) & 0xfff) + 1 + } + + #[test] + fn test_cache_topology_sharing_matches_package() { + const LEAF: u32 = 0x8000_001d; + // Host-like AMD cache leaf: L1d/L1i per 2 threads, L2 per 2 threads, and + // an L3 shared by 16 threads (a whole CCX) -- the host span the guest + // must NOT inherit on a vNUMA layout. + let mut cpuid = vec![ + cache_entry(LEAF, 0, 1, 1, 2), // L1 data + cache_entry(LEAF, 1, 2, 1, 2), // L1 instruction + cache_entry(LEAF, 2, 3, 2, 2), // L2 unified + cache_entry(LEAF, 3, 3, 3, 16), // L3 unified + ]; + + // 1 thread/core, 2 cores/die, 1 die/package, 2 packages => 4 vCPUs with + // 2 vCPUs per package (one vNUMA node). + update_cpuid_cache_topology(&mut cpuid, 1, 2, 1, CpuVendor::AMD); + + // Per-core caches collapse to the core's thread count (1 here). + assert_eq!(cache_sharing(&cpuid, LEAF, 0), 1); + assert_eq!(cache_sharing(&cpuid, LEAF, 1), 1); + assert_eq!(cache_sharing(&cpuid, LEAF, 2), 1); + // The LLC is shared across the whole package (== the vNUMA node). + assert_eq!(cache_sharing(&cpuid, LEAF, 3), 2); + } + + #[test] + fn test_cache_topology_smt_and_intel_leaf() { + const LEAF: u32 = 0x4; + // Intel deterministic cache leaf with an oversized host L3 span. + let mut cpuid = vec![ + cache_entry(LEAF, 0, 1, 1, 2), // L1 data + cache_entry(LEAF, 1, 2, 1, 2), // L1 instruction + cache_entry(LEAF, 2, 3, 2, 2), // L2 unified + cache_entry(LEAF, 3, 3, 3, 64), // L3 unified + ]; + + // 2 threads/core, 4 cores/die, 1 die/package, 2 packages => 8 vCPUs with + // 4 vCPUs per package. + update_cpuid_cache_topology(&mut cpuid, 2, 4, 1, CpuVendor::Intel); + + // Per-core caches share across SMT siblings (2 threads). + assert_eq!(cache_sharing(&cpuid, LEAF, 0), 2); + assert_eq!(cache_sharing(&cpuid, LEAF, 1), 2); + assert_eq!(cache_sharing(&cpuid, LEAF, 2), 2); + // The LLC spans the package: 2 threads * 4 cores * 1 die = 8. + assert_eq!(cache_sharing(&cpuid, LEAF, 3), 8); + } } From ef7a922cbb1b11d49249de83bdae96707407c7a6 Mon Sep 17 00:00:00 2001 From: Steven Noonan Date: Mon, 29 Jun 2026 11:48:57 -0700 Subject: [PATCH 4/7] arch, vmm: vNUMA-aware x2APIC IDs for unbalanced NUMA layouts cloud-hypervisor derives a vCPU's x2APIC ID from cpu_id / (threads*cores*dies), so it can only carve equal-sized packages. An unbalanced vNUMA layout (e.g. 4+3 vCPUs) therefore cannot map one guest package per node, which is what keeps a node's last-level cache from appearing to span nodes. Add vcpu_x2apic_id(), mirroring Xen's guest_vcpu_x2apic_id: each node occupies a power-of-two APIC-ID window sized to the largest node (encoded as cores_per_die); a vCPU's ID is (node << pkg_shift) | offset, and under-filled nodes leave the tail of their window unused. For balanced layouts this is bit-identical to the uniform encoding. Compute every vCPU's APIC ID once in CpuManager from the NUMA layout and feed it to the vlapic, MADT, SRAT, the AML _MAT and the per-vCPU CPUID topology leaves so all interfaces agree. Relax the topology-count check to allow over-provisioned APIC space (total >= max_vcpus). Signed-off-by: Steven Noonan --- arch/src/x86_64/mod.rs | 150 ++++++++++++++++++++++++++++++++++++++--- vmm/src/acpi.rs | 2 +- vmm/src/config.rs | 26 ++++++- vmm/src/cpu.rs | 44 +++++++++--- 4 files changed, 199 insertions(+), 23 deletions(-) diff --git a/arch/src/x86_64/mod.rs b/arch/src/x86_64/mod.rs index 736adbe91a..bc40710719 100644 --- a/arch/src/x86_64/mod.rs +++ b/arch/src/x86_64/mod.rs @@ -237,6 +237,58 @@ pub fn get_max_x2apic_id(topology: (u16, u16, u16, u16)) -> u32 { ) } +/// Per-vCPU x2APIC ID for a (possibly unbalanced) vNUMA layout. +/// +/// [`get_x2apic_id`] derives the package from `cpu_id / (threads*cores*dies)`, +/// i.e. it can only carve equal-sized packages -- so it cannot place an +/// unbalanced NUMA layout (e.g. 4+3 vCPUs) such that each guest package maps to +/// one node. This mirrors Xen's `guest_vcpu_x2apic_id`: each node occupies a +/// power-of-two-sized window of APIC IDs (sized to the largest node, which the +/// caller encodes as `cores_per_die`), and a vCPU's ID is +/// `(node << pkg_shift) | (offset_within_node << thread_width)`. Under-filled +/// nodes leave the tail of their window unused. +/// +/// The node index is the proximity domain (guest NUMA id) and the offset is the +/// vCPU's position in that node's CPU list, so any vCPU->node assignment works. +/// Falls back to the uniform encoding when there is no usable multi-node layout +/// (no topology, <=1 node, or an unassigned vCPU); for a balanced layout the +/// window encoding is bit-identical to the uniform one anyway. +pub fn vcpu_x2apic_id( + cpu_id: u32, + topology: Option<(u16, u16, u16, u16)>, + numa_nodes: &crate::NumaNodes, +) -> u32 { + let Some((threads, cores, dies, _packages)) = topology else { + return get_x2apic_id(cpu_id, topology); + }; + if numa_nodes.len() <= 1 { + return get_x2apic_id(cpu_id, topology); + } + + // Locate the vCPU's node (proximity domain) and its offset within that + // node's CPU list. + let mut found: Option<(u32, u32)> = None; + for (node_id, node) in numa_nodes.iter() { + if let Some(pos) = node.cpus.iter().position(|&c| c == cpu_id) { + found = Some((*node_id, pos as u32)); + break; + } + } + let Some((node_id, offset)) = found else { + return get_x2apic_id(cpu_id, topology); + }; + + // Package window shift, matching the field widths get_x2apic_id / + // update_cpuid_topology derive from the same tuple, so the APIC IDs agree + // with the CPUID topology leaves the guest reads. + let thread_width = u16::BITS - (threads - 1).leading_zeros(); + let core_width = u16::BITS - (cores - 1).leading_zeros(); + let die_width = u16::BITS - (dies - 1).leading_zeros(); + let pkg_shift = thread_width + core_width + die_width; + + (node_id << pkg_shift) | (offset << thread_width) +} + #[derive(Copy, Clone, Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize)] pub enum CpuidReg { EAX, @@ -1082,8 +1134,11 @@ pub fn configure_vcpu( topology: (u16, u16, u16, u16), nested: bool, setup_registers: bool, + // Precomputed x2APIC ID for vNUMA-aware (possibly unbalanced) layouts; when + // None the uniform encoding is derived from `topology` as before. + x2apic_id_override: Option, ) -> super::Result<()> { - let x2apic_id = get_x2apic_id(id, Some(topology)); + let x2apic_id = x2apic_id_override.unwrap_or_else(|| get_x2apic_id(id, Some(topology))); // Per vCPU CPUID changes; common are handled via generate_common_cpuid() let mut cpuid = cpuid; @@ -1119,7 +1174,14 @@ pub fn configure_vcpu( assert!(apic_id_patched); update_cpuid_topology( - &mut cpuid, topology.0, topology.1, topology.2, topology.3, cpu_vendor, id, + &mut cpuid, + topology.0, + topology.1, + topology.2, + topology.3, + cpu_vendor, + id, + x2apic_id_override, ); // The TSC frequency CPUID leaf should not be included when running with HyperV emulation @@ -1593,6 +1655,7 @@ fn max_addressable_ids(width: u32, mask: u32) -> u32 { ((1u64 << width) - 1).min(mask as u64) as u32 } +#[allow(clippy::too_many_arguments)] fn update_cpuid_topology( cpuid: &mut Vec, threads_per_core: u16, @@ -1601,11 +1664,14 @@ fn update_cpuid_topology( packages: u16, cpu_vendor: CpuVendor, id: u32, + x2apic_id_override: Option, ) { - let x2apic_id = get_x2apic_id( - id, - Some((threads_per_core, cores_per_die, dies_per_package, packages)), - ); + let x2apic_id = x2apic_id_override.unwrap_or_else(|| { + get_x2apic_id( + id, + Some((threads_per_core, cores_per_die, dies_per_package, packages)), + ) + }); // Note: the topology defined here is per "package" (~NUMA node). let thread_width = u16::BITS - (threads_per_core - 1).leading_zeros(); @@ -2101,7 +2167,7 @@ mod tests { host_cache_subleaf(0x4, 4, 0, 0, 0), ]; let host_l3 = cpuid[3].eax; - update_cpuid_topology(&mut cpuid, 2, 60, 1, 2, CpuVendor::Intel, 0); + update_cpuid_topology(&mut cpuid, 2, 60, 1, 2, CpuVendor::Intel, 0, None); // L1 and L2 are shared across the 2 threads of a core assert_eq!(sharing_field(&cpuid, 0x4, 0), 1); @@ -2160,7 +2226,7 @@ mod tests { host_cache_subleaf(0x8000_001d, 4, 0, 0, 0), ]; let host_l3 = cpuid[3].eax; - update_cpuid_topology(&mut cpuid, 2, 96, 1, 2, CpuVendor::AMD, 0); + update_cpuid_topology(&mut cpuid, 2, 96, 1, 2, CpuVendor::AMD, 0, None); assert_eq!(sharing_field(&cpuid, 0x8000_001d, 0), 1); assert_eq!(sharing_field(&cpuid, 0x8000_001d, 1), 1); @@ -2206,7 +2272,7 @@ mod tests { (63 << CACHE_SHARING_SHIFT) | CACHE_SELF_INIT, ), ]; - update_cpuid_topology(&mut cpuid, 2, 8, 12, 1, CpuVendor::AMD, 0); + update_cpuid_topology(&mut cpuid, 2, 8, 12, 1, CpuVendor::AMD, 0, None); // L1 and L2 stay per core even with multiple dies assert_eq!(sharing_field(&cpuid, 0x8000_001d, 0), 1); @@ -2227,7 +2293,7 @@ mod tests { 3, (0xfff << CACHE_SHARING_SHIFT) | CACHE_SELF_INIT, )]; - update_cpuid_topology(&mut cpuid, 2, 3000, 1, 1, CpuVendor::Intel, 0); + update_cpuid_topology(&mut cpuid, 2, 3000, 1, 1, CpuVendor::Intel, 0, None); assert_eq!(sharing_field(&cpuid, 0x4, 3), 0xfff); } @@ -2284,4 +2350,68 @@ mod tests { // The LLC spans the package: 2 threads * 4 cores * 1 die = 8. assert_eq!(cache_sharing(&cpuid, LEAF, 3), 8); } + + fn numa_nodes(groups: &[&[u32]]) -> crate::NumaNodes { + let mut nodes = crate::NumaNodes::new(); + for (i, cpus) in groups.iter().enumerate() { + nodes.insert( + i as u32, + crate::NumaNode { + cpus: cpus.to_vec(), + ..Default::default() + }, + ); + } + nodes + } + + #[test] + fn vcpu_x2apic_id_balanced_matches_uniform() { + // For a balanced layout the window encoding is bit-identical to the + // uniform one, so existing (balanced) guests are unaffected. + let topo = (1u16, 2u16, 1u16, 2u16); + let nodes = numa_nodes(&[&[0, 1], &[2, 3]]); + for cpu in 0..4 { + assert_eq!( + vcpu_x2apic_id(cpu, Some(topo), &nodes), + get_x2apic_id(cpu, Some(topo)), + ); + } + } + + #[test] + fn vcpu_x2apic_id_unbalanced_two_nodes() { + // 3 + 4 vCPUs: cores_per_die = largest node = 4 -> pkg_shift 2. Node 0 + // (the short one) leaves APIC ID 3 unused. + let topo = (1u16, 4u16, 1u16, 2u16); + let nodes = numa_nodes(&[&[0, 1, 2], &[3, 4, 5, 6]]); + let ids: Vec = (0..7) + .map(|c| vcpu_x2apic_id(c, Some(topo), &nodes)) + .collect(); + assert_eq!(ids, vec![0, 1, 2, 4, 5, 6, 7]); + } + + #[test] + fn vcpu_x2apic_id_unbalanced_three_nodes() { + // 3 + 2 + 2 vCPUs: cores_per_die = 3 -> pkg_shift 2. Each node's window + // is 4 APIC IDs wide; the two short nodes leave their tails unused. + let topo = (1u16, 3u16, 1u16, 3u16); + let nodes = numa_nodes(&[&[0, 1, 2], &[3, 4], &[5, 6]]); + let ids: Vec = (0..7) + .map(|c| vcpu_x2apic_id(c, Some(topo), &nodes)) + .collect(); + assert_eq!(ids, vec![0, 1, 2, 4, 5, 8, 9]); + } + + #[test] + fn vcpu_x2apic_id_single_node_falls_back() { + let topo = (1u16, 4u16, 1u16, 1u16); + let nodes = numa_nodes(&[&[0, 1, 2, 3]]); + for cpu in 0..4 { + assert_eq!( + vcpu_x2apic_id(cpu, Some(topo), &nodes), + get_x2apic_id(cpu, Some(topo)), + ); + } + } } diff --git a/vmm/src/acpi.rs b/vmm/src/acpi.rs index 1f8b8e5706..44e48bf61b 100644 --- a/vmm/src/acpi.rs +++ b/vmm/src/acpi.rs @@ -483,7 +483,7 @@ fn create_srat_table( for cpu in &node.cpus { #[cfg(target_arch = "x86_64")] - let x2apic_id = arch::x86_64::get_x2apic_id(*cpu, topology); + let x2apic_id = arch::x86_64::vcpu_x2apic_id(*cpu, topology, numa_nodes); #[cfg(target_arch = "aarch64")] let x2apic_id = *cpu; diff --git a/vmm/src/config.rs b/vmm/src/config.rs index a5c9174ce9..a565f5c3b1 100644 --- a/vmm/src/config.rs +++ b/vmm/src/config.rs @@ -3714,11 +3714,16 @@ impl VmConfig { return Err(ValidationError::CpuTopologyDiesPerPackage); } + // The topology must describe at least as many slots as there are + // vCPUs. It may describe more: a vNUMA layout sizes cores_per_die to + // the largest node and gives every node a package window of that + // size, so unbalanced nodes leave the tail of their window empty + // (their APIC IDs are placed accordingly -- see vcpu_x2apic_id). let total: u32 = (t.threads_per_core as u32) * (t.cores_per_die as u32) * (t.dies_per_package as u32) * (t.packages as u32); - if total != self.cpus.max_vcpus { + if total < self.cpus.max_vcpus { return Err(ValidationError::CpuTopologyCount); } } @@ -6343,12 +6348,15 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}" Err(ValidationError::CpusMaxLowerThanBoot(16, 32)) ); + // A topology describing fewer slots than max_vcpus is rejected. (More + // slots than vCPUs is allowed: a vNUMA layout over-provisions package + // windows for unbalanced nodes.) let mut invalid_config = valid_config.clone(); invalid_config.cpus.max_vcpus = 16; invalid_config.cpus.boot_vcpus = 16; invalid_config.cpus.topology = Some(CpuTopology { - threads_per_core: 2, - cores_per_die: 8, + threads_per_core: 1, + cores_per_die: 4, dies_per_package: 1, packages: 2, }); @@ -6357,6 +6365,18 @@ id=\"{id}\",pci_segment={pci_segment},queue_sizes={queue_sizes}" Err(ValidationError::CpuTopologyCount) ); + // Over-provisioned topology (more slots than vCPUs) is now valid. + let mut over_provisioned = valid_config.clone(); + over_provisioned.cpus.max_vcpus = 16; + over_provisioned.cpus.boot_vcpus = 16; + over_provisioned.cpus.topology = Some(CpuTopology { + threads_per_core: 2, + cores_per_die: 8, + dies_per_package: 1, + packages: 2, + }); + over_provisioned.validate().unwrap(); + let mut still_valid_config = valid_config.clone(); still_valid_config.cpus.max_vcpus = 8; still_valid_config.cpus.boot_vcpus = 8; diff --git a/vmm/src/cpu.rs b/vmm/src/cpu.rs index 0e41280670..809a720723 100644 --- a/vmm/src/cpu.rs +++ b/vmm/src/cpu.rs @@ -539,6 +539,7 @@ impl Vcpu { /// * `kernel_entry_point` - Kernel entry point address in guest memory and boot protocol used. /// * `guest_memory` - Guest memory. /// * `cpuid` - (x86_64) CpuId, wrapper over the `kvm_cpuid2` structure. + #[allow(clippy::too_many_arguments)] pub fn configure( &mut self, boot_setup: Option<(EntryPoint, &GuestMemoryAtomic)>, @@ -547,6 +548,7 @@ impl Vcpu { #[cfg(target_arch = "x86_64")] topology: (u16, u16, u16, u16), #[cfg(target_arch = "x86_64")] nested: bool, #[cfg(feature = "igvm")] igvm_enabled: bool, + #[cfg(target_arch = "x86_64")] x2apic_id: Option, ) -> Result<()> { #[cfg(target_arch = "aarch64")] self.finalize(&[])?; @@ -578,6 +580,7 @@ impl Vcpu { topology, nested, setup_registers, + x2apic_id, ) .map_err(Error::VcpuConfiguration)?; } @@ -785,6 +788,11 @@ pub struct CpuManager { acpi_address: Option, proximity_domain_per_cpu: BTreeMap, affinity: BTreeMap>, + // Per-vCPU x2APIC IDs, indexed by vCPU id. Precomputed once from the NUMA + // layout + topology so MADT, SRAT, the vlapic and CPUID all agree, including + // for unbalanced vNUMA layouts the uniform encoding cannot express. + #[cfg(target_arch = "x86_64")] + vcpu_to_apicid: Vec, dynamic: bool, hypervisor: Arc, #[cfg(feature = "sev_snp")] @@ -968,6 +976,25 @@ impl CpuManager { BTreeMap::new() }; + // Resolve each vCPU's x2APIC ID once from the NUMA layout + topology. + // For a multi-node layout this yields the per-node APIC window (so an + // unbalanced split still maps each guest package to one node); otherwise + // it is the uniform encoding. All APIC consumers below read this. + #[cfg(target_arch = "x86_64")] + let vcpu_to_apicid: Vec = { + let topology = config.topology.as_ref().map(|t| { + ( + t.threads_per_core, + t.cores_per_die, + t.dies_per_package, + t.packages, + ) + }); + (0..config.max_vcpus) + .map(|cpu| arch::x86_64::vcpu_x2apic_id(cpu, topology, numa_nodes)) + .collect() + }; + #[cfg(feature = "tdx")] let dynamic = !tdx_enabled; #[cfg(not(feature = "tdx"))] @@ -1022,6 +1049,8 @@ impl CpuManager { acpi_address: None, proximity_domain_per_cpu, affinity, + #[cfg(target_arch = "x86_64")] + vcpu_to_apicid, dynamic, hypervisor, #[cfg(feature = "sev_snp")] @@ -1038,9 +1067,7 @@ impl CpuManager { debug!("Creating vCPU: cpu_id = {cpu_id}"); #[cfg(target_arch = "x86_64")] - let topology = self.get_vcpu_topology(); - #[cfg(target_arch = "x86_64")] - let x2apic_id = arch::x86_64::get_x2apic_id(cpu_id, topology); + let x2apic_id = self.vcpu_to_apicid[cpu_id as usize]; #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] let x2apic_id = cpu_id; @@ -1115,6 +1142,7 @@ impl CpuManager { self.config.nested, #[cfg(feature = "igvm")] self.igvm_enabled, + Some(self.vcpu_to_apicid[vcpu.id as usize]), )?; #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] @@ -1816,7 +1844,7 @@ impl CpuManager { madt.write(36, layout::APIC_START.0); for cpu in 0..self.config.max_vcpus { - let x2apic_id = arch::x86_64::get_x2apic_id(cpu, self.get_vcpu_topology()); + let x2apic_id = self.vcpu_to_apicid[cpu as usize]; let lapic = LocalX2Apic { r#type: acpi::ACPI_X2APIC_PROCESSOR, @@ -2445,7 +2473,7 @@ struct Cpu { proximity_domain: u32, dynamic: bool, #[cfg(target_arch = "x86_64")] - topology: Option<(u16, u16, u16, u16)>, + x2apic_id: u32, } #[cfg(target_arch = "x86_64")] @@ -2457,7 +2485,7 @@ const MADT_CPU_ONLINE_CAPABLE_FLAG: usize = 1; impl Cpu { #[cfg(target_arch = "x86_64")] fn generate_mat(&self) -> Vec { - let x2apic_id = arch::x86_64::get_x2apic_id(self.cpu_id, self.topology); + let x2apic_id = self.x2apic_id; LocalX2Apic { r#type: acpi::ACPI_X2APIC_PROCESSOR, @@ -2762,8 +2790,6 @@ impl Aml for CpuManager { }; let mut cpu_data_inner: Vec<&dyn Aml> = vec![&hid, &uid, &methods]; - #[cfg(target_arch = "x86_64")] - let topology = self.get_vcpu_topology(); let mut cpu_devices = Vec::new(); for cpu_id in 0..self.config.max_vcpus { let proximity_domain = *self.proximity_domain_per_cpu.get(&cpu_id).unwrap_or(&0); @@ -2772,7 +2798,7 @@ impl Aml for CpuManager { proximity_domain, dynamic: self.dynamic, #[cfg(target_arch = "x86_64")] - topology, + x2apic_id: self.vcpu_to_apicid[cpu_id as usize], }; cpu_devices.push(cpu_device); From e414c7b3e7dd4c886a4c523d71791ce1ead2e4d0 Mon Sep 17 00:00:00 2001 From: Steven Noonan Date: Mon, 29 Jun 2026 15:06:31 -0700 Subject: [PATCH 5/7] arch: x86_64: clear SSBD CPUID bit when SPEC_CTRL is absent KVM_GET_SUPPORTED_CPUID can advertise CPUID(7,0).EDX[31] (SSBD via IA32_SPEC_CTRL) without EDX[26] (SPEC_CTRL) on AMD hosts, where speculation control is exposed through leaf 0x8000_0008 instead. The guest kernel's cpuid-deps check then warns: x86 CPU feature dependency check failure: CPU0 has '18*32+31' enabled but '18*32+26' disabled. Enforce the dependency in the guest CPUID: drop EDX[31] when EDX[26] is clear. Real SSBD stays available to the guest via 0x8000_0008.EBX[24]. Signed-off-by: Steven Noonan --- arch/src/x86_64/mod.rs | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/arch/src/x86_64/mod.rs b/arch/src/x86_64/mod.rs index bc40710719..eb7fe34fe2 100644 --- a/arch/src/x86_64/mod.rs +++ b/arch/src/x86_64/mod.rs @@ -946,14 +946,31 @@ fn required_common_cpuid_updates( for entry in cpuid.as_mut_slice().iter_mut() { #[allow(unused_unsafe)] match entry.function { - // Clear AMX related bits if the AMX feature is not enabled - 0x7 if !config.amx => { - if entry.index == 0 { - entry.edx &= !((1 << AMX_BF16) | (1 << AMX_TILE) | (1 << AMX_INT8)); + 0x7 => { + // Clear AMX related bits if the AMX feature is not enabled + if !config.amx { + if entry.index == 0 { + entry.edx &= !((1 << AMX_BF16) | (1 << AMX_TILE) | (1 << AMX_INT8)); + } + if entry.index == 1 { + entry.eax &= !(1 << AMX_FP16); + entry.edx &= !(1 << AMX_COMPLEX); + } } - if entry.index == 1 { - entry.eax &= !(1 << AMX_FP16); - entry.edx &= !(1 << AMX_COMPLEX); + // Enforce the IA32_SPEC_CTRL CPUID dependency the guest kernel + // checks: EDX[31] (SSBD via SPEC_CTRL) requires EDX[26] + // (SPEC_CTRL). KVM_GET_SUPPORTED_CPUID can advertise SSBD + // without SPEC_CTRL on AMD -- where spec-control is exposed via + // leaf 0x8000_0008 instead -- which trips a "feature dependency + // check failure" warning in the guest. Drop the dependent bit + // when its base is absent; real SSBD remains available through + // 0x8000_0008.EBX[24]. + if entry.index == 0 { + const SPEC_CTRL: u32 = 1 << 26; + const SPEC_CTRL_SSBD: u32 = 1 << 31; + if (entry.edx & SPEC_CTRL) == 0 { + entry.edx &= !SPEC_CTRL_SSBD; + } } } // Tile Information (purely AMX related). From 4943d564e7c7b474c57f022e26a1c97098e04fa1 Mon Sep 17 00:00:00 2001 From: Luca Di Maio Date: Tue, 13 Oct 2026 16:47:07 +0200 Subject: [PATCH 6/7] block: flush the image with fdatasync instead of a full fsync A guest flush asks that written data survive a crash. fdatasync gives that: the data plus the metadata needed to read it back. fsync also commits the timestamps, and on ext4 every O_DIRECT write updates the image mtime, so each guest flush forced a host journal commit on top of the device flush. This was already in QEMU already. Signed-off-by: Luca Di Maio --- block/src/formats/raw/engine_sync.rs | 3 ++- block/src/io/async_io/aio_data_io.rs | 5 +++-- block/src/io/async_io/uring_data_io.rs | 5 ++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/block/src/formats/raw/engine_sync.rs b/block/src/formats/raw/engine_sync.rs index 05481fc93a..bef693fe00 100644 --- a/block/src/formats/raw/engine_sync.rs +++ b/block/src/formats/raw/engine_sync.rs @@ -67,8 +67,9 @@ impl AsyncIo for RawSync { } fn fsync(&mut self, user_data: Option) -> AsyncIoResult<()> { + // A guest flush needs the data and the metadata to read it back, not the timestamps. // SAFETY: FFI call - let result = unsafe { libc::fsync(self.raw_file.as_raw_fd() as libc::c_int) }; + let result = unsafe { libc::fdatasync(self.raw_file.as_raw_fd() as libc::c_int) }; if result < 0 { return Err(AsyncIoError::Fsync(io::Error::last_os_error())); } diff --git a/block/src/io/async_io/aio_data_io.rs b/block/src/io/async_io/aio_data_io.rs index 85ea4df618..c29a82b177 100644 --- a/block/src/io/async_io/aio_data_io.rs +++ b/block/src/io/async_io/aio_data_io.rs @@ -90,7 +90,8 @@ impl AioDataIo { Ok(()) } - /// Submits an fsync operation carrying `user_data`. + /// Submits an fdatasync operation carrying `user_data`: a guest flush needs the data and the + /// metadata to read it back, not the timestamps. pub fn submit_fsync(&mut self, fd: RawFd, user_data: u64) -> io::Result<()> { if self.in_flight.contains_key(&user_data) { return Err(duplicate_user_data_error(user_data)); @@ -98,7 +99,7 @@ impl AioDataIo { let mut iocb = aio::IoControlBlock { aio_fildes: fd.as_raw_fd() as u32, - aio_lio_opcode: aio::IOCB_CMD_FSYNC as u16, + aio_lio_opcode: aio::IOCB_CMD_FDSYNC as u16, aio_data: user_data, aio_flags: aio::IOCB_FLAG_RESFD, aio_resfd: self.completions.notifier().as_raw_fd() as u32, diff --git a/block/src/io/async_io/uring_data_io.rs b/block/src/io/async_io/uring_data_io.rs index dca0ab8296..b391ae7924 100644 --- a/block/src/io/async_io/uring_data_io.rs +++ b/block/src/io/async_io/uring_data_io.rs @@ -197,11 +197,14 @@ impl UringDataIo { self.submit_kernel_entry(user_data, &opcode::Nop::new().build().user_data(user_data)) } - /// Submits an fsync operation carrying `user_data`. + /// Submits an fdatasync operation carrying `user_data`: a guest flush needs the data and the + /// metadata to read it back, not the timestamps, and a full fsync of a file whose mtime every + /// write touches forces a host journal commit per flush. pub fn submit_fsync(&mut self, fd: RawFd, user_data: u64) -> io::Result<()> { self.submit_kernel_entry( user_data, &opcode::Fsync::new(types::Fd(fd)) + .flags(types::FsyncFlags::DATASYNC) .build() .user_data(user_data), ) From 400f4f5d75d5ddef116fd6c98dffaa145b30bbdc Mon Sep 17 00:00:00 2001 From: Asher Feldman Date: Thu, 15 Oct 2026 17:02:25 -0700 Subject: [PATCH 7/7] build: Bump rustls to 0.23.46 0.23.46 fixes RUSTSEC-2026-0346, RUSTSEC-2026-0347, RUSTSEC-2026-0348 and RUSTSEC-2026-0349, which cargo audit reports for 0.23.45. Signed-off-by: Asher Feldman --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 30e7a2c73d..5d7be66a34 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2115,9 +2115,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "log", "once_cell",