diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d262ddce21..ffd58ed6cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -299,19 +299,23 @@ jobs: if: matrix.arch == 'x86_64' || matrix.arch == 'riscv64' - run: cargo xtask ci rs --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.rs_flags }} --package httpd --features ci,hermit/dhcpv4,hermit/rtl8139 qemu ${{ matrix.qemu_flags }} --devices rtl8139 if: matrix.arch != 'riscv64' - - run: cargo xtask ci rs --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.rs_flags }} --package httpd --no-default-features --features ci,hermit/dhcpv4,hermit/tcp,hermit/gem-net qemu ${{ matrix.qemu_flags }} --devices cadence-gem + - run: cargo xtask ci rs --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.rs_flags }} --package httpd --no-default-features --features ci,hermit/dhcpv4,hermit/tcp,hermit/gem-net,hermit/riscv-plic qemu ${{ matrix.qemu_flags }} --devices cadence-gem + if: matrix.arch == 'riscv64' + - run: cargo xtask ci rs --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.rs_flags }} --package hello_world qemu ${{ matrix.qemu_flags }} --machine virt-aia-aplic + if: matrix.arch == 'riscv64' + - run: cargo xtask ci rs --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.rs_flags }} --package hello_world --features hermit/riscv-plic qemu ${{ matrix.qemu_flags }} --machine virt if: matrix.arch == 'riscv64' - run: cargo clean working-directory: . - run: cargo xtask ci rs --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.rs_flags }} --package testudp --features hermit/udp,hermit/dhcpv4,hermit/virtio-net qemu ${{ matrix.qemu_flags }} --devices virtio-net-pci - run: cargo xtask ci rs --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.rs_flags }} --package testudp --features hermit/udp,hermit/dhcpv4,hermit/rtl8139 qemu ${{ matrix.qemu_flags }} --devices rtl8139 if: matrix.arch != 'riscv64' - - run: cargo xtask ci rs --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.rs_flags }} --package testudp --no-default-features --features hermit/udp,hermit/dhcpv4,hermit/gem-net qemu ${{ matrix.qemu_flags }} --devices cadence-gem + - run: cargo xtask ci rs --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.rs_flags }} --package testudp --no-default-features --features hermit/udp,hermit/dhcpv4,hermit/gem-net,hermit/riscv-plic qemu ${{ matrix.qemu_flags }} --devices cadence-gem if: matrix.arch == 'riscv64' - run: cargo xtask ci rs --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.rs_flags }} --package miotcp --features hermit/dhcpv4,hermit/virtio-net qemu ${{ matrix.qemu_flags }} --devices virtio-net-pci - run: cargo xtask ci rs --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.rs_flags }} --package miotcp --features hermit/dhcpv4,hermit/rtl8139 qemu ${{ matrix.qemu_flags }} --devices rtl8139 if: matrix.arch != 'riscv64' - - run: cargo xtask ci rs --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.rs_flags }} --package miotcp --no-default-features --features hermit/dhcpv4,hermit/tcp,hermit/gem-net qemu ${{ matrix.qemu_flags }} --devices cadence-gem + - run: cargo xtask ci rs --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.rs_flags }} --package miotcp --no-default-features --features hermit/dhcpv4,hermit/tcp,hermit/gem-net,hermit/riscv-plic qemu ${{ matrix.qemu_flags }} --devices cadence-gem if: matrix.arch == 'riscv64' - run: cargo xtask ci rs --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.rs_flags }} --package poll --features hermit/dhcpv4,hermit/virtio-net qemu ${{ matrix.qemu_flags }} --devices virtio-net-pci - run: cargo xtask ci rs --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.rs_flags }} --package poll --features hermit/dhcpv4,hermit/rtl8139 qemu ${{ matrix.qemu_flags }} --devices rtl8139 diff --git a/Cargo.lock b/Cargo.lock index 37471af913..9b7580a6f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -277,6 +277,17 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "bitfield-struct" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ca6739863c590881f038d033a146c51ddae239186a4327014839fd864f44ed5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -887,6 +898,7 @@ dependencies = [ "async-executor", "async-lock", "bit_field", + "bitfield-struct 0.13.0", "bitflags 2.13.1", "build-time", "built", @@ -1557,8 +1569,7 @@ dependencies = [ [[package]] name = "riscv" version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e42cdafa0aa3f0f956b7993cace26de5dafff7bb4f2c5b1dcb2c3723f4267a4f" +source = "git+https://github.com/rust-embedded/riscv?branch=master#0e18338b3dbd7a4b9b89a1674457aa6442023fa2" dependencies = [ "critical-section", "embedded-hal", @@ -1570,8 +1581,7 @@ dependencies = [ [[package]] name = "riscv-macros" version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ad9f6dda08d90d091bc0743967cab369b8fe39ea74b26783ba2ca5c0ce39e86" +source = "git+https://github.com/rust-embedded/riscv?branch=master#0e18338b3dbd7a4b9b89a1674457aa6442023fa2" dependencies = [ "proc-macro2", "quote", @@ -1581,8 +1591,7 @@ dependencies = [ [[package]] name = "riscv-types" version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3f2ad9f15a07f4a0e1677124f9120ce7e83ab7e1ca7186af0ca9da529b62e80" +source = "git+https://github.com/rust-embedded/riscv?branch=master#0e18338b3dbd7a4b9b89a1674457aa6442023fa2" [[package]] name = "rustix" @@ -2006,7 +2015,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd2f32d50a7e480738d6e43ea2048cd1c34cc33362e55d87d1eebd02bd2f563f" dependencies = [ "allocator-api2 0.3.1", - "bitfield-struct", + "bitfield-struct 0.11.0", "bitflags 2.13.1", "endian-num", "num_enum", diff --git a/Cargo.toml b/Cargo.toml index cb0c93dedb..b0af75cb58 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -162,6 +162,14 @@ semihosting = ["dep:semihosting"] ## _application processor_s (AP) from the _boot-strap processor_ (BSP). smp = ["acpi"] +## Enable drivers for legacy platform level interrupt controller (PLIC). +## +## If this flag is not set, the advanced platform level interrupt controller (APLIC) +## and the incoming message signaled interrupt controller (IMSIC) will be used. +## +## This flag is only evaluated for riscv64. +riscv-plic = [] + #! ### Network Features ## Enables TCP support. @@ -421,9 +429,11 @@ memory_addresses = { version = "0.4", default-features = false, features = ["aar semihosting = { version = "0.1", optional = true } [target.'cfg(target_arch = "riscv64")'.dependencies] +bitfield-struct = "0.13.0" fdt = { version = "0.1", features = ["pretty-printing"] } memory_addresses = { version = "0.4", default-features = false, features = ["riscv64"] } -riscv = "0.16" +#TODO: use upstream version once changes merged and published +riscv = { version = "0.16", git = "https://github.com/rust-embedded/riscv", branch = "master" } sbi-rt = "0.0.4" semihosting = { version = "0.1", optional = true } tock-registers = { version = "0.10", optional = true } diff --git a/build.rs b/build.rs index 27a06c95ae..b37206aa33 100644 --- a/build.rs +++ b/build.rs @@ -10,6 +10,8 @@ use llvm_tools::LlvmTools; fn main() -> Result<()> { built::write_built_file().unwrap(); + configure_msix_support(); + if env::var("CARGO_CFG_TARGET_ARCH").unwrap() == "x86_64" && env::var_os("CARGO_FEATURE_SMP").is_some() { @@ -19,6 +21,18 @@ fn main() -> Result<()> { Ok(()) } +fn configure_msix_support() { + println!("cargo:rustc-check-cfg=cfg(msix_supported)"); + + let has_pci = env::var_os("CARGO_FEATURE_PCI").is_some(); + let has_plic = env::var_os("CARGO_FEATURE_RISCV_PLIC").is_some(); + let target_arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap_or_default(); + + if has_pci && (target_arch == "x86_64" || (target_arch == "riscv64" && !has_plic)) { + println!("cargo:rustc-cfg=msix_supported"); + } +} + fn assemble_x86_64_smp_boot() -> Result<()> { let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap()); diff --git a/src/arch/riscv64/kernel/core_local.rs b/src/arch/riscv64/kernel/core_local.rs index 65b7e5b9ed..b9ed705dcd 100644 --- a/src/arch/riscv64/kernel/core_local.rs +++ b/src/arch/riscv64/kernel/core_local.rs @@ -10,6 +10,8 @@ use hermit_sync::InterruptTicketMutex; use hermit_sync::{RawRwSpinLock, RawSpinMutex}; use crate::arch::kernel::CPU_ONLINE; +#[cfg(not(feature = "riscv-plic"))] +use crate::arch::kernel::interrupts::MsiController; #[cfg(feature = "smp")] use crate::scheduler::SchedulerInput; use crate::scheduler::{CoreId, PerCoreScheduler}; @@ -26,6 +28,8 @@ pub struct CoreLocal { /// Queues to handle incoming requests from the other cores #[cfg(feature = "smp")] pub scheduler_input: InterruptTicketMutex, + #[cfg(not(feature = "riscv-plic"))] + msi_controller: Cell<*mut MsiController>, } impl CoreLocal { @@ -44,6 +48,8 @@ impl CoreLocal { ex: StaticLocalExecutor::new(), #[cfg(feature = "smp")] scheduler_input: InterruptTicketMutex::new(SchedulerInput::new()), + #[cfg(not(feature = "riscv-plic"))] + msi_controller: Cell::new(ptr::null_mut()), }; let this = if core_id == 0 { take_static::take_static! { @@ -87,3 +93,13 @@ pub fn set_core_scheduler(scheduler: *mut PerCoreScheduler) { pub(crate) fn ex() -> &'static StaticLocalExecutor { &CoreLocal::get().ex } + +#[cfg(not(feature = "riscv-plic"))] +pub(crate) fn msi_controller() -> Option<&'static mut MsiController> { + unsafe { CoreLocal::get().msi_controller.get().as_mut() } +} + +#[cfg(not(feature = "riscv-plic"))] +pub(crate) fn set_msi_controller(msi_controller: *mut MsiController) { + CoreLocal::get().msi_controller.set(msi_controller); +} diff --git a/src/arch/riscv64/kernel/devicetree.rs b/src/arch/riscv64/kernel/devicetree.rs index 145bb9c77a..db00c2eba2 100644 --- a/src/arch/riscv64/kernel/devicetree.rs +++ b/src/arch/riscv64/kernel/devicetree.rs @@ -1,15 +1,25 @@ +#[cfg(not(feature = "riscv-plic"))] +use alloc::collections::BTreeMap; +#[cfg(not(feature = "riscv-plic"))] +use alloc::vec::Vec; #[cfg(all(feature = "virtio", not(feature = "pci")))] use core::ptr::NonNull; use memory_addresses::PhysAddr; #[cfg(all(feature = "gem-net", not(feature = "pci")))] use memory_addresses::VirtAddr; +#[cfg(not(feature = "riscv-plic"))] +use riscv::interrupt::Interrupt; #[cfg(all(feature = "virtio", not(feature = "pci")))] use virtio::mmio::{DeviceRegisters, DeviceRegistersVolatileFieldAccess}; #[cfg(all(feature = "virtio", not(feature = "pci")))] use volatile::VolatileRef; +use crate::arch::kernel::interrupts::EXTERNAL_INTERRUPT_CONTROLLER; +#[cfg(feature = "riscv-plic")] use crate::arch::kernel::interrupts::init_plic; +#[cfg(not(feature = "riscv-plic"))] +use crate::arch::kernel::interrupts::{init_aplic, init_interrupt_files}; #[cfg(all( any( feature = "virtio-fs", @@ -28,6 +38,7 @@ use crate::arch::kernel::mmio::MmioDriver; not(feature = "pci") ))] use crate::arch::kernel::mmio::register_driver; +#[cfg(all(any(feature = "virtio", feature = "gem-net"), not(feature = "pci")))] use crate::arch::mm::paging::{self, PageSize}; use crate::drivers::InterruptHandlerMap; #[cfg(all(feature = "gem-net", not(feature = "pci")))] @@ -48,47 +59,238 @@ use crate::drivers::virtio::transport::mmio::VirtioDriver; use crate::env::{self, FdtStartInfo}; #[cfg(all(any(feature = "gem-net", feature = "virtio-net"), not(feature = "pci")))] use crate::executor::device::NETWORK_DEVICE; +#[cfg(all(feature = "virtio", not(feature = "pci")))] +use crate::mm::PageRangeAllocator; -static mut PLATFORM_MODEL: Model = Model::Unknown; - +#[cfg(feature = "riscv-plic")] enum Model { Fux40, Virt, Unknown, } +pub enum InterruptType { + /// Default or unspecified type + None = 0, + /// Low to high edge sensitive type enabled + LowToHighEdge = 1, + /// Active low level sensitive type enabled + ActiveLowLevel = 2, + /// Active high level sensitive type enabled + ActiveHighLevel = 4, + /// High to low edge sensitive type enabled + HighToLowEdge = 8, +} +impl From for InterruptType { + fn from(value: u32) -> Self { + match value { + 0 => Self::None, + 1 => Self::LowToHighEdge, + 2 => Self::ActiveLowLevel, + 4 => Self::ActiveHighLevel, + 8 => Self::HighToLowEdge, + _ => panic!("invalid InterruptType bits"), + } + } +} + /// Inits variables based on the device tree /// This function should only be called once -pub fn init() { - debug!("Init devicetree"); +pub fn init_interrupt_controller() { let Some(fdt) = env::start_info().fdt() else { return; }; - let model = fdt - .find_node("/") - .unwrap() - .property("compatible") - .expect("compatible not found in FDT") - .as_str() - .unwrap(); - - let platform_model = if model.contains("riscv-virtio") { - Model::Virt - } else if model.contains("sifive,hifive-unmatched-a00") - || model.contains("sifive,hifive-unleashed-a00") - || model.contains("sifive,fu740") - || model.contains("sifive,fu540") - { - Model::Fux40 - } else { - warn!("Unknown platform, guessing PLIC context 1"); - Model::Unknown - }; - unsafe { - PLATFORM_MODEL = platform_model; + #[cfg(not(feature = "riscv-plic"))] + if let Some(imsic_node) = find_imsic(&fdt) { + let imsic_region = imsic_node + .reg() + .expect("Reg property for imsic not found in FDT") + .next() + .unwrap(); + let addr = PhysAddr::from(imsic_region.starting_address.addr()); + let size = imsic_region.size.unwrap(); + + let interrupt_cells = 1; + let mut num_harts = 0; + + // Build a mapping from interrupt-controller phandle to hart-id + let mut intc_to_hart = BTreeMap::new(); + for cpu_node in fdt.find_node("/cpus").unwrap().children() { + if !cpu_node + .compatible() + .is_some_and(|c| c.all().any(|x| x == "riscv")) + { + continue; + } + + // Assumes cpu has only one child node which is the interrupt-controller + let intc_node = cpu_node + .children() + .next() + .expect("No child node found for cpu node in FDT"); + assert!( + intc_node + .compatible() + .is_some_and(|c| c.all().any(|x| x == "riscv,cpu-intc")), + "Child node of cpu node is not compatible with riscv,cpu-intc" + ); + assert!( + intc_node + .interrupt_cells() + .is_some_and(|c| c == interrupt_cells) + ); + + let intc_phandle = intc_node.property("phandle").unwrap().as_usize().unwrap(); + let hart_id = cpu_node.property("reg").unwrap().as_usize().unwrap(); + + intc_to_hart.insert(intc_phandle, hart_id); + + num_harts += 1; + } + + let mut num_interrupt_files = 0; + let interrupts_extended = imsic_node.property("interrupts-extended").unwrap().value; + let size_per_entry = (1 + interrupt_cells) * size_of::(); + + // Build a mapping from hart-id to index of the interrupt file region + let mut interrupt_file_indices: Vec = Vec::new(); + interrupt_file_indices.resize_with(interrupts_extended.len() / size_per_entry, || 0); + for (index, entry) in interrupts_extended.chunks_exact(size_per_entry).enumerate() { + let irq_type = u32::from_be_bytes(entry[4..8].try_into().unwrap()); + if irq_type != Interrupt::SupervisorExternal as u32 { + continue; + } + + let intc_phandle = u32::from_be_bytes(entry[0..4].try_into().unwrap()); + let hart_id = intc_to_hart + .get(&(intc_phandle as usize)) + .expect("No cpu node found for interrupt-controller phandle in FDT"); + + interrupt_file_indices[*hart_id] = index; + + num_interrupt_files += 1; + } + assert!( + num_interrupt_files == num_harts, + "Number of interrupt files does not match number of harts in FDT. AMP is not supported." + ); + + debug!("Found IMSIC at {addr:p}, size: {size:#x}, num_harts: {num_harts}"); + init_interrupt_files(addr, size, interrupt_file_indices); + } + + #[cfg(not(feature = "riscv-plic"))] + if let Some(aplic_node) = find_aplic(&fdt) { + let aplic_region = aplic_node + .reg() + .expect("Reg property for APLIC not found in FDT") + .next() + .unwrap(); + let addr = PhysAddr::from(aplic_region.starting_address.addr()); + let size = aplic_region.size.unwrap(); + let msi_delivery = aplic_node.property("msi-parent").is_some(); + + debug!("Found APLIC at {addr:p}, size: {size:#x}, msi_delivery: {msi_delivery:?}"); + init_aplic(addr, size, msi_delivery); + } + + #[cfg(feature = "riscv-plic")] + if let Some(plic_node) = fdt.find_compatible(&["sifive,plic-1.0.0"]) { + debug!("Found external interrupt controller"); + let plic_region = plic_node + .reg() + .expect("Reg property for PLIC not found in FDT") + .next() + .unwrap(); + + let plic_region_start = PhysAddr::from(plic_region.starting_address.addr()); + let plic_region_size = plic_region.size.unwrap(); + debug!("Init PLIC at {plic_region_start:p}, size: {plic_region_size:x}"); + + let model = fdt + .find_node("/") + .unwrap() + .property("compatible") + .expect("compatible not found in FDT") + .as_str() + .unwrap(); + + let platform_model = if model.contains("riscv-virtio") { + Model::Virt + } else if model.contains("sifive,hifive-unmatched-a00") + || model.contains("sifive,hifive-unleashed-a00") + || model.contains("sifive,fu740") + || model.contains("sifive,fu540") + { + Model::Fux40 + } else { + warn!("Unknown platform, guessing PLIC context 1"); + Model::Unknown + }; + info!("Model: {model}"); + + // TODO: Determine correct context via devicetree and allow more than one context + let context = match platform_model { + Model::Virt | Model::Unknown => 1, + Model::Fux40 => 2, + }; + init_plic(plic_region_start, plic_region_size, context); + } + + if EXTERNAL_INTERRUPT_CONTROLLER.lock().is_none() { + warn!("No external interrupt controller found"); + } +} + +#[cfg(not(feature = "riscv-plic"))] +pub fn msi_supported_vectors() -> Option { + let fdt = env::start_info().fdt()?; + let imsic_node = find_imsic(&fdt)?; + + imsic_node.property("riscv,num-ids")?.as_usize() +} + +#[cfg(not(feature = "riscv-plic"))] +fn find_imsic<'a>(fdt: &'a fdt::Fdt<'_>) -> Option> { + let mut node = fdt.find_compatible(&["riscv,imsics"])?; + + // Different interrupts domains, including m-mode domains, show up as different nodes. + // We expect a hierarchy of one m-mode domain and one s-mode domain as described in + // 'The RISC-V Advanced Interrupt Architecture', Version 1, Figure 4.2 + if node.property("status").and_then(|p| p.as_str()) == Some("disabled") { + let phandle = node.property("riscv,children")?.as_usize()?; + node = fdt.find_phandle(phandle as u32)?; + + // Ensure the S-mode domain is actually enabled + assert!( + node.property("status").and_then(|p| p.as_str()) != Some("disabled"), + "Referenced s-mode interrupt domain is not enabled in FDT" + ); + } + + Some(node) +} + +#[cfg(not(feature = "riscv-plic"))] +fn find_aplic<'a>(fdt: &'a fdt::Fdt<'_>) -> Option> { + let mut node = fdt.find_compatible(&["riscv,aplic"])?; + + // Different interrupts domains, including m-mode domains, show up as different nodes. + // We expect a hierarchy of one m-mode domain and one s-mode domain as described in + // 'The RISC-V Advanced Interrupt Architecture', Version 1, Figure 4.2 + if node.property("status").and_then(|p| p.as_str()) == Some("disabled") { + let phandle = node.property("riscv,children")?.as_usize()?; + node = fdt.find_phandle(phandle as u32)?; + + // Ensure the S-mode domain is actually enabled + assert!( + node.property("status").and_then(|p| p.as_str()) != Some("disabled"), + "Referenced s-mode interrupt domain is not enabled in FDT" + ); } - info!("Model: {model}"); + + Some(node) } #[cfg_attr( @@ -102,37 +304,6 @@ pub fn init_drivers(handlers: &mut InterruptHandlerMap) { if let Some(fdt) = env::start_info().fdt() { debug!("Init drivers using devicetree"); - // Init PLIC first - if let Some(plic_node) = fdt.find_compatible(&["sifive,plic-1.0.0"]) { - debug!("Found interrupt controller"); - let plic_region = plic_node - .reg() - .expect("Reg property for PLIC not found in FDT") - .next() - .unwrap(); - - let plic_region_start = PhysAddr::from(plic_region.starting_address.addr()); - debug!( - "Init PLIC at {:p}, size: {:x}", - plic_region_start, - plic_region.size.unwrap() - ); - assert!( - plic_region.size.unwrap() < usize::try_from(paging::HugePageSize::SIZE).unwrap() - ); - - paging::identity_map::(plic_region_start); - - // TODO: Determine correct context via devicetree and allow more than one context - let context = unsafe { - match PLATFORM_MODEL { - Model::Virt | Model::Unknown => 1, - Model::Fux40 => 2, - } - }; - init_plic(plic_region.starting_address, context); - } - // Init GEM #[cfg(all(feature = "gem-net", not(feature = "pci")))] if let Some(gem_node) = fdt.find_compatible(&["sifive,fu540-c000-gem"]) { @@ -148,6 +319,19 @@ pub fn init_drivers(handlers: &mut InterruptHandlerMap) { .expect("interrupts property for GEM not found in FDT") .next() .unwrap(); + let parent_interrupt_cells = gem_node + .interrupt_parent() + .expect("interrupt-parent node for virtio mmio not found in FDT") + .interrupt_cells() + .expect("#interrupt-cells property for virtio mmio missing or invalid"); + let (irq_number, source_mode) = match parent_interrupt_cells { + 1 => (irq as u32, 0), + 2 => ((irq >> 32) as u32, irq as u32), + _ => { + panic!("Unsupported #interrupt-cells value: {parent_interrupt_cells}"); + } + }; + let mac = gem_node .property("local-mac-address") .expect("local-mac-address property for GEM not found in FDT") @@ -177,12 +361,22 @@ pub fn init_drivers(handlers: &mut InterruptHandlerMap) { paging::identity_map::(gem_region_start); match gem::init_device( VirtAddr::new(gem_region_start.as_u64()), - irq.try_into().unwrap(), + irq_number.try_into().unwrap(), phy_addr, <[u8; 6]>::try_from(mac).expect("MAC with invalid length"), handlers, ) { - Ok(drv) => *NETWORK_DEVICE.lock() = Some(drv), + Ok(drv) => { + EXTERNAL_INTERRUPT_CONTROLLER + .lock() + .as_mut() + .unwrap() + .set_interrupt_source_mode( + irq_number.try_into().unwrap(), + source_mode.into(), + ); + *NETWORK_DEVICE.lock() = Some(drv); + } Err(err) => error!("Could not initialize GEM driver: {err}"), } } @@ -211,6 +405,18 @@ pub fn init_drivers(handlers: &mut InterruptHandlerMap) { .expect("interrupts property for virtio mmio not found in FDT") .next() .unwrap(); + let parent_interrupt_cells = virtio_node + .interrupt_parent() + .expect("interrupt-parent node for virtio mmio not found in FDT") + .interrupt_cells() + .expect("#interrupt-cells property for virtio mmio missing or invalid"); + let (irq_number, source_mode) = match parent_interrupt_cells { + 1 => (irq as u32, 0), + 2 => ((irq >> 32) as u32, irq as u32), + _ => { + panic!("Unsupported #interrupt-cells value: {parent_interrupt_cells}"); + } + }; let virtio_region_start = PhysAddr::from(virtio_region.starting_address.expose_provenance()); @@ -247,7 +453,7 @@ pub fn init_drivers(handlers: &mut InterruptHandlerMap) { if cfg!(debug_assertions) { use free_list::PageRange; - use crate::mm::{FrameAlloc, PageRangeAllocator}; + use crate::mm::FrameAlloc; let start = virtio_region.starting_address.addr(); let len = virtio_region.size.unwrap(); @@ -258,33 +464,49 @@ pub fn init_drivers(handlers: &mut InterruptHandlerMap) { debug!("Found virtio {id:?} at {mmio:p}"); - match mmio_virtio::init_device(mmio, irq.try_into().unwrap(), handlers) { + let drv = match mmio_virtio::init_device(mmio, irq_number.try_into().unwrap(), handlers) + { + Ok(drv) => drv, + Err(DriverError::InitVirtioDevFail(VirtioError::DevNotSupported(0))) => { + continue; + } + Err(err) => { + error!("Could not initialize virtio-mmio device: {err}"); + continue; + } + }; + + EXTERNAL_INTERRUPT_CONTROLLER + .lock() + .as_mut() + .unwrap() + .set_interrupt_source_mode(irq_number.try_into().unwrap(), source_mode.into()); + + match drv { #[cfg(feature = "virtio-console")] - Ok(VirtioDriver::Console(drv)) => crate::console::switch_to_virtio(*drv), + VirtioDriver::Console(drv) => crate::console::switch_to_virtio(*drv), #[cfg(feature = "virtio-fs")] - Ok(VirtioDriver::Fs(drv)) => { + VirtioDriver::Fs(drv) => { register_driver(MmioDriver::VirtioFs(hermit_sync::InterruptSpinMutex::new( *drv, ))); } #[cfg(feature = "virtio-net")] - Ok(VirtioDriver::Net(drv)) => { + VirtioDriver::Net(drv) => { *NETWORK_DEVICE.lock() = Some(*drv); } #[cfg(feature = "virtio-rng")] - Ok(VirtioDriver::Rng(drv)) => { + VirtioDriver::Rng(drv) => { register_driver(MmioDriver::VirtioRng(hermit_sync::InterruptSpinMutex::new( *drv, ))); } #[cfg(feature = "virtio-vsock")] - Ok(VirtioDriver::Vsock(drv)) => { + VirtioDriver::Vsock(drv) => { register_driver(MmioDriver::VirtioVsock( hermit_sync::InterruptSpinMutex::new(*drv), )); } - Err(DriverError::InitVirtioDevFail(VirtioError::DevNotSupported(0))) => (), - Err(err) => error!("Could not initialize virtio-mmio device: {err}"), } } } diff --git a/src/arch/riscv64/kernel/interrupts.rs b/src/arch/riscv64/kernel/interrupts.rs deleted file mode 100644 index 4298428e02..0000000000 --- a/src/arch/riscv64/kernel/interrupts.rs +++ /dev/null @@ -1,310 +0,0 @@ -use alloc::vec::Vec; -use core::mem::offset_of; -use core::num::NonZeroU16; -use core::ptr::NonNull; - -use ahash::RandomState; -use bit_field::BitField; -use hashbrown::HashMap; -use hermit_sync::{InterruptTicketMutex, OnceCell, SpinMutex}; -use riscv::asm::wfi; -use riscv::interrupt::{Exception, Interrupt, Trap}; -use riscv::register::{scause, sie, sip, sstatus, stval}; -use trapframe::TrapFrame; -use volatile::access::{NoAccess, ReadOnly}; -use volatile::{VolatileFieldAccess, VolatilePtr, VolatileRef}; - -use crate::drivers::InterruptHandlerMap; -use crate::scheduler; - -const NUMBER_OF_SOURCES: usize = 1024; -const NUMBER_OF_CONTEXTS: usize = 15871; - -const INTERRUPT_PENDING_BITS_OFFSET: usize = 0x00_1000; -const INTERRUPT_ENABLE_BITS_OFFSET: usize = 0x00_2000; -const CONTEXT_BASED_REGISTERS: usize = 0x20_0000; - -type SourceBitArray = [u32; NUMBER_OF_SOURCES / (u32::BITS as usize)]; - -#[repr(C, align(4096))] -#[derive(VolatileFieldAccess)] -struct ContextBasedRegisters { - priority_threshold: u32, - claim_or_complete: u32, -} - -#[repr(C)] -#[derive(VolatileFieldAccess)] -struct Plic { - #[access(NoAccess)] - _reserved0: u32, - interrupt_priorities: [u32; NUMBER_OF_SOURCES - 1], - #[access(ReadOnly)] - interrupt_pending_bits: SourceBitArray, - #[access(NoAccess)] - _reserved3: [u32; (INTERRUPT_ENABLE_BITS_OFFSET - 0x00_1080) / size_of::()], - interrupt_enable_bits: [SourceBitArray; NUMBER_OF_CONTEXTS], - #[access(NoAccess)] - _reserved2: [u32; (CONTEXT_BASED_REGISTERS - 0x1f_2000) / size_of::()], - context_based_registers: [ContextBasedRegisters; NUMBER_OF_CONTEXTS], -} - -const _: () = assert!(offset_of!(Plic, interrupt_pending_bits) == INTERRUPT_PENDING_BITS_OFFSET); -const _: () = assert!(offset_of!(Plic, interrupt_enable_bits) == INTERRUPT_ENABLE_BITS_OFFSET); -const _: () = assert!(offset_of!(Plic, context_based_registers) == CONTEXT_BASED_REGISTERS); - -trait PlicVolatileMemberAccess<'a> { - fn interrupt_priority(self, source: NonZeroU16) -> VolatilePtr<'a, u32>; - fn context_based_register(self, context: u16) -> VolatilePtr<'a, ContextBasedRegisters>; - fn set_enable_bit(self, context: u16, source: NonZeroU16, value: bool); -} - -impl<'a> PlicVolatileMemberAccess<'a> for VolatilePtr<'a, Plic> { - fn interrupt_priority(self, source: NonZeroU16) -> VolatilePtr<'a, u32> { - unsafe { - self.interrupt_priorities().map(|slice| { - slice - .cast() - .offset(isize::try_from(source.get()).unwrap() - 1) - }) - } - } - - fn context_based_register(self, context: u16) -> VolatilePtr<'a, ContextBasedRegisters> { - unsafe { - self.context_based_registers() - .map(|slice| slice.cast().offset(isize::try_from(context).unwrap())) - } - } - - fn set_enable_bit(self, context: u16, source: NonZeroU16, value: bool) { - let source = usize::from(source.get()); - unsafe { - self.interrupt_enable_bits() - .map(|slice| { - slice - .cast::() - .offset(isize::try_from(context).unwrap()) - }) - .map(|context_slice| { - context_slice - .cast::() - .offset((source / 32).try_into().unwrap()) - }) - .update(|mut word| { - word.set_bit(source % 32, value); - word - }); - }; - } -} - -static PLIC: SpinMutex>> = SpinMutex::new(None); - -/// PLIC context for new interrupt handlers -static PLIC_CONTEXT: OnceCell = OnceCell::new(); - -/// PLIC context for new interrupt handlers -static CURRENT_INTERRUPTS: SpinMutex> = SpinMutex::new(Vec::new()); - -static INTERRUPT_HANDLERS: OnceCell = OnceCell::new(); - -/// Init Interrupts -pub(crate) fn install() { - unsafe { - // Install trap handler - trapframe::init(); - // Enable external interrupts - sie::set_sext(); - } -} - -/// Init PLIC -pub(crate) fn init_plic(base: *const u8, context: u16) { - *PLIC.lock() = - Some(unsafe { VolatileRef::new(NonNull::new(base.cast::().cast_mut()).unwrap()) }); - PLIC_CONTEXT.set(context).unwrap(); -} - -/// Enable Interrupts -#[inline] -pub(crate) fn enable() { - unsafe { - sstatus::set_sie(); - } -} - -static IRQ_NAMES: InterruptTicketMutex> = - InterruptTicketMutex::new(HashMap::with_hasher(RandomState::with_seeds(0, 0, 0, 0))); - -#[allow(dead_code)] -pub(crate) fn add_irq_name(irq_number: u8, name: &'static str) { - debug!("Register name \"{name}\" for interrupt {irq_number}"); - IRQ_NAMES.lock().insert(irq_number, name); -} - -/// Waits for the next interrupt (Only Supervisor-level software/timer interrupt for now) -/// and calls the specific handler -#[inline] -pub(crate) fn enable_and_wait() { - unsafe { - //Enable Supervisor-level software interrupts - sie::set_ssoft(); - //sie::set_sext(); - debug!("Wait {:x?}", sie::read()); - loop { - wfi(); - // Interrupts are disabled at this point, so a pending interrupt will - // resume the execution. We still have to check if a interrupt is pending - // because the WFI instruction could be implemented as NOP (The RISC-V Instruction Set ManualVolume II: Privileged Architecture) - - let pending_interrupts = sip::read(); - - // trace!("sip: {:x?}", pending_interrupts); - #[cfg(feature = "smp")] - if pending_interrupts.ssoft() { - //Clear Supervisor-level software interrupt - core::arch::asm!( - "csrc sip, {ssoft_mask}", - ssoft_mask = in(reg) 0x2, - ); - trace!("SOFT"); - //Disable Supervisor-level software interrupt - sie::clear_ssoft(); - crate::arch::kernel::scheduler::wakeup_handler(); - break; - } - - if pending_interrupts.sext() { - trace!("EXT"); - external_handler(); - break; - } - - if pending_interrupts.stimer() { - // // Disable Supervisor-level software interrupt, wakeup not needed - // sie::clear_ssoft(); - - debug!("sip: {pending_interrupts:x?}"); - trace!("TIMER"); - crate::arch::kernel::scheduler::timer_handler(); - break; - } - } - } -} - -/// Disable Interrupts -#[inline] -pub(crate) fn disable() { - unsafe { sstatus::clear_sie() }; -} - -/// Currently not needed because we use the trapframe crate -pub(crate) fn install_handlers(handlers: InterruptHandlerMap) { - for irq_number in handlers.keys() { - let mut plic_guard = PLIC.lock(); - let plic_ptr = plic_guard.as_mut().unwrap().as_mut_ptr(); - let context = *PLIC_CONTEXT.get().unwrap(); - let source = NonZeroU16::new(u16::from(*irq_number)).unwrap(); - - // Set priority to 7 (highest on FU740) - plic_ptr.interrupt_priority(source).write(1); - // Set Threshold to 0 (lowest) - plic_ptr - .context_based_register(context) - .priority_threshold() - .write(0); - // Enable irq for context - plic_ptr.set_enable_bit(context, source, true); - } - - INTERRUPT_HANDLERS.set(handlers).unwrap(); -} - -// Derived from rCore: https://github.com/rcore-os/rCore -/// Dispatch and handle interrupt. -/// -/// This function is called from `trap.S` which is in the trapframe crate. -#[unsafe(no_mangle)] -pub extern "C" fn trap_handler(tf: &mut TrapFrame) { - let scause = scause::read(); - let cause = scause.cause(); - let cause = Trap::::try_from(cause).unwrap(); - let stval = stval::read(); - let sepc = tf.sepc; - trace!("Interrupt: {cause:?}"); - trace!("tf = {tf:x?} "); - trace!("stval = {stval:x}"); - trace!("sepc = {sepc:x}"); - trace!("SSTATUS FS = {:?}", sstatus::read().fs()); - - match cause { - Trap::Interrupt(Interrupt::SupervisorExternal) => external_handler(), - #[cfg(feature = "smp")] - Trap::Interrupt(Interrupt::SupervisorSoft) => { - crate::arch::kernel::scheduler::wakeup_handler(); - } - Trap::Interrupt(Interrupt::SupervisorTimer) => { - crate::arch::kernel::scheduler::timer_handler(); - } - cause => { - error!("Interrupt: {cause:?}"); - error!("tf = {tf:x?} "); - error!("stval = {stval:x}"); - error!("sepc = {sepc:x}"); - error!("SSTATUS FS = {:?}", sstatus::read().fs()); - scheduler::abort(); - } - } - trace!("Interrupt end"); -} - -/// Handles external interrupts -fn external_handler() { - use crate::arch::kernel::core_local::core_scheduler; - use crate::scheduler::PerCoreSchedulerExt; - - // Claim interrupt - let mut plic_guard = PLIC.lock(); - let plic_ptr = plic_guard.as_mut().unwrap().as_mut_ptr(); - let context = *PLIC_CONTEXT.get().unwrap(); - let claim_ptr = plic_ptr.context_based_register(context).claim_or_complete(); - let irq = claim_ptr.read(); - - if irq != 0 { - debug!("External INT: {irq}"); - let mut cur_int = CURRENT_INTERRUPTS.lock(); - cur_int.push(irq); - if cur_int.len() > 1 { - warn!("More than one external interrupt is pending!"); - } - // Release lock early - drop(cur_int); - - // Call handler - if let Some(handlers) = INTERRUPT_HANDLERS.get() - && let Some(queue) = handlers.get(&u8::try_from(irq).unwrap()) - { - for handler in queue.iter() { - handler(); - } - } - crate::executor::run(); - - core_scheduler().reschedule(); - - // Complete interrupt after handling - claim_ptr.write(irq); - - // Remove from active interrupts - let mut cur_int = CURRENT_INTERRUPTS.lock(); - if let Some(active_irq) = cur_int.pop() - && active_irq != irq - { - warn!("Interrupt mismatch during EOI!"); - } - } -} - -pub(crate) fn print_statistics() {} diff --git a/src/arch/riscv64/kernel/interrupts/aplic.rs b/src/arch/riscv64/kernel/interrupts/aplic.rs new file mode 100644 index 0000000000..9dc0a804e3 --- /dev/null +++ b/src/arch/riscv64/kernel/interrupts/aplic.rs @@ -0,0 +1,590 @@ +//! Advanced Platform-Level Interrupt Controller (APLIC) driver for RISC-V. +//! +//! [RISC-V Advanced Interrupt Architecture]: https://github.com/riscv/riscv-aia/releases/download/20250312/riscv-interrupts-20250312.pdf + +use core::mem::offset_of; +use core::num::NonZeroU16; +use core::ptr::NonNull; + +use align_address::Align; +use bitfield_struct::bitfield; +use free_list::PageLayout; +use memory_addresses::{PhysAddr, VirtAddr}; +use volatile::access::{NoAccess, ReadOnly, ReadWrite, WriteOnly}; +use volatile::{VolatileFieldAccess, VolatileRef}; + +use crate::arch::kernel::devicetree::InterruptType as DeviceTreeInterruptType; +use crate::arch::kernel::interrupts::{EXTERNAL_INTERRUPT_CONTROLLER, ExternalInterruptController}; +use crate::arch::mm::paging::{self, BasePageSize, PageSize, PageTableEntryFlags}; +use crate::arch::riscv64::kernel::core_local::{core_id, msi_controller}; +use crate::mm::{PageAlloc, PageRangeAllocator}; + +#[bitfield(u32)] +struct DomainConfig { + // BE - Big Endian (0 = little endian, 1 = big endian) + #[bits(1)] + big_endian: bool, + + #[bits(1)] + __: bool, + + // DM - Domain Mode (0 = direct delivery, 1 = msi delivery) + #[bits(1)] + msi_delivery: bool, + + #[bits(5)] + __: u8, + + // IE - Interrupt Enable (0 = disabled, 1 = enabled) + #[bits(1)] + interrupt_enable: bool, + + #[bits(23)] + __: u32, +} + +#[bitfield(u32)] +struct SourceConfigRegister { + #[bits(10)] + __payload: u16, + + #[bits(1)] + delegated: u8, + + #[bits(21)] + __: u32, +} + +#[derive(Debug)] +#[repr(u8)] +pub(crate) enum SourceMode { + // Inactive in this domain (and not delegated) + Inactive = 0, + // Active, detached from the source wire + Detached = 1, + // Edge-sensitive; interrupt asserted on rising edge + Edge1 = 4, + // Edge-sensitive; interrupt asserted on falling edge + Edge0 = 5, + // Level-sensitive; interrupt asserted when high + Level1 = 6, + // Level-sensitive; interrupt asserted when low + Level0 = 7, +} +impl SourceMode { + const fn into_bits(self) -> u8 { + self as _ + } + + const fn from_bits(bits: u8) -> Self { + match bits { + 0 => Self::Inactive, + 1 => Self::Detached, + 4 => Self::Edge1, + 5 => Self::Edge0, + 6 => Self::Level1, + 7 => Self::Level0, + _ => panic!("invalid SourceMode bits"), + } + } +} +impl From for SourceMode { + fn from(value: DeviceTreeInterruptType) -> Self { + match value { + DeviceTreeInterruptType::None => Self::Inactive, + DeviceTreeInterruptType::LowToHighEdge => Self::Edge1, + DeviceTreeInterruptType::ActiveLowLevel => Self::Level0, + DeviceTreeInterruptType::ActiveHighLevel => Self::Level1, + DeviceTreeInterruptType::HighToLowEdge => Self::Edge0, + } + } +} + +#[bitfield(u32)] +struct SourceConfigSourceMode { + #[bits(3)] + mode: SourceMode, + + #[bits(7)] + __: u8, + + #[bits(1)] + delegated: u8, + + #[bits(21)] + __: u32, +} + +#[bitfield(u32)] +struct SourceConfigDelegated { + #[bits(10)] + child_index: u16, + + #[bits(1)] + delegated: u8, + + #[bits(21)] + __: u32, +} + +#[bitfield(u32)] +struct TargetDirectDelivery { + // Priority number for interrupt source + // + // Lower values indicate higher priority. The maximum priority is 0, and the minimum + // priority is 255. + #[bits(8)] + priority: u8, + + #[bits(10)] + __: u16, + + // Hart to which the interrupt for this source will be delivered + #[bits(14)] + hart_index: u16, +} + +#[bitfield(u32)] +struct TargetMsiDelivery { + // External interrupt identity + #[bits(11)] + eiid: u16, + + #[bits(1)] + __: u8, + + // Number of the target hart’s guest interrupt file to which MSIs will be sent. + // Only relevant if domain’s harts implement hypervisor extension. + #[bits(6)] + __guest_index: u8, + + // Hart to which the interrupt for this source will be forwarded + #[bits(14)] + hart_index: u16, +} + +#[bitfield(u32)] +struct TargetRegister { + #[bits(18)] + __payload: u32, + + // Target hart index + #[bits(14)] + hart_index: u16, +} + +#[repr(C)] +#[derive(VolatileFieldAccess)] +struct AplicControlRegion { + // Domain configuration + // Bit 8: IE - Interrupt Enable + // Bit 2: DM - Domain Mode (0 = direct delivery, 1 = msi delivery) + // Bit 0: BE - Big Endian + #[access(ReadWrite)] + domaincfg: DomainConfig, + #[access(ReadWrite)] + sourcecfg: [SourceConfigRegister; 1023], + #[access(NoAccess)] + _reserved_sourcecfg: [u32; (0x1bc0 - 0x1000) / size_of::()], + + // MSI address configuration registers (m-mode only) + #[access(NoAccess)] + _mmsiaddrcfg: u32, + #[access(NoAccess)] + _mmsiaddrcfgh: u32, + #[access(NoAccess)] + _smsiaddrcfg: u32, + #[access(NoAccess)] + _smsiaddrcfgh: u32, + #[access(NoAccess)] + _reserved_msiaddrcfg: [u32; (0x1c00 - 0x1bd0) / size_of::()], + + // Set interrupt pending bits + #[access(ReadWrite)] + _setip: [u32; 32], + #[access(NoAccess)] + _reserved_setip: [u32; (0x1cdc - 0x1c80) / size_of::()], + // Set interrupt pending bit number + #[access(WriteOnly)] + _setipnum: u32, + #[access(NoAccess)] + _reserved_setipnum: [u32; (0x1d00 - 0x1ce0) / size_of::()], + + // Clear interrupt pending bits + #[access(ReadWrite)] + _in_clrip: [u32; 32], + #[access(NoAccess)] + _reserved_in_clrip: [u32; (0x1ddc - 0x1d80) / size_of::()], + // Clear interrupt pending bit number + #[access(WriteOnly)] + _clripnum: u32, + #[access(NoAccess)] + _reserved_clripnum: [u32; (0x1e00 - 0x1de0) / size_of::()], + + // Set interrupt enable bits + #[access(ReadWrite)] + _setie: [u32; 32], + #[access(NoAccess)] + _reserved_setie: [u32; (0x1edc - 0x1e80) / size_of::()], + // Set interrupt enable bit number + #[access(WriteOnly)] + setienum: u32, + #[access(NoAccess)] + _reserved_setienum: [u32; (0x1f00 - 0x1ee0) / size_of::()], + + // Clear interrupt enable bits + #[access(WriteOnly)] + _clrie: [u32; 32], + #[access(NoAccess)] + _reserved_clrie: [u32; (0x1fdc - 0x1f80) / size_of::()], + // Clear interrupt enable bit number + #[access(WriteOnly)] + clrienum: u32, + #[access(NoAccess)] + _reserved_clrienum: [u32; (0x2000 - 0x1fe0) / size_of::()], + + // Set interrupt pending bit by number (little-endian) + #[access(WriteOnly)] + _setipnum_le: u32, + // Set interrupt pending bit by number (big-endian) + #[access(WriteOnly)] + _setipnum_be: u32, + #[access(NoAccess)] + _reserved_setipnum_be: [u32; (0x3000 - 0x2008) / size_of::()], + + // Generate MSI + #[access(ReadWrite)] + _genmsi: u32, + + // Interrupt Targets + target: [TargetRegister; 1023], +} +const _: () = assert!(offset_of!(AplicControlRegion, _mmsiaddrcfg) == 0x1bc0); +const _: () = assert!(offset_of!(AplicControlRegion, _smsiaddrcfg) == 0x1bc8); +const _: () = assert!(offset_of!(AplicControlRegion, _setip) == 0x1c00); +const _: () = assert!(offset_of!(AplicControlRegion, _setipnum) == 0x1cdc); +const _: () = assert!(offset_of!(AplicControlRegion, _in_clrip) == 0x1d00); +const _: () = assert!(offset_of!(AplicControlRegion, _clripnum) == 0x1ddc); +const _: () = assert!(offset_of!(AplicControlRegion, _setie) == 0x1e00); +const _: () = assert!(offset_of!(AplicControlRegion, setienum) == 0x1edc); +const _: () = assert!(offset_of!(AplicControlRegion, _clrie) == 0x1f00); +const _: () = assert!(offset_of!(AplicControlRegion, clrienum) == 0x1fdc); +const _: () = assert!(offset_of!(AplicControlRegion, _setipnum_le) == 0x2000); +const _: () = assert!(offset_of!(AplicControlRegion, _genmsi) == 0x3000); +const _: () = assert!(size_of::() == 0x4000); + +#[bitfield(u32)] +struct TopInterrupt { + #[bits(8)] + priority: u8, + + #[bits(8)] + __: u8, + + #[bits(10)] + identity: u16, + + #[bits(6)] + __: u8, +} + +#[repr(C)] +#[derive(VolatileFieldAccess)] +struct InterruptDeliveryControl { + // Interrupt delivery enable (0 = disabled, 1 = enabled) + #[access(ReadWrite)] + idelivery: u32, + + // Interrupt force (0 = no effect, 1 = force interrupt) + // For testing only. + #[access(ReadWrite)] + iforce: u32, + + // Interrupt enable threshold (0 = all interrupts enabled, n = interrupts with priority > n enabled) + #[access(ReadWrite)] + ithreshold: u32, + + #[access(NoAccess)] + _reserved: [u32; 3], + + // Top interrupt + #[access(ReadOnly)] + topi: TopInterrupt, + + // Claim top interrupt + #[access(ReadOnly)] + claimi: TopInterrupt, +} +const _: () = assert!(offset_of!(InterruptDeliveryControl, topi) == 0x18); +const _: () = assert!(offset_of!(InterruptDeliveryControl, claimi) == 0x1c); +const _: () = assert!(size_of::() == 0x20); + +const APLIC_DIRECT_DELIVERY_MODE_MAX_HARTS: u16 = 512; + +// In direct delivery mode the number of harts is limited to 512. +type InterruptDeliveryControlArray = + [InterruptDeliveryControl; APLIC_DIRECT_DELIVERY_MODE_MAX_HARTS as usize]; + +pub(crate) struct Aplic { + control_region: VolatileRef<'static, AplicControlRegion>, + interrupt_delivery_control: VolatileRef<'static, InterruptDeliveryControlArray>, + ipriolen: u8, +} + +impl Aplic { + fn new( + control_region: VolatileRef<'static, AplicControlRegion>, + interrupt_delivery_control: VolatileRef<'static, InterruptDeliveryControlArray>, + ) -> Self { + Self { + control_region, + interrupt_delivery_control, + ipriolen: 0, + } + } + + fn init(&mut self, msi_delivery: bool) { + let aplic_ptr = self.control_region.as_mut_ptr(); + let mut domaincfg = aplic_ptr.domaincfg().read(); + if domaincfg.big_endian() { + domaincfg.set_big_endian(false); + aplic_ptr.domaincfg().write(domaincfg); + assert!( + !aplic_ptr.domaincfg().read().big_endian(), + "Only little-endian is supported" + ); + } + + let aplic_ptr = self.control_region.as_mut_ptr(); + let mut domaincfg = aplic_ptr.domaincfg().read(); + + if domaincfg.msi_delivery() != msi_delivery { + domaincfg.set_msi_delivery(msi_delivery); + aplic_ptr.domaincfg().write(domaincfg); + assert!( + aplic_ptr.domaincfg().read().msi_delivery() == msi_delivery, + "APLIC does not support the desired delivery mode." + ); + } + + let aplic_ptr = self.control_region.as_mut_ptr(); + aplic_ptr.domaincfg().update(|mut cfg| { + cfg.set_interrupt_enable(true); + cfg + }); + + if !msi_delivery { + self.ipriolen = self.probe_ipriolen(); + let hart_idc = unsafe { + self.interrupt_delivery_control + .as_mut_ptr() + .map(|control| control.cast().offset(Aplic::get_hart_index() as isize)) + }; + hart_idc.idelivery().write(1); + } + } + + /// Determines the number of implemented interrupt priority bits (IPRIOLEN) + /// by probing the first source's target WARL register. + fn probe_ipriolen(&mut self) -> u8 { + let saved_source_mode = self.get_interrupt_source_mode(1); + self.set_interrupt_source_mode(1, SourceMode::Detached); + let target = unsafe { + self.control_region + .as_mut_ptr() + .target() + .map(|slice| slice.cast::()) + }; + let saved_target_reg = target.read(); + target.write(TargetRegister::from( + TargetDirectDelivery::new() + .with_hart_index(Aplic::get_hart_index()) + .with_priority(0xff) + .into_bits(), + )); + let readback = TargetDirectDelivery::from(target.read().into_bits()).priority(); + target.write(saved_target_reg); + self.set_interrupt_source_mode(1, saved_source_mode); + + let ipriolen = (u8::BITS - readback.leading_zeros()) as u8; + assert!( + (1..=8).contains(&ipriolen), + "APLIC IPRIOLEN must be between 1 and 8, but probed {ipriolen}" + ); + ipriolen + } + + fn get_hart_index() -> u16 { + // The core identifier and the hart index of an core in an interrupt domain are not necessarily the same. + // The hart index can be extracted from the devicetree as following. + // 1. Find all core nodes cpu@X + // 2. For each core find core local interrupter node ( = "riscv,cpu-intc") and get its phandle + // 3. Find aplic node ( = "riscv,aplic", != "disabled") for active interrupt domain + // 4. Property is a list of tuples with the following format: ... + // For supervisor external interrupts interrupt-specifier = 0x9 + // For machine external interrupts interrupt-specifier = 0xb + // 5. The index of the tuple in the list is the hart index of the core in the interrupt domain. + // + // The core identifier is identical to the interrupt domain hart index if + // - the core identifier are continuous and start with 0 and + // - the interrupt-extended property of the aplic node is ordered by core identifier. + // On QEMU virt machine these assumptions hold true. + + let core_id: u16 = core_id().try_into().unwrap(); + assert!( + core_id < APLIC_DIRECT_DELIVERY_MODE_MAX_HARTS, + "APLIC direct delivery mode supports only {APLIC_DIRECT_DELIVERY_MODE_MAX_HARTS} harts, but core_id is {core_id}" + ); + core_id + } + + pub fn set_enable_bit(&mut self, irq_number: u16, value: bool) { + if value { + self.control_region + .as_mut_ptr() + .setienum() + .write(u32::from(irq_number)); + } else { + self.control_region + .as_mut_ptr() + .clrienum() + .write(u32::from(irq_number)); + } + + if let Some(msi_controller) = msi_controller() { + // As long as IRQ numbers are reused as EIID (external interrupt identifier), we can use + // the irq_number directly as EIID. + let eiid = irq_number; + msi_controller.set_interrupt_enable(NonZeroU16::new(eiid).unwrap(), value); + } + } + + pub fn set_interrupt_source_mode(&mut self, irq_number: u16, mode: SourceMode) { + let sourcecfg = unsafe { + self.control_region.as_mut_ptr().sourcecfg().map(|slice| { + slice + .cast() + .offset(isize::try_from(irq_number).unwrap() - 1) + }) + }; + sourcecfg.write(SourceConfigRegister::from( + SourceConfigSourceMode::new().with_mode(mode).into_bits(), + )); + } + + pub fn get_interrupt_source_mode(&mut self, irq_number: u16) -> SourceMode { + let sourcecfg = unsafe { + self.control_region.as_mut_ptr().sourcecfg().map(|slice| { + slice + .cast() + .offset(isize::try_from(irq_number).unwrap() - 1) + }) + }; + let sourcecfg_value: u32 = sourcecfg.read(); + SourceConfigSourceMode::from(sourcecfg_value).mode() + } + + pub fn set_interrupt_priority(&mut self, irq_number: u16, priority: u8) { + let new_value = if msi_controller().is_some() { + TargetMsiDelivery::new() + .with_hart_index(Aplic::get_hart_index()) + .with_eiid(irq_number) + .into_bits() + } else { + let max_priority = u8::MAX >> (8 - self.ipriolen); + TargetDirectDelivery::new() + .with_hart_index(Aplic::get_hart_index()) + .with_priority(priority.min(max_priority)) + .into_bits() + }; + + let target = unsafe { + self.control_region.as_mut_ptr().target().map(|slice| { + slice + .cast() + .offset(isize::try_from(irq_number).unwrap() - 1) + }) + }; + target.write(TargetRegister::from(new_value)); + } + + pub fn set_priority_threshold(&mut self, threshold: u8) { + if let Some(msi_controller) = msi_controller() { + msi_controller.set_interrupt_priority_threshold(threshold); + } else { + let hart_idc = unsafe { + self.interrupt_delivery_control + .as_mut_ptr() + .map(|control| control.cast().offset(Aplic::get_hart_index() as isize)) + }; + hart_idc.ithreshold().write(u32::from(threshold)); + } + } + + pub fn claim_interrupt(&mut self) -> Option { + trace!("Claiming interrupt from APLIC"); + if let Some(msi_controller) = msi_controller() { + msi_controller.claim_interrupt() + } else { + let hart_idc = unsafe { + self.interrupt_delivery_control + .as_mut_ptr() + .map(|control| control.cast().offset(Aplic::get_hart_index() as isize)) + }; + let claimi = hart_idc.claimi().read(); + NonZeroU16::new(claimi.identity()) + } + } + + pub fn complete_interrupt(&mut self, _irq_number: u16) { + if let Some(msi_controller) = msi_controller() { + msi_controller.complete_interrupt(NonZeroU16::new(_irq_number).unwrap()); + } else { + // reading claimi register automatically completes the interrupt + } + } +} + +pub fn init_aplic(addr: PhysAddr, size: usize, msi_delivery: bool) { + assert!( + addr.is_aligned_to(BasePageSize::SIZE), + "Aplic control region is not page aligned" + ); + assert!( + size == 32 * 1024, + "Expected 32 KiB control region for APLIC in direct delivery mode" + ); + + let layout = PageLayout::from_size(size).unwrap(); + let page_range = PageAlloc::allocate(layout).unwrap(); + let control_region_addr = VirtAddr::from(page_range.start()); + + let mut flags = PageTableEntryFlags::empty(); + flags.device().normal().writable().execute_disable(); + paging::map::( + control_region_addr, + addr, + size / usize::try_from(BasePageSize::SIZE).unwrap(), + flags, + ); + + let control_region = unsafe { + VolatileRef::new( + NonNull::new(control_region_addr.as_mut_ptr::()).unwrap(), + ) + }; + + let interrupt_delivery_control = unsafe { + VolatileRef::new( + NonNull::new( + VirtAddr::from(control_region_addr.as_u64() + 0x4000) + .as_mut_ptr::(), + ) + .unwrap(), + ) + }; + let mut aplic = Aplic::new(control_region, interrupt_delivery_control); + aplic.init(msi_delivery); + + *EXTERNAL_INTERRUPT_CONTROLLER.lock() = Some(ExternalInterruptController::Aplic(aplic)); +} diff --git a/src/arch/riscv64/kernel/interrupts/imsic.rs b/src/arch/riscv64/kernel/interrupts/imsic.rs new file mode 100644 index 0000000000..aacee98afd --- /dev/null +++ b/src/arch/riscv64/kernel/interrupts/imsic.rs @@ -0,0 +1,205 @@ +//! Message Signaled Interrupt Controller (IMSIC) driver for RISC-V. +//! +//! [RISC-V Advanced Interrupt Architecture]: https://github.com/riscv/riscv-aia/releases/download/20250312/riscv-interrupts-20250312.pdf + +use alloc::boxed::Box; +use alloc::vec::Vec; +use core::num::NonZeroU16; +use core::ptr::NonNull; + +use align_address::Align; +use memory_addresses::{PhysAddr, VirtAddr}; +use riscv::register::{sireg, siselect, stopei}; +use volatile::access::{NoAccess, WriteOnly}; +use volatile::{VolatileFieldAccess, VolatileRef}; + +use crate::arch::kernel::interrupts::MSI_EIID_WAKEUP; +use crate::arch::mm::paging::{self, BasePageSize, PageSize}; +use crate::arch::riscv64::kernel::core_local::set_msi_controller; +use crate::init_cell::InitCell; + +// Address of interrupt files for each hart index by hart_id. +// Use HARTS_AVAILABLE to map CpuId to hart_id. +pub(crate) static INTERRUPT_FILES: InitCell> = InitCell::new(Vec::new()); + +#[repr(C)] +#[derive(VolatileFieldAccess)] +pub(crate) struct InterruptFile { + #[access(WriteOnly)] + seteipnum_le: u32, + + #[access(WriteOnly)] + _seteipnum_be: u32, + + #[access(NoAccess)] + __: [u32; 0x3fe], +} +const _: () = assert!(size_of::() == 0x1000); + +#[repr(usize)] +enum Eidelivery { + // Interrupt delivery disabled + _Disabled = 0, + + // Interrupt delivery from the interrupt file is enabled + ViaInterruptFile = 1, + + // Interrupt delivery from a PLIC or APLIC is enabled. + // Support is option. + _ViaExternalController = 0x4000_0000, +} + +#[repr(usize)] +enum ISelect { + // Interrupt delivery mode. + Eidelivery = 0x70, + + // Interrupt priority threshold + Eithreshold = 0x72, + + // Interrupt ending bits + _Eip0 = 0x80, + _Eip63 = 0xbf, + + // Interrupt enable bits + Eie0 = 0xc0, + Eie63 = 0xff, +} + +pub(crate) struct Imsic { + max_vectors: u16, +} + +// IRQ numbers are reused as EIID (external interrupt identifier aka msi vector). +// This greatly simplifies support for IMSIC: +// - No per core msi pool allocator +// - No per core mapping from EIID to IRQ number +// - No per core handler map +// This work if the following assumptions are true: +// - The range of EIIDs supported by the IMSICs is a superset of the range of IRQ numbers +// supported by the APLIC. +// This adds the following limitations: +// - IPIs might be more expensive if they collide with other IRQs. +impl Imsic { + fn new(max_vectors: u16) -> Self { + Self { max_vectors } + } + + fn read(&mut self, index: usize) -> usize { + assert!(index & 1 == 0, "If XLEN=64, the index must be even"); + unsafe { + siselect::write(siselect::Siselect::from_bits(index)); + } + sireg::read().bits() + } + + fn write(&mut self, index: usize, value: usize) { + unsafe { + siselect::write(siselect::Siselect::from_bits(index)); + sireg::write(sireg::Sireg::from_bits(value)); + } + } + + fn set_interrupt_delivery_mode(&mut self, mode: Eidelivery) { + self.write(ISelect::Eidelivery as usize, mode as usize); + } + + pub fn set_interrupt_priority_threshold(&mut self, threshold: u8) { + assert!( + threshold == 0 || threshold >= MSI_EIID_WAKEUP as u8, + "IPIs shall not be masked by the priority threshold" + ); + self.write(ISelect::Eithreshold as usize, threshold as usize); + } + + pub fn set_interrupt_enable(&mut self, eiid: NonZeroU16, value: bool) { + assert!(eiid.get() < self.max_vectors); + + let eiid = eiid.get() as usize; + let eie_index = ISelect::Eie0 as usize + ((eiid / 64) * 2); + assert!( + eie_index <= ISelect::Eie63 as usize, + "Interrupt number {eiid} is out of range for Imsic" + ); + let bit_position = eiid % 64; + let current_value = self.read(eie_index); + if value { + self.write(eie_index, current_value | (1 << bit_position)); + } else { + self.write(eie_index, current_value & !(1 << bit_position)); + } + } + + pub fn claim_interrupt(&mut self) -> Option { + unsafe { stopei::read_clear() } + .iid() + .try_into() + .ok() + .and_then(NonZeroU16::new) + } + + pub fn complete_interrupt(&mut self, _eiid: NonZeroU16) { + // atomic read and write of stopic register automatically completes the interrupt + } + + pub fn set_ipi(&mut self, hart_id: usize, eiid: NonZeroU16) { + assert!(eiid.get() < self.max_vectors); + let interrupt_file_addr = INTERRUPT_FILES.get().unwrap()[hart_id]; + let mut interrupt_file = + unsafe { VolatileRef::new(NonNull::new(interrupt_file_addr.as_mut_ptr()).unwrap()) }; + interrupt_file + .as_mut_ptr() + .seteipnum_le() + .write(u32::from(eiid.get())); + } + + #[cfg(all(msix_supported, feature = "virtio"))] + pub fn get_physical_interrupt_file_address(&mut self, hart_id: usize) -> PhysAddr { + // Interrupts files are identity mapped. Virtual address == physical address. + let interrupt_file_addr = INTERRUPT_FILES.get().unwrap()[hart_id]; + PhysAddr::from(interrupt_file_addr.as_u64()) + } +} + +pub fn init_interrupt_files(addr: PhysAddr, size: usize, interrupt_file_indices: Vec) { + assert!( + addr.is_aligned_to(BasePageSize::SIZE), + "Imsic control region is not page aligned" + ); + assert!( + size.is_multiple_of(usize::try_from(BasePageSize::SIZE).unwrap()), + "Imsic control region size is not a multiple of a page" + ); + + let page_size = usize::try_from(BasePageSize::SIZE).unwrap(); + let num_pages = size / page_size; + for page in 0..num_pages { + let phys_addr = addr + page * page_size; + paging::identity_map::(phys_addr); + } + + let interrupt_file_base_addr = VirtAddr::from(addr.as_u64()); + INTERRUPT_FILES.with(|files| { + *files.unwrap() = interrupt_file_indices + .into_iter() + .map(|index| { + let hart_addr = interrupt_file_base_addr + (index * size_of::()); + assert!(hart_addr < interrupt_file_base_addr + size); + hart_addr + }) + .collect(); + }); + INTERRUPT_FILES.finalize(); +} + +pub(crate) fn init_imsic(max_vectors: u16) { + let mut imsic = Box::new(Imsic::new(max_vectors)); + + imsic.set_interrupt_delivery_mode(Eidelivery::ViaInterruptFile); + + // Enable MSI used for IPI + #[cfg(feature = "smp")] + imsic.set_interrupt_enable(NonZeroU16::new(MSI_EIID_WAKEUP).unwrap(), true); + + set_msi_controller(Box::into_raw(imsic)); +} diff --git a/src/arch/riscv64/kernel/interrupts/mod.rs b/src/arch/riscv64/kernel/interrupts/mod.rs new file mode 100644 index 0000000000..041c1e2f5c --- /dev/null +++ b/src/arch/riscv64/kernel/interrupts/mod.rs @@ -0,0 +1,353 @@ +use core::num::NonZeroU16; + +use ahash::RandomState; +use hashbrown::HashMap; +use hermit_sync::{InterruptTicketMutex, OnceCell, SpinMutex}; +use riscv::asm::wfi; +use riscv::interrupt::{Exception, Interrupt, Trap}; +use riscv::register::{scause, sie, sip, sstatus, stval}; +use trapframe::TrapFrame; + +use crate::arch::kernel::HARTS_AVAILABLE; +use crate::arch::kernel::devicetree::InterruptType as DeviceTreeInterruptType; +#[cfg(not(feature = "riscv-plic"))] +use crate::arch::riscv64::kernel::core_local::msi_controller; +#[cfg(not(feature = "riscv-plic"))] +use crate::arch::riscv64::kernel::devicetree::msi_supported_vectors; +use crate::drivers::InterruptHandlerMap; +use crate::scheduler; +use crate::scheduler::CoreId; + +#[cfg(not(feature = "riscv-plic"))] +mod imsic; +#[cfg(not(feature = "riscv-plic"))] +pub(crate) use imsic::init_interrupt_files; +#[cfg(not(feature = "riscv-plic"))] +use imsic::{Imsic, init_imsic}; + +#[cfg(not(feature = "riscv-plic"))] +mod aplic; +#[cfg(not(feature = "riscv-plic"))] +pub(crate) use aplic::init_aplic; +#[cfg(not(feature = "riscv-plic"))] +use aplic::{Aplic, SourceMode}; + +#[cfg(feature = "riscv-plic")] +mod plic; +#[cfg(feature = "riscv-plic")] +use plic::Plic; +#[cfg(feature = "riscv-plic")] +pub(crate) use plic::init_plic; + +pub(crate) static EXTERNAL_INTERRUPT_CONTROLLER: SpinMutex> = + SpinMutex::new(None); + +static INTERRUPT_HANDLERS: OnceCell = OnceCell::new(); + +#[cfg(not(feature = "riscv-plic"))] +const MSI_EIID_WAKEUP: u16 = 2; + +#[cfg(not(feature = "riscv-plic"))] +pub type MsiController = Imsic; + +pub(crate) enum ExternalInterruptController { + #[cfg(feature = "riscv-plic")] + Plic(Plic), + #[cfg(not(feature = "riscv-plic"))] + Aplic(Aplic), +} + +impl ExternalInterruptController { + fn enable_interrupt(&mut self, irq_number: u16) { + match self { + #[cfg(feature = "riscv-plic")] + Self::Plic(plic) => plic.set_enable_bit(irq_number, true), + #[cfg(not(feature = "riscv-plic"))] + Self::Aplic(aplic) => aplic.set_enable_bit(irq_number, true), + } + } + + #[cfg_attr(not(any(feature = "virtio", feature = "pci")), allow(dead_code))] + pub fn set_interrupt_source_mode( + &mut self, + _irq_number: u16, + _irq_type: DeviceTreeInterruptType, + ) { + match self { + #[cfg(feature = "riscv-plic")] + Self::Plic(_plic) => { /* noop */ } + #[cfg(not(feature = "riscv-plic"))] + Self::Aplic(aplic) => { + aplic.set_interrupt_source_mode(_irq_number, SourceMode::from(_irq_type)); + } + } + } + + fn set_interrupt_priority(&mut self, irq_number: u16, priority: u8) { + match self { + #[cfg(feature = "riscv-plic")] + Self::Plic(plic) => plic.set_interrupt_priority(irq_number, priority), + #[cfg(not(feature = "riscv-plic"))] + Self::Aplic(aplic) => aplic.set_interrupt_priority(irq_number, priority), + } + } + + fn set_priority_threshold(&mut self, threshold: u8) { + match self { + #[cfg(feature = "riscv-plic")] + Self::Plic(plic) => plic.set_priority_threshold(threshold), + #[cfg(not(feature = "riscv-plic"))] + Self::Aplic(aplic) => aplic.set_priority_threshold(threshold), + } + } + + fn claim_interrupt(&mut self) -> Option { + match self { + #[cfg(feature = "riscv-plic")] + Self::Plic(plic) => plic.claim_interrupt(), + #[cfg(not(feature = "riscv-plic"))] + Self::Aplic(aplic) => aplic.claim_interrupt(), + } + } + + fn complete_interrupt(&mut self, irq_number: u16) { + match self { + #[cfg(feature = "riscv-plic")] + Self::Plic(plic) => plic.complete_interrupt(irq_number), + #[cfg(not(feature = "riscv-plic"))] + Self::Aplic(aplic) => aplic.complete_interrupt(irq_number), + } + } +} + +/// Init Interrupts +pub(crate) fn install() { + #[cfg(not(feature = "riscv-plic"))] + if let Some(max_vectors) = msi_supported_vectors() { + init_imsic(max_vectors.try_into().unwrap()); + } + + unsafe { + // Install trap handler + trapframe::init(); + // Enable external interrupts + sie::set_sext(); + } +} + +/// Enable Interrupts +#[inline] +pub(crate) fn enable() { + unsafe { + sstatus::set_sie(); + } +} + +static IRQ_NAMES: InterruptTicketMutex> = + InterruptTicketMutex::new(HashMap::with_hasher(RandomState::with_seeds(0, 0, 0, 0))); + +#[allow(dead_code)] +pub(crate) fn add_irq_name(irq_number: u8, name: &'static str) { + debug!("Register name \"{name}\" for interrupt {irq_number}"); + IRQ_NAMES.lock().insert(irq_number, name); +} + +/// Waits for the next interrupt (Only Supervisor-level software/timer interrupt for now) +/// and calls the specific handler +#[inline] +pub(crate) fn enable_and_wait() { + unsafe { + //Enable Supervisor-level software interrupts + sie::set_ssoft(); + //sie::set_sext(); + debug!("Wait {:x?}", sie::read()); + loop { + wfi(); + // Interrupts are disabled at this point, so a pending interrupt will + // resume the execution. We still have to check if a interrupt is pending + // because the WFI instruction could be implemented as NOP (The RISC-V Instruction Set ManualVolume II: Privileged Architecture) + + let pending_interrupts = sip::read(); + + // trace!("sip: {:x?}", pending_interrupts); + #[cfg(feature = "smp")] + if pending_interrupts.ssoft() { + //Clear Supervisor-level software interrupt + core::arch::asm!( + "csrc sip, {ssoft_mask}", + ssoft_mask = in(reg) 0x2, + ); + trace!("SOFT"); + //Disable Supervisor-level software interrupt + sie::clear_ssoft(); + crate::arch::kernel::scheduler::wakeup_handler(); + break; + } + + if pending_interrupts.sext() { + trace!("EXT"); + external_handler(); + break; + } + + if pending_interrupts.stimer() { + // // Disable Supervisor-level software interrupt, wakeup not needed + // sie::clear_ssoft(); + + debug!("sip: {pending_interrupts:x?}"); + trace!("TIMER"); + crate::arch::kernel::scheduler::timer_handler(); + break; + } + } + } +} + +/// Disable Interrupts +#[inline] +pub(crate) fn disable() { + unsafe { sstatus::clear_sie() }; +} + +/// Currently not needed because we use the trapframe crate +#[cfg_attr( + not(all(not(feature = "riscv-plic"), feature = "smp")), + expect(unused_mut) +)] +pub(crate) fn install_handlers(mut handlers: InterruptHandlerMap) { + let mut ctrl_guard = EXTERNAL_INTERRUPT_CONTROLLER.lock(); + let ctrl = ctrl_guard.as_mut().unwrap(); + + // FIXME: For each MSI-X vector assigned to a pci device, the external interrupt with the same + // number is enabled too. This might incur unnecessary cpu traps. + + for irq_number in handlers.keys() { + // Set priority to 255 (lowest priority) + ctrl.set_interrupt_priority(u16::from(*irq_number), u8::MAX); + ctrl.enable_interrupt(u16::from(*irq_number)); + } + ctrl.set_priority_threshold(0); + + // Register MSI handler for IPIs + #[cfg(all(not(feature = "riscv-plic"), feature = "smp"))] + if msi_controller().is_some() { + handlers + .entry(MSI_EIID_WAKEUP.try_into().unwrap()) + .or_default() + .push_back(|| { + crate::arch::kernel::scheduler::wakeup_handler(); + }); + } + + INTERRUPT_HANDLERS.set(handlers).unwrap(); +} + +// Derived from rCore: https://github.com/rcore-os/rCore +/// Dispatch and handle interrupt. +/// +/// This function is called from `trap.S` which is in the trapframe crate. +#[unsafe(no_mangle)] +pub extern "C" fn trap_handler(tf: &mut TrapFrame) { + let scause = scause::read(); + let cause = scause.cause(); + let cause = Trap::::try_from(cause).unwrap(); + let stval = stval::read(); + let sepc = tf.sepc; + trace!("Interrupt: {cause:?}"); + trace!("tf = {tf:x?} "); + trace!("stval = {stval:x}"); + trace!("sepc = {sepc:x}"); + trace!("SSTATUS FS = {:?}", sstatus::read().fs()); + + match cause { + Trap::Interrupt(Interrupt::SupervisorExternal) => external_handler(), + #[cfg(feature = "smp")] + Trap::Interrupt(Interrupt::SupervisorSoft) => { + crate::arch::kernel::scheduler::wakeup_handler(); + } + Trap::Interrupt(Interrupt::SupervisorTimer) => { + crate::arch::kernel::scheduler::timer_handler(); + } + cause => { + error!("Interrupt: {cause:?}"); + error!("tf = {tf:x?} "); + error!("stval = {stval:x}"); + error!("sepc = {sepc:x}"); + error!("SSTATUS FS = {:?}", sstatus::read().fs()); + scheduler::abort(); + } + } + trace!("Interrupt end"); +} + +/// Handles external interrupts +fn external_handler() { + use crate::arch::kernel::core_local::core_scheduler; + use crate::scheduler::PerCoreSchedulerExt; + + let irq = { + #[cfg(not(feature = "riscv-plic"))] + if let Some(msi) = msi_controller() { + msi.claim_interrupt() + } else { + EXTERNAL_INTERRUPT_CONTROLLER + .lock() + .as_mut() + .unwrap() + .claim_interrupt() + } + + #[cfg(feature = "riscv-plic")] + EXTERNAL_INTERRUPT_CONTROLLER + .lock() + .as_mut() + .unwrap() + .claim_interrupt() + }; + let Some(irq) = irq else { return }; + + let irq_number = irq.get(); + debug!("External INT: {irq_number}"); + if let Some(handlers) = INTERRUPT_HANDLERS.get() + && let Ok(irq_u8) = u8::try_from(irq_number) + && let Some(queue) = handlers.get(&irq_u8) + { + for handler in queue.iter() { + handler(); + } + } + + crate::executor::run(); + + #[cfg(not(feature = "riscv-plic"))] + if let Some(msi) = msi_controller() { + msi.complete_interrupt(irq); + } else { + EXTERNAL_INTERRUPT_CONTROLLER + .lock() + .as_mut() + .unwrap() + .complete_interrupt(irq_number); + } + #[cfg(feature = "riscv-plic")] + EXTERNAL_INTERRUPT_CONTROLLER + .lock() + .as_mut() + .unwrap() + .complete_interrupt(irq_number); + + core_scheduler().reschedule(); +} + +pub(crate) fn print_statistics() {} + +pub fn wakeup_core(core_to_wakeup: CoreId) { + let hart_id = HARTS_AVAILABLE.finalize()[core_to_wakeup as usize]; + debug!("Wakeup core: {core_to_wakeup} , hart_id: {hart_id}"); + #[cfg(not(feature = "riscv-plic"))] + if let Some(imsic) = msi_controller() { + imsic.set_ipi(hart_id, NonZeroU16::new(MSI_EIID_WAKEUP).unwrap()); + return; + } + sbi_rt::send_ipi(sbi_rt::HartMask::from_mask_base(0b1, hart_id)); +} diff --git a/src/arch/riscv64/kernel/interrupts/plic.rs b/src/arch/riscv64/kernel/interrupts/plic.rs new file mode 100644 index 0000000000..b50b46e4b6 --- /dev/null +++ b/src/arch/riscv64/kernel/interrupts/plic.rs @@ -0,0 +1,147 @@ +//! Platform-Level Interrupt Controller (PLIC) driver for RISC-V. +//! +//! [RISC-V PLIC Specification]: https://github.com/riscv/riscv-plic-spec/releases/download/1.0.0/riscv-plic-1.0.0.pdf + +use core::mem::offset_of; +use core::num::NonZeroU16; +use core::ptr::NonNull; + +use bit_field::BitField; +use memory_addresses::{PhysAddr, VirtAddr}; +use volatile::access::{NoAccess, ReadOnly}; +use volatile::{VolatileFieldAccess, VolatileRef}; + +use crate::arch::kernel::interrupts::{EXTERNAL_INTERRUPT_CONTROLLER, ExternalInterruptController}; +use crate::arch::mm::paging::{self, PageSize}; + +const NUMBER_OF_SOURCES: usize = 1024; +const NUMBER_OF_CONTEXTS: usize = 15871; + +const INTERRUPT_PENDING_BITS_OFFSET: usize = 0x00_1000; +const INTERRUPT_ENABLE_BITS_OFFSET: usize = 0x00_2000; +const CONTEXT_BASED_REGISTERS: usize = 0x20_0000; + +type SourceBitArray = [u32; NUMBER_OF_SOURCES / (u32::BITS as usize)]; + +#[repr(C, align(4096))] +#[derive(VolatileFieldAccess)] +struct ContextBasedRegisters { + priority_threshold: u32, + claim_or_complete: u32, +} + +#[repr(C)] +#[derive(VolatileFieldAccess)] +struct PlicControlRegion { + #[access(NoAccess)] + _reserved0: u32, + interrupt_priorities: [u32; NUMBER_OF_SOURCES - 1], + #[access(ReadOnly)] + interrupt_pending_bits: SourceBitArray, + #[access(NoAccess)] + _reserved3: [u32; (INTERRUPT_ENABLE_BITS_OFFSET - 0x00_1080) / size_of::()], + interrupt_enable_bits: [SourceBitArray; NUMBER_OF_CONTEXTS], + #[access(NoAccess)] + _reserved2: [u32; (CONTEXT_BASED_REGISTERS - 0x1f_2000) / size_of::()], + context_based_registers: [ContextBasedRegisters; NUMBER_OF_CONTEXTS], +} + +const _: () = + assert!(offset_of!(PlicControlRegion, interrupt_pending_bits) == INTERRUPT_PENDING_BITS_OFFSET); +const _: () = + assert!(offset_of!(PlicControlRegion, interrupt_enable_bits) == INTERRUPT_ENABLE_BITS_OFFSET); +const _: () = + assert!(offset_of!(PlicControlRegion, context_based_registers) == CONTEXT_BASED_REGISTERS); + +pub(crate) struct Plic { + control_region: VolatileRef<'static, PlicControlRegion>, + context: u16, +} + +impl Plic { + pub fn set_enable_bit(&mut self, irq_number: u16, value: bool) { + let source = NonZeroU16::new(irq_number).unwrap(); + let source_idx = usize::from(source.get()); + let plic_ptr = self.control_region.as_mut_ptr(); + unsafe { + plic_ptr + .interrupt_enable_bits() + .map(|slice| { + slice + .cast::() + .offset(isize::try_from(self.context).unwrap()) + }) + .map(|context_slice| { + context_slice + .cast::() + .offset((source_idx / 32).try_into().unwrap()) + }) + .update(|mut word| { + word.set_bit(source_idx % 32, value); + word + }); + } + } + + pub fn set_interrupt_priority(&mut self, irq_number: u16, priority: u8) { + let source = NonZeroU16::new(irq_number).unwrap(); + let plic_ptr = self.control_region.as_mut_ptr(); + unsafe { + plic_ptr + .interrupt_priorities() + .map(|slice| { + slice + .cast() + .offset(isize::try_from(source.get()).unwrap() - 1) + }) + .write(u32::from(priority)); + } + } + + pub fn set_priority_threshold(&mut self, threshold: u8) { + let plic_ptr = self.control_region.as_mut_ptr(); + unsafe { + plic_ptr + .context_based_registers() + .map(|slice| slice.cast().offset(isize::try_from(self.context).unwrap())) + .priority_threshold() + .write(u32::from(threshold)); + } + } + + pub fn claim_interrupt(&mut self) -> Option { + let plic_ptr = self.control_region.as_mut_ptr(); + unsafe { + let irq = plic_ptr + .context_based_registers() + .map(|slice| slice.cast().offset(isize::try_from(self.context).unwrap())) + .claim_or_complete() + .read(); + NonZeroU16::new(irq as u16) + } + } + + pub fn complete_interrupt(&mut self, irq_number: u16) { + let plic_ptr = self.control_region.as_mut_ptr(); + unsafe { + plic_ptr + .context_based_registers() + .map(|slice| slice.cast().offset(isize::try_from(self.context).unwrap())) + .claim_or_complete() + .write(u32::from(irq_number)); + } + } +} + +pub fn init_plic(addr: PhysAddr, size: usize, context: u16) { + assert!(size < usize::try_from(paging::HugePageSize::SIZE).unwrap()); + paging::identity_map::(addr); + let base = VirtAddr::from(addr.as_u64()); + let control_region = + unsafe { VolatileRef::new(NonNull::new(base.as_mut_ptr::()).unwrap()) }; + let plic = Plic { + control_region, + context, + }; + *EXTERNAL_INTERRUPT_CONTROLLER.lock() = Some(ExternalInterruptController::Plic(plic)); +} diff --git a/src/arch/riscv64/kernel/mod.rs b/src/arch/riscv64/kernel/mod.rs index 492772be1b..41dadd873e 100644 --- a/src/arch/riscv64/kernel/mod.rs +++ b/src/arch/riscv64/kernel/mod.rs @@ -17,9 +17,10 @@ use core::sync::atomic::{AtomicPtr, AtomicU32, AtomicU64, Ordering}; use free_list::PageLayout; use riscv::register::sstatus; -pub(crate) use self::processor::{set_oneshot_timer, wakeup_core}; +pub(crate) use self::processor::set_oneshot_timer; use crate::arch::kernel::core_local::core_id; pub use crate::arch::kernel::devicetree::init_drivers; +pub use crate::arch::kernel::interrupts::wakeup_core; use crate::arch::kernel::processor::lsb; use crate::config::KERNEL_STACK_SIZE; use crate::env::{self, FdtStartInfo}; @@ -79,10 +80,10 @@ pub fn get_current_boot_id() -> u32 { /// Real Boot Processor initialization as soon as we have put the first Welcome message on the screen. pub fn boot_processor_init() { - devicetree::init(); crate::mm::init(); crate::mm::print_information(); env::init(); + devicetree::init_interrupt_controller(); interrupts::install(); #[cfg(feature = "pci")] pci::init(); diff --git a/src/arch/riscv64/kernel/pci.rs b/src/arch/riscv64/kernel/pci.rs index 6db2c37c5e..111d9f1383 100644 --- a/src/arch/riscv64/kernel/pci.rs +++ b/src/arch/riscv64/kernel/pci.rs @@ -11,6 +11,7 @@ use pci_types::{ PciHeader, }; +use crate::arch::riscv64::kernel::interrupts::EXTERNAL_INTERRUPT_CONTROLLER; use crate::arch::riscv64::mm::paging::{self, BasePageSize, PageSize, PageTableEntryFlags}; use crate::drivers::pci::{PCI_DEVICES, PciDevice}; use crate::env::{self, FdtStartInfo}; @@ -229,7 +230,7 @@ impl PciSpaceAllocator { /// Iterator for PCI Device Enumeration /// -/// Reference: PCI LOCAL BUS SPECIFICATION, REV. 3.0, Chapter 6 +// Reference: PCI LOCAL BUS SPECIFICATION, REV. 3.0, Chapter 6 struct PciDeviceIterator<'a, T: ConfigRegionAccess> { access: &'a T, max_bus: u16, @@ -352,11 +353,27 @@ fn detect_interrupt_line( (value_slice, residual_slice) = residual_slice.split_at(size_of::()); let interrupt_parent_phandle = u32::from_be_bytes(value_slice.try_into().unwrap()); let interrupt_parent_node = fdt.find_phandle(interrupt_parent_phandle).unwrap(); - assert!(interrupt_parent_node.cell_sizes().address_cells == 0); - assert!(interrupt_parent_node.interrupt_cells().unwrap() == 1); - - (value_slice, residual_slice) = residual_slice.split_at(size_of::()); - let parent_interrupt_specifier = u32::from_be_bytes(value_slice.try_into().unwrap()); + let parent_interrupt_cells = interrupt_parent_node.interrupt_cells().unwrap(); + let (irq_number, source_mode) = match parent_interrupt_cells { + 1 => { + (value_slice, residual_slice) = residual_slice.split_at(size_of::()); + let irq_number = u32::from_be_bytes(value_slice.try_into().unwrap()); + (irq_number, 0) + } + 2 => { + (value_slice, residual_slice) = residual_slice.split_at(size_of::()); + let irq_number = u32::from_be_bytes(value_slice.try_into().unwrap()); + (value_slice, residual_slice) = residual_slice.split_at(size_of::()); + let source_mode = u32::from_be_bytes(value_slice.try_into().unwrap()); + (irq_number, source_mode) + } + _ => panic!("Unsupported #interrupt-cells value: {parent_interrupt_cells}"), + }; + EXTERNAL_INTERRUPT_CONTROLLER + .lock() + .as_mut() + .unwrap() + .set_interrupt_source_mode(irq_number.try_into().unwrap(), source_mode.into()); let key_interrupt_map = [ _child_unit_address_high & interrupt_map_mask[0], @@ -371,7 +388,7 @@ fn detect_interrupt_line( u32::from(interrupt_pin), ]; if key_interrupt_map == key_device { - return Some(InterruptLine::try_from(parent_interrupt_specifier).unwrap()); + return Some(InterruptLine::try_from(irq_number).unwrap()); } } diff --git a/src/arch/riscv64/kernel/processor.rs b/src/arch/riscv64/kernel/processor.rs index a0aaaf7c32..afafc375c5 100644 --- a/src/arch/riscv64/kernel/processor.rs +++ b/src/arch/riscv64/kernel/processor.rs @@ -3,8 +3,7 @@ use core::num::NonZeroU64; use riscv::register::{sie, sstatus, time}; -use crate::arch::kernel::{HARTS_AVAILABLE, get_timebase_freq}; -use crate::scheduler::CoreId; +use crate::arch::kernel::get_timebase_freq; /// Current FPU state. Saved at context switch when changed #[repr(C, packed)] @@ -286,9 +285,3 @@ pub fn set_oneshot_timer(wakeup_time: Option) { sbi_rt::set_timer(next_time); } - -pub fn wakeup_core(core_to_wakeup: CoreId) { - let hart_id = HARTS_AVAILABLE.finalize()[core_to_wakeup as usize]; - debug!("Wakeup core: {core_to_wakeup} , hart_id: {hart_id}"); - sbi_rt::send_ipi(sbi_rt::HartMask::from_mask_base(0b1, hart_id)); -} diff --git a/src/drivers/console.rs b/src/drivers/console.rs index 2719664559..84c19e39a7 100644 --- a/src/drivers/console.rs +++ b/src/drivers/console.rs @@ -217,10 +217,7 @@ impl VirtioConsoleDriver { /// Handle interrupt and acknowledge interrupt pub fn handle_interrupt(&mut self) { - #[cfg_attr( - not(all(feature = "pci", target_arch = "x86_64")), - expect(irrefutable_let_patterns) - )] + #[cfg_attr(not(msix_supported), expect(irrefutable_let_patterns))] let InterruptCapability::IsrStatus(isr_stat) = &mut self.caps_coll.int_cap else { panic!("MSI-X vectors should be configured to the interrupt type-specific handlers.") }; @@ -306,7 +303,7 @@ impl super::virtio::VirtioDriver for VirtioConsoleDriver { crate::arch::kernel::interrupts::add_irq_name(irq, "virtio"); info!("Virtio interrupt handler at line {irq}"); } - #[cfg(all(feature = "pci", target_arch = "x86_64"))] + #[cfg(msix_supported)] InterruptCapability::Msix(msix_table) => { let handle_queue_interrupt: fn() = Self::handle_queue_interrupt; caps_coll.com_cfg.register_msix_vectors( diff --git a/src/drivers/fs.rs b/src/drivers/fs.rs index 68d282fe1d..557bfdc9f0 100644 --- a/src/drivers/fs.rs +++ b/src/drivers/fs.rs @@ -108,7 +108,7 @@ impl super::virtio::VirtioDriver for VirtioFsDriver { crate::arch::kernel::interrupts::add_irq_name(irq, "virtio"); info!("Virtio interrupt handler at line {irq}"); } - #[cfg(all(feature = "pci", target_arch = "x86_64"))] + #[cfg(msix_supported)] InterruptCapability::Msix(msix_table) => { use core::iter; @@ -146,10 +146,7 @@ impl super::virtio::VirtioDriver for VirtioFsDriver { impl VirtioFsDriver { pub fn handle_interrupt(&mut self) { - #[cfg_attr( - not(all(feature = "pci", target_arch = "x86_64")), - expect(irrefutable_let_patterns) - )] + #[cfg_attr(not(msix_supported), expect(irrefutable_let_patterns))] let InterruptCapability::IsrStatus(isr_stat) = &mut self.caps_coll.int_cap else { panic!("MSI-X vectors should be configured to the interrupt type-specific handlers.") }; diff --git a/src/drivers/net/virtio.rs b/src/drivers/net/virtio.rs index d3a936098d..6a981a1967 100644 --- a/src/drivers/net/virtio.rs +++ b/src/drivers/net/virtio.rs @@ -553,10 +553,7 @@ impl NetworkDriver for VirtioNetDriver { } fn handle_interrupt(&mut self) { - #[cfg_attr( - not(all(feature = "pci", target_arch = "x86_64")), - expect(irrefutable_let_patterns) - )] + #[cfg_attr(not(msix_supported), expect(irrefutable_let_patterns))] let InterruptCapability::IsrStatus(isr_stat) = &mut self.caps_coll.int_cap else { panic!("MSI-X vectors should be configured to the interrupt type-specific handlers.") }; @@ -882,10 +879,7 @@ impl crate::drivers::virtio::VirtioDriver for VirtioNetDriver { debug!("Using RX buffer size of {}", dev_spec_init.0.buf_size); recv_vqs = Some(dev_spec_init.0); send_vqs = Some(dev_spec_init.1); - #[cfg_attr( - not(all(feature = "pci", target_arch = "x86_64")), - expect(unused_variables) - )] + #[cfg_attr(not(msix_supported), expect(unused_variables))] let num_vqs = dev_spec_init.2; ctrl_vq = Some(dev_spec_init.3); send_capacity = Some(dev_spec_init.4); @@ -902,7 +896,7 @@ impl crate::drivers::virtio::VirtioDriver for VirtioNetDriver { crate::arch::kernel::interrupts::add_irq_name(irq, "virtio"); info!("Virtio interrupt handler at line {irq}"); } - #[cfg(all(feature = "pci", target_arch = "x86_64"))] + #[cfg(msix_supported)] InterruptCapability::Msix(msix_table) => { let recv_vqs = (0..num_vqs).step_by(2); let send_vqs = (1..num_vqs).step_by(2); diff --git a/src/drivers/pci.rs b/src/drivers/pci.rs index b9b50a0cd2..86fcd1e6c9 100644 --- a/src/drivers/pci.rs +++ b/src/drivers/pci.rs @@ -536,7 +536,7 @@ pub(crate) mod msix { fn configure(&mut self, index: u16, vector: u8); } - #[cfg(target_arch = "x86_64")] + #[cfg(msix_supported)] impl MsixTableVolatileElementAccess for volatile::VolatileRef<'_, [TableEntry]> { /// Configures the [TableEntry] at the given index of the MSI-X table to trigger the vector provided. /// @@ -546,9 +546,22 @@ pub(crate) mod msix { fn configure(&mut self, index: u16, vector: u8) { use bit_field::BitField; + #[cfg(target_arch = "x86_64")] // Intel 64 and IA-32 Architectures Software Developer’s Manual volume 3 section 12.11.2.1. core::assert_matches!(vector, 0x10..=VECTOR_MAX); + #[cfg(target_arch = "x86_64")] + // Format described in Intel 64 and IA-32 Architectures Software Developer’s Manual volume 3 section 12.11.2. + let addr = 0xfee0_0000u64; + #[cfg(target_arch = "riscv64")] + let addr = { + use crate::arch::kernel::core_local::msi_controller; + msi_controller() + .unwrap() + .get_physical_interrupt_file_address(0) + .as_u64() + }; + let msix_entry = unsafe { self.as_mut_ptr() .map(|table| table.cast().offset(index.try_into().unwrap())) @@ -560,10 +573,12 @@ pub(crate) mod msix { .vector_control() .update(|mut control| *control.set_bit(0, true)); - // Format described in Intel 64 and IA-32 Architectures Software Developer’s Manual volume 3 section 12.11.2. msix_entry .message_address() - .update(|mut addr_low| *addr_low.set_bits(20..32, 0xfee)); + .update(|mut addr_low| *addr_low.set_bits(0..32, addr as u32)); + msix_entry + .message_upper_address() + .update(|mut addr_high| *addr_high.set_bits(0..32, (addr >> 32) as u32)); msix_entry .message_data() .update(|mut data| *data.set_bits(0..8, u32::from(vector))); diff --git a/src/drivers/virtio/transport/mod.rs b/src/drivers/virtio/transport/mod.rs index 466f6d3b24..0c439b0313 100644 --- a/src/drivers/virtio/transport/mod.rs +++ b/src/drivers/virtio/transport/mod.rs @@ -10,7 +10,7 @@ pub mod pci; pub(crate) enum InterruptCapability { IsrStatus(IsrStatus), - #[cfg(all(feature = "pci", target_arch = "x86_64"))] + #[cfg(msix_supported)] Msix(volatile::VolatileRef<'static, [crate::drivers::pci::msix::TableEntry]>), } diff --git a/src/drivers/virtio/transport/pci.rs b/src/drivers/virtio/transport/pci.rs index 96d63beaa4..95760f6952 100644 --- a/src/drivers/virtio/transport/pci.rs +++ b/src/drivers/virtio/transport/pci.rs @@ -19,6 +19,8 @@ use virtio::{DeviceStatus, le16, le32}; use volatile::access::ReadOnly; use volatile::{VolatilePtr, VolatileRef}; +#[cfg(all(msix_supported, target_arch = "riscv64"))] +use crate::arch::kernel::core_local::msi_controller; use crate::arch::kernel::pci::PciConfigRegion; use crate::drivers::InterruptHandlerMap; #[cfg(feature = "virtio-console")] @@ -34,9 +36,9 @@ use crate::drivers::fs::VirtioFsDriver; use crate::drivers::net::virtio::VirtioNetDriver; use crate::drivers::pci::PciDevice; use crate::drivers::pci::error::PciError; -#[cfg(all(feature = "pci", target_arch = "x86_64"))] +#[cfg(msix_supported)] use crate::drivers::pci::msix; -#[cfg(target_arch = "x86_64")] +#[cfg(msix_supported)] use crate::drivers::pci::msix::MsixTableVolatileElementAccess; #[cfg(feature = "virtio-rng")] use crate::drivers::rng::VirtioRngDriver; @@ -181,7 +183,7 @@ pub struct VqCfgHandler<'a> { raw: VolatileRef<'a, CommonCfg>, } -#[cfg_attr(not(target_arch = "x86_64"), expect(unused))] +#[cfg_attr(not(msix_supported), expect(unused))] pub(crate) const NO_VECTOR: u16 = 0xffff; impl VqCfgHandler<'_> { @@ -235,7 +237,7 @@ impl VqCfgHandler<'_> { .write(addr.as_u64().into()); } - #[cfg(target_arch = "x86_64")] + #[cfg(msix_supported)] fn set_queue_msix_vector(&mut self, index: u16) -> Result<(), ()> { self.select_queue(); let queue_msix_vector = self.raw.as_mut_ptr().queue_msix_vector(); @@ -362,8 +364,7 @@ impl ComCfg { status.contains(DeviceStatus::DEVICE_NEEDS_RESET) } } - -#[cfg(target_arch = "x86_64")] +#[cfg(msix_supported)] impl ComCfg { fn set_config_msix_vector(&mut self, index: u16) -> Result<(), ()> { let config_msix_vector = self.com_cfg.as_mut_ptr().config_msix_vector(); @@ -388,8 +389,13 @@ impl ComCfg { ) { // One for the device config irq. let needed_irqs = 1 + queue_handlers.len(); + + #[cfg(target_arch = "x86_64")] // We will need to map the IRQ number to the vector number by adding 32. const IRQ_RANGE: core::ops::RangeInclusive = 0..=(msix::VECTOR_MAX - 32); + #[cfg(target_arch = "riscv64")] + const IRQ_RANGE: core::ops::RangeInclusive = 1..=(msix::VECTOR_MAX); + let mut free_irqs = IRQ_RANGE .filter(|v| !handlers.contains_key(v)) // If we do not have enough free IRQs, fall back to using any @@ -405,6 +411,9 @@ impl ComCfg { .entry(config_irq) .or_default() .push_back(config_handler); + #[cfg(not(target_arch = "x86_64"))] + msix_table.configure(TABLE_CONFIG_INDEX, config_irq); + #[cfg(target_arch = "x86_64")] msix_table.configure(TABLE_CONFIG_INDEX, config_irq + 32); self.set_config_msix_vector(TABLE_CONFIG_INDEX).unwrap(); crate::arch::kernel::interrupts::add_irq_name(config_irq, "virtio config"); @@ -413,6 +422,9 @@ impl ComCfg { for (((queues, handler), irq), table_queue_index) in queue_handlers.zip(free_irqs).zip(1..) { handlers.entry(irq).or_default().push_back(handler); + #[cfg(not(target_arch = "x86_64"))] + msix_table.configure(table_queue_index, irq); + #[cfg(target_arch = "x86_64")] msix_table.configure(table_queue_index, irq + 32); for i in queues { self.select_vq(i) @@ -584,7 +596,7 @@ pub(crate) fn map_caps( let mut notif_cfg = None; let mut isr_cfg = None; let mut dev_cfg_list = Vec::new(); - #[cfg(target_arch = "x86_64")] + #[cfg(msix_supported)] let mut msix_table = None; let bar_mappings = device.memory_map_bars(true); @@ -641,7 +653,7 @@ pub(crate) fn map_caps( CapCfgType::Isr => { let cond = isr_cfg.is_none(); // We prefer MSI-X over ISR Status. - #[cfg(target_arch = "x86_64")] + #[cfg(msix_supported)] let cond = cond && msix_table.is_none(); if cond { match pci_cap.map_cap_cfg() { @@ -665,9 +677,15 @@ pub(crate) fn map_caps( _ => continue, } } - // We can currently only make use of MSI-X on x86_64. - #[cfg(target_arch = "x86_64")] + #[cfg(msix_supported)] PciCapability::MsiX(mut msix_capability) => { + #[cfg(target_arch = "riscv64")] + { + if msi_controller().is_none() { + continue; + } + } + msix_capability.set_enabled(true, device.access()); // the capability should provide a valid BAR ID and "[t]he BAR [...] must map @@ -693,9 +711,20 @@ pub(crate) fn map_caps( } } - let isr_cfg = cfg_select! { - target_arch = "x86_64" => msix_table.map(InterruptCapability::Msix), - _ => None, + let isr_cfg = { + #[cfg(msix_supported)] + { + #[cfg(target_arch = "riscv64")] + let has_msix = msi_controller().is_some(); + #[cfg(not(target_arch = "riscv64"))] + let has_msix = true; + + msix_table + .filter(|_| has_msix) + .map(InterruptCapability::Msix) + } + #[cfg(not(msix_supported))] + None } .or(isr_cfg.map(InterruptCapability::IsrStatus)); @@ -703,7 +732,7 @@ pub(crate) fn map_caps( Some(InterruptCapability::IsrStatus(_)) => { info!("The device will use legacy interrupts."); } - #[cfg(target_arch = "x86_64")] + #[cfg(msix_supported)] Some(InterruptCapability::Msix(_)) => { info!("Found MSI-X capability. The device will use message signaled interrupts."); } diff --git a/src/drivers/vsock.rs b/src/drivers/vsock.rs index 765234782c..4295c9bbdd 100644 --- a/src/drivers/vsock.rs +++ b/src/drivers/vsock.rs @@ -273,10 +273,7 @@ impl VirtioVsockDriver { } pub fn handle_interrupt(&mut self) { - #[cfg_attr( - not(all(feature = "pci", target_arch = "x86_64")), - expect(irrefutable_let_patterns) - )] + #[cfg_attr(not(msix_supported), expect(irrefutable_let_patterns))] let InterruptCapability::IsrStatus(isr_stat) = &mut self.caps_coll.int_cap else { panic!("MSI-X vectors should be configured to the interrupt type-specific handlers.") }; @@ -376,7 +373,7 @@ impl super::virtio::VirtioDriver for VirtioVsockDriver { crate::arch::kernel::interrupts::add_irq_name(irq, "virtio"); info!("Virtio interrupt handler at line {irq}"); } - #[cfg(all(feature = "pci", target_arch = "x86_64"))] + #[cfg(msix_supported)] InterruptCapability::Msix(msix_table) => { // The no-op handler allows the processor to receive an interrupt and reschedule. // FIXME: replace with a function to wake the vsock task waker once it is not woken unconditionally. diff --git a/src/executor/network.rs b/src/executor/network.rs index a233f427a2..c87d4aacf9 100644 --- a/src/executor/network.rs +++ b/src/executor/network.rs @@ -52,12 +52,7 @@ pub(crate) fn network_handler() { } } -#[cfg(all( - feature = "virtio-net", - not(feature = "rtl8139"), - feature = "pci", - target_arch = "x86_64" -))] +#[cfg(all(feature = "virtio-net", not(feature = "rtl8139"), msix_supported))] pub(crate) fn network_device_configuration_handler() { if let Ok(nic) = NIC.lock().as_nic_mut() { nic.handle_device_configuration_interrupt(); @@ -404,12 +399,7 @@ impl<'a> NetworkInterface<'a> { self.get_inner_device().handle_interrupt(); } - #[cfg(all( - feature = "virtio-net", - not(feature = "rtl8139"), - feature = "pci", - target_arch = "x86_64" - ))] + #[cfg(all(feature = "virtio-net", not(feature = "rtl8139"), msix_supported))] fn handle_device_configuration_interrupt(&mut self) { self.get_inner_device() .handle_device_configuration_interrupt(); diff --git a/src/init_cell.rs b/src/init_cell.rs index c2646e0134..206b4dee6e 100644 --- a/src/init_cell.rs +++ b/src/init_cell.rs @@ -1,5 +1,6 @@ #![cfg_attr( not(any( + all(target_arch = "riscv64", not(feature = "riscv-plic")), feature = "virtio-vsock", feature = "virtio-fs", feature = "virtio-rng" diff --git a/xtask/src/ci/qemu.rs b/xtask/src/ci/qemu.rs index 70201ac8de..3322d7e4dc 100644 --- a/xtask/src/ci/qemu.rs +++ b/xtask/src/ci/qemu.rs @@ -52,6 +52,37 @@ pub struct Qemu { /// Arguments to pass to QEMU and Hermit, separated by another `--`. #[arg(last = true)] qemu_and_hermit_args: Vec, + + /// Select the RISC-V machine type. + #[arg(long, value_enum, default_value_t = RiscvMachine::VirtAiaAplicImsic)] + machine: RiscvMachine, +} + +#[derive(ValueEnum, PartialEq, Eq, Clone, Copy)] +#[value(rename_all = "kebab-case")] +pub enum RiscvMachine { + /// QEMU `sifive_u`, used for CadenceGem. + SifiveU, + + /// QEMU `virt`, which exercises the legacy PLIC path. + Virt, + + /// QEMU `virt,aia=aplic`, which exercises the APLIC path. + VirtAiaAplic, + + /// QEMU `virt,aia=aplic,imsic`, which exercises the APLIC in msi-delivery mode. + VirtAiaAplicImsic, +} + +impl RiscvMachine { + fn qemu_machine(self) -> &'static str { + match self { + Self::SifiveU => "sifive_u", + Self::Virt => "virt", + Self::VirtAiaAplic => "virt,aia=aplic", + Self::VirtAiaAplicImsic => "virt,aia=aplic-imsic", + } + } } #[derive(ValueEnum, PartialEq, Eq, Clone, Copy)] @@ -322,9 +353,9 @@ impl Qemu { } else if arch == Arch::Riscv64 { // CadenceGem requires sifive_u let machine = if self.devices.contains(&Device::CadenceGem) { - "sifive_u" + RiscvMachine::SifiveU.qemu_machine() } else { - "virt" + self.machine.qemu_machine() }; let opensbi_paths = &[ @@ -399,7 +430,8 @@ impl Qemu { // possibly because it requires sifive_u as the machine. if !self.devices.contains(&Device::CadenceGem) { cpu_args.push("-cpu".to_owned()); - cpu_args.push("rv64".to_owned()); + // FIXME: replace with rva23s64 once ci reaches qemu 10.1 + cpu_args.push("max".to_owned()); } }